diff options
Diffstat (limited to 'drivers/dma')
34 files changed, 2112 insertions, 336 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index de469821bc1b..f2b2c4e87aef 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -107,6 +107,13 @@ config AT_HDMAC help Support the Atmel AHB DMA controller. +config AT_XDMAC + tristate "Atmel XDMA support" + depends on ARCH_AT91 + select DMA_ENGINE + help + Support the Atmel XDMA controller. + config FSL_DMA tristate "Freescale Elo series DMA support" depends on FSL_SOC @@ -395,12 +402,12 @@ config XILINX_VDMA config DMA_SUN6I tristate "Allwinner A31 SoCs DMA support" - depends on MACH_SUN6I || COMPILE_TEST + depends on MACH_SUN6I || MACH_SUN8I || COMPILE_TEST depends on RESET_CONTROLLER select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help - Support for the DMA engine for Allwinner A31 SoCs. + Support for the DMA engine first found in Allwinner A31 SoCs. config NBPFAXI_DMA tristate "Renesas Type-AXI NBPF DMA support" diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index cb626c179911..2022b5451377 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -16,6 +16,7 @@ obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/ obj-$(CONFIG_MV_XOR) += mv_xor.o obj-$(CONFIG_DW_DMAC_CORE) += dw/ obj-$(CONFIG_AT_HDMAC) += at_hdmac.o +obj-$(CONFIG_AT_XDMAC) += at_xdmac.o obj-$(CONFIG_MX3_IPU) += ipu/ obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o obj-$(CONFIG_SH_DMAE_BASE) += sh/ diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index e34024b000a4..1364d00881dd 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -2164,7 +2164,6 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) __func__, ret); goto out_no_memcpy; } - pl08x->memcpy.chancnt = ret; /* Register slave channels */ ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave, @@ -2175,7 +2174,6 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) __func__, ret); goto out_no_slave; } - pl08x->slave.chancnt = ret; ret = dma_async_device_register(&pl08x->memcpy); if (ret) { diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c new file mode 100644 index 000000000000..b60d77a22df6 --- /dev/null +++ b/drivers/dma/at_xdmac.c @@ -0,0 +1,1524 @@ +/* + * Driver for the Atmel Extensible DMA Controller (aka XDMAC on AT91 systems) + * + * Copyright (C) 2014 Atmel Corporation + * + * Author: Ludovic Desroches <ludovic.desroches@atmel.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <asm/barrier.h> +#include <dt-bindings/dma/at91.h> +#include <linux/clk.h> +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/pm.h> + +#include "dmaengine.h" + +/* Global registers */ +#define AT_XDMAC_GTYPE 0x00 /* Global Type Register */ +#define AT_XDMAC_NB_CH(i) (((i) & 0x1F) + 1) /* Number of Channels Minus One */ +#define AT_XDMAC_FIFO_SZ(i) (((i) >> 5) & 0x7FF) /* Number of Bytes */ +#define AT_XDMAC_NB_REQ(i) ((((i) >> 16) & 0x3F) + 1) /* Number of Peripheral Requests Minus One */ +#define AT_XDMAC_GCFG 0x04 /* Global Configuration Register */ +#define AT_XDMAC_GWAC 0x08 /* Global Weighted Arbiter Configuration Register */ +#define AT_XDMAC_GIE 0x0C /* Global Interrupt Enable Register */ +#define AT_XDMAC_GID 0x10 /* Global Interrupt Disable Register */ +#define AT_XDMAC_GIM 0x14 /* Global Interrupt Mask Register */ +#define AT_XDMAC_GIS 0x18 /* Global Interrupt Status Register */ +#define AT_XDMAC_GE 0x1C /* Global Channel Enable Register */ +#define AT_XDMAC_GD 0x20 /* Global Channel Disable Register */ +#define AT_XDMAC_GS 0x24 /* Global Channel Status Register */ +#define AT_XDMAC_GRS 0x28 /* Global Channel Read Suspend Register */ +#define AT_XDMAC_GWS 0x2C /* Global Write Suspend Register */ +#define AT_XDMAC_GRWS 0x30 /* Global Channel Read Write Suspend Register */ +#define AT_XDMAC_GRWR 0x34 /* Global Channel Read Write Resume Register */ +#define AT_XDMAC_GSWR 0x38 /* Global Channel Software Request Register */ +#define AT_XDMAC_GSWS 0x3C /* Global channel Software Request Status Register */ +#define AT_XDMAC_GSWF 0x40 /* Global Channel Software Flush Request Register */ +#define AT_XDMAC_VERSION 0xFFC /* XDMAC Version Register */ + +/* Channel relative registers offsets */ +#define AT_XDMAC_CIE 0x00 /* Channel Interrupt Enable Register */ +#define AT_XDMAC_CIE_BIE BIT(0) /* End of Block Interrupt Enable Bit */ +#define AT_XDMAC_CIE_LIE BIT(1) /* End of Linked List Interrupt Enable Bit */ +#define AT_XDMAC_CIE_DIE BIT(2) /* End of Disable Interrupt Enable Bit */ +#define AT_XDMAC_CIE_FIE BIT(3) /* End of Flush Interrupt Enable Bit */ +#define AT_XDMAC_CIE_RBEIE BIT(4) /* Read Bus Error Interrupt Enable Bit */ +#define AT_XDMAC_CIE_WBEIE BIT(5) /* Write Bus Error Interrupt Enable Bit */ +#define AT_XDMAC_CIE_ROIE BIT(6) /* Request Overflow Interrupt Enable Bit */ +#define AT_XDMAC_CID 0x04 /* Channel Interrupt Disable Register */ +#define AT_XDMAC_CID_BID BIT(0) /* End of Block Interrupt Disable Bit */ +#define AT_XDMAC_CID_LID BIT(1) /* End of Linked List Interrupt Disable Bit */ +#define AT_XDMAC_CID_DID BIT(2) /* End of Disable Interrupt Disable Bit */ +#define AT_XDMAC_CID_FID BIT(3) /* End of Flush Interrupt Disable Bit */ +#define AT_XDMAC_CID_RBEID BIT(4) /* Read Bus Error Interrupt Disable Bit */ +#define AT_XDMAC_CID_WBEID BIT(5) /* Write Bus Error Interrupt Disable Bit */ +#define AT_XDMAC_CID_ROID BIT(6) /* Request Overflow Interrupt Disable Bit */ +#define AT_XDMAC_CIM 0x08 /* Channel Interrupt Mask Register */ +#define AT_XDMAC_CIM_BIM BIT(0) /* End of Block Interrupt Mask Bit */ +#define AT_XDMAC_CIM_LIM BIT(1) /* End of Linked List Interrupt Mask Bit */ +#define AT_XDMAC_CIM_DIM BIT(2) /* End of Disable Interrupt Mask Bit */ +#define AT_XDMAC_CIM_FIM BIT(3) /* End of Flush Interrupt Mask Bit */ +#define AT_XDMAC_CIM_RBEIM BIT(4) /* Read Bus Error Interrupt Mask Bit */ +#define AT_XDMAC_CIM_WBEIM BIT(5) /* Write Bus Error Interrupt Mask Bit */ +#define AT_XDMAC_CIM_ROIM BIT(6) /* Request Overflow Interrupt Mask Bit */ +#define AT_XDMAC_CIS 0x0C /* Channel Interrupt Status Register */ +#define AT_XDMAC_CIS_BIS BIT(0) /* End of Block Interrupt Status Bit */ +#define AT_XDMAC_CIS_LIS BIT(1) /* End of Linked List Interrupt Status Bit */ +#define AT_XDMAC_CIS_DIS BIT(2) /* End of Disable Interrupt Status Bit */ +#define AT_XDMAC_CIS_FIS BIT(3) /* End of Flush Interrupt Status Bit */ +#define AT_XDMAC_CIS_RBEIS BIT(4) /* Read Bus Error Interrupt Status Bit */ +#define AT_XDMAC_CIS_WBEIS BIT(5) /* Write Bus Error Interrupt Status Bit */ +#define AT_XDMAC_CIS_ROIS BIT(6) /* Request Overflow Interrupt Status Bit */ +#define AT_XDMAC_CSA 0x10 /* Channel Source Address Register */ +#define AT_XDMAC_CDA 0x14 /* Channel Destination Address Register */ +#define AT_XDMAC_CNDA 0x18 /* Channel Next Descriptor Address Register */ +#define AT_XDMAC_CNDA_NDAIF(i) ((i) & 0x1) /* Channel x Next Descriptor Interface */ +#define AT_XDMAC_CNDA_NDA(i) ((i) & 0xfffffffc) /* Channel x Next Descriptor Address */ +#define AT_XDMAC_CNDC 0x1C /* Channel Next Descriptor Control Register */ +#define AT_XDMAC_CNDC_NDE (0x1 << 0) /* Channel x Next Descriptor Enable */ +#define AT_XDMAC_CNDC_NDSUP (0x1 << 1) /* Channel x Next Descriptor Source Update */ +#define AT_XDMAC_CNDC_NDDUP (0x1 << 2) /* Channel x Next Descriptor Destination Update */ +#define AT_XDMAC_CNDC_NDVIEW_NDV0 (0x0 << 3) /* Channel x Next Descriptor View 0 */ +#define AT_XDMAC_CNDC_NDVIEW_NDV1 (0x1 << 3) /* Channel x Next Descriptor View 1 */ +#define AT_XDMAC_CNDC_NDVIEW_NDV2 (0x2 << 3) /* Channel x Next Descriptor View 2 */ +#define AT_XDMAC_CNDC_NDVIEW_NDV3 (0x3 << 3) /* Channel x Next Descriptor View 3 */ +#define AT_XDMAC_CUBC 0x20 /* Channel Microblock Control Register */ +#define AT_XDMAC_CBC 0x24 /* Channel Block Control Register */ +#define AT_XDMAC_CC 0x28 /* Channel Configuration Register */ +#define AT_XDMAC_CC_TYPE (0x1 << 0) /* Channel Transfer Type */ +#define AT_XDMAC_CC_TYPE_MEM_TRAN (0x0 << 0) /* Memory to Memory Transfer */ +#define AT_XDMAC_CC_TYPE_PER_TRAN (0x1 << 0) /* Peripheral to Memory or Memory to Peripheral Transfer */ +#define AT_XDMAC_CC_MBSIZE_MASK (0x3 << 1) +#define AT_XDMAC_CC_MBSIZE_SINGLE (0x0 << 1) +#define AT_XDMAC_CC_MBSIZE_FOUR (0x1 << 1) +#define AT_XDMAC_CC_MBSIZE_EIGHT (0x2 << 1) +#define AT_XDMAC_CC_MBSIZE_SIXTEEN (0x3 << 1) +#define AT_XDMAC_CC_DSYNC (0x1 << 4) /* Channel Synchronization */ +#define AT_XDMAC_CC_DSYNC_PER2MEM (0x0 << 4) +#define AT_XDMAC_CC_DSYNC_MEM2PER (0x1 << 4) +#define AT_XDMAC_CC_PROT (0x1 << 5) /* Channel Protection */ +#define AT_XDMAC_CC_PROT_SEC (0x0 << 5) +#define AT_XDMAC_CC_PROT_UNSEC (0x1 << 5) +#define AT_XDMAC_CC_SWREQ (0x1 << 6) /* Channel Software Request Trigger */ +#define AT_XDMAC_CC_SWREQ_HWR_CONNECTED (0x0 << 6) +#define AT_XDMAC_CC_SWREQ_SWR_CONNECTED (0x1 << 6) +#define AT_XDMAC_CC_MEMSET (0x1 << 7) /* Channel Fill Block of memory */ +#define AT_XDMAC_CC_MEMSET_NORMAL_MODE (0x0 << 7) +#define AT_XDMAC_CC_MEMSET_HW_MODE (0x1 << 7) +#define AT_XDMAC_CC_CSIZE(i) ((0x7 & (i)) << 8) /* Channel Chunk Size */ +#define AT_XDMAC_CC_DWIDTH_OFFSET 11 +#define AT_XDMAC_CC_DWIDTH_MASK (0x3 << AT_XDMAC_CC_DWIDTH_OFFSET) +#define AT_XDMAC_CC_DWIDTH(i) ((0x3 & (i)) << AT_XDMAC_CC_DWIDTH_OFFSET) /* Channel Data Width */ +#define AT_XDMAC_CC_DWIDTH_BYTE 0x0 +#define AT_XDMAC_CC_DWIDTH_HALFWORD 0x1 +#define AT_XDMAC_CC_DWIDTH_WORD 0x2 +#define AT_XDMAC_CC_DWIDTH_DWORD 0x3 +#define AT_XDMAC_CC_SIF(i) ((0x1 & (i)) << 13) /* Channel Source Interface Identifier */ +#define AT_XDMAC_CC_DIF(i) ((0x1 & (i)) << 14) /* Channel Destination Interface Identifier */ +#define AT_XDMAC_CC_SAM_MASK (0x3 << 16) /* Channel Source Addressing Mode */ +#define AT_XDMAC_CC_SAM_FIXED_AM (0x0 << 16) +#define AT_XDMAC_CC_SAM_INCREMENTED_AM (0x1 << 16) +#define AT_XDMAC_CC_SAM_UBS_AM (0x2 << 16) +#define AT_XDMAC_CC_SAM_UBS_DS_AM (0x3 << 16) +#define AT_XDMAC_CC_DAM_MASK (0x3 << 18) /* Channel Source Addressing Mode */ +#define AT_XDMAC_CC_DAM_FIXED_AM (0x0 << 18) +#define AT_XDMAC_CC_DAM_INCREMENTED_AM (0x1 << 18) +#define AT_XDMAC_CC_DAM_UBS_AM (0x2 << 18) +#define AT_XDMAC_CC_DAM_UBS_DS_AM (0x3 << 18) +#define AT_XDMAC_CC_INITD (0x1 << 21) /* Channel Initialization Terminated (read only) */ +#define AT_XDMAC_CC_INITD_TERMINATED (0x0 << 21) +#define AT_XDMAC_CC_INITD_IN_PROGRESS (0x1 << 21) +#define AT_XDMAC_CC_RDIP (0x1 << 22) /* Read in Progress (read only) */ +#define AT_XDMAC_CC_RDIP_DONE (0x0 << 22) +#define AT_XDMAC_CC_RDIP_IN_PROGRESS (0x1 << 22) +#define AT_XDMAC_CC_WRIP (0x1 << 23) /* Write in Progress (read only) */ +#define AT_XDMAC_CC_WRIP_DONE (0x0 << 23) +#define AT_XDMAC_CC_WRIP_IN_PROGRESS (0x1 << 23) +#define AT_XDMAC_CC_PERID(i) (0x7f & (h) << 24) /* Channel Peripheral Identifier */ +#define AT_XDMAC_CDS_MSP 0x2C /* Channel Data Stride Memory Set Pattern */ +#define AT_XDMAC_CSUS 0x30 /* Channel Source Microblock Stride */ +#define AT_XDMAC_CDUS 0x34 /* Channel Destination Microblock Stride */ + +#define AT_XDMAC_CHAN_REG_BASE 0x50 /* Channel registers base address */ + +/* Microblock control members */ +#define AT_XDMAC_MBR_UBC_UBLEN_MAX 0xFFFFFFUL /* Maximum Microblock Length */ +#define AT_XDMAC_MBR_UBC_NDE (0x1 << 24) /* Next Descriptor Enable */ +#define AT_XDMAC_MBR_UBC_NSEN (0x1 << 25) /* Next Descriptor Source Update */ +#define AT_XDMAC_MBR_UBC_NDEN (0x1 << 26) /* Next Descriptor Destination Update */ +#define AT_XDMAC_MBR_UBC_NDV0 (0x0 << 27) /* Next Descriptor View 0 */ +#define AT_XDMAC_MBR_UBC_NDV1 (0x1 << 27) /* Next Descriptor View 1 */ +#define AT_XDMAC_MBR_UBC_NDV2 (0x2 << 27) /* Next Descriptor View 2 */ +#define AT_XDMAC_MBR_UBC_NDV3 (0x3 << 27) /* Next Descriptor View 3 */ + +#define AT_XDMAC_MAX_CHAN 0x20 + +enum atc_status { + AT_XDMAC_CHAN_IS_CYCLIC = 0, + AT_XDMAC_CHAN_IS_PAUSED, +}; + +/* ----- Channels ----- */ +struct at_xdmac_chan { + struct dma_chan chan; + void __iomem *ch_regs; + u32 mask; /* Channel Mask */ + u32 cfg[3]; /* Channel Configuration Register */ + #define AT_XDMAC_CUR_CFG 0 /* Current channel conf */ + #define AT_XDMAC_DEV_TO_MEM_CFG 1 /* Predifined dev to mem channel conf */ + #define AT_XDMAC_MEM_TO_DEV_CFG 2 /* Predifined mem to dev channel conf */ + u8 perid; /* Peripheral ID */ + u8 perif; /* Peripheral Interface */ + u8 memif; /* Memory Interface */ + u32 per_src_addr; + u32 per_dst_addr; + u32 save_cim; + u32 save_cnda; + u32 save_cndc; + unsigned long status; + struct tasklet_struct tasklet; + + spinlock_t lock; + + struct list_head xfers_list; + struct list_head free_descs_list; +}; + + +/* ----- Controller ----- */ +struct at_xdmac { + struct dma_device dma; + void __iomem *regs; + int irq; + struct clk *clk; + u32 save_gim; + u32 save_gs; + struct dma_pool *at_xdmac_desc_pool; + struct at_xdmac_chan chan[0]; +}; + + +/* ----- Descriptors ----- */ + +/* Linked List Descriptor */ +struct at_xdmac_lld { + dma_addr_t mbr_nda; /* Next Descriptor Member */ + u32 mbr_ubc; /* Microblock Control Member */ + dma_addr_t mbr_sa; /* Source Address Member */ + dma_addr_t mbr_da; /* Destination Address Member */ + u32 mbr_cfg; /* Configuration Register */ +}; + + +struct at_xdmac_desc { + struct at_xdmac_lld lld; + enum dma_transfer_direction direction; + struct dma_async_tx_descriptor tx_dma_desc; + struct list_head desc_node; + /* Following members are only used by the first descriptor */ + bool active_xfer; + unsigned int xfer_size; + struct list_head descs_list; + struct list_head xfer_node; +}; + +static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb) +{ + return atxdmac->regs + (AT_XDMAC_CHAN_REG_BASE + chan_nb * 0x40); +} + +#define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg)) +#define at_xdmac_write(atxdmac, reg, value) \ + writel_relaxed((value), (atxdmac)->regs + (reg)) + +#define at_xdmac_chan_read(atchan, reg) readl_relaxed((atchan)->ch_regs + (reg)) +#define at_xdmac_chan_write(atchan, reg, value) writel_relaxed((value), (atchan)->ch_regs + (reg)) + +static inline struct at_xdmac_chan *to_at_xdmac_chan(struct dma_chan *dchan) +{ + return container_of(dchan, struct at_xdmac_chan, chan); +} + +static struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} + +static inline struct at_xdmac *to_at_xdmac(struct dma_device *ddev) +{ + return container_of(ddev, struct at_xdmac, dma); +} + +static inline struct at_xdmac_desc *txd_to_at_desc(struct dma_async_tx_descriptor *txd) +{ + return container_of(txd, struct at_xdmac_desc, tx_dma_desc); +} + +static inline int at_xdmac_chan_is_cyclic(struct at_xdmac_chan *atchan) +{ + return test_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status); +} + +static inline int at_xdmac_chan_is_paused(struct at_xdmac_chan *atchan) +{ + return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); +} + +static inline int at_xdmac_csize(u32 maxburst) +{ + int csize; + + csize = ffs(maxburst) - 1; + if (csize > 4) + csize = -EINVAL; + + return csize; +}; + +static inline u8 at_xdmac_get_dwidth(u32 cfg) +{ + return (cfg & AT_XDMAC_CC_DWIDTH_MASK) >> AT_XDMAC_CC_DWIDTH_OFFSET; +}; + +static unsigned int init_nr_desc_per_channel = 64; +module_param(init_nr_desc_per_channel, uint, 0644); +MODULE_PARM_DESC(init_nr_desc_per_channel, + "initial descriptors per channel (default: 64)"); + + +static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan) +{ + return at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask; +} + +static void at_xdmac_off(struct at_xdmac *atxdmac) +{ + at_xdmac_write(atxdmac, AT_XDMAC_GD, -1L); + + /* Wait that all chans are disabled. */ + while (at_xdmac_read(atxdmac, AT_XDMAC_GS)) + cpu_relax(); + + at_xdmac_write(atxdmac, AT_XDMAC_GID, -1L); +} + +/* Call with lock hold. */ +static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan, + struct at_xdmac_desc *first) +{ + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); + u32 reg; + + dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first); + + if (at_xdmac_chan_is_enabled(atchan)) + return; + + /* Set transfer as active to not try to start it again. */ + first->active_xfer = true; + + /* Tell xdmac where to get the first descriptor. */ + reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys) + | AT_XDMAC_CNDA_NDAIF(atchan->memif); + at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg); + + /* + * When doing memory to memory transfer we need to use the next + * descriptor view 2 since some fields of the configuration register + * depend on transfer size and src/dest addresses. + */ + if (is_slave_direction(first->direction)) { + reg = AT_XDMAC_CNDC_NDVIEW_NDV1; + if (first->direction == DMA_MEM_TO_DEV) + atchan->cfg[AT_XDMAC_CUR_CFG] = + atchan->cfg[AT_XDMAC_MEM_TO_DEV_CFG]; + else + atchan->cfg[AT_XDMAC_CUR_CFG] = + atchan->cfg[AT_XDMAC_DEV_TO_MEM_CFG]; + at_xdmac_chan_write(atchan, AT_XDMAC_CC, + atchan->cfg[AT_XDMAC_CUR_CFG]); + } else { + /* + * No need to write AT_XDMAC_CC reg, it will be done when the + * descriptor is fecthed. + */ + reg = AT_XDMAC_CNDC_NDVIEW_NDV2; + } + + reg |= AT_XDMAC_CNDC_NDDUP + | AT_XDMAC_CNDC_NDSUP + | AT_XDMAC_CNDC_NDE; + at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg); + + dev_vdbg(chan2dev(&atchan->chan), + "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n", + __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC), + at_xdmac_chan_read(atchan, AT_XDMAC_CNDA), + at_xdmac_chan_read(atchan, AT_XDMAC_CNDC), + at_xdmac_chan_read(atchan, AT_XDMAC_CSA), + at_xdmac_chan_read(atchan, AT_XDMAC_CDA), + at_xdmac_chan_read(atchan, AT_XDMAC_CUBC)); + + at_xdmac_chan_write(atchan, AT_XDMAC_CID, 0xffffffff); + reg = AT_XDMAC_CIE_RBEIE | AT_XDMAC_CIE_WBEIE | AT_XDMAC_CIE_ROIE; + /* + * There is no end of list when doing cyclic dma, we need to get + * an interrupt after each periods. + */ + if (at_xdmac_chan_is_cyclic(atchan)) + at_xdmac_chan_write(atchan, AT_XDMAC_CIE, + reg | AT_XDMAC_CIE_BIE); + else + at_xdmac_chan_write(atchan, AT_XDMAC_CIE, + reg | AT_XDMAC_CIE_LIE); + at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask); + dev_vdbg(chan2dev(&atchan->chan), + "%s: enable channel (0x%08x)\n", __func__, atchan->mask); + wmb(); + at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask); + + dev_vdbg(chan2dev(&atchan->chan), + "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n", + __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC), + at_xdmac_chan_read(atchan, AT_XDMAC_CNDA), + at_xdmac_chan_read(atchan, AT_XDMAC_CNDC), + at_xdmac_chan_read(atchan, AT_XDMAC_CSA), + at_xdmac_chan_read(atchan, AT_XDMAC_CDA), + at_xdmac_chan_read(atchan, AT_XDMAC_CUBC)); + +} + +static dma_cookie_t at_xdmac_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct at_xdmac_desc *desc = txd_to_at_desc(tx); + struct at_xdmac_chan *atchan = to_at_xdmac_chan(tx->chan); + dma_cookie_t cookie; + + spin_lock_bh(&atchan->lock); + cookie = dma_cookie_assign(tx); + + dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n", + __func__, atchan, desc); + list_add_tail(&desc->xfer_node, &atchan->xfers_list); + if (list_is_singular(&atchan->xfers_list)) + at_xdmac_start_xfer(atchan, desc); + + spin_unlock_bh(&atchan->lock); + return cookie; +} + +static struct at_xdmac_desc *at_xdmac_alloc_desc(struct dma_chan *chan, + gfp_t gfp_flags) +{ + struct at_xdmac_desc *desc; + struct at_xdmac *atxdmac = to_at_xdmac(chan->device); + dma_addr_t phys; + + desc = dma_pool_alloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys); + if (desc) { + memset(desc, 0, sizeof(*desc)); + INIT_LIST_HEAD(&desc->descs_list); + dma_async_tx_descriptor_init(&desc->tx_dma_desc, chan); + desc->tx_dma_desc.tx_submit = at_xdmac_tx_submit; + desc->tx_dma_desc.phys = phys; + } + + return desc; +} + +/* Call must be protected by lock. */ +static struct at_xdmac_desc *at_xdmac_get_desc(struct at_xdmac_chan *atchan) +{ + struct at_xdmac_desc *desc; + + if (list_empty(&atchan->free_descs_list)) { + desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT); + } else { + desc = list_first_entry(&atchan->free_descs_list, + struct at_xdmac_desc, desc_node); + list_del(&desc->desc_node); + desc->active_xfer = false; + } + + return desc; +} + +static struct dma_chan *at_xdmac_xlate(struct of_phandle_args *dma_spec, + struct of_dma *of_dma) +{ + struct at_xdmac *atxdmac = of_dma->of_dma_data; + struct at_xdmac_chan *atchan; + struct dma_chan *chan; + struct device *dev = atxdmac->dma.dev; + + if (dma_spec->args_count != 1) { + dev_err(dev, "dma phandler args: bad number of args\n"); + return NULL; + } + + chan = dma_get_any_slave_channel(&atxdmac->dma); + if (!chan) { + dev_err(dev, "can't get a dma channel\n"); + return NULL; + } + + atchan = to_at_xdmac_chan(chan); + atchan->memif = AT91_XDMAC_DT_GET_MEM_IF(dma_spec->args[0]); + atchan->perif = AT91_XDMAC_DT_GET_PER_IF(dma_spec->args[0]); + atchan->perid = AT91_XDMAC_DT_GET_PERID(dma_spec->args[0]); + dev_dbg(dev, "chan dt cfg: memif=%u perif=%u perid=%u\n", + atchan->memif, atchan->perif, atchan->perid); + + return chan; +} + +static int at_xdmac_set_slave_config(struct dma_chan *chan, + struct dma_slave_config *sconfig) +{ + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + u8 dwidth; + int csize; + + atchan->cfg[AT_XDMAC_DEV_TO_MEM_CFG] = + AT91_XDMAC_DT_PERID(atchan->perid) + | AT_XDMAC_CC_DAM_INCREMENTED_AM + | AT_XDMAC_CC_SAM_FIXED_AM + | AT_XDMAC_CC_DIF(atchan->memif) + | AT_XDMAC_CC_SIF(atchan->perif) + | AT_XDMAC_CC_SWREQ_HWR_CONNECTED + | AT_XDMAC_CC_DSYNC_PER2MEM + | AT_XDMAC_CC_MBSIZE_SIXTEEN + | AT_XDMAC_CC_TYPE_PER_TRAN; + csize = at_xdmac_csize(sconfig->src_maxburst); + if (csize < 0) { + dev_err(chan2dev(chan), "invalid src maxburst value\n"); + return -EINVAL; + } + atchan->cfg[AT_XDMAC_DEV_TO_MEM_CFG] |= AT_XDMAC_CC_CSIZE(csize); + dwidth = ffs(sconfig->src_addr_width) - 1; + atchan->cfg[AT_XDMAC_DEV_TO_MEM_CFG] |= AT_XDMAC_CC_DWIDTH(dwidth); + + + atchan->cfg[AT_XDMAC_MEM_TO_DEV_CFG] = + AT91_XDMAC_DT_PERID(atchan->perid) + | AT_XDMAC_CC_DAM_FIXED_AM + | AT_XDMAC_CC_SAM_INCREMENTED_AM + | AT_XDMAC_CC_DIF(atchan->perif) + | AT_XDMAC_CC_SIF(atchan->memif) + | AT_XDMAC_CC_SWREQ_HWR_CONNECTED + | AT_XDMAC_CC_DSYNC_MEM2PER + | AT_XDMAC_CC_MBSIZE_SIXTEEN + | AT_XDMAC_CC_TYPE_PER_TRAN; + csize = at_xdmac_csize(sconfig->dst_maxburst); + if (csize < 0) { + dev_err(chan2dev(chan), "invalid src maxburst value\n"); + return -EINVAL; + } + atchan->cfg[AT_XDMAC_MEM_TO_DEV_CFG] |= AT_XDMAC_CC_CSIZE(csize); + dwidth = ffs(sconfig->dst_addr_width) - 1; + atchan->cfg[AT_XDMAC_MEM_TO_DEV_CFG] |= AT_XDMAC_CC_DWIDTH(dwidth); + + /* Src and dst addr are needed to configure the link list descriptor. */ + atchan->per_src_addr = sconfig->src_addr; + atchan->per_dst_addr = sconfig->dst_addr; + + dev_dbg(chan2dev(chan), + "%s: cfg[dev2mem]=0x%08x, cfg[mem2dev]=0x%08x, per_src_addr=0x%08x, per_dst_addr=0x%08x\n", + __func__, atchan->cfg[AT_XDMAC_DEV_TO_MEM_CFG], + atchan->cfg[AT_XDMAC_MEM_TO_DEV_CFG], + atchan->per_src_addr, atchan->per_dst_addr); + + return 0; +} + +static struct dma_async_tx_descriptor * +at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + struct at_xdmac_desc *first = NULL, *prev = NULL; + struct scatterlist *sg; + int i; + u32 cfg; + unsigned int xfer_size = 0; + + if (!sgl) + return NULL; + + if (!is_slave_direction(direction)) { + dev_err(chan2dev(chan), "invalid DMA direction\n"); + return NULL; + } + + dev_dbg(chan2dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n", + __func__, sg_len, + direction == DMA_MEM_TO_DEV ? "to device" : "from device", + flags); + + /* Protect dma_sconfig field that can be modified by set_slave_conf. */ + spin_lock_bh(&atchan->lock); + + /* Prepare descriptors. */ + for_each_sg(sgl, sg, sg_len, i) { + struct at_xdmac_desc *desc = NULL; + u32 len, mem; + + len = sg_dma_len(sg); + mem = sg_dma_address(sg); + if (unlikely(!len)) { + dev_err(chan2dev(chan), "sg data length is zero\n"); + spin_unlock_bh(&atchan->lock); + return NULL; + } + dev_dbg(chan2dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n", + __func__, i, len, mem); + + desc = at_xdmac_get_desc(atchan); + if (!desc) { + dev_err(chan2dev(chan), "can't get descriptor\n"); + if (first) + list_splice_init(&first->descs_list, &atchan->free_descs_list); + spin_unlock_bh(&atchan->lock); + return NULL; + } + + /* Linked list descriptor setup. */ + if (direction == DMA_DEV_TO_MEM) { + desc->lld.mbr_sa = atchan->per_src_addr; + desc->lld.mbr_da = mem; + cfg = atchan->cfg[AT_XDMAC_DEV_TO_MEM_CFG]; + } else { + desc->lld.mbr_sa = mem; + desc->lld.mbr_da = atchan->per_dst_addr; + cfg = atchan->cfg[AT_XDMAC_MEM_TO_DEV_CFG]; + } + desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV1 /* next descriptor view */ + | AT_XDMAC_MBR_UBC_NDEN /* next descriptor dst parameter update */ + | AT_XDMAC_MBR_UBC_NSEN /* next descriptor src parameter update */ + | (i == sg_len - 1 ? 0 : AT_XDMAC_MBR_UBC_NDE) /* descriptor fetch */ + | len / (1 << at_xdmac_get_dwidth(cfg)); /* microblock length */ + dev_dbg(chan2dev(chan), + "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n", + __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc); + + /* Chain lld. */ + if (prev) { + prev->lld.mbr_nda = desc->tx_dma_desc.phys; + dev_dbg(chan2dev(chan), + "%s: chain lld: prev=0x%p, mbr_nda=%pad\n", + __func__, prev, &prev->lld.mbr_nda); + } + + prev = desc; + if (!first) + first = desc; + + dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", + __func__, desc, first); + list_add_tail(&desc->desc_node, &first->descs_list); + xfer_size += len; + } + + spin_unlock_bh(&atchan->lock); + + first->tx_dma_desc.flags = flags; + first->xfer_size = xfer_size; + first->direction = direction; + + return &first->tx_dma_desc; +} + +static struct dma_async_tx_descriptor * +at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, + size_t buf_len, size_t period_len, + enum dma_transfer_direction direction, + unsigned long flags) +{ + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + struct at_xdmac_desc *first = NULL, *prev = NULL; + unsigned int periods = buf_len / period_len; + int i; + u32 cfg; + + dev_dbg(chan2dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n", + __func__, &buf_addr, buf_len, period_len, + direction == DMA_MEM_TO_DEV ? "mem2per" : "per2mem", flags); + + if (!is_slave_direction(direction)) { + dev_err(chan2dev(chan), "invalid DMA direction\n"); + return NULL; + } + + if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) { + dev_err(chan2dev(chan), "channel currently used\n"); + return NULL; + } + + for (i = 0; i < periods; i++) { + struct at_xdmac_desc *desc = NULL; + + spin_lock_bh(&atchan->lock); + desc = at_xdmac_get_desc(atchan); + if (!desc) { + dev_err(chan2dev(chan), "can't get descriptor\n"); + if (first) + list_splice_init(&first->descs_list, &atchan->free_descs_list); + spin_unlock_bh(&atchan->lock); + return NULL; + } + spin_unlock_bh(&atchan->lock); + dev_dbg(chan2dev(chan), + "%s: desc=0x%p, tx_dma_desc.phys=%pad\n", + __func__, desc, &desc->tx_dma_desc.phys); + + if (direction == DMA_DEV_TO_MEM) { + desc->lld.mbr_sa = atchan->per_src_addr; + desc->lld.mbr_da = buf_addr + i * period_len; + cfg = atchan->cfg[AT_XDMAC_DEV_TO_MEM_CFG]; + } else { + desc->lld.mbr_sa = buf_addr + i * period_len; + desc->lld.mbr_da = atchan->per_dst_addr; + cfg = atchan->cfg[AT_XDMAC_MEM_TO_DEV_CFG]; + } + desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV1 + | AT_XDMAC_MBR_UBC_NDEN + | AT_XDMAC_MBR_UBC_NSEN + | AT_XDMAC_MBR_UBC_NDE + | period_len >> at_xdmac_get_dwidth(cfg); + + dev_dbg(chan2dev(chan), + "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n", + __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc); + + /* Chain lld. */ + if (prev) { + prev->lld.mbr_nda = desc->tx_dma_desc.phys; + dev_dbg(chan2dev(chan), + "%s: chain lld: prev=0x%p, mbr_nda=%pad\n", + __func__, prev, &prev->lld.mbr_nda); + } + + prev = desc; + if (!first) + first = desc; + + dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", + __func__, desc, first); + list_add_tail(&desc->desc_node, &first->descs_list); + } + + prev->lld.mbr_nda = first->tx_dma_desc.phys; + dev_dbg(chan2dev(chan), + "%s: chain lld: prev=0x%p, mbr_nda=%pad\n", + __func__, prev, &prev->lld.mbr_nda); + first->tx_dma_desc.flags = flags; + first->xfer_size = buf_len; + first->direction = direction; + + return &first->tx_dma_desc; +} + +static struct dma_async_tx_descriptor * +at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + struct at_xdmac_desc *first = NULL, *prev = NULL; + size_t remaining_size = len, xfer_size = 0, ublen; + dma_addr_t src_addr = src, dst_addr = dest; + u32 dwidth; + /* + * WARNING: We don't know the direction, it involves we can't + * dynamically set the source and dest interface so we have to use the + * same one. Only interface 0 allows EBI access. Hopefully we can + * access DDR through both ports (at least on SAMA5D4x), so we can use + * the same interface for source and dest, that solves the fact we + * don't know the direction. + */ + u32 chan_cc = AT_XDMAC_CC_DAM_INCREMENTED_AM + | AT_XDMAC_CC_SAM_INCREMENTED_AM + | AT_XDMAC_CC_DIF(0) + | AT_XDMAC_CC_SIF(0) + | AT_XDMAC_CC_MBSIZE_SIXTEEN + | AT_XDMAC_CC_TYPE_MEM_TRAN; + + dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, len=%zd, flags=0x%lx\n", + __func__, &src, &dest, len, flags); + + if (unlikely(!len)) + return NULL; + + /* + * Check address alignment to select the greater data width we can use. + * Some XDMAC implementations don't provide dword transfer, in this + * case selecting dword has the same behavior as selecting word transfers. + */ + if (!((src_addr | dst_addr) & 7)) { + dwidth = AT_XDMAC_CC_DWIDTH_DWORD; + dev_dbg(chan2dev(chan), "%s: dwidth: double word\n", __func__); + } else if (!((src_addr | dst_addr) & 3)) { + dwidth = AT_XDMAC_CC_DWIDTH_WORD; + dev_dbg(chan2dev(chan), "%s: dwidth: word\n", __func__); + } else if (!((src_addr | dst_addr) & 1)) { + dwidth = AT_XDMAC_CC_DWIDTH_HALFWORD; + dev_dbg(chan2dev(chan), "%s: dwidth: half word\n", __func__); + } else { + dwidth = AT_XDMAC_CC_DWIDTH_BYTE; + dev_dbg(chan2dev(chan), "%s: dwidth: byte\n", __func__); + } + + /* Prepare descriptors. */ + while (remaining_size) { + struct at_xdmac_desc *desc = NULL; + + dev_dbg(chan2dev(chan), "%s: remaining_size=%zu\n", __func__, remaining_size); + + spin_lock_bh(&atchan->lock); + desc = at_xdmac_get_desc(atchan); + spin_unlock_bh(&atchan->lock); + if (!desc) { + dev_err(chan2dev(chan), "can't get descriptor\n"); + if (first) + list_splice_init(&first->descs_list, &atchan->free_descs_list); + return NULL; + } + + /* Update src and dest addresses. */ + src_addr += xfer_size; + dst_addr += xfer_size; + + if (remaining_size >= AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth) + xfer_size = AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth; + else + xfer_size = remaining_size; + + dev_dbg(chan2dev(chan), "%s: xfer_size=%zu\n", __func__, xfer_size); + + /* Check remaining length and change data width if needed. */ + if (!((src_addr | dst_addr | xfer_size) & 7)) { + dwidth = AT_XDMAC_CC_DWIDTH_DWORD; + dev_dbg(chan2dev(chan), "%s: dwidth: double word\n", __func__); + } else if (!((src_addr | dst_addr | xfer_size) & 3)) { + dwidth = AT_XDMAC_CC_DWIDTH_WORD; + dev_dbg(chan2dev(chan), "%s: dwidth: word\n", __func__); + } else if (!((src_addr | dst_addr | xfer_size) & 1)) { + dwidth = AT_XDMAC_CC_DWIDTH_HALFWORD; + dev_dbg(chan2dev(chan), "%s: dwidth: half word\n", __func__); + } else if ((src_addr | dst_addr | xfer_size) & 1) { + dwidth = AT_XDMAC_CC_DWIDTH_BYTE; + dev_dbg(chan2dev(chan), "%s: dwidth: byte\n", __func__); + } + chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth); + + ublen = xfer_size >> dwidth; + remaining_size -= xfer_size; + + desc->lld.mbr_sa = src_addr; + desc->lld.mbr_da = dst_addr; + desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2 + | AT_XDMAC_MBR_UBC_NDEN + | AT_XDMAC_MBR_UBC_NSEN + | (remaining_size ? AT_XDMAC_MBR_UBC_NDE : 0) + | ublen; + desc->lld.mbr_cfg = chan_cc; + + dev_dbg(chan2dev(chan), + "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n", + __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc, desc->lld.mbr_cfg); + + /* Chain lld. */ + if (prev) { + prev->lld.mbr_nda = desc->tx_dma_desc.phys; + dev_dbg(chan2dev(chan), + "%s: chain lld: prev=0x%p, mbr_nda=0x%08x\n", + __func__, prev, prev->lld.mbr_nda); + } + + prev = desc; + if (!first) + first = desc; + + dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n", + __func__, desc, first); + list_add_tail(&desc->desc_node, &first->descs_list); + } + + first->tx_dma_desc.flags = flags; + first->xfer_size = len; + + return &first->tx_dma_desc; +} + +static enum dma_status +at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); + struct at_xdmac_desc *desc, *_desc; + struct list_head *descs_list; + enum dma_status ret; + int residue; + u32 cur_nda, mask, value; + u8 dwidth = at_xdmac_get_dwidth(atchan->cfg[AT_XDMAC_CUR_CFG]); + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + if (!txstate) + return ret; + + spin_lock_bh(&atchan->lock); + + desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node); + + /* + * If the transfer has not been started yet, don't need to compute the + * residue, it's the transfer length. + */ + if (!desc->active_xfer) { + dma_set_residue(txstate, desc->xfer_size); + spin_unlock_bh(&atchan->lock); + return ret; + } + + residue = desc->xfer_size; + /* + * Flush FIFO: only relevant when the transfer is source peripheral + * synchronized. + */ + mask = AT_XDMAC_CC_TYPE | AT_XDMAC_CC_DSYNC; + value = AT_XDMAC_CC_TYPE_PER_TRAN | AT_XDMAC_CC_DSYNC_PER2MEM; + if ((atchan->cfg[AT_XDMAC_CUR_CFG] & mask) == value) { + at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask); + while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS)) + cpu_relax(); + } + + cur_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc; + /* + * Remove size of all microblocks already transferred and the current + * one. Then add the remaining size to transfer of the current + * microblock. + */ + descs_list = &desc->descs_list; + list_for_each_entry_safe(desc, _desc, descs_list, desc_node) { + residue -= (desc->lld.mbr_ubc & 0xffffff) << dwidth; + if ((desc->lld.mbr_nda & 0xfffffffc) == cur_nda) + break; + } + residue += at_xdmac_chan_read(atchan, AT_XDMAC_CUBC) << dwidth; + + spin_unlock_bh(&atchan->lock); + + dma_set_residue(txstate, residue); + + dev_dbg(chan2dev(chan), + "%s: desc=0x%p, tx_dma_desc.phys=%pad, tx_status=%d, cookie=%d, residue=%d\n", + __func__, desc, &desc->tx_dma_desc.phys, ret, cookie, residue); + + return ret; +} + +/* Call must be protected by lock. */ +static void at_xdmac_remove_xfer(struct at_xdmac_chan *atchan, + struct at_xdmac_desc *desc) +{ + dev_dbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); + + /* + * Remove the transfer from the transfer list then move the transfer + * descriptors into the free descriptors list. + */ + list_del(&desc->xfer_node); + list_splice_init(&desc->descs_list, &atchan->free_descs_list); +} + +static void at_xdmac_advance_work(struct at_xdmac_chan *atchan) +{ + struct at_xdmac_desc *desc; + + spin_lock_bh(&atchan->lock); + + /* + * If channel is enabled, do nothing, advance_work will be triggered + * after the interruption. + */ + if (!at_xdmac_chan_is_enabled(atchan) && !list_empty(&atchan->xfers_list)) { + desc = list_first_entry(&atchan->xfers_list, + struct at_xdmac_desc, + xfer_node); + dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); + if (!desc->active_xfer) + at_xdmac_start_xfer(atchan, desc); + } + + spin_unlock_bh(&atchan->lock); +} + +static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan) +{ + struct at_xdmac_desc *desc; + struct dma_async_tx_descriptor *txd; + + desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node); + txd = &desc->tx_dma_desc; + + if (txd->callback && (txd->flags & DMA_PREP_INTERRUPT)) + txd->callback(txd->callback_param); +} + +static void at_xdmac_tasklet(unsigned long data) +{ + struct at_xdmac_chan *atchan = (struct at_xdmac_chan *)data; + struct at_xdmac_desc *desc; + u32 error_mask; + + dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08lx\n", + __func__, atchan->status); + + error_mask = AT_XDMAC_CIS_RBEIS + | AT_XDMAC_CIS_WBEIS + | AT_XDMAC_CIS_ROIS; + + if (at_xdmac_chan_is_cyclic(atchan)) { + at_xdmac_handle_cyclic(atchan); + } else if ((atchan->status & AT_XDMAC_CIS_LIS) + || (atchan->status & error_mask)) { + struct dma_async_tx_descriptor *txd; + + if (atchan->status & AT_XDMAC_CIS_RBEIS) + dev_err(chan2dev(&atchan->chan), "read bus error!!!"); + if (atchan->status & AT_XDMAC_CIS_WBEIS) + dev_err(chan2dev(&atchan->chan), "write bus error!!!"); + if (atchan->status & AT_XDMAC_CIS_ROIS) + dev_err(chan2dev(&atchan->chan), "request overflow error!!!"); + + spin_lock_bh(&atchan->lock); + desc = list_first_entry(&atchan->xfers_list, + struct at_xdmac_desc, + xfer_node); + dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); + BUG_ON(!desc->active_xfer); + + txd = &desc->tx_dma_desc; + + at_xdmac_remove_xfer(atchan, desc); + spin_unlock_bh(&atchan->lock); + + if (!at_xdmac_chan_is_cyclic(atchan)) { + dma_cookie_complete(txd); + if (txd->callback && (txd->flags & DMA_PREP_INTERRUPT)) + txd->callback(txd->callback_param); + } + + dma_run_dependencies(txd); + + at_xdmac_advance_work(atchan); + } +} + +static irqreturn_t at_xdmac_interrupt(int irq, void *dev_id) +{ + struct at_xdmac *atxdmac = (struct at_xdmac *)dev_id; + struct at_xdmac_chan *atchan; + u32 imr, status, pending; + u32 chan_imr, chan_status; + int i, ret = IRQ_NONE; + + do { + imr = at_xdmac_read(atxdmac, AT_XDMAC_GIM); + status = at_xdmac_read(atxdmac, AT_XDMAC_GIS); + pending = status & imr; + + dev_vdbg(atxdmac->dma.dev, + "%s: status=0x%08x, imr=0x%08x, pending=0x%08x\n", + __func__, status, imr, pending); + + if (!pending) + break; + + /* We have to find which channel has generated the interrupt. */ + for (i = 0; i < atxdmac->dma.chancnt; i++) { + if (!((1 << i) & pending)) + continue; + + atchan = &atxdmac->chan[i]; + chan_imr = at_xdmac_chan_read(atchan, AT_XDMAC_CIM); + chan_status = at_xdmac_chan_read(atchan, AT_XDMAC_CIS); + atchan->status = chan_status & chan_imr; + dev_vdbg(atxdmac->dma.dev, + "%s: chan%d: imr=0x%x, status=0x%x\n", + __func__, i, chan_imr, chan_status); + dev_vdbg(chan2dev(&atchan->chan), + "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n", + __func__, + at_xdmac_chan_read(atchan, AT_XDMAC_CC), + at_xdmac_chan_read(atchan, AT_XDMAC_CNDA), + at_xdmac_chan_read(atchan, AT_XDMAC_CNDC), + at_xdmac_chan_read(atchan, AT_XDMAC_CSA), + at_xdmac_chan_read(atchan, AT_XDMAC_CDA), + at_xdmac_chan_read(atchan, AT_XDMAC_CUBC)); + + if (atchan->status & (AT_XDMAC_CIS_RBEIS | AT_XDMAC_CIS_WBEIS)) + at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); + + tasklet_schedule(&atchan->tasklet); + ret = IRQ_HANDLED; + } + + } while (pending); + + return ret; +} + +static void at_xdmac_issue_pending(struct dma_chan *chan) +{ + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + + dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__); + + if (!at_xdmac_chan_is_cyclic(atchan)) + at_xdmac_advance_work(atchan); + + return; +} + +static int at_xdmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + struct at_xdmac_desc *desc, *_desc; + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); + int ret = 0; + + dev_dbg(chan2dev(chan), "%s: cmd=%d\n", __func__, cmd); + + spin_lock_bh(&atchan->lock); + + switch (cmd) { + case DMA_PAUSE: + at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask); + set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); + break; + + case DMA_RESUME: + if (!at_xdmac_chan_is_paused(atchan)) + break; + + at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask); + clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); + break; + + case DMA_TERMINATE_ALL: + at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); + while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask) + cpu_relax(); + + /* Cancel all pending transfers. */ + list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) + at_xdmac_remove_xfer(atchan, desc); + + clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status); + break; + + case DMA_SLAVE_CONFIG: + ret = at_xdmac_set_slave_config(chan, + (struct dma_slave_config *)arg); + break; + + default: + dev_err(chan2dev(chan), + "unmanaged or unknown dma control cmd: %d\n", cmd); + ret = -ENXIO; + } + + spin_unlock_bh(&atchan->lock); + + return ret; +} + +static int at_xdmac_alloc_chan_resources(struct dma_chan *chan) +{ + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + struct at_xdmac_desc *desc; + int i; + + spin_lock_bh(&atchan->lock); + + if (at_xdmac_chan_is_enabled(atchan)) { + dev_err(chan2dev(chan), + "can't allocate channel resources (channel enabled)\n"); + i = -EIO; + goto spin_unlock; + } + + if (!list_empty(&atchan->free_descs_list)) { + dev_err(chan2dev(chan), + "can't allocate channel resources (channel not free from a previous use)\n"); + i = -EIO; + goto spin_unlock; + } + + for (i = 0; i < init_nr_desc_per_channel; i++) { + desc = at_xdmac_alloc_desc(chan, GFP_ATOMIC); + if (!desc) { + dev_warn(chan2dev(chan), + "only %d descriptors have been allocated\n", i); + break; + } + list_add_tail(&desc->desc_node, &atchan->free_descs_list); + } + + dma_cookie_init(chan); + + dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i); + +spin_unlock: + spin_unlock_bh(&atchan->lock); + return i; +} + +static void at_xdmac_free_chan_resources(struct dma_chan *chan) +{ + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + struct at_xdmac *atxdmac = to_at_xdmac(chan->device); + struct at_xdmac_desc *desc, *_desc; + + list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) { + dev_dbg(chan2dev(chan), "%s: freeing descriptor %p\n", __func__, desc); + list_del(&desc->desc_node); + dma_pool_free(atxdmac->at_xdmac_desc_pool, desc, desc->tx_dma_desc.phys); + } + + return; +} + +#define AT_XDMAC_DMA_BUSWIDTHS\ + (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) |\ + BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |\ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |\ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |\ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) + +static int at_xdmac_device_slave_caps(struct dma_chan *dchan, + struct dma_slave_caps *caps) +{ + + caps->src_addr_widths = AT_XDMAC_DMA_BUSWIDTHS; + caps->dstn_addr_widths = AT_XDMAC_DMA_BUSWIDTHS; + caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + caps->cmd_pause = true; + caps->cmd_terminate = true; + caps->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + + return 0; +} + +#ifdef CONFIG_PM +static int atmel_xdmac_prepare(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct at_xdmac *atxdmac = platform_get_drvdata(pdev); + struct dma_chan *chan, *_chan; + + list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + + /* Wait for transfer completion, except in cyclic case. */ + if (at_xdmac_chan_is_enabled(atchan) && !at_xdmac_chan_is_cyclic(atchan)) + return -EAGAIN; + } + return 0; +} +#else +# define atmel_xdmac_prepare NULL +#endif + +#ifdef CONFIG_PM_SLEEP +static int atmel_xdmac_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct at_xdmac *atxdmac = platform_get_drvdata(pdev); + struct dma_chan *chan, *_chan; + + list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + + if (at_xdmac_chan_is_cyclic(atchan)) { + if (!at_xdmac_chan_is_paused(atchan)) + at_xdmac_control(chan, DMA_PAUSE, 0); + atchan->save_cim = at_xdmac_chan_read(atchan, AT_XDMAC_CIM); + atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA); + atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC); + } + } + atxdmac->save_gim = at_xdmac_read(atxdmac, AT_XDMAC_GIM); + + at_xdmac_off(atxdmac); + clk_disable_unprepare(atxdmac->clk); + return 0; +} + +static int atmel_xdmac_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct at_xdmac *atxdmac = platform_get_drvdata(pdev); + struct at_xdmac_chan *atchan; + struct dma_chan *chan, *_chan; + int i; + u32 cfg; + + clk_prepare_enable(atxdmac->clk); + + /* Clear pending interrupts. */ + for (i = 0; i < atxdmac->dma.chancnt; i++) { + atchan = &atxdmac->chan[i]; + while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS)) + cpu_relax(); + } + + at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim); + at_xdmac_write(atxdmac, AT_XDMAC_GE, atxdmac->save_gs); + list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { + atchan = to_at_xdmac_chan(chan); + cfg = atchan->cfg[AT_XDMAC_CUR_CFG]; + at_xdmac_chan_write(atchan, AT_XDMAC_CC, cfg); + if (at_xdmac_chan_is_cyclic(atchan)) { + at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda); + at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc); + at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim); + wmb(); + at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask); + } + } + return 0; +} +#endif /* CONFIG_PM_SLEEP */ + +static int at_xdmac_probe(struct platform_device *pdev) +{ + struct resource *res; + struct at_xdmac *atxdmac; + int irq, size, nr_channels, i, ret; + void __iomem *base; + u32 reg; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -EINVAL; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + /* + * Read number of xdmac channels, read helper function can't be used + * since atxdmac is not yet allocated and we need to know the number + * of channels to do the allocation. + */ + reg = readl_relaxed(base + AT_XDMAC_GTYPE); + nr_channels = AT_XDMAC_NB_CH(reg); + if (nr_channels > AT_XDMAC_MAX_CHAN) { + dev_err(&pdev->dev, "invalid number of channels (%u)\n", + nr_channels); + return -EINVAL; + } + + size = sizeof(*atxdmac); + size += nr_channels * sizeof(struct at_xdmac_chan); + atxdmac = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); + if (!atxdmac) { + dev_err(&pdev->dev, "can't allocate at_xdmac structure\n"); + return -ENOMEM; + } + + atxdmac->regs = base; + atxdmac->irq = irq; + + atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk"); + if (IS_ERR(atxdmac->clk)) { + dev_err(&pdev->dev, "can't get dma_clk\n"); + return PTR_ERR(atxdmac->clk); + } + + /* Do not use dev res to prevent races with tasklet */ + ret = request_irq(atxdmac->irq, at_xdmac_interrupt, 0, "at_xdmac", atxdmac); + if (ret) { + dev_err(&pdev->dev, "can't request irq\n"); + return ret; + } + + ret = clk_prepare_enable(atxdmac->clk); + if (ret) { + dev_err(&pdev->dev, "can't prepare or enable clock\n"); + goto err_free_irq; + } + + atxdmac->at_xdmac_desc_pool = + dmam_pool_create(dev_name(&pdev->dev), &pdev->dev, + sizeof(struct at_xdmac_desc), 4, 0); + if (!atxdmac->at_xdmac_desc_pool) { + dev_err(&pdev->dev, "no memory for descriptors dma pool\n"); + ret = -ENOMEM; + goto err_clk_disable; + } + + dma_cap_set(DMA_CYCLIC, atxdmac->dma.cap_mask); + dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask); + dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask); + /* + * Without DMA_PRIVATE the driver is not able to allocate more than + * one channel, second allocation fails in private_candidate. + */ + dma_cap_set(DMA_PRIVATE, atxdmac->dma.cap_mask); + atxdmac->dma.dev = &pdev->dev; + atxdmac->dma.device_alloc_chan_resources = at_xdmac_alloc_chan_resources; + atxdmac->dma.device_free_chan_resources = at_xdmac_free_chan_resources; + atxdmac->dma.device_tx_status = at_xdmac_tx_status; + atxdmac->dma.device_issue_pending = at_xdmac_issue_pending; + atxdmac->dma.device_prep_dma_cyclic = at_xdmac_prep_dma_cyclic; + atxdmac->dma.device_prep_dma_memcpy = at_xdmac_prep_dma_memcpy; + atxdmac->dma.device_prep_slave_sg = at_xdmac_prep_slave_sg; + atxdmac->dma.device_control = at_xdmac_control; + atxdmac->dma.device_slave_caps = at_xdmac_device_slave_caps; + + /* Disable all chans and interrupts. */ + at_xdmac_off(atxdmac); + + /* Init channels. */ + INIT_LIST_HEAD(&atxdmac->dma.channels); + for (i = 0; i < nr_channels; i++) { + struct at_xdmac_chan *atchan = &atxdmac->chan[i]; + + atchan->chan.device = &atxdmac->dma; + list_add_tail(&atchan->chan.device_node, + &atxdmac->dma.channels); + + atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i); + atchan->mask = 1 << i; + + spin_lock_init(&atchan->lock); + INIT_LIST_HEAD(&atchan->xfers_list); + INIT_LIST_HEAD(&atchan->free_descs_list); + tasklet_init(&atchan->tasklet, at_xdmac_tasklet, + (unsigned long)atchan); + + /* Clear pending interrupts. */ + while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS)) + cpu_relax(); + } + platform_set_drvdata(pdev, atxdmac); + + ret = dma_async_device_register(&atxdmac->dma); + if (ret) { + dev_err(&pdev->dev, "fail to register DMA engine device\n"); + goto err_clk_disable; + } + + ret = of_dma_controller_register(pdev->dev.of_node, + at_xdmac_xlate, atxdmac); + if (ret) { + dev_err(&pdev->dev, "could not register of dma controller\n"); + goto err_dma_unregister; + } + + dev_info(&pdev->dev, "%d channels, mapped at 0x%p\n", + nr_channels, atxdmac->regs); + + return 0; + +err_dma_unregister: + dma_async_device_unregister(&atxdmac->dma); +err_clk_disable: + clk_disable_unprepare(atxdmac->clk); +err_free_irq: + free_irq(atxdmac->irq, atxdmac->dma.dev); + return ret; +} + +static int at_xdmac_remove(struct platform_device *pdev) +{ + struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev); + int i; + + at_xdmac_off(atxdmac); + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&atxdmac->dma); + clk_disable_unprepare(atxdmac->clk); + + synchronize_irq(atxdmac->irq); + + free_irq(atxdmac->irq, atxdmac->dma.dev); + + for (i = 0; i < atxdmac->dma.chancnt; i++) { + struct at_xdmac_chan *atchan = &atxdmac->chan[i]; + + tasklet_kill(&atchan->tasklet); + at_xdmac_free_chan_resources(&atchan->chan); + } + + return 0; +} + +static const struct dev_pm_ops atmel_xdmac_dev_pm_ops = { + .prepare = atmel_xdmac_prepare, + SET_LATE_SYSTEM_SLEEP_PM_OPS(atmel_xdmac_suspend, atmel_xdmac_resume) +}; + +static const struct of_device_id atmel_xdmac_dt_ids[] = { + { + .compatible = "atmel,sama5d4-dma", + }, { + /* sentinel */ + } +}; +MODULE_DEVICE_TABLE(of, atmel_xdmac_dt_ids); + +static struct platform_driver at_xdmac_driver = { + .probe = at_xdmac_probe, + .remove = at_xdmac_remove, + .driver = { + .name = "at_xdmac", + .owner = THIS_MODULE, + .of_match_table = of_match_ptr(atmel_xdmac_dt_ids), + .pm = &atmel_xdmac_dev_pm_ops, + } +}; + +static int __init at_xdmac_init(void) +{ + return platform_driver_probe(&at_xdmac_driver, at_xdmac_probe); +} +subsys_initcall(at_xdmac_init); + +MODULE_DESCRIPTION("Atmel Extended DMA Controller driver"); +MODULE_AUTHOR("Ludovic Desroches <ludovic.desroches@atmel.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/bcm2835-dma.c b/drivers/dma/bcm2835-dma.c index 815dbdfdb195..918b7b3f766f 100644 --- a/drivers/dma/bcm2835-dma.c +++ b/drivers/dma/bcm2835-dma.c @@ -525,8 +525,6 @@ static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id, int irq) vchan_init(&c->vc, &d->ddev); INIT_LIST_HEAD(&c->node); - d->ddev.chancnt++; - c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id); c->ch = chan_id; c->irq_number = irq; diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c index cf7633f6d474..b743adf56465 100644 --- a/drivers/dma/cppi41.c +++ b/drivers/dma/cppi41.c @@ -1,3 +1,4 @@ +#include <linux/delay.h> #include <linux/dmaengine.h> #include <linux/dma-mapping.h> #include <linux/platform_device.h> @@ -567,7 +568,7 @@ static int cppi41_tear_down_chan(struct cppi41_channel *c) reg |= GCR_TEARDOWN; cppi_writel(reg, c->gcr_reg); c->td_queued = 1; - c->td_retry = 100; + c->td_retry = 500; } if (!c->td_seen || !c->td_desc_seen) { @@ -603,12 +604,16 @@ static int cppi41_tear_down_chan(struct cppi41_channel *c) * descriptor before the TD we fetch it from enqueue, it has to be * there waiting for us. */ - if (!c->td_seen && c->td_retry) + if (!c->td_seen && c->td_retry) { + udelay(1); return -EAGAIN; - + } WARN_ON(!c->td_retry); + if (!c->td_desc_seen) { desc_phys = cppi41_pop_desc(cdd, c->q_num); + if (!desc_phys) + desc_phys = cppi41_pop_desc(cdd, c->q_comp_num); WARN_ON(!desc_phys); } diff --git a/drivers/dma/dma-jz4740.c b/drivers/dma/dma-jz4740.c index 8605ed10c96a..bdeafeefa5f6 100644 --- a/drivers/dma/dma-jz4740.c +++ b/drivers/dma/dma-jz4740.c @@ -563,10 +563,9 @@ static int jz4740_dma_probe(struct platform_device *pdev) dd->device_prep_dma_cyclic = jz4740_dma_prep_dma_cyclic; dd->device_control = jz4740_dma_control; dd->dev = &pdev->dev; - dd->chancnt = JZ_DMA_NR_CHANS; INIT_LIST_HEAD(&dd->channels); - for (i = 0; i < dd->chancnt; i++) { + for (i = 0; i < JZ_DMA_NR_CHANS; i++) { chan = &dmadev->chan[i]; chan->id = i; chan->vchan.desc_free = jz4740_dma_desc_free; diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 24bfaf0b92ba..e057935e3023 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -330,8 +330,7 @@ static int __init dma_channel_table_init(void) if (err) { pr_err("initialization failure\n"); for_each_dma_cap_mask(cap, dma_cap_mask_all) - if (channel_table[cap]) - free_percpu(channel_table[cap]); + free_percpu(channel_table[cap]); } return err; diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index 244722170410..380478562b7d 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -22,6 +22,7 @@ #include <linux/mm.h> #include <linux/module.h> #include <linux/slab.h> +#include <linux/pm_runtime.h> #include "../dmaengine.h" #include "internal.h" @@ -1504,6 +1505,9 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) dw->regs = chip->regs; chip->dw = dw; + pm_runtime_enable(chip->dev); + pm_runtime_get_sync(chip->dev); + dw_params = dma_read_byaddr(chip->regs, DW_PARAMS); autocfg = dw_params >> DW_PARAMS_EN & 0x1; @@ -1667,11 +1671,14 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n", nr_channels); + pm_runtime_put_sync_suspend(chip->dev); + return 0; err_dma_register: free_irq(chip->irq, dw); err_pdata: + pm_runtime_put_sync_suspend(chip->dev); return err; } EXPORT_SYMBOL_GPL(dw_dma_probe); @@ -1681,6 +1688,8 @@ int dw_dma_remove(struct dw_dma_chip *chip) struct dw_dma *dw = chip->dw; struct dw_dma_chan *dwc, *_dwc; + pm_runtime_get_sync(chip->dev); + dw_dma_off(dw); dma_async_device_unregister(&dw->dma); @@ -1693,6 +1702,8 @@ int dw_dma_remove(struct dw_dma_chip *chip) channel_clear_bit(dw, CH_EN, dwc->mask); } + pm_runtime_put_sync_suspend(chip->dev); + pm_runtime_disable(chip->dev); return 0; } EXPORT_SYMBOL_GPL(dw_dma_remove); diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c index fd455979cc9f..b969206439b7 100644 --- a/drivers/dma/edma.c +++ b/drivers/dma/edma.c @@ -1106,52 +1106,14 @@ bool edma_filter_fn(struct dma_chan *chan, void *param) } EXPORT_SYMBOL(edma_filter_fn); -static struct platform_device *pdev0, *pdev1; - -static const struct platform_device_info edma_dev_info0 = { - .name = "edma-dma-engine", - .id = 0, - .dma_mask = DMA_BIT_MASK(32), -}; - -static const struct platform_device_info edma_dev_info1 = { - .name = "edma-dma-engine", - .id = 1, - .dma_mask = DMA_BIT_MASK(32), -}; - static int edma_init(void) { - int ret = platform_driver_register(&edma_driver); - - if (ret == 0) { - pdev0 = platform_device_register_full(&edma_dev_info0); - if (IS_ERR(pdev0)) { - platform_driver_unregister(&edma_driver); - ret = PTR_ERR(pdev0); - goto out; - } - } - - if (!of_have_populated_dt() && EDMA_CTLRS == 2) { - pdev1 = platform_device_register_full(&edma_dev_info1); - if (IS_ERR(pdev1)) { - platform_driver_unregister(&edma_driver); - platform_device_unregister(pdev0); - ret = PTR_ERR(pdev1); - } - } - -out: - return ret; + return platform_driver_register(&edma_driver); } subsys_initcall(edma_init); static void __exit edma_exit(void) { - platform_device_unregister(pdev0); - if (pdev1) - platform_device_unregister(pdev1); platform_driver_unregister(&edma_driver); } module_exit(edma_exit); diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c index a38e5d24db0d..e9ebb89e1711 100644 --- a/drivers/dma/fsl-edma.c +++ b/drivers/dma/fsl-edma.c @@ -118,17 +118,17 @@ BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) struct fsl_edma_hw_tcd { - u32 saddr; - u16 soff; - u16 attr; - u32 nbytes; - u32 slast; - u32 daddr; - u16 doff; - u16 citer; - u32 dlast_sga; - u16 csr; - u16 biter; + __le32 saddr; + __le16 soff; + __le16 attr; + __le32 nbytes; + __le32 slast; + __le32 daddr; + __le16 doff; + __le16 citer; + __le32 dlast_sga; + __le16 csr; + __le16 biter; }; struct fsl_edma_sw_tcd { @@ -175,18 +175,12 @@ struct fsl_edma_engine { }; /* - * R/W functions for big- or little-endian registers - * the eDMA controller's endian is independent of the CPU core's endian. + * R/W functions for big- or little-endian registers: + * The eDMA controller's endian is independent of the CPU core's endian. + * For the big-endian IP module, the offset for 8-bit or 16-bit registers + * should also be swapped opposite to that in little-endian IP. */ -static u16 edma_readw(struct fsl_edma_engine *edma, void __iomem *addr) -{ - if (edma->big_endian) - return ioread16be(addr); - else - return ioread16(addr); -} - static u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) { if (edma->big_endian) @@ -197,13 +191,18 @@ static u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) static void edma_writeb(struct fsl_edma_engine *edma, u8 val, void __iomem *addr) { - iowrite8(val, addr); + /* swap the reg offset for these in big-endian mode */ + if (edma->big_endian) + iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3)); + else + iowrite8(val, addr); } static void edma_writew(struct fsl_edma_engine *edma, u16 val, void __iomem *addr) { + /* swap the reg offset for these in big-endian mode */ if (edma->big_endian) - iowrite16be(val, addr); + iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2)); else iowrite16(val, addr); } @@ -254,13 +253,12 @@ static void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR; ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux]; + slot = EDMAMUX_CHCFG_SOURCE(slot); if (enable) - edma_writeb(fsl_chan->edma, - EDMAMUX_CHCFG_ENBL | EDMAMUX_CHCFG_SOURCE(slot), - muxaddr + ch_off); + iowrite8(EDMAMUX_CHCFG_ENBL | slot, muxaddr + ch_off); else - edma_writeb(fsl_chan->edma, EDMAMUX_CHCFG_DIS, muxaddr + ch_off); + iowrite8(EDMAMUX_CHCFG_DIS, muxaddr + ch_off); } static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth addr_width) @@ -286,9 +284,8 @@ static void fsl_edma_free_desc(struct virt_dma_desc *vdesc) fsl_desc = to_fsl_edma_desc(vdesc); for (i = 0; i < fsl_desc->n_tcds; i++) - dma_pool_free(fsl_desc->echan->tcd_pool, - fsl_desc->tcd[i].vtcd, - fsl_desc->tcd[i].ptcd); + dma_pool_free(fsl_desc->echan->tcd_pool, fsl_desc->tcd[i].vtcd, + fsl_desc->tcd[i].ptcd); kfree(fsl_desc); } @@ -363,8 +360,8 @@ static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, /* calculate the total size in this desc */ for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) - len += edma_readl(fsl_chan->edma, &(edesc->tcd[i].vtcd->nbytes)) - * edma_readw(fsl_chan->edma, &(edesc->tcd[i].vtcd->biter)); + len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes) + * le16_to_cpu(edesc->tcd[i].vtcd->biter); if (!in_progress) return len; @@ -376,17 +373,15 @@ static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, /* figure out the finished and calculate the residue */ for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { - size = edma_readl(fsl_chan->edma, &(edesc->tcd[i].vtcd->nbytes)) - * edma_readw(fsl_chan->edma, &(edesc->tcd[i].vtcd->biter)); + size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes) + * le16_to_cpu(edesc->tcd[i].vtcd->biter); if (dir == DMA_MEM_TO_DEV) - dma_addr = edma_readl(fsl_chan->edma, - &(edesc->tcd[i].vtcd->saddr)); + dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr); else - dma_addr = edma_readl(fsl_chan->edma, - &(edesc->tcd[i].vtcd->daddr)); + dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr); len -= size; - if (cur_addr > dma_addr && cur_addr < dma_addr + size) { + if (cur_addr >= dma_addr && cur_addr < dma_addr + size) { len += dma_addr + size - cur_addr; break; } @@ -424,55 +419,67 @@ static enum dma_status fsl_edma_tx_status(struct dma_chan *chan, return fsl_chan->status; } -static void fsl_edma_set_tcd_params(struct fsl_edma_chan *fsl_chan, - u32 src, u32 dst, u16 attr, u16 soff, u32 nbytes, - u32 slast, u16 citer, u16 biter, u32 doff, u32 dlast_sga, - u16 csr) +static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, + struct fsl_edma_hw_tcd *tcd) { + struct fsl_edma_engine *edma = fsl_chan->edma; void __iomem *addr = fsl_chan->edma->membase; u32 ch = fsl_chan->vchan.chan.chan_id; /* - * TCD parameters have been swapped in fill_tcd_params(), - * so just write them to registers in the cpu endian here + * TCD parameters are stored in struct fsl_edma_hw_tcd in little + * endian format. However, we need to load the TCD registers in + * big- or little-endian obeying the eDMA engine model endian. */ - writew(0, addr + EDMA_TCD_CSR(ch)); - writel(src, addr + EDMA_TCD_SADDR(ch)); - writel(dst, addr + EDMA_TCD_DADDR(ch)); - writew(attr, addr + EDMA_TCD_ATTR(ch)); - writew(soff, addr + EDMA_TCD_SOFF(ch)); - writel(nbytes, addr + EDMA_TCD_NBYTES(ch)); - writel(slast, addr + EDMA_TCD_SLAST(ch)); - writew(citer, addr + EDMA_TCD_CITER(ch)); - writew(biter, addr + EDMA_TCD_BITER(ch)); - writew(doff, addr + EDMA_TCD_DOFF(ch)); - writel(dlast_sga, addr + EDMA_TCD_DLAST_SGA(ch)); - writew(csr, addr + EDMA_TCD_CSR(ch)); -} - -static void fill_tcd_params(struct fsl_edma_engine *edma, - struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, - u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, - u16 biter, u16 doff, u32 dlast_sga, bool major_int, - bool disable_req, bool enable_sg) + edma_writew(edma, 0, addr + EDMA_TCD_CSR(ch)); + edma_writel(edma, le32_to_cpu(tcd->saddr), addr + EDMA_TCD_SADDR(ch)); + edma_writel(edma, le32_to_cpu(tcd->daddr), addr + EDMA_TCD_DADDR(ch)); + + edma_writew(edma, le16_to_cpu(tcd->attr), addr + EDMA_TCD_ATTR(ch)); + edma_writew(edma, le16_to_cpu(tcd->soff), addr + EDMA_TCD_SOFF(ch)); + + edma_writel(edma, le32_to_cpu(tcd->nbytes), addr + EDMA_TCD_NBYTES(ch)); + edma_writel(edma, le32_to_cpu(tcd->slast), addr + EDMA_TCD_SLAST(ch)); + + edma_writew(edma, le16_to_cpu(tcd->citer), addr + EDMA_TCD_CITER(ch)); + edma_writew(edma, le16_to_cpu(tcd->biter), addr + EDMA_TCD_BITER(ch)); + edma_writew(edma, le16_to_cpu(tcd->doff), addr + EDMA_TCD_DOFF(ch)); + + edma_writel(edma, le32_to_cpu(tcd->dlast_sga), addr + EDMA_TCD_DLAST_SGA(ch)); + + edma_writew(edma, le16_to_cpu(tcd->csr), addr + EDMA_TCD_CSR(ch)); +} + +static inline +void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, + u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, + u16 biter, u16 doff, u32 dlast_sga, bool major_int, + bool disable_req, bool enable_sg) { u16 csr = 0; /* - * eDMA hardware SGs require the TCD parameters stored in memory - * the same endian as the eDMA module so that they can be loaded - * automatically by the engine + * eDMA hardware SGs require the TCDs to be stored in little + * endian format irrespective of the register endian model. + * So we put the value in little endian in memory, waiting + * for fsl_edma_set_tcd_regs doing the swap. */ - edma_writel(edma, src, &(tcd->saddr)); - edma_writel(edma, dst, &(tcd->daddr)); - edma_writew(edma, attr, &(tcd->attr)); - edma_writew(edma, EDMA_TCD_SOFF_SOFF(soff), &(tcd->soff)); - edma_writel(edma, EDMA_TCD_NBYTES_NBYTES(nbytes), &(tcd->nbytes)); - edma_writel(edma, EDMA_TCD_SLAST_SLAST(slast), &(tcd->slast)); - edma_writew(edma, EDMA_TCD_CITER_CITER(citer), &(tcd->citer)); - edma_writew(edma, EDMA_TCD_DOFF_DOFF(doff), &(tcd->doff)); - edma_writel(edma, EDMA_TCD_DLAST_SGA_DLAST_SGA(dlast_sga), &(tcd->dlast_sga)); - edma_writew(edma, EDMA_TCD_BITER_BITER(biter), &(tcd->biter)); + tcd->saddr = cpu_to_le32(src); + tcd->daddr = cpu_to_le32(dst); + + tcd->attr = cpu_to_le16(attr); + + tcd->soff = cpu_to_le16(EDMA_TCD_SOFF_SOFF(soff)); + + tcd->nbytes = cpu_to_le32(EDMA_TCD_NBYTES_NBYTES(nbytes)); + tcd->slast = cpu_to_le32(EDMA_TCD_SLAST_SLAST(slast)); + + tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer)); + tcd->doff = cpu_to_le16(EDMA_TCD_DOFF_DOFF(doff)); + + tcd->dlast_sga = cpu_to_le32(EDMA_TCD_DLAST_SGA_DLAST_SGA(dlast_sga)); + + tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter)); if (major_int) csr |= EDMA_TCD_CSR_INT_MAJOR; @@ -482,7 +489,7 @@ static void fill_tcd_params(struct fsl_edma_engine *edma, if (enable_sg) csr |= EDMA_TCD_CSR_E_SG; - edma_writew(edma, csr, &(tcd->csr)); + tcd->csr = cpu_to_le16(csr); } static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan, @@ -558,9 +565,9 @@ static struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( doff = fsl_chan->fsc.addr_width; } - fill_tcd_params(fsl_chan->edma, fsl_desc->tcd[i].vtcd, src_addr, - dst_addr, fsl_chan->fsc.attr, soff, nbytes, 0, - iter, iter, doff, last_sg, true, false, true); + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, dst_addr, + fsl_chan->fsc.attr, soff, nbytes, 0, iter, + iter, doff, last_sg, true, false, true); dma_buf_next += period_len; } @@ -607,16 +614,16 @@ static struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( iter = sg_dma_len(sg) / nbytes; if (i < sg_len - 1) { last_sg = fsl_desc->tcd[(i + 1)].ptcd; - fill_tcd_params(fsl_chan->edma, fsl_desc->tcd[i].vtcd, - src_addr, dst_addr, fsl_chan->fsc.attr, - soff, nbytes, 0, iter, iter, doff, last_sg, - false, false, true); + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, + dst_addr, fsl_chan->fsc.attr, soff, + nbytes, 0, iter, iter, doff, last_sg, + false, false, true); } else { last_sg = 0; - fill_tcd_params(fsl_chan->edma, fsl_desc->tcd[i].vtcd, - src_addr, dst_addr, fsl_chan->fsc.attr, - soff, nbytes, 0, iter, iter, doff, last_sg, - true, true, false); + fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, + dst_addr, fsl_chan->fsc.attr, soff, + nbytes, 0, iter, iter, doff, last_sg, + true, true, false); } } @@ -625,17 +632,13 @@ static struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( static void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) { - struct fsl_edma_hw_tcd *tcd; struct virt_dma_desc *vdesc; vdesc = vchan_next_desc(&fsl_chan->vchan); if (!vdesc) return; fsl_chan->edesc = to_fsl_edma_desc(vdesc); - tcd = fsl_chan->edesc->tcd[0].vtcd; - fsl_edma_set_tcd_params(fsl_chan, tcd->saddr, tcd->daddr, tcd->attr, - tcd->soff, tcd->nbytes, tcd->slast, tcd->citer, - tcd->biter, tcd->doff, tcd->dlast_sga, tcd->csr); + fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); fsl_edma_enable_request(fsl_chan); fsl_chan->status = DMA_IN_PROGRESS; } diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index de3da031ff18..38821cdf862b 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -1337,7 +1337,6 @@ static int fsl_dma_chan_probe(struct fsldma_device *fdev, /* Add the channel to DMA device channel list */ list_add_tail(&chan->common.device_node, &fdev->common.channels); - fdev->common.chancnt++; dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible, chan->irq != NO_IRQ ? chan->irq : fdev->irq); diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 88afc48c2ca7..d0df198f62e9 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -729,6 +729,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, case IMX_DMATYPE_CSPI: case IMX_DMATYPE_EXT: case IMX_DMATYPE_SSI: + case IMX_DMATYPE_SAI: per_2_emi = sdma->script_addrs->app_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_app_addr; break; @@ -1287,7 +1288,8 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) unsigned short *ram_code; if (!fw) { - dev_err(sdma->dev, "firmware not found\n"); + dev_info(sdma->dev, "external firmware not found, using ROM firmware\n"); + /* In this case we just use the ROM firmware. */ return; } @@ -1346,7 +1348,7 @@ static int sdma_get_firmware(struct sdma_engine *sdma, return ret; } -static int __init sdma_init(struct sdma_engine *sdma) +static int sdma_init(struct sdma_engine *sdma) { int i, ret; dma_addr_t ccb_phys; diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 895f869d6c2c..32eae38291e5 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -1265,9 +1265,17 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) op = IOAT_OP_XOR; dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE); + if (dma_mapping_error(dev, dest_dma)) + goto dma_unmap; + for (i = 0; i < IOAT_NUM_SRC_TEST; i++) + dma_srcs[i] = DMA_ERROR_CODE; + for (i = 0; i < IOAT_NUM_SRC_TEST; i++) { dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_srcs[i])) + goto dma_unmap; + } tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs, IOAT_NUM_SRC_TEST, PAGE_SIZE, DMA_PREP_INTERRUPT); @@ -1298,7 +1306,6 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) goto dma_unmap; } - dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); for (i = 0; i < IOAT_NUM_SRC_TEST; i++) dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); @@ -1313,6 +1320,8 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) } dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); + dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); + /* skip validate if the capability is not present */ if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) goto free_resources; @@ -1327,8 +1336,13 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) xor_val_result = 1; for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) + dma_srcs[i] = DMA_ERROR_CODE; + for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_srcs[i])) + goto dma_unmap; + } tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, &xor_val_result, DMA_PREP_INTERRUPT); @@ -1374,8 +1388,13 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) xor_val_result = 0; for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) + dma_srcs[i] = DMA_ERROR_CODE; + for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_srcs[i])) + goto dma_unmap; + } tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, &xor_val_result, DMA_PREP_INTERRUPT); @@ -1417,14 +1436,18 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) goto free_resources; dma_unmap: if (op == IOAT_OP_XOR) { - dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); + if (dest_dma != DMA_ERROR_CODE) + dma_unmap_page(dev, dest_dma, PAGE_SIZE, + DMA_FROM_DEVICE); for (i = 0; i < IOAT_NUM_SRC_TEST; i++) - dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, - DMA_TO_DEVICE); + if (dma_srcs[i] != DMA_ERROR_CODE) + dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, + DMA_TO_DEVICE); } else if (op == IOAT_OP_XOR_VAL) { for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) - dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, - DMA_TO_DEVICE); + if (dma_srcs[i] != DMA_ERROR_CODE) + dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, + DMA_TO_DEVICE); } free_resources: dma->device_free_chan_resources(dma_chan); diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c index 9349021cb74c..a1de14ab2c51 100644 --- a/drivers/dma/k3dma.c +++ b/drivers/dma/k3dma.c @@ -722,7 +722,6 @@ static int k3_dma_probe(struct platform_device *op) d->slave.device_issue_pending = k3_dma_issue_pending; d->slave.device_control = k3_dma_control; d->slave.copy_align = DMA_ALIGN; - d->slave.chancnt = d->dma_requests; /* init virtual channel */ d->chans = devm_kzalloc(&op->dev, @@ -787,6 +786,7 @@ static int k3_dma_remove(struct platform_device *op) return 0; } +#ifdef CONFIG_PM_SLEEP static int k3_dma_suspend(struct device *dev) { struct k3_dma_dev *d = dev_get_drvdata(dev); @@ -816,6 +816,7 @@ static int k3_dma_resume(struct device *dev) k3_dma_enable_dma(d, true); return 0; } +#endif static SIMPLE_DEV_PM_OPS(k3_dma_pmops, k3_dma_suspend, k3_dma_resume); diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c index 586d52c86477..01bec4023de2 100644 --- a/drivers/dma/mpc512x_dma.c +++ b/drivers/dma/mpc512x_dma.c @@ -885,6 +885,7 @@ static int mpc_dma_probe(struct platform_device *op) struct resource res; ulong regs_start, regs_size; int retval, i; + u8 chancnt; mdma = devm_kzalloc(dev, sizeof(struct mpc_dma), GFP_KERNEL); if (!mdma) { @@ -956,10 +957,6 @@ static int mpc_dma_probe(struct platform_device *op) dma = &mdma->dma; dma->dev = dev; - if (mdma->is_mpc8308) - dma->chancnt = MPC8308_DMACHAN_MAX; - else - dma->chancnt = MPC512x_DMACHAN_MAX; dma->device_alloc_chan_resources = mpc_dma_alloc_chan_resources; dma->device_free_chan_resources = mpc_dma_free_chan_resources; dma->device_issue_pending = mpc_dma_issue_pending; @@ -972,7 +969,12 @@ static int mpc_dma_probe(struct platform_device *op) dma_cap_set(DMA_MEMCPY, dma->cap_mask); dma_cap_set(DMA_SLAVE, dma->cap_mask); - for (i = 0; i < dma->chancnt; i++) { + if (mdma->is_mpc8308) + chancnt = MPC8308_DMACHAN_MAX; + else + chancnt = MPC512x_DMACHAN_MAX; + + for (i = 0; i < chancnt; i++) { mchan = &mdma->channels[i]; mchan->chan.device = dma; diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c index a7a36835a233..d7d61e1a01c3 100644 --- a/drivers/dma/nbpfaxi.c +++ b/drivers/dma/nbpfaxi.c @@ -1479,7 +1479,7 @@ static struct platform_device_id nbpf_ids[] = { }; MODULE_DEVICE_TABLE(platform, nbpf_ids); -#ifdef CONFIG_PM_RUNTIME +#ifdef CONFIG_PM static int nbpf_runtime_suspend(struct device *dev) { struct nbpf_device *nbpf = platform_get_drvdata(to_platform_device(dev)); diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 7ff73a249502..c0016a68b446 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -1074,8 +1074,6 @@ static int omap_dma_chan_init(struct omap_dmadev *od, int dma_sig) vchan_init(&c->vc, &od->ddev); INIT_LIST_HEAD(&c->node); - od->ddev.chancnt++; - return 0; } diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index 9f9ca9fe5ce6..6e0e47d76b23 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -997,7 +997,7 @@ static void pch_dma_remove(struct pci_dev *pdev) #define PCI_DEVICE_ID_ML7831_DMA1_8CH 0x8810 #define PCI_DEVICE_ID_ML7831_DMA2_4CH 0x8815 -const struct pci_device_id pch_dma_id_table[] = { +static const struct pci_device_id pch_dma_id_table[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_EG20T_PCH_DMA_8CH), 8 }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_EG20T_PCH_DMA_4CH), 4 }, { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA1_8CH), 8}, /* UART Video */ diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 4839bfa74a10..bdf40b530032 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -27,6 +27,7 @@ #include <linux/of.h> #include <linux/of_dma.h> #include <linux/err.h> +#include <linux/pm_runtime.h> #include "dmaengine.h" #define PL330_MAX_CHAN 8 @@ -265,13 +266,16 @@ static unsigned cmd_line; #define NR_DEFAULT_DESC 16 +/* Delay for runtime PM autosuspend, ms */ +#define PL330_AUTOSUSPEND_DELAY 20 + /* Populated by the PL330 core driver for DMA API driver's info */ struct pl330_config { u32 periph_id; #define DMAC_MODE_NS (1 << 0) unsigned int mode; unsigned int data_bus_width:10; /* In number of bits */ - unsigned int data_buf_dep:10; + unsigned int data_buf_dep:11; unsigned int num_chan:4; unsigned int num_peri:6; u32 peri_ns; @@ -1958,6 +1962,7 @@ static void pl330_tasklet(unsigned long data) struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data; struct dma_pl330_desc *desc, *_dt; unsigned long flags; + bool power_down = false; spin_lock_irqsave(&pch->lock, flags); @@ -1972,10 +1977,17 @@ static void pl330_tasklet(unsigned long data) /* Try to submit a req imm. next to the last completed cookie */ fill_queue(pch); - /* Make sure the PL330 Channel thread is active */ - spin_lock(&pch->thread->dmac->lock); - _start(pch->thread); - spin_unlock(&pch->thread->dmac->lock); + if (list_empty(&pch->work_list)) { + spin_lock(&pch->thread->dmac->lock); + _stop(pch->thread); + spin_unlock(&pch->thread->dmac->lock); + power_down = true; + } else { + /* Make sure the PL330 Channel thread is active */ + spin_lock(&pch->thread->dmac->lock); + _start(pch->thread); + spin_unlock(&pch->thread->dmac->lock); + } while (!list_empty(&pch->completed_list)) { dma_async_tx_callback callback; @@ -1990,6 +2002,12 @@ static void pl330_tasklet(unsigned long data) if (pch->cyclic) { desc->status = PREP; list_move_tail(&desc->node, &pch->work_list); + if (power_down) { + spin_lock(&pch->thread->dmac->lock); + _start(pch->thread); + spin_unlock(&pch->thread->dmac->lock); + power_down = false; + } } else { desc->status = FREE; list_move_tail(&desc->node, &pch->dmac->desc_pool); @@ -2004,6 +2022,12 @@ static void pl330_tasklet(unsigned long data) } } spin_unlock_irqrestore(&pch->lock, flags); + + /* If work list empty, power down */ + if (power_down) { + pm_runtime_mark_last_busy(pch->dmac->ddma.dev); + pm_runtime_put_autosuspend(pch->dmac->ddma.dev); + } } bool pl330_filter(struct dma_chan *chan, void *param) @@ -2073,6 +2097,7 @@ static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned switch (cmd) { case DMA_TERMINATE_ALL: + pm_runtime_get_sync(pl330->ddma.dev); spin_lock_irqsave(&pch->lock, flags); spin_lock(&pl330->lock); @@ -2099,10 +2124,15 @@ static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned dma_cookie_complete(&desc->txd); } + if (!list_empty(&pch->work_list)) + pm_runtime_put(pl330->ddma.dev); + list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool); list_splice_tail_init(&pch->work_list, &pl330->desc_pool); list_splice_tail_init(&pch->completed_list, &pl330->desc_pool); spin_unlock_irqrestore(&pch->lock, flags); + pm_runtime_mark_last_busy(pl330->ddma.dev); + pm_runtime_put_autosuspend(pl330->ddma.dev); break; case DMA_SLAVE_CONFIG: slave_config = (struct dma_slave_config *)arg; @@ -2138,6 +2168,7 @@ static void pl330_free_chan_resources(struct dma_chan *chan) tasklet_kill(&pch->task); + pm_runtime_get_sync(pch->dmac->ddma.dev); spin_lock_irqsave(&pch->lock, flags); pl330_release_channel(pch->thread); @@ -2147,6 +2178,8 @@ static void pl330_free_chan_resources(struct dma_chan *chan) list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool); spin_unlock_irqrestore(&pch->lock, flags); + pm_runtime_mark_last_busy(pch->dmac->ddma.dev); + pm_runtime_put_autosuspend(pch->dmac->ddma.dev); } static enum dma_status @@ -2162,6 +2195,15 @@ static void pl330_issue_pending(struct dma_chan *chan) unsigned long flags; spin_lock_irqsave(&pch->lock, flags); + if (list_empty(&pch->work_list)) { + /* + * Warn on nothing pending. Empty submitted_list may + * break our pm_runtime usage counter as it is + * updated on work_list emptiness status. + */ + WARN_ON(list_empty(&pch->submitted_list)); + pm_runtime_get_sync(pch->dmac->ddma.dev); + } list_splice_tail_init(&pch->submitted_list, &pch->work_list); spin_unlock_irqrestore(&pch->lock, flags); @@ -2336,7 +2378,7 @@ static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len) int burst_len; burst_len = pl330->pcfg.data_bus_width / 8; - burst_len *= pl330->pcfg.data_buf_dep; + burst_len *= pl330->pcfg.data_buf_dep / pl330->pcfg.num_chan; burst_len >>= desc->rqcfg.brst_size; /* src/dst_burst_len can't be more than 16 */ @@ -2459,16 +2501,25 @@ pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, /* Select max possible burst size */ burst = pl330->pcfg.data_bus_width / 8; - while (burst > 1) { - if (!(len % burst)) - break; + /* + * Make sure we use a burst size that aligns with all the memcpy + * parameters because our DMA programming algorithm doesn't cope with + * transfers which straddle an entry in the DMA device's MFIFO. + */ + while ((src | dst | len) & (burst - 1)) burst /= 2; - } desc->rqcfg.brst_size = 0; while (burst != (1 << desc->rqcfg.brst_size)) desc->rqcfg.brst_size++; + /* + * If burst size is smaller than bus width then make sure we only + * transfer one at a time to avoid a burst stradling an MFIFO entry. + */ + if (desc->rqcfg.brst_size * 8 < pl330->pcfg.data_bus_width) + desc->rqcfg.brst_len = 1; + desc->rqcfg.brst_len = get_burst_len(desc, len); desc->txd.flags = flags; @@ -2585,6 +2636,46 @@ static int pl330_dma_device_slave_caps(struct dma_chan *dchan, return 0; } +/* + * Runtime PM callbacks are provided by amba/bus.c driver. + * + * It is assumed here that IRQ safe runtime PM is chosen in probe and amba + * bus driver will only disable/enable the clock in runtime PM callbacks. + */ +static int __maybe_unused pl330_suspend(struct device *dev) +{ + struct amba_device *pcdev = to_amba_device(dev); + + pm_runtime_disable(dev); + + if (!pm_runtime_status_suspended(dev)) { + /* amba did not disable the clock */ + amba_pclk_disable(pcdev); + } + amba_pclk_unprepare(pcdev); + + return 0; +} + +static int __maybe_unused pl330_resume(struct device *dev) +{ + struct amba_device *pcdev = to_amba_device(dev); + int ret; + + ret = amba_pclk_prepare(pcdev); + if (ret) + return ret; + + if (!pm_runtime_status_suspended(dev)) + ret = amba_pclk_enable(pcdev); + + pm_runtime_enable(dev); + + return ret; +} + +static SIMPLE_DEV_PM_OPS(pl330_pm, pl330_suspend, pl330_resume); + static int pl330_probe(struct amba_device *adev, const struct amba_id *id) { @@ -2610,6 +2701,9 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) return -ENOMEM; } + pd = &pl330->ddma; + pd->dev = &adev->dev; + pl330->mcbufsz = pdat ? pdat->mcbuf_sz : 0; res = &adev->res; @@ -2646,7 +2740,6 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) if (!add_desc(pl330, GFP_KERNEL, NR_DEFAULT_DESC)) dev_warn(&adev->dev, "unable to allocate desc\n"); - pd = &pl330->ddma; INIT_LIST_HEAD(&pd->channels); /* Initialize channel parameters */ @@ -2683,7 +2776,6 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) list_add_tail(&pch->chan.device_node, &pd->channels); } - pd->dev = &adev->dev; if (pdat) { pd->cap_mask = pdat->cap_mask; } else { @@ -2732,12 +2824,18 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) dev_info(&adev->dev, - "Loaded driver for PL330 DMAC-%d\n", adev->periphid); + "Loaded driver for PL330 DMAC-%x\n", adev->periphid); dev_info(&adev->dev, "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n", pcfg->data_buf_dep, pcfg->data_bus_width / 8, pcfg->num_chan, pcfg->num_peri, pcfg->num_events); + pm_runtime_irq_safe(&adev->dev); + pm_runtime_use_autosuspend(&adev->dev); + pm_runtime_set_autosuspend_delay(&adev->dev, PL330_AUTOSUSPEND_DELAY); + pm_runtime_mark_last_busy(&adev->dev); + pm_runtime_put_autosuspend(&adev->dev); + return 0; probe_err3: /* Idle the DMAC */ @@ -2764,6 +2862,8 @@ static int pl330_remove(struct amba_device *adev) struct pl330_dmac *pl330 = amba_get_drvdata(adev); struct dma_pl330_chan *pch, *_p; + pm_runtime_get_noresume(pl330->ddma.dev); + if (adev->dev.of_node) of_dma_controller_free(adev->dev.of_node); @@ -2802,6 +2902,7 @@ static struct amba_driver pl330_driver = { .drv = { .owner = THIS_MODULE, .name = "dma-pl330", + .pm = &pl330_pm, }, .id_table = pl330_ids, .probe = pl330_probe, @@ -2810,6 +2911,6 @@ static struct amba_driver pl330_driver = { module_amba_driver(pl330_driver); -MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>"); +MODULE_AUTHOR("Jaswinder Singh <jassisinghbrar@gmail.com>"); MODULE_DESCRIPTION("API Driver for PL330 DMAC"); MODULE_LICENSE("GPL"); diff --git a/drivers/dma/qcom_bam_dma.c b/drivers/dma/qcom_bam_dma.c index 9d7b5542397c..3122a99ec06b 100644 --- a/drivers/dma/qcom_bam_dma.c +++ b/drivers/dma/qcom_bam_dma.c @@ -79,35 +79,97 @@ struct bam_async_desc { struct bam_desc_hw desc[0]; }; -#define BAM_CTRL 0x0000 -#define BAM_REVISION 0x0004 -#define BAM_SW_REVISION 0x0080 -#define BAM_NUM_PIPES 0x003C -#define BAM_TIMER 0x0040 -#define BAM_TIMER_CTRL 0x0044 -#define BAM_DESC_CNT_TRSHLD 0x0008 -#define BAM_IRQ_SRCS 0x000C -#define BAM_IRQ_SRCS_MSK 0x0010 -#define BAM_IRQ_SRCS_UNMASKED 0x0030 -#define BAM_IRQ_STTS 0x0014 -#define BAM_IRQ_CLR 0x0018 -#define BAM_IRQ_EN 0x001C -#define BAM_CNFG_BITS 0x007C -#define BAM_IRQ_SRCS_EE(ee) (0x0800 + ((ee) * 0x80)) -#define BAM_IRQ_SRCS_MSK_EE(ee) (0x0804 + ((ee) * 0x80)) -#define BAM_P_CTRL(pipe) (0x1000 + ((pipe) * 0x1000)) -#define BAM_P_RST(pipe) (0x1004 + ((pipe) * 0x1000)) -#define BAM_P_HALT(pipe) (0x1008 + ((pipe) * 0x1000)) -#define BAM_P_IRQ_STTS(pipe) (0x1010 + ((pipe) * 0x1000)) -#define BAM_P_IRQ_CLR(pipe) (0x1014 + ((pipe) * 0x1000)) -#define BAM_P_IRQ_EN(pipe) (0x1018 + ((pipe) * 0x1000)) -#define BAM_P_EVNT_DEST_ADDR(pipe) (0x182C + ((pipe) * 0x1000)) -#define BAM_P_EVNT_REG(pipe) (0x1818 + ((pipe) * 0x1000)) -#define BAM_P_SW_OFSTS(pipe) (0x1800 + ((pipe) * 0x1000)) -#define BAM_P_DATA_FIFO_ADDR(pipe) (0x1824 + ((pipe) * 0x1000)) -#define BAM_P_DESC_FIFO_ADDR(pipe) (0x181C + ((pipe) * 0x1000)) -#define BAM_P_EVNT_TRSHLD(pipe) (0x1828 + ((pipe) * 0x1000)) -#define BAM_P_FIFO_SIZES(pipe) (0x1820 + ((pipe) * 0x1000)) +enum bam_reg { + BAM_CTRL, + BAM_REVISION, + BAM_NUM_PIPES, + BAM_DESC_CNT_TRSHLD, + BAM_IRQ_SRCS, + BAM_IRQ_SRCS_MSK, + BAM_IRQ_SRCS_UNMASKED, + BAM_IRQ_STTS, + BAM_IRQ_CLR, + BAM_IRQ_EN, + BAM_CNFG_BITS, + BAM_IRQ_SRCS_EE, + BAM_IRQ_SRCS_MSK_EE, + BAM_P_CTRL, + BAM_P_RST, + BAM_P_HALT, + BAM_P_IRQ_STTS, + BAM_P_IRQ_CLR, + BAM_P_IRQ_EN, + BAM_P_EVNT_DEST_ADDR, + BAM_P_EVNT_REG, + BAM_P_SW_OFSTS, + BAM_P_DATA_FIFO_ADDR, + BAM_P_DESC_FIFO_ADDR, + BAM_P_EVNT_GEN_TRSHLD, + BAM_P_FIFO_SIZES, +}; + +struct reg_offset_data { + u32 base_offset; + unsigned int pipe_mult, evnt_mult, ee_mult; +}; + +static const struct reg_offset_data bam_v1_3_reg_info[] = { + [BAM_CTRL] = { 0x0F80, 0x00, 0x00, 0x00 }, + [BAM_REVISION] = { 0x0F84, 0x00, 0x00, 0x00 }, + [BAM_NUM_PIPES] = { 0x0FBC, 0x00, 0x00, 0x00 }, + [BAM_DESC_CNT_TRSHLD] = { 0x0F88, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS] = { 0x0F8C, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_MSK] = { 0x0F90, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_UNMASKED] = { 0x0FB0, 0x00, 0x00, 0x00 }, + [BAM_IRQ_STTS] = { 0x0F94, 0x00, 0x00, 0x00 }, + [BAM_IRQ_CLR] = { 0x0F98, 0x00, 0x00, 0x00 }, + [BAM_IRQ_EN] = { 0x0F9C, 0x00, 0x00, 0x00 }, + [BAM_CNFG_BITS] = { 0x0FFC, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_EE] = { 0x1800, 0x00, 0x00, 0x80 }, + [BAM_IRQ_SRCS_MSK_EE] = { 0x1804, 0x00, 0x00, 0x80 }, + [BAM_P_CTRL] = { 0x0000, 0x80, 0x00, 0x00 }, + [BAM_P_RST] = { 0x0004, 0x80, 0x00, 0x00 }, + [BAM_P_HALT] = { 0x0008, 0x80, 0x00, 0x00 }, + [BAM_P_IRQ_STTS] = { 0x0010, 0x80, 0x00, 0x00 }, + [BAM_P_IRQ_CLR] = { 0x0014, 0x80, 0x00, 0x00 }, + [BAM_P_IRQ_EN] = { 0x0018, 0x80, 0x00, 0x00 }, + [BAM_P_EVNT_DEST_ADDR] = { 0x102C, 0x00, 0x40, 0x00 }, + [BAM_P_EVNT_REG] = { 0x1018, 0x00, 0x40, 0x00 }, + [BAM_P_SW_OFSTS] = { 0x1000, 0x00, 0x40, 0x00 }, + [BAM_P_DATA_FIFO_ADDR] = { 0x1024, 0x00, 0x40, 0x00 }, + [BAM_P_DESC_FIFO_ADDR] = { 0x101C, 0x00, 0x40, 0x00 }, + [BAM_P_EVNT_GEN_TRSHLD] = { 0x1028, 0x00, 0x40, 0x00 }, + [BAM_P_FIFO_SIZES] = { 0x1020, 0x00, 0x40, 0x00 }, +}; + +static const struct reg_offset_data bam_v1_4_reg_info[] = { + [BAM_CTRL] = { 0x0000, 0x00, 0x00, 0x00 }, + [BAM_REVISION] = { 0x0004, 0x00, 0x00, 0x00 }, + [BAM_NUM_PIPES] = { 0x003C, 0x00, 0x00, 0x00 }, + [BAM_DESC_CNT_TRSHLD] = { 0x0008, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS] = { 0x000C, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_MSK] = { 0x0010, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_UNMASKED] = { 0x0030, 0x00, 0x00, 0x00 }, + [BAM_IRQ_STTS] = { 0x0014, 0x00, 0x00, 0x00 }, + [BAM_IRQ_CLR] = { 0x0018, 0x00, 0x00, 0x00 }, + [BAM_IRQ_EN] = { 0x001C, 0x00, 0x00, 0x00 }, + [BAM_CNFG_BITS] = { 0x007C, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_EE] = { 0x0800, 0x00, 0x00, 0x80 }, + [BAM_IRQ_SRCS_MSK_EE] = { 0x0804, 0x00, 0x00, 0x80 }, + [BAM_P_CTRL] = { 0x1000, 0x1000, 0x00, 0x00 }, + [BAM_P_RST] = { 0x1004, 0x1000, 0x00, 0x00 }, + [BAM_P_HALT] = { 0x1008, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_STTS] = { 0x1010, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_CLR] = { 0x1014, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_EN] = { 0x1018, 0x1000, 0x00, 0x00 }, + [BAM_P_EVNT_DEST_ADDR] = { 0x102C, 0x00, 0x1000, 0x00 }, + [BAM_P_EVNT_REG] = { 0x1018, 0x00, 0x1000, 0x00 }, + [BAM_P_SW_OFSTS] = { 0x1000, 0x00, 0x1000, 0x00 }, + [BAM_P_DATA_FIFO_ADDR] = { 0x1824, 0x00, 0x1000, 0x00 }, + [BAM_P_DESC_FIFO_ADDR] = { 0x181C, 0x00, 0x1000, 0x00 }, + [BAM_P_EVNT_GEN_TRSHLD] = { 0x1828, 0x00, 0x1000, 0x00 }, + [BAM_P_FIFO_SIZES] = { 0x1820, 0x00, 0x1000, 0x00 }, +}; /* BAM CTRL */ #define BAM_SW_RST BIT(0) @@ -297,6 +359,8 @@ struct bam_device { /* execution environment ID, from DT */ u32 ee; + const struct reg_offset_data *layout; + struct clk *bamclk; int irq; @@ -305,6 +369,23 @@ struct bam_device { }; /** + * bam_addr - returns BAM register address + * @bdev: bam device + * @pipe: pipe instance (ignored when register doesn't have multiple instances) + * @reg: register enum + */ +static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe, + enum bam_reg reg) +{ + const struct reg_offset_data r = bdev->layout[reg]; + + return bdev->regs + r.base_offset + + r.pipe_mult * pipe + + r.evnt_mult * pipe + + r.ee_mult * bdev->ee; +} + +/** * bam_reset_channel - Reset individual BAM DMA channel * @bchan: bam channel * @@ -317,8 +398,8 @@ static void bam_reset_channel(struct bam_chan *bchan) lockdep_assert_held(&bchan->vc.lock); /* reset channel */ - writel_relaxed(1, bdev->regs + BAM_P_RST(bchan->id)); - writel_relaxed(0, bdev->regs + BAM_P_RST(bchan->id)); + writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST)); + writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST)); /* don't allow cpu to reorder BAM register accesses done after this */ wmb(); @@ -347,17 +428,18 @@ static void bam_chan_init_hw(struct bam_chan *bchan, * because we allocated 1 more descriptor (8 bytes) than we can use */ writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)), - bdev->regs + BAM_P_DESC_FIFO_ADDR(bchan->id)); - writel_relaxed(BAM_DESC_FIFO_SIZE, bdev->regs + - BAM_P_FIFO_SIZES(bchan->id)); + bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR)); + writel_relaxed(BAM_DESC_FIFO_SIZE, + bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES)); /* enable the per pipe interrupts, enable EOT, ERR, and INT irqs */ - writel_relaxed(P_DEFAULT_IRQS_EN, bdev->regs + BAM_P_IRQ_EN(bchan->id)); + writel_relaxed(P_DEFAULT_IRQS_EN, + bam_addr(bdev, bchan->id, BAM_P_IRQ_EN)); /* unmask the specific pipe and EE combo */ - val = readl_relaxed(bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); val |= BIT(bchan->id); - writel_relaxed(val, bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); /* don't allow cpu to reorder the channel enable done below */ wmb(); @@ -367,7 +449,7 @@ static void bam_chan_init_hw(struct bam_chan *bchan, if (dir == DMA_DEV_TO_MEM) val |= P_DIRECTION; - writel_relaxed(val, bdev->regs + BAM_P_CTRL(bchan->id)); + writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL)); bchan->initialized = 1; @@ -432,12 +514,12 @@ static void bam_free_chan(struct dma_chan *chan) bchan->fifo_virt = NULL; /* mask irq for pipe/channel */ - val = readl_relaxed(bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); val &= ~BIT(bchan->id); - writel_relaxed(val, bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); /* disable irq */ - writel_relaxed(0, bdev->regs + BAM_P_IRQ_EN(bchan->id)); + writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN)); } /** @@ -583,14 +665,14 @@ static int bam_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, switch (cmd) { case DMA_PAUSE: spin_lock_irqsave(&bchan->vc.lock, flag); - writel_relaxed(1, bdev->regs + BAM_P_HALT(bchan->id)); + writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT)); bchan->paused = 1; spin_unlock_irqrestore(&bchan->vc.lock, flag); break; case DMA_RESUME: spin_lock_irqsave(&bchan->vc.lock, flag); - writel_relaxed(0, bdev->regs + BAM_P_HALT(bchan->id)); + writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT)); bchan->paused = 0; spin_unlock_irqrestore(&bchan->vc.lock, flag); break; @@ -626,7 +708,7 @@ static u32 process_channel_irqs(struct bam_device *bdev) unsigned long flags; struct bam_async_desc *async_desc; - srcs = readl_relaxed(bdev->regs + BAM_IRQ_SRCS_EE(bdev->ee)); + srcs = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_EE)); /* return early if no pipe/channel interrupts are present */ if (!(srcs & P_IRQ)) @@ -639,11 +721,9 @@ static u32 process_channel_irqs(struct bam_device *bdev) continue; /* clear pipe irq */ - pipe_stts = readl_relaxed(bdev->regs + - BAM_P_IRQ_STTS(i)); + pipe_stts = readl_relaxed(bam_addr(bdev, i, BAM_P_IRQ_STTS)); - writel_relaxed(pipe_stts, bdev->regs + - BAM_P_IRQ_CLR(i)); + writel_relaxed(pipe_stts, bam_addr(bdev, i, BAM_P_IRQ_CLR)); spin_lock_irqsave(&bchan->vc.lock, flags); async_desc = bchan->curr_txd; @@ -694,12 +774,12 @@ static irqreturn_t bam_dma_irq(int irq, void *data) tasklet_schedule(&bdev->task); if (srcs & BAM_IRQ) - clr_mask = readl_relaxed(bdev->regs + BAM_IRQ_STTS); + clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS)); /* don't allow reorder of the various accesses to the BAM registers */ mb(); - writel_relaxed(clr_mask, bdev->regs + BAM_IRQ_CLR); + writel_relaxed(clr_mask, bam_addr(bdev, 0, BAM_IRQ_CLR)); return IRQ_HANDLED; } @@ -763,7 +843,7 @@ static void bam_apply_new_config(struct bam_chan *bchan, else maxburst = bchan->slave.dst_maxburst; - writel_relaxed(maxburst, bdev->regs + BAM_DESC_CNT_TRSHLD); + writel_relaxed(maxburst, bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); bchan->reconfigure = 0; } @@ -830,7 +910,7 @@ static void bam_start_dma(struct bam_chan *bchan) /* ensure descriptor writes and dma start not reordered */ wmb(); writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw), - bdev->regs + BAM_P_EVNT_REG(bchan->id)); + bam_addr(bdev, bchan->id, BAM_P_EVNT_REG)); } /** @@ -918,43 +998,44 @@ static int bam_init(struct bam_device *bdev) u32 val; /* read revision and configuration information */ - val = readl_relaxed(bdev->regs + BAM_REVISION) >> NUM_EES_SHIFT; + val = readl_relaxed(bam_addr(bdev, 0, BAM_REVISION)) >> NUM_EES_SHIFT; val &= NUM_EES_MASK; /* check that configured EE is within range */ if (bdev->ee >= val) return -EINVAL; - val = readl_relaxed(bdev->regs + BAM_NUM_PIPES); + val = readl_relaxed(bam_addr(bdev, 0, BAM_NUM_PIPES)); bdev->num_channels = val & BAM_NUM_PIPES_MASK; /* s/w reset bam */ /* after reset all pipes are disabled and idle */ - val = readl_relaxed(bdev->regs + BAM_CTRL); + val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL)); val |= BAM_SW_RST; - writel_relaxed(val, bdev->regs + BAM_CTRL); + writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); val &= ~BAM_SW_RST; - writel_relaxed(val, bdev->regs + BAM_CTRL); + writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); /* make sure previous stores are visible before enabling BAM */ wmb(); /* enable bam */ val |= BAM_EN; - writel_relaxed(val, bdev->regs + BAM_CTRL); + writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); /* set descriptor threshhold, start with 4 bytes */ - writel_relaxed(DEFAULT_CNT_THRSHLD, bdev->regs + BAM_DESC_CNT_TRSHLD); + writel_relaxed(DEFAULT_CNT_THRSHLD, + bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); /* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */ - writel_relaxed(BAM_CNFG_BITS_DEFAULT, bdev->regs + BAM_CNFG_BITS); + writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS)); /* enable irqs for errors */ writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN, - bdev->regs + BAM_IRQ_EN); + bam_addr(bdev, 0, BAM_IRQ_EN)); /* unmask global bam interrupt */ - writel_relaxed(BAM_IRQ_MSK, bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); return 0; } @@ -969,9 +1050,18 @@ static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan, bchan->vc.desc_free = bam_dma_free_desc; } +static const struct of_device_id bam_of_match[] = { + { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info }, + { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info }, + {} +}; + +MODULE_DEVICE_TABLE(of, bam_of_match); + static int bam_dma_probe(struct platform_device *pdev) { struct bam_device *bdev; + const struct of_device_id *match; struct resource *iores; int ret, i; @@ -981,6 +1071,14 @@ static int bam_dma_probe(struct platform_device *pdev) bdev->dev = &pdev->dev; + match = of_match_node(bam_of_match, pdev->dev.of_node); + if (!match) { + dev_err(&pdev->dev, "Unsupported BAM module\n"); + return -ENODEV; + } + + bdev->layout = match->data; + iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); bdev->regs = devm_ioremap_resource(&pdev->dev, iores); if (IS_ERR(bdev->regs)) @@ -1084,7 +1182,7 @@ static int bam_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&bdev->common); /* mask all interrupts for this execution environment */ - writel_relaxed(0, bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + writel_relaxed(0, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); devm_free_irq(bdev->dev, bdev->irq, bdev); @@ -1104,12 +1202,6 @@ static int bam_dma_remove(struct platform_device *pdev) return 0; } -static const struct of_device_id bam_of_match[] = { - { .compatible = "qcom,bam-v1.4.0", }, - {} -}; -MODULE_DEVICE_TABLE(of, bam_of_match); - static struct platform_driver bam_dma_driver = { .probe = bam_dma_probe, .remove = bam_dma_remove, diff --git a/drivers/dma/sa11x0-dma.c b/drivers/dma/sa11x0-dma.c index 94f2be30412e..96bb62c39c41 100644 --- a/drivers/dma/sa11x0-dma.c +++ b/drivers/dma/sa11x0-dma.c @@ -829,7 +829,6 @@ static int sa11x0_dma_init_dmadev(struct dma_device *dmadev, { unsigned i; - dmadev->chancnt = ARRAY_SIZE(chan_desc); INIT_LIST_HEAD(&dmadev->channels); dmadev->dev = dev; dmadev->device_alloc_chan_resources = sa11x0_dma_alloc_chan_resources; @@ -838,7 +837,7 @@ static int sa11x0_dma_init_dmadev(struct dma_device *dmadev, dmadev->device_tx_status = sa11x0_dma_tx_status; dmadev->device_issue_pending = sa11x0_dma_issue_pending; - for (i = 0; i < dmadev->chancnt; i++) { + for (i = 0; i < ARRAY_SIZE(chan_desc); i++) { struct sa11x0_dma_chan *c; c = kzalloc(sizeof(*c), GFP_KERNEL); diff --git a/drivers/dma/sh/rcar-audmapp.c b/drivers/dma/sh/rcar-audmapp.c index 02979d5199d0..d95bbdd721f4 100644 --- a/drivers/dma/sh/rcar-audmapp.c +++ b/drivers/dma/sh/rcar-audmapp.c @@ -253,7 +253,6 @@ static int audmapp_chan_probe(struct platform_device *pdev, static void audmapp_chan_remove(struct audmapp_device *audev) { - struct dma_device *dma_dev = &audev->shdma_dev.dma_dev; struct shdma_chan *schan; int i; @@ -261,7 +260,6 @@ static void audmapp_chan_remove(struct audmapp_device *audev) BUG_ON(!schan); shdma_chan_remove(schan); } - dma_dev->chancnt = 0; } static struct dma_chan *audmapp_of_xlate(struct of_phandle_args *dma_spec, diff --git a/drivers/dma/sh/rcar-hpbdma.c b/drivers/dma/sh/rcar-hpbdma.c index fe83320d7b57..20a6f6f2a018 100644 --- a/drivers/dma/sh/rcar-hpbdma.c +++ b/drivers/dma/sh/rcar-hpbdma.c @@ -619,7 +619,6 @@ error: static void hpb_dmae_chan_remove(struct hpb_dmae_device *hpbdev) { - struct dma_device *dma_dev = &hpbdev->shdma_dev.dma_dev; struct shdma_chan *schan; int i; @@ -628,7 +627,6 @@ static void hpb_dmae_chan_remove(struct hpb_dmae_device *hpbdev) shdma_chan_remove(schan); } - dma_dev->chancnt = 0; } static int hpb_dmae_remove(struct platform_device *pdev) diff --git a/drivers/dma/sh/shdma-base.c b/drivers/dma/sh/shdma-base.c index 42d497416196..3a2adb131d46 100644 --- a/drivers/dma/sh/shdma-base.c +++ b/drivers/dma/sh/shdma-base.c @@ -391,6 +391,8 @@ static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all) dev_dbg(schan->dev, "Bring down channel %d\n", schan->id); pm_runtime_put(schan->dev); schan->pm_state = SHDMA_PM_ESTABLISHED; + } else if (schan->pm_state == SHDMA_PM_PENDING) { + shdma_chan_xfer_ld_queue(schan); } } } @@ -951,7 +953,7 @@ void shdma_chan_probe(struct shdma_dev *sdev, /* Add the channel to DMA device channel list */ list_add_tail(&schan->dma_chan.device_node, &sdev->dma_dev.channels); - sdev->schan[sdev->dma_dev.chancnt++] = schan; + sdev->schan[id] = schan; } EXPORT_SYMBOL(shdma_chan_probe); diff --git a/drivers/dma/sh/shdmac.c b/drivers/dma/sh/shdmac.c index 1b1d252f938e..aec8a84784a4 100644 --- a/drivers/dma/sh/shdmac.c +++ b/drivers/dma/sh/shdmac.c @@ -572,7 +572,6 @@ err_no_irq: static void sh_dmae_chan_remove(struct sh_dmae_device *shdev) { - struct dma_device *dma_dev = &shdev->shdma_dev.dma_dev; struct shdma_chan *schan; int i; @@ -581,7 +580,6 @@ static void sh_dmae_chan_remove(struct sh_dmae_device *shdev) shdma_chan_remove(schan); } - dma_dev->chancnt = 0; } static void sh_dmae_shutdown(struct platform_device *pdev) diff --git a/drivers/dma/sh/sudmac.c b/drivers/dma/sh/sudmac.c index 2d7a480aa749..6da2eaa6c294 100644 --- a/drivers/dma/sh/sudmac.c +++ b/drivers/dma/sh/sudmac.c @@ -295,7 +295,6 @@ err_no_irq: static void sudmac_chan_remove(struct sudmac_device *su_dev) { - struct dma_device *dma_dev = &su_dev->shdma_dev.dma_dev; struct shdma_chan *schan; int i; @@ -304,7 +303,6 @@ static void sudmac_chan_remove(struct sudmac_device *su_dev) shdma_chan_remove(schan); } - dma_dev->chancnt = 0; } static dma_addr_t sudmac_slave_addr(struct shdma_chan *schan) diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c index 981880ffe276..3492a5f91d31 100644 --- a/drivers/dma/sirf-dma.c +++ b/drivers/dma/sirf-dma.c @@ -735,7 +735,6 @@ static int sirfsoc_dma_probe(struct platform_device *op) dma = &sdma->dma; dma->dev = dev; - dma->chancnt = SIRFSOC_DMA_CHANNELS; dma->device_alloc_chan_resources = sirfsoc_dma_alloc_chan_resources; dma->device_free_chan_resources = sirfsoc_dma_free_chan_resources; @@ -752,7 +751,7 @@ static int sirfsoc_dma_probe(struct platform_device *op) dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); dma_cap_set(DMA_PRIVATE, dma->cap_mask); - for (i = 0; i < dma->chancnt; i++) { + for (i = 0; i < SIRFSOC_DMA_CHANNELS; i++) { schan = &sdma->channels[i]; schan->chan.device = dma; @@ -835,6 +834,7 @@ static int sirfsoc_dma_runtime_resume(struct device *dev) return 0; } +#ifdef CONFIG_PM_SLEEP static int sirfsoc_dma_pm_suspend(struct device *dev) { struct sirfsoc_dma *sdma = dev_get_drvdata(dev); @@ -916,6 +916,7 @@ static int sirfsoc_dma_pm_resume(struct device *dev) return 0; } +#endif static const struct dev_pm_ops sirfsoc_dma_pm_ops = { SET_RUNTIME_PM_OPS(sirfsoc_dma_runtime_suspend, sirfsoc_dma_runtime_resume, NULL) diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 7de218c3ed91..15d49461c0d2 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -3051,7 +3051,7 @@ static int dma40_runtime_resume(struct device *dev) static const struct dev_pm_ops dma40_pm_ops = { SET_LATE_SYSTEM_SLEEP_PM_OPS(dma40_suspend, dma40_resume) - SET_PM_RUNTIME_PM_OPS(dma40_runtime_suspend, + SET_RUNTIME_PM_OPS(dma40_runtime_suspend, dma40_runtime_resume, NULL) }; @@ -3432,6 +3432,7 @@ static int __init d40_lcla_allocate(struct d40_base *base) d40_err(base->dev, "Failed to allocate %d pages.\n", base->lcla_pool.pages); + ret = -ENOMEM; for (j = 0; j < i; j++) free_pages(page_list[j], base->lcla_pool.pages); diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index 3aa10b328254..159f1736a16f 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -18,6 +18,7 @@ #include <linux/interrupt.h> #include <linux/module.h> #include <linux/of_dma.h> +#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/reset.h> #include <linux/slab.h> @@ -26,24 +27,6 @@ #include "virt-dma.h" /* - * There's 16 physical channels that can work in parallel. - * - * However we have 30 different endpoints for our requests. - * - * Since the channels are able to handle only an unidirectional - * transfer, we need to allocate more virtual channels so that - * everyone can grab one channel. - * - * Some devices can't work in both direction (mostly because it - * wouldn't make sense), so we have a bit fewer virtual channels than - * 2 channels per endpoints. - */ - -#define NR_MAX_CHANNELS 16 -#define NR_MAX_REQUESTS 30 -#define NR_MAX_VCHANS 53 - -/* * Common registers */ #define DMA_IRQ_EN(x) ((x) * 0x04) @@ -60,6 +43,12 @@ #define DMA_STAT 0x30 /* + * sun8i specific registers + */ +#define SUN8I_DMA_GATE 0x20 +#define SUN8I_DMA_GATE_ENABLE 0x4 + +/* * Channels specific registers */ #define DMA_CHAN_ENABLE 0x00 @@ -102,6 +91,19 @@ #define DRQ_SDRAM 1 /* + * Hardware channels / ports representation + * + * The hardware is used in several SoCs, with differing numbers + * of channels and endpoints. This structure ties those numbers + * to a certain compatible string. + */ +struct sun6i_dma_config { + u32 nr_max_channels; + u32 nr_max_requests; + u32 nr_max_vchans; +}; + +/* * Hardware representation of the LLI * * The hardware will be fed the physical address of this structure, @@ -159,6 +161,7 @@ struct sun6i_dma_dev { struct dma_pool *pool; struct sun6i_pchan *pchans; struct sun6i_vchan *vchans; + const struct sun6i_dma_config *cfg; }; static struct device *chan2dev(struct dma_chan *chan) @@ -230,30 +233,25 @@ static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev, readl(pchan->base + DMA_CHAN_CUR_PARA)); } -static inline int convert_burst(u32 maxburst, u8 *burst) +static inline s8 convert_burst(u32 maxburst) { switch (maxburst) { case 1: - *burst = 0; - break; + return 0; case 8: - *burst = 2; - break; + return 2; default: return -EINVAL; } - - return 0; } -static inline int convert_buswidth(enum dma_slave_buswidth addr_width, u8 *width) +static inline s8 convert_buswidth(enum dma_slave_buswidth addr_width) { if ((addr_width < DMA_SLAVE_BUSWIDTH_1_BYTE) || (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES)) return -EINVAL; - *width = addr_width >> 1; - return 0; + return addr_width >> 1; } static void *sun6i_dma_lli_add(struct sun6i_dma_lli *prev, @@ -284,26 +282,25 @@ static inline int sun6i_dma_cfg_lli(struct sun6i_dma_lli *lli, struct dma_slave_config *config) { u8 src_width, dst_width, src_burst, dst_burst; - int ret; if (!config) return -EINVAL; - ret = convert_burst(config->src_maxburst, &src_burst); - if (ret) - return ret; + src_burst = convert_burst(config->src_maxburst); + if (src_burst) + return src_burst; - ret = convert_burst(config->dst_maxburst, &dst_burst); - if (ret) - return ret; + dst_burst = convert_burst(config->dst_maxburst); + if (dst_burst) + return dst_burst; - ret = convert_buswidth(config->src_addr_width, &src_width); - if (ret) - return ret; + src_width = convert_buswidth(config->src_addr_width); + if (src_width) + return src_width; - ret = convert_buswidth(config->dst_addr_width, &dst_width); - if (ret) - return ret; + dst_width = convert_buswidth(config->dst_addr_width); + if (dst_width) + return dst_width; lli->cfg = DMA_CHAN_CFG_SRC_BURST(src_burst) | DMA_CHAN_CFG_SRC_WIDTH(src_width) | @@ -432,6 +429,7 @@ static int sun6i_dma_start_desc(struct sun6i_vchan *vchan) static void sun6i_dma_tasklet(unsigned long data) { struct sun6i_dma_dev *sdev = (struct sun6i_dma_dev *)data; + const struct sun6i_dma_config *cfg = sdev->cfg; struct sun6i_vchan *vchan; struct sun6i_pchan *pchan; unsigned int pchan_alloc = 0; @@ -459,7 +457,7 @@ static void sun6i_dma_tasklet(unsigned long data) } spin_lock_irq(&sdev->lock); - for (pchan_idx = 0; pchan_idx < NR_MAX_CHANNELS; pchan_idx++) { + for (pchan_idx = 0; pchan_idx < cfg->nr_max_channels; pchan_idx++) { pchan = &sdev->pchans[pchan_idx]; if (pchan->vchan || list_empty(&sdev->pending)) @@ -480,7 +478,7 @@ static void sun6i_dma_tasklet(unsigned long data) } spin_unlock_irq(&sdev->lock); - for (pchan_idx = 0; pchan_idx < NR_MAX_CHANNELS; pchan_idx++) { + for (pchan_idx = 0; pchan_idx < cfg->nr_max_channels; pchan_idx++) { if (!(pchan_alloc & BIT(pchan_idx))) continue; @@ -502,7 +500,7 @@ static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) int i, j, ret = IRQ_NONE; u32 status; - for (i = 0; i < 2; i++) { + for (i = 0; i < sdev->cfg->nr_max_channels / DMA_IRQ_CHAN_NR; i++) { status = readl(sdev->base + DMA_IRQ_STAT(i)); if (!status) continue; @@ -512,7 +510,7 @@ static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) writel(status, sdev->base + DMA_IRQ_STAT(i)); - for (j = 0; (j < 8) && status; j++) { + for (j = 0; (j < DMA_IRQ_CHAN_NR) && status; j++) { if (status & DMA_IRQ_QUEUE) { pchan = sdev->pchans + j; vchan = pchan->vchan; @@ -525,7 +523,7 @@ static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) } } - status = status >> 4; + status = status >> DMA_IRQ_CHAN_WIDTH; } if (!atomic_read(&sdev->tasklet_shutdown)) @@ -542,11 +540,10 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( { struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); struct sun6i_vchan *vchan = to_sun6i_vchan(chan); - struct dma_slave_config *sconfig = &vchan->cfg; struct sun6i_dma_lli *v_lli; struct sun6i_desc *txd; dma_addr_t p_lli; - int ret; + s8 burst, width; dev_dbg(chan2dev(chan), "%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n", @@ -565,14 +562,21 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( goto err_txd_free; } - ret = sun6i_dma_cfg_lli(v_lli, src, dest, len, sconfig); - if (ret) - goto err_dma_free; + v_lli->src = src; + v_lli->dst = dest; + v_lli->len = len; + v_lli->para = NORMAL_WAIT; + burst = convert_burst(8); + width = convert_buswidth(DMA_SLAVE_BUSWIDTH_4_BYTES); v_lli->cfg |= DMA_CHAN_CFG_SRC_DRQ(DRQ_SDRAM) | DMA_CHAN_CFG_DST_DRQ(DRQ_SDRAM) | DMA_CHAN_CFG_DST_LINEAR_MODE | - DMA_CHAN_CFG_SRC_LINEAR_MODE; + DMA_CHAN_CFG_SRC_LINEAR_MODE | + DMA_CHAN_CFG_SRC_BURST(burst) | + DMA_CHAN_CFG_SRC_WIDTH(width) | + DMA_CHAN_CFG_DST_BURST(burst) | + DMA_CHAN_CFG_DST_WIDTH(width); sun6i_dma_lli_add(NULL, v_lli, p_lli, txd); @@ -580,8 +584,6 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( return vchan_tx_prep(&vchan->vc, &txd->vd, flags); -err_dma_free: - dma_pool_free(sdev->pool, v_lli, p_lli); err_txd_free: kfree(txd); return NULL; @@ -817,7 +819,7 @@ static struct dma_chan *sun6i_dma_of_xlate(struct of_phandle_args *dma_spec, struct dma_chan *chan; u8 port = dma_spec->args[0]; - if (port > NR_MAX_REQUESTS) + if (port > sdev->cfg->nr_max_requests) return NULL; chan = dma_get_any_slave_channel(&sdev->slave); @@ -850,7 +852,7 @@ static inline void sun6i_dma_free(struct sun6i_dma_dev *sdev) { int i; - for (i = 0; i < NR_MAX_VCHANS; i++) { + for (i = 0; i < sdev->cfg->nr_max_vchans; i++) { struct sun6i_vchan *vchan = &sdev->vchans[i]; list_del(&vchan->vc.chan.device_node); @@ -858,8 +860,48 @@ static inline void sun6i_dma_free(struct sun6i_dma_dev *sdev) } } +/* + * For A31: + * + * There's 16 physical channels that can work in parallel. + * + * However we have 30 different endpoints for our requests. + * + * Since the channels are able to handle only an unidirectional + * transfer, we need to allocate more virtual channels so that + * everyone can grab one channel. + * + * Some devices can't work in both direction (mostly because it + * wouldn't make sense), so we have a bit fewer virtual channels than + * 2 channels per endpoints. + */ + +static struct sun6i_dma_config sun6i_a31_dma_cfg = { + .nr_max_channels = 16, + .nr_max_requests = 30, + .nr_max_vchans = 53, +}; + +/* + * The A23 only has 8 physical channels, a maximum DRQ port id of 24, + * and a total of 37 usable source and destination endpoints. + */ + +static struct sun6i_dma_config sun8i_a23_dma_cfg = { + .nr_max_channels = 8, + .nr_max_requests = 24, + .nr_max_vchans = 37, +}; + +static struct of_device_id sun6i_dma_match[] = { + { .compatible = "allwinner,sun6i-a31-dma", .data = &sun6i_a31_dma_cfg }, + { .compatible = "allwinner,sun8i-a23-dma", .data = &sun8i_a23_dma_cfg }, + { /* sentinel */ } +}; + static int sun6i_dma_probe(struct platform_device *pdev) { + const struct of_device_id *device; struct sun6i_dma_dev *sdc; struct resource *res; int ret, i; @@ -868,6 +910,11 @@ static int sun6i_dma_probe(struct platform_device *pdev) if (!sdc) return -ENOMEM; + device = of_match_device(sun6i_dma_match, &pdev->dev); + if (!device) + return -ENODEV; + sdc->cfg = device->data; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); sdc->base = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(sdc->base)) @@ -914,30 +961,30 @@ static int sun6i_dma_probe(struct platform_device *pdev) sdc->slave.device_prep_slave_sg = sun6i_dma_prep_slave_sg; sdc->slave.device_prep_dma_memcpy = sun6i_dma_prep_dma_memcpy; sdc->slave.device_control = sun6i_dma_control; - sdc->slave.chancnt = NR_MAX_VCHANS; + sdc->slave.copy_align = 4; sdc->slave.dev = &pdev->dev; - sdc->pchans = devm_kcalloc(&pdev->dev, NR_MAX_CHANNELS, + sdc->pchans = devm_kcalloc(&pdev->dev, sdc->cfg->nr_max_channels, sizeof(struct sun6i_pchan), GFP_KERNEL); if (!sdc->pchans) return -ENOMEM; - sdc->vchans = devm_kcalloc(&pdev->dev, NR_MAX_VCHANS, + sdc->vchans = devm_kcalloc(&pdev->dev, sdc->cfg->nr_max_vchans, sizeof(struct sun6i_vchan), GFP_KERNEL); if (!sdc->vchans) return -ENOMEM; tasklet_init(&sdc->task, sun6i_dma_tasklet, (unsigned long)sdc); - for (i = 0; i < NR_MAX_CHANNELS; i++) { + for (i = 0; i < sdc->cfg->nr_max_channels; i++) { struct sun6i_pchan *pchan = &sdc->pchans[i]; pchan->idx = i; pchan->base = sdc->base + 0x100 + i * 0x40; } - for (i = 0; i < NR_MAX_VCHANS; i++) { + for (i = 0; i < sdc->cfg->nr_max_vchans; i++) { struct sun6i_vchan *vchan = &sdc->vchans[i]; INIT_LIST_HEAD(&vchan->node); @@ -977,6 +1024,15 @@ static int sun6i_dma_probe(struct platform_device *pdev) goto err_dma_unregister; } + /* + * sun8i variant requires us to toggle a dma gating register, + * as seen in Allwinner's SDK. This register is not documented + * in the A23 user manual. + */ + if (of_device_is_compatible(pdev->dev.of_node, + "allwinner,sun8i-a23-dma")) + writel(SUN8I_DMA_GATE_ENABLE, sdc->base + SUN8I_DMA_GATE); + return 0; err_dma_unregister: @@ -1009,11 +1065,6 @@ static int sun6i_dma_remove(struct platform_device *pdev) return 0; } -static struct of_device_id sun6i_dma_match[] = { - { .compatible = "allwinner,sun6i-a31-dma" }, - { /* sentinel */ } -}; - static struct platform_driver sun6i_dma_driver = { .probe = sun6i_dma_probe, .remove = sun6i_dma_remove, diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index a6c0b3f68393..d8450c3f35f0 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -1587,7 +1587,7 @@ static int tegra_dma_pm_resume(struct device *dev) #endif static const struct dev_pm_ops tegra_dma_dev_pm_ops = { -#ifdef CONFIG_PM_RUNTIME +#ifdef CONFIG_PM .runtime_suspend = tegra_dma_runtime_suspend, .runtime_resume = tegra_dma_runtime_resume, #endif diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c index 17686caf64d5..0659ec9c4488 100644 --- a/drivers/dma/txx9dmac.c +++ b/drivers/dma/txx9dmac.c @@ -76,7 +76,7 @@ static void channel64_write_CHAR(const struct txx9dmac_chan *dc, dma_addr_t val) static void channel64_clear_CHAR(const struct txx9dmac_chan *dc) { -#if defined(CONFIG_32BIT) && !defined(CONFIG_64BIT_PHYS_ADDR) +#if defined(CONFIG_32BIT) && !defined(CONFIG_PHYS_ADDR_T_64BIT) channel64_writel(dc, CHAR, 0); channel64_writel(dc, __pad_CHAR, 0); #else diff --git a/drivers/dma/txx9dmac.h b/drivers/dma/txx9dmac.h index f5a760598882..f6517b928bab 100644 --- a/drivers/dma/txx9dmac.h +++ b/drivers/dma/txx9dmac.h @@ -67,7 +67,7 @@ static inline bool txx9_dma_have_SMPCHN(void) /* Hardware register definitions. */ struct txx9dmac_cregs { -#if defined(CONFIG_32BIT) && !defined(CONFIG_64BIT_PHYS_ADDR) +#if defined(CONFIG_32BIT) && !defined(CONFIG_PHYS_ADDR_T_64BIT) TXX9_DMA_REG32(CHAR); /* Chain Address Register */ #else u64 CHAR; /* Chain Address Register */ @@ -201,7 +201,7 @@ static inline bool is_dmac64(const struct txx9dmac_chan *dc) #ifdef TXX9_DMA_USE_SIMPLE_CHAIN /* Hardware descriptor definition. (for simple-chain) */ struct txx9dmac_hwdesc { -#if defined(CONFIG_32BIT) && !defined(CONFIG_64BIT_PHYS_ADDR) +#if defined(CONFIG_32BIT) && !defined(CONFIG_PHYS_ADDR_T_64BIT) TXX9_DMA_REG32(CHAR); #else u64 CHAR; diff --git a/drivers/dma/xilinx/xilinx_vdma.c b/drivers/dma/xilinx/xilinx_vdma.c index a6e64767186e..4a3a8f3137b3 100644 --- a/drivers/dma/xilinx/xilinx_vdma.c +++ b/drivers/dma/xilinx/xilinx_vdma.c @@ -942,6 +942,9 @@ xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, if (!xt->numf || !xt->sgl[0].size) return NULL; + if (xt->frame_size != 1) + return NULL; + /* Allocate a transaction descriptor. */ desc = xilinx_vdma_alloc_tx_descriptor(chan); if (!desc) @@ -960,7 +963,7 @@ xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, hw = &segment->hw; hw->vsize = xt->numf; hw->hsize = xt->sgl[0].size; - hw->stride = xt->sgl[0].icg << + hw->stride = (xt->sgl[0].icg + xt->sgl[0].size) << XILINX_VDMA_FRMDLY_STRIDE_STRIDE_SHIFT; hw->stride |= chan->config.frm_dly << XILINX_VDMA_FRMDLY_STRIDE_FRMDLY_SHIFT; @@ -971,9 +974,11 @@ xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, hw->buf_addr = xt->src_start; /* Link the previous next descriptor to current */ - prev = list_last_entry(&desc->segments, - struct xilinx_vdma_tx_segment, node); - prev->hw.next_desc = segment->phys; + if (!list_empty(&desc->segments)) { + prev = list_last_entry(&desc->segments, + struct xilinx_vdma_tx_segment, node); + prev->hw.next_desc = segment->phys; + } /* Insert the segment into the descriptor segments list. */ list_add_tail(&segment->node, &desc->segments); |