summaryrefslogtreecommitdiff
path: root/drivers/dma
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma')
-rw-r--r--drivers/dma/Kconfig2
-rw-r--r--drivers/dma/amba-pl08x.c86
-rw-r--r--drivers/dma/bcm2835-dma.c604
-rw-r--r--drivers/dma/dmaengine.c37
-rw-r--r--drivers/dma/dw/core.c525
-rw-r--r--drivers/dma/dw/pci.c5
-rw-r--r--drivers/dma/dw/platform.c40
-rw-r--r--drivers/dma/dw/regs.h56
-rw-r--r--drivers/dma/edma.c11
-rw-r--r--drivers/dma/fsldma.c3
-rw-r--r--drivers/dma/hsu/hsu.c8
-rw-r--r--drivers/dma/hsu/hsu.h4
-rw-r--r--drivers/dma/ioat/init.c17
-rw-r--r--drivers/dma/ioat/registers.h7
-rw-r--r--drivers/dma/mmp_pdma.c3
-rw-r--r--drivers/dma/mpc512x_dma.c174
-rw-r--r--drivers/dma/mv_xor.c98
-rw-r--r--drivers/dma/mv_xor.h1
-rw-r--r--drivers/dma/of-dma.c12
19 files changed, 1088 insertions, 605 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index d96d87c56f2e..67b37ce94143 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -332,7 +332,7 @@ config MPC512X_DMA
config MV_XOR
bool "Marvell XOR engine support"
- depends on PLAT_ORION
+ depends on PLAT_ORION || ARCH_MVEBU || COMPILE_TEST
select DMA_ENGINE
select DMA_ENGINE_RAID
select ASYNC_TX_ENABLE_CHANNEL_SWITCH
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index 9b42c0588550..81db1c4811ce 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -107,16 +107,20 @@ struct pl08x_driver_data;
/**
* struct vendor_data - vendor-specific config parameters for PL08x derivatives
* @channels: the number of channels available in this variant
+ * @signals: the number of request signals available from the hardware
* @dualmaster: whether this version supports dual AHB masters or not.
* @nomadik: whether the channels have Nomadik security extension bits
* that need to be checked for permission before use and some registers are
* missing
* @pl080s: whether this version is a PL080S, which has separate register and
* LLI word for transfer size.
+ * @max_transfer_size: the maximum single element transfer size for this
+ * PL08x variant.
*/
struct vendor_data {
u8 config_offset;
u8 channels;
+ u8 signals;
bool dualmaster;
bool nomadik;
bool pl080s;
@@ -235,7 +239,7 @@ struct pl08x_dma_chan {
struct virt_dma_chan vc;
struct pl08x_phy_chan *phychan;
const char *name;
- const struct pl08x_channel_data *cd;
+ struct pl08x_channel_data *cd;
struct dma_slave_config cfg;
struct pl08x_txd *at;
struct pl08x_driver_data *host;
@@ -1909,6 +1913,12 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
if (slave) {
chan->cd = &pl08x->pd->slave_channels[i];
+ /*
+ * Some implementations have muxed signals, whereas some
+ * use a mux in front of the signals and need dynamic
+ * assignment of signals.
+ */
+ chan->signal = i;
pl08x_dma_slave_init(chan);
} else {
chan->cd = &pl08x->pd->memcpy_channel;
@@ -2050,40 +2060,33 @@ static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec,
struct of_dma *ofdma)
{
struct pl08x_driver_data *pl08x = ofdma->of_dma_data;
- struct pl08x_channel_data *data;
- struct pl08x_dma_chan *chan;
struct dma_chan *dma_chan;
+ struct pl08x_dma_chan *plchan;
if (!pl08x)
return NULL;
- if (dma_spec->args_count != 2)
+ if (dma_spec->args_count != 2) {
+ dev_err(&pl08x->adev->dev,
+ "DMA channel translation requires two cells\n");
return NULL;
+ }
dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]);
- if (dma_chan)
- return dma_get_slave_channel(dma_chan);
-
- chan = devm_kzalloc(pl08x->slave.dev, sizeof(*chan) + sizeof(*data),
- GFP_KERNEL);
- if (!chan)
+ if (!dma_chan) {
+ dev_err(&pl08x->adev->dev,
+ "DMA slave channel not found\n");
return NULL;
+ }
- data = (void *)&chan[1];
- data->bus_id = "(none)";
- data->periph_buses = dma_spec->args[1];
-
- chan->cd = data;
- chan->host = pl08x;
- chan->slave = true;
- chan->name = data->bus_id;
- chan->state = PL08X_CHAN_IDLE;
- chan->signal = dma_spec->args[0];
- chan->vc.desc_free = pl08x_desc_free;
-
- vchan_init(&chan->vc, &pl08x->slave);
+ plchan = to_pl08x_chan(dma_chan);
+ dev_dbg(&pl08x->adev->dev,
+ "translated channel for signal %d\n",
+ dma_spec->args[0]);
- return dma_get_slave_channel(&chan->vc.chan);
+ /* Augment channel data for applicable AHB buses */
+ plchan->cd->periph_buses = dma_spec->args[1];
+ return dma_get_slave_channel(dma_chan);
}
static int pl08x_of_probe(struct amba_device *adev,
@@ -2091,9 +2094,11 @@ static int pl08x_of_probe(struct amba_device *adev,
struct device_node *np)
{
struct pl08x_platform_data *pd;
+ struct pl08x_channel_data *chanp = NULL;
u32 cctl_memcpy = 0;
u32 val;
int ret;
+ int i;
pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL);
if (!pd)
@@ -2195,6 +2200,27 @@ static int pl08x_of_probe(struct amba_device *adev,
/* Use the buses that can access memory, obviously */
pd->memcpy_channel.periph_buses = pd->mem_buses;
+ /*
+ * Allocate channel data for all possible slave channels (one
+ * for each possible signal), channels will then be allocated
+ * for a device and have it's AHB interfaces set up at
+ * translation time.
+ */
+ chanp = devm_kcalloc(&adev->dev,
+ pl08x->vd->signals,
+ sizeof(struct pl08x_channel_data),
+ GFP_KERNEL);
+ if (!chanp)
+ return -ENOMEM;
+
+ pd->slave_channels = chanp;
+ for (i = 0; i < pl08x->vd->signals; i++) {
+ /* chanp->periph_buses will be assigned at translation */
+ chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i);
+ chanp++;
+ }
+ pd->num_slave_channels = pl08x->vd->signals;
+
pl08x->pd = pd;
return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate,
@@ -2234,6 +2260,10 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
goto out_no_pl08x;
}
+ /* Assign useful pointers to the driver state */
+ pl08x->adev = adev;
+ pl08x->vd = vd;
+
/* Initialize memcpy engine */
dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);
pl08x->memcpy.dev = &adev->dev;
@@ -2284,10 +2314,6 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
}
}
- /* Assign useful pointers to the driver state */
- pl08x->adev = adev;
- pl08x->vd = vd;
-
/* By default, AHB1 only. If dualmaster, from platform */
pl08x->lli_buses = PL08X_AHB1;
pl08x->mem_buses = PL08X_AHB1;
@@ -2438,6 +2464,7 @@ out_no_pl08x:
static struct vendor_data vendor_pl080 = {
.config_offset = PL080_CH_CONFIG,
.channels = 8,
+ .signals = 16,
.dualmaster = true,
.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
};
@@ -2445,6 +2472,7 @@ static struct vendor_data vendor_pl080 = {
static struct vendor_data vendor_nomadik = {
.config_offset = PL080_CH_CONFIG,
.channels = 8,
+ .signals = 32,
.dualmaster = true,
.nomadik = true,
.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
@@ -2453,6 +2481,7 @@ static struct vendor_data vendor_nomadik = {
static struct vendor_data vendor_pl080s = {
.config_offset = PL080S_CH_CONFIG,
.channels = 8,
+ .signals = 32,
.pl080s = true,
.max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK,
};
@@ -2460,6 +2489,7 @@ static struct vendor_data vendor_pl080s = {
static struct vendor_data vendor_pl081 = {
.config_offset = PL080_CH_CONFIG,
.channels = 2,
+ .signals = 16,
.dualmaster = false,
.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
};
diff --git a/drivers/dma/bcm2835-dma.c b/drivers/dma/bcm2835-dma.c
index 996c4b00d323..6149b27c33ad 100644
--- a/drivers/dma/bcm2835-dma.c
+++ b/drivers/dma/bcm2835-dma.c
@@ -46,6 +46,9 @@
#include "virt-dma.h"
+#define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 14
+#define BCM2835_DMA_CHAN_NAME_SIZE 8
+
struct bcm2835_dmadev {
struct dma_device ddev;
spinlock_t lock;
@@ -73,7 +76,6 @@ struct bcm2835_chan {
struct list_head node;
struct dma_slave_config cfg;
- bool cyclic;
unsigned int dreq;
int ch;
@@ -82,6 +84,9 @@ struct bcm2835_chan {
void __iomem *chan_base;
int irq_number;
+ unsigned int irq_flags;
+
+ bool is_lite_channel;
};
struct bcm2835_desc {
@@ -89,47 +94,104 @@ struct bcm2835_desc {
struct virt_dma_desc vd;
enum dma_transfer_direction dir;
- struct bcm2835_cb_entry *cb_list;
-
unsigned int frames;
size_t size;
+
+ bool cyclic;
+
+ struct bcm2835_cb_entry cb_list[];
};
#define BCM2835_DMA_CS 0x00
#define BCM2835_DMA_ADDR 0x04
+#define BCM2835_DMA_TI 0x08
#define BCM2835_DMA_SOURCE_AD 0x0c
#define BCM2835_DMA_DEST_AD 0x10
-#define BCM2835_DMA_NEXTCB 0x1C
+#define BCM2835_DMA_LEN 0x14
+#define BCM2835_DMA_STRIDE 0x18
+#define BCM2835_DMA_NEXTCB 0x1c
+#define BCM2835_DMA_DEBUG 0x20
/* DMA CS Control and Status bits */
-#define BCM2835_DMA_ACTIVE BIT(0)
-#define BCM2835_DMA_INT BIT(2)
+#define BCM2835_DMA_ACTIVE BIT(0) /* activate the DMA */
+#define BCM2835_DMA_END BIT(1) /* current CB has ended */
+#define BCM2835_DMA_INT BIT(2) /* interrupt status */
+#define BCM2835_DMA_DREQ BIT(3) /* DREQ state */
#define BCM2835_DMA_ISPAUSED BIT(4) /* Pause requested or not active */
#define BCM2835_DMA_ISHELD BIT(5) /* Is held by DREQ flow control */
-#define BCM2835_DMA_ERR BIT(8)
+#define BCM2835_DMA_WAITING_FOR_WRITES BIT(6) /* waiting for last
+ * AXI-write to ack
+ */
+#define BCM2835_DMA_ERR BIT(8)
+#define BCM2835_DMA_PRIORITY(x) ((x & 15) << 16) /* AXI priority */
+#define BCM2835_DMA_PANIC_PRIORITY(x) ((x & 15) << 20) /* panic priority */
+/* current value of TI.BCM2835_DMA_WAIT_RESP */
+#define BCM2835_DMA_WAIT_FOR_WRITES BIT(28)
+#define BCM2835_DMA_DIS_DEBUG BIT(29) /* disable debug pause signal */
#define BCM2835_DMA_ABORT BIT(30) /* Stop current CB, go to next, WO */
#define BCM2835_DMA_RESET BIT(31) /* WO, self clearing */
+/* Transfer information bits - also bcm2835_cb.info field */
#define BCM2835_DMA_INT_EN BIT(0)
+#define BCM2835_DMA_TDMODE BIT(1) /* 2D-Mode */
+#define BCM2835_DMA_WAIT_RESP BIT(3) /* wait for AXI-write to be acked */
#define BCM2835_DMA_D_INC BIT(4)
-#define BCM2835_DMA_D_DREQ BIT(6)
+#define BCM2835_DMA_D_WIDTH BIT(5) /* 128bit writes if set */
+#define BCM2835_DMA_D_DREQ BIT(6) /* enable DREQ for destination */
+#define BCM2835_DMA_D_IGNORE BIT(7) /* ignore destination writes */
#define BCM2835_DMA_S_INC BIT(8)
-#define BCM2835_DMA_S_DREQ BIT(10)
-
-#define BCM2835_DMA_PER_MAP(x) ((x) << 16)
+#define BCM2835_DMA_S_WIDTH BIT(9) /* 128bit writes if set */
+#define BCM2835_DMA_S_DREQ BIT(10) /* enable SREQ for source */
+#define BCM2835_DMA_S_IGNORE BIT(11) /* ignore source reads - read 0 */
+#define BCM2835_DMA_BURST_LENGTH(x) ((x & 15) << 12)
+#define BCM2835_DMA_PER_MAP(x) ((x & 31) << 16) /* REQ source */
+#define BCM2835_DMA_WAIT(x) ((x & 31) << 21) /* add DMA-wait cycles */
+#define BCM2835_DMA_NO_WIDE_BURSTS BIT(26) /* no 2 beat write bursts */
+
+/* debug register bits */
+#define BCM2835_DMA_DEBUG_LAST_NOT_SET_ERR BIT(0)
+#define BCM2835_DMA_DEBUG_FIFO_ERR BIT(1)
+#define BCM2835_DMA_DEBUG_READ_ERR BIT(2)
+#define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_SHIFT 4
+#define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_BITS 4
+#define BCM2835_DMA_DEBUG_ID_SHIFT 16
+#define BCM2835_DMA_DEBUG_ID_BITS 9
+#define BCM2835_DMA_DEBUG_STATE_SHIFT 16
+#define BCM2835_DMA_DEBUG_STATE_BITS 9
+#define BCM2835_DMA_DEBUG_VERSION_SHIFT 25
+#define BCM2835_DMA_DEBUG_VERSION_BITS 3
+#define BCM2835_DMA_DEBUG_LITE BIT(28)
+
+/* shared registers for all dma channels */
+#define BCM2835_DMA_INT_STATUS 0xfe0
+#define BCM2835_DMA_ENABLE 0xff0
#define BCM2835_DMA_DATA_TYPE_S8 1
#define BCM2835_DMA_DATA_TYPE_S16 2
#define BCM2835_DMA_DATA_TYPE_S32 4
#define BCM2835_DMA_DATA_TYPE_S128 16
-#define BCM2835_DMA_BULK_MASK BIT(0)
-#define BCM2835_DMA_FIQ_MASK (BIT(2) | BIT(3))
-
/* Valid only for channels 0 - 14, 15 has its own base address */
#define BCM2835_DMA_CHAN(n) ((n) << 8) /* Base address */
#define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n))
+/* the max dma length for different channels */
+#define MAX_DMA_LEN SZ_1G
+#define MAX_LITE_DMA_LEN (SZ_64K - 4)
+
+static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c)
+{
+ /* lite and normal channels have different max frame length */
+ return c->is_lite_channel ? MAX_LITE_DMA_LEN : MAX_DMA_LEN;
+}
+
+/* how many frames of max_len size do we need to transfer len bytes */
+static inline size_t bcm2835_dma_frames_for_length(size_t len,
+ size_t max_len)
+{
+ return DIV_ROUND_UP(len, max_len);
+}
+
static inline struct bcm2835_dmadev *to_bcm2835_dma_dev(struct dma_device *d)
{
return container_of(d, struct bcm2835_dmadev, ddev);
@@ -146,19 +208,209 @@ static inline struct bcm2835_desc *to_bcm2835_dma_desc(
return container_of(t, struct bcm2835_desc, vd.tx);
}
-static void bcm2835_dma_desc_free(struct virt_dma_desc *vd)
+static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc)
{
- struct bcm2835_desc *desc = container_of(vd, struct bcm2835_desc, vd);
- int i;
+ size_t i;
for (i = 0; i < desc->frames; i++)
dma_pool_free(desc->c->cb_pool, desc->cb_list[i].cb,
desc->cb_list[i].paddr);
- kfree(desc->cb_list);
kfree(desc);
}
+static void bcm2835_dma_desc_free(struct virt_dma_desc *vd)
+{
+ bcm2835_dma_free_cb_chain(
+ container_of(vd, struct bcm2835_desc, vd));
+}
+
+static void bcm2835_dma_create_cb_set_length(
+ struct bcm2835_chan *chan,
+ struct bcm2835_dma_cb *control_block,
+ size_t len,
+ size_t period_len,
+ size_t *total_len,
+ u32 finalextrainfo)
+{
+ size_t max_len = bcm2835_dma_max_frame_length(chan);
+
+ /* set the length taking lite-channel limitations into account */
+ control_block->length = min_t(u32, len, max_len);
+
+ /* finished if we have no period_length */
+ if (!period_len)
+ return;
+
+ /*
+ * period_len means: that we need to generate
+ * transfers that are terminating at every
+ * multiple of period_len - this is typically
+ * used to set the interrupt flag in info
+ * which is required during cyclic transfers
+ */
+
+ /* have we filled in period_length yet? */
+ if (*total_len + control_block->length < period_len)
+ return;
+
+ /* calculate the length that remains to reach period_length */
+ control_block->length = period_len - *total_len;
+
+ /* reset total_length for next period */
+ *total_len = 0;
+
+ /* add extrainfo bits in info */
+ control_block->info |= finalextrainfo;
+}
+
+static inline size_t bcm2835_dma_count_frames_for_sg(
+ struct bcm2835_chan *c,
+ struct scatterlist *sgl,
+ unsigned int sg_len)
+{
+ size_t frames = 0;
+ struct scatterlist *sgent;
+ unsigned int i;
+ size_t plength = bcm2835_dma_max_frame_length(c);
+
+ for_each_sg(sgl, sgent, sg_len, i)
+ frames += bcm2835_dma_frames_for_length(
+ sg_dma_len(sgent), plength);
+
+ return frames;
+}
+
+/**
+ * bcm2835_dma_create_cb_chain - create a control block and fills data in
+ *
+ * @chan: the @dma_chan for which we run this
+ * @direction: the direction in which we transfer
+ * @cyclic: it is a cyclic transfer
+ * @info: the default info bits to apply per controlblock
+ * @frames: number of controlblocks to allocate
+ * @src: the src address to assign (if the S_INC bit is set
+ * in @info, then it gets incremented)
+ * @dst: the dst address to assign (if the D_INC bit is set
+ * in @info, then it gets incremented)
+ * @buf_len: the full buffer length (may also be 0)
+ * @period_len: the period length when to apply @finalextrainfo
+ * in addition to the last transfer
+ * this will also break some control-blocks early
+ * @finalextrainfo: additional bits in last controlblock
+ * (or when period_len is reached in case of cyclic)
+ * @gfp: the GFP flag to use for allocation
+ */
+static struct bcm2835_desc *bcm2835_dma_create_cb_chain(
+ struct dma_chan *chan, enum dma_transfer_direction direction,
+ bool cyclic, u32 info, u32 finalextrainfo, size_t frames,
+ dma_addr_t src, dma_addr_t dst, size_t buf_len,
+ size_t period_len, gfp_t gfp)
+{
+ struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
+ size_t len = buf_len, total_len;
+ size_t frame;
+ struct bcm2835_desc *d;
+ struct bcm2835_cb_entry *cb_entry;
+ struct bcm2835_dma_cb *control_block;
+
+ if (!frames)
+ return NULL;
+
+ /* allocate and setup the descriptor. */
+ d = kzalloc(sizeof(*d) + frames * sizeof(struct bcm2835_cb_entry),
+ gfp);
+ if (!d)
+ return NULL;
+
+ d->c = c;
+ d->dir = direction;
+ d->cyclic = cyclic;
+
+ /*
+ * Iterate over all frames, create a control block
+ * for each frame and link them together.
+ */
+ for (frame = 0, total_len = 0; frame < frames; d->frames++, frame++) {
+ cb_entry = &d->cb_list[frame];
+ cb_entry->cb = dma_pool_alloc(c->cb_pool, gfp,
+ &cb_entry->paddr);
+ if (!cb_entry->cb)
+ goto error_cb;
+
+ /* fill in the control block */
+ control_block = cb_entry->cb;
+ control_block->info = info;
+ control_block->src = src;
+ control_block->dst = dst;
+ control_block->stride = 0;
+ control_block->next = 0;
+ /* set up length in control_block if requested */
+ if (buf_len) {
+ /* calculate length honoring period_length */
+ bcm2835_dma_create_cb_set_length(
+ c, control_block,
+ len, period_len, &total_len,
+ cyclic ? finalextrainfo : 0);
+
+ /* calculate new remaining length */
+ len -= control_block->length;
+ }
+
+ /* link this the last controlblock */
+ if (frame)
+ d->cb_list[frame - 1].cb->next = cb_entry->paddr;
+
+ /* update src and dst and length */
+ if (src && (info & BCM2835_DMA_S_INC))
+ src += control_block->length;
+ if (dst && (info & BCM2835_DMA_D_INC))
+ dst += control_block->length;
+
+ /* Length of total transfer */
+ d->size += control_block->length;
+ }
+
+ /* the last frame requires extra flags */
+ d->cb_list[d->frames - 1].cb->info |= finalextrainfo;
+
+ /* detect a size missmatch */
+ if (buf_len && (d->size != buf_len))
+ goto error_cb;
+
+ return d;
+error_cb:
+ bcm2835_dma_free_cb_chain(d);
+
+ return NULL;
+}
+
+static void bcm2835_dma_fill_cb_chain_with_sg(
+ struct dma_chan *chan,
+ enum dma_transfer_direction direction,
+ struct bcm2835_cb_entry *cb,
+ struct scatterlist *sgl,
+ unsigned int sg_len)
+{
+ struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
+ size_t max_len = bcm2835_dma_max_frame_length(c);
+ unsigned int i, len;
+ dma_addr_t addr;
+ struct scatterlist *sgent;
+
+ for_each_sg(sgl, sgent, sg_len, i) {
+ for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent);
+ len > 0;
+ addr += cb->cb->length, len -= cb->cb->length, cb++) {
+ if (direction == DMA_DEV_TO_MEM)
+ cb->cb->dst = addr;
+ else
+ cb->cb->src = addr;
+ cb->cb->length = min(len, max_len);
+ }
+ }
+}
+
static int bcm2835_dma_abort(void __iomem *chan_base)
{
unsigned long cs;
@@ -218,6 +470,15 @@ static irqreturn_t bcm2835_dma_callback(int irq, void *data)
struct bcm2835_desc *d;
unsigned long flags;
+ /* check the shared interrupt */
+ if (c->irq_flags & IRQF_SHARED) {
+ /* check if the interrupt is enabled */
+ flags = readl(c->chan_base + BCM2835_DMA_CS);
+ /* if not set then we are not the reason for the irq */
+ if (!(flags & BCM2835_DMA_INT))
+ return IRQ_NONE;
+ }
+
spin_lock_irqsave(&c->vc.lock, flags);
/* Acknowledge interrupt */
@@ -226,12 +487,18 @@ static irqreturn_t bcm2835_dma_callback(int irq, void *data)
d = c->desc;
if (d) {
- /* TODO Only works for cyclic DMA */
- vchan_cyclic_callback(&d->vd);
- }
+ if (d->cyclic) {
+ /* call the cyclic callback */
+ vchan_cyclic_callback(&d->vd);
- /* Keep the DMA engine running */
- writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS);
+ /* Keep the DMA engine running */
+ writel(BCM2835_DMA_ACTIVE,
+ c->chan_base + BCM2835_DMA_CS);
+ } else {
+ vchan_cookie_complete(&c->desc->vd);
+ bcm2835_dma_start_desc(c);
+ }
+ }
spin_unlock_irqrestore(&c->vc.lock, flags);
@@ -252,8 +519,8 @@ static int bcm2835_dma_alloc_chan_resources(struct dma_chan *chan)
return -ENOMEM;
}
- return request_irq(c->irq_number,
- bcm2835_dma_callback, 0, "DMA IRQ", c);
+ return request_irq(c->irq_number, bcm2835_dma_callback,
+ c->irq_flags, "DMA IRQ", c);
}
static void bcm2835_dma_free_chan_resources(struct dma_chan *chan)
@@ -339,8 +606,6 @@ static void bcm2835_dma_issue_pending(struct dma_chan *chan)
struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
unsigned long flags;
- c->cyclic = true; /* Nothing else is implemented */
-
spin_lock_irqsave(&c->vc.lock, flags);
if (vchan_issue_pending(&c->vc) && !c->desc)
bcm2835_dma_start_desc(c);
@@ -348,122 +613,160 @@ static void bcm2835_dma_issue_pending(struct dma_chan *chan)
spin_unlock_irqrestore(&c->vc.lock, flags);
}
-static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic(
- struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
- size_t period_len, enum dma_transfer_direction direction,
- unsigned long flags)
+struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy(
+ struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
+ size_t len, unsigned long flags)
{
struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
- enum dma_slave_buswidth dev_width;
struct bcm2835_desc *d;
- dma_addr_t dev_addr;
- unsigned int es, sync_type;
- unsigned int frame;
- int i;
+ u32 info = BCM2835_DMA_D_INC | BCM2835_DMA_S_INC;
+ u32 extra = BCM2835_DMA_INT_EN | BCM2835_DMA_WAIT_RESP;
+ size_t max_len = bcm2835_dma_max_frame_length(c);
+ size_t frames;
+
+ /* if src, dst or len is not given return with an error */
+ if (!src || !dst || !len)
+ return NULL;
+
+ /* calculate number of frames */
+ frames = bcm2835_dma_frames_for_length(len, max_len);
+
+ /* allocate the CB chain - this also fills in the pointers */
+ d = bcm2835_dma_create_cb_chain(chan, DMA_MEM_TO_MEM, false,
+ info, extra, frames,
+ src, dst, len, 0, GFP_KERNEL);
+ if (!d)
+ return NULL;
+
+ return vchan_tx_prep(&c->vc, &d->vd, flags);
+}
+
+static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg(
+ struct dma_chan *chan,
+ struct scatterlist *sgl, unsigned int sg_len,
+ enum dma_transfer_direction direction,
+ unsigned long flags, void *context)
+{
+ struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
+ struct bcm2835_desc *d;
+ dma_addr_t src = 0, dst = 0;
+ u32 info = BCM2835_DMA_WAIT_RESP;
+ u32 extra = BCM2835_DMA_INT_EN;
+ size_t frames;
- /* Grab configuration */
if (!is_slave_direction(direction)) {
- dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
+ dev_err(chan->device->dev,
+ "%s: bad direction?\n", __func__);
return NULL;
}
+ if (c->dreq != 0)
+ info |= BCM2835_DMA_PER_MAP(c->dreq);
+
if (direction == DMA_DEV_TO_MEM) {
- dev_addr = c->cfg.src_addr;
- dev_width = c->cfg.src_addr_width;
- sync_type = BCM2835_DMA_S_DREQ;
+ if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
+ return NULL;
+ src = c->cfg.src_addr;
+ info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;
} else {
- dev_addr = c->cfg.dst_addr;
- dev_width = c->cfg.dst_addr_width;
- sync_type = BCM2835_DMA_D_DREQ;
+ if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
+ return NULL;
+ dst = c->cfg.dst_addr;
+ info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;
}
- /* Bus width translates to the element size (ES) */
- switch (dev_width) {
- case DMA_SLAVE_BUSWIDTH_4_BYTES:
- es = BCM2835_DMA_DATA_TYPE_S32;
- break;
- default:
- return NULL;
- }
+ /* count frames in sg list */
+ frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len);
- /* Now allocate and setup the descriptor. */
- d = kzalloc(sizeof(*d), GFP_NOWAIT);
+ /* allocate the CB chain */
+ d = bcm2835_dma_create_cb_chain(chan, direction, false,
+ info, extra,
+ frames, src, dst, 0, 0,
+ GFP_KERNEL);
if (!d)
return NULL;
- d->c = c;
- d->dir = direction;
- d->frames = buf_len / period_len;
+ /* fill in frames with scatterlist pointers */
+ bcm2835_dma_fill_cb_chain_with_sg(chan, direction, d->cb_list,
+ sgl, sg_len);
- d->cb_list = kcalloc(d->frames, sizeof(*d->cb_list), GFP_KERNEL);
- if (!d->cb_list) {
- kfree(d);
+ return vchan_tx_prep(&c->vc, &d->vd, flags);
+}
+
+static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic(
+ struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
+ size_t period_len, enum dma_transfer_direction direction,
+ unsigned long flags)
+{
+ struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
+ struct bcm2835_desc *d;
+ dma_addr_t src, dst;
+ u32 info = BCM2835_DMA_WAIT_RESP;
+ u32 extra = BCM2835_DMA_INT_EN;
+ size_t max_len = bcm2835_dma_max_frame_length(c);
+ size_t frames;
+
+ /* Grab configuration */
+ if (!is_slave_direction(direction)) {
+ dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
return NULL;
}
- /* Allocate memory for control blocks */
- for (i = 0; i < d->frames; i++) {
- struct bcm2835_cb_entry *cb_entry = &d->cb_list[i];
- cb_entry->cb = dma_pool_zalloc(c->cb_pool, GFP_ATOMIC,
- &cb_entry->paddr);
- if (!cb_entry->cb)
- goto error_cb;
+ if (!buf_len) {
+ dev_err(chan->device->dev,
+ "%s: bad buffer length (= 0)\n", __func__);
+ return NULL;
}
/*
- * Iterate over all frames, create a control block
- * for each frame and link them together.
+ * warn if buf_len is not a multiple of period_len - this may leed
+ * to unexpected latencies for interrupts and thus audiable clicks
*/
- for (frame = 0; frame < d->frames; frame++) {
- struct bcm2835_dma_cb *control_block = d->cb_list[frame].cb;
-
- /* Setup adresses */
- if (d->dir == DMA_DEV_TO_MEM) {
- control_block->info = BCM2835_DMA_D_INC;
- control_block->src = dev_addr;
- control_block->dst = buf_addr + frame * period_len;
- } else {
- control_block->info = BCM2835_DMA_S_INC;
- control_block->src = buf_addr + frame * period_len;
- control_block->dst = dev_addr;
- }
+ if (buf_len % period_len)
+ dev_warn_once(chan->device->dev,
+ "%s: buffer_length (%zd) is not a multiple of period_len (%zd)\n",
+ __func__, buf_len, period_len);
- /* Enable interrupt */
- control_block->info |= BCM2835_DMA_INT_EN;
+ /* Setup DREQ channel */
+ if (c->dreq != 0)
+ info |= BCM2835_DMA_PER_MAP(c->dreq);
- /* Setup synchronization */
- if (sync_type != 0)
- control_block->info |= sync_type;
+ if (direction == DMA_DEV_TO_MEM) {
+ if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
+ return NULL;
+ src = c->cfg.src_addr;
+ dst = buf_addr;
+ info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;
+ } else {
+ if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)
+ return NULL;
+ dst = c->cfg.dst_addr;
+ src = buf_addr;
+ info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;
+ }
- /* Setup DREQ channel */
- if (c->dreq != 0)
- control_block->info |=
- BCM2835_DMA_PER_MAP(c->dreq);
+ /* calculate number of frames */
+ frames = /* number of periods */
+ DIV_ROUND_UP(buf_len, period_len) *
+ /* number of frames per period */
+ bcm2835_dma_frames_for_length(period_len, max_len);
- /* Length of a frame */
- control_block->length = period_len;
- d->size += control_block->length;
+ /*
+ * allocate the CB chain
+ * note that we need to use GFP_NOWAIT, as the ALSA i2s dmaengine
+ * implementation calls prep_dma_cyclic with interrupts disabled.
+ */
+ d = bcm2835_dma_create_cb_chain(chan, direction, true,
+ info, extra,
+ frames, src, dst, buf_len,
+ period_len, GFP_NOWAIT);
+ if (!d)
+ return NULL;
- /*
- * Next block is the next frame.
- * This DMA engine driver currently only supports cyclic DMA.
- * Therefore, wrap around at number of frames.
- */
- control_block->next = d->cb_list[((frame + 1) % d->frames)].paddr;
- }
+ /* wrap around into a loop */
+ d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr;
return vchan_tx_prep(&c->vc, &d->vd, flags);
-error_cb:
- i--;
- for (; i >= 0; i--) {
- struct bcm2835_cb_entry *cb_entry = &d->cb_list[i];
-
- dma_pool_free(c->cb_pool, cb_entry->cb, cb_entry->paddr);
- }
-
- kfree(d->cb_list);
- kfree(d);
- return NULL;
}
static int bcm2835_dma_slave_config(struct dma_chan *chan,
@@ -529,7 +832,8 @@ static int bcm2835_dma_terminate_all(struct dma_chan *chan)
return 0;
}
-static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id, int irq)
+static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id,
+ int irq, unsigned int irq_flags)
{
struct bcm2835_chan *c;
@@ -544,6 +848,12 @@ static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id, int irq)
c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id);
c->ch = chan_id;
c->irq_number = irq;
+ c->irq_flags = irq_flags;
+
+ /* check in DEBUG register if this is a LITE channel */
+ if (readl(c->chan_base + BCM2835_DMA_DEBUG) &
+ BCM2835_DMA_DEBUG_LITE)
+ c->is_lite_channel = true;
return 0;
}
@@ -587,9 +897,11 @@ static int bcm2835_dma_probe(struct platform_device *pdev)
struct resource *res;
void __iomem *base;
int rc;
- int i;
- int irq;
+ int i, j;
+ int irq[BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED + 1];
+ int irq_flags;
uint32_t chans_available;
+ char chan_name[BCM2835_DMA_CHAN_NAME_SIZE];
if (!pdev->dev.dma_mask)
pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
@@ -615,16 +927,22 @@ static int bcm2835_dma_probe(struct platform_device *pdev)
dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask);
dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
+ dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
+ dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources;
od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources;
od->ddev.device_tx_status = bcm2835_dma_tx_status;
od->ddev.device_issue_pending = bcm2835_dma_issue_pending;
od->ddev.device_prep_dma_cyclic = bcm2835_dma_prep_dma_cyclic;
+ od->ddev.device_prep_slave_sg = bcm2835_dma_prep_slave_sg;
+ od->ddev.device_prep_dma_memcpy = bcm2835_dma_prep_dma_memcpy;
od->ddev.device_config = bcm2835_dma_slave_config;
od->ddev.device_terminate_all = bcm2835_dma_terminate_all;
od->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
od->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
- od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+ od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
+ BIT(DMA_MEM_TO_MEM);
+ od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
od->ddev.dev = &pdev->dev;
INIT_LIST_HEAD(&od->ddev.channels);
spin_lock_init(&od->lock);
@@ -640,22 +958,48 @@ static int bcm2835_dma_probe(struct platform_device *pdev)
goto err_no_dma;
}
- /*
- * Do not use the FIQ and BULK channels,
- * because they are used by the GPU.
- */
- chans_available &= ~(BCM2835_DMA_FIQ_MASK | BCM2835_DMA_BULK_MASK);
+ /* get irqs for each channel that we support */
+ for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {
+ /* skip masked out channels */
+ if (!(chans_available & (1 << i))) {
+ irq[i] = -1;
+ continue;
+ }
- for (i = 0; i < pdev->num_resources; i++) {
- irq = platform_get_irq(pdev, i);
- if (irq < 0)
- break;
+ /* get the named irq */
+ snprintf(chan_name, sizeof(chan_name), "dma%i", i);
+ irq[i] = platform_get_irq_byname(pdev, chan_name);
+ if (irq[i] >= 0)
+ continue;
- if (chans_available & (1 << i)) {
- rc = bcm2835_dma_chan_init(od, i, irq);
- if (rc)
- goto err_no_dma;
- }
+ /* legacy device tree case handling */
+ dev_warn_once(&pdev->dev,
+ "missing interrupt-names property in device tree - legacy interpretation is used\n");
+ /*
+ * in case of channel >= 11
+ * use the 11th interrupt and that is shared
+ */
+ irq[i] = platform_get_irq(pdev, i < 11 ? i : 11);
+ }
+
+ /* get irqs for each channel */
+ for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {
+ /* skip channels without irq */
+ if (irq[i] < 0)
+ continue;
+
+ /* check if there are other channels that also use this irq */
+ irq_flags = 0;
+ for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++)
+ if ((i != j) && (irq[j] == irq[i])) {
+ irq_flags = IRQF_SHARED;
+ break;
+ }
+
+ /* initialize the channel */
+ rc = bcm2835_dma_chan_init(od, i, irq[i], irq_flags);
+ if (rc)
+ goto err_no_dma;
}
dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i);
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 0cb259c59916..8c9f45fd55fc 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -289,7 +289,7 @@ enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
do {
status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
- pr_err("%s: timeout!\n", __func__);
+ dev_err(chan->device->dev, "%s: timeout!\n", __func__);
return DMA_ERROR;
}
if (status != DMA_IN_PROGRESS)
@@ -482,7 +482,8 @@ int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
device = chan->device;
/* check if the channel supports slave transactions */
- if (!test_bit(DMA_SLAVE, device->cap_mask.bits))
+ if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) ||
+ test_bit(DMA_CYCLIC, device->cap_mask.bits)))
return -ENXIO;
/*
@@ -518,7 +519,7 @@ static struct dma_chan *private_candidate(const dma_cap_mask_t *mask,
struct dma_chan *chan;
if (mask && !__dma_device_satisfies_mask(dev, mask)) {
- pr_debug("%s: wrong capabilities\n", __func__);
+ dev_dbg(dev->dev, "%s: wrong capabilities\n", __func__);
return NULL;
}
/* devices with multiple channels need special handling as we need to
@@ -533,12 +534,12 @@ static struct dma_chan *private_candidate(const dma_cap_mask_t *mask,
list_for_each_entry(chan, &dev->channels, device_node) {
if (chan->client_count) {
- pr_debug("%s: %s busy\n",
+ dev_dbg(dev->dev, "%s: %s busy\n",
__func__, dma_chan_name(chan));
continue;
}
if (fn && !fn(chan, fn_param)) {
- pr_debug("%s: %s filter said false\n",
+ dev_dbg(dev->dev, "%s: %s filter said false\n",
__func__, dma_chan_name(chan));
continue;
}
@@ -567,11 +568,12 @@ static struct dma_chan *find_candidate(struct dma_device *device,
if (err) {
if (err == -ENODEV) {
- pr_debug("%s: %s module removed\n", __func__,
- dma_chan_name(chan));
+ dev_dbg(device->dev, "%s: %s module removed\n",
+ __func__, dma_chan_name(chan));
list_del_rcu(&device->global_node);
} else
- pr_debug("%s: failed to get %s: (%d)\n",
+ dev_dbg(device->dev,
+ "%s: failed to get %s: (%d)\n",
__func__, dma_chan_name(chan), err);
if (--device->privatecnt == 0)
@@ -602,7 +604,8 @@ struct dma_chan *dma_get_slave_channel(struct dma_chan *chan)
device->privatecnt++;
err = dma_chan_get(chan);
if (err) {
- pr_debug("%s: failed to get %s: (%d)\n",
+ dev_dbg(chan->device->dev,
+ "%s: failed to get %s: (%d)\n",
__func__, dma_chan_name(chan), err);
chan = NULL;
if (--device->privatecnt == 0)
@@ -814,8 +817,9 @@ void dmaengine_get(void)
list_del_rcu(&device->global_node);
break;
} else if (err)
- pr_debug("%s: failed to get %s: (%d)\n",
- __func__, dma_chan_name(chan), err);
+ dev_dbg(chan->device->dev,
+ "%s: failed to get %s: (%d)\n",
+ __func__, dma_chan_name(chan), err);
}
}
@@ -862,12 +866,12 @@ static bool device_has_all_tx_types(struct dma_device *device)
return false;
#endif
- #if defined(CONFIG_ASYNC_MEMCPY) || defined(CONFIG_ASYNC_MEMCPY_MODULE)
+ #if IS_ENABLED(CONFIG_ASYNC_MEMCPY)
if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))
return false;
#endif
- #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE)
+ #if IS_ENABLED(CONFIG_ASYNC_XOR)
if (!dma_has_cap(DMA_XOR, device->cap_mask))
return false;
@@ -877,7 +881,7 @@ static bool device_has_all_tx_types(struct dma_device *device)
#endif
#endif
- #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE)
+ #if IS_ENABLED(CONFIG_ASYNC_PQ)
if (!dma_has_cap(DMA_PQ, device->cap_mask))
return false;
@@ -1222,8 +1226,9 @@ dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
while (tx->cookie == -EBUSY) {
if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
- pr_err("%s timeout waiting for descriptor submission\n",
- __func__);
+ dev_err(tx->chan->device->dev,
+ "%s timeout waiting for descriptor submission\n",
+ __func__);
return DMA_ERROR;
}
cpu_relax();
diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c
index 5ad0ec1f0e29..edf053f73a49 100644
--- a/drivers/dma/dw/core.c
+++ b/drivers/dma/dw/core.c
@@ -45,22 +45,19 @@
DW_DMA_MSIZE_16; \
u8 _dmsize = _is_slave ? _sconfig->dst_maxburst : \
DW_DMA_MSIZE_16; \
+ u8 _dms = (_dwc->direction == DMA_MEM_TO_DEV) ? \
+ _dwc->p_master : _dwc->m_master; \
+ u8 _sms = (_dwc->direction == DMA_DEV_TO_MEM) ? \
+ _dwc->p_master : _dwc->m_master; \
\
(DWC_CTLL_DST_MSIZE(_dmsize) \
| DWC_CTLL_SRC_MSIZE(_smsize) \
| DWC_CTLL_LLP_D_EN \
| DWC_CTLL_LLP_S_EN \
- | DWC_CTLL_DMS(_dwc->dst_master) \
- | DWC_CTLL_SMS(_dwc->src_master)); \
+ | DWC_CTLL_DMS(_dms) \
+ | DWC_CTLL_SMS(_sms)); \
})
-/*
- * Number of descriptors to allocate for each channel. This should be
- * made configurable somehow; preferably, the clients (at least the
- * ones using slave transfers) should be able to give us a hint.
- */
-#define NR_DESCS_PER_CHANNEL 64
-
/* The set of bus widths supported by the DMA controller */
#define DW_DMA_BUSWIDTHS \
BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
@@ -80,76 +77,78 @@ static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
return to_dw_desc(dwc->active_list.next);
}
-static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
+static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
{
- struct dw_desc *desc, *_desc;
- struct dw_desc *ret = NULL;
- unsigned int i = 0;
- unsigned long flags;
+ struct dw_desc *desc = txd_to_dw_desc(tx);
+ struct dw_dma_chan *dwc = to_dw_dma_chan(tx->chan);
+ dma_cookie_t cookie;
+ unsigned long flags;
spin_lock_irqsave(&dwc->lock, flags);
- list_for_each_entry_safe(desc, _desc, &dwc->free_list, desc_node) {
- i++;
- if (async_tx_test_ack(&desc->txd)) {
- list_del(&desc->desc_node);
- ret = desc;
- break;
- }
- dev_dbg(chan2dev(&dwc->chan), "desc %p not ACKed\n", desc);
- }
+ cookie = dma_cookie_assign(tx);
+
+ /*
+ * REVISIT: We should attempt to chain as many descriptors as
+ * possible, perhaps even appending to those already submitted
+ * for DMA. But this is hard to do in a race-free manner.
+ */
+
+ list_add_tail(&desc->desc_node, &dwc->queue);
spin_unlock_irqrestore(&dwc->lock, flags);
+ dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n",
+ __func__, desc->txd.cookie);
- dev_vdbg(chan2dev(&dwc->chan), "scanned %u descriptors on freelist\n", i);
+ return cookie;
+}
- return ret;
+static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
+{
+ struct dw_dma *dw = to_dw_dma(dwc->chan.device);
+ struct dw_desc *desc;
+ dma_addr_t phys;
+
+ desc = dma_pool_zalloc(dw->desc_pool, GFP_ATOMIC, &phys);
+ if (!desc)
+ return NULL;
+
+ dwc->descs_allocated++;
+ INIT_LIST_HEAD(&desc->tx_list);
+ dma_async_tx_descriptor_init(&desc->txd, &dwc->chan);
+ desc->txd.tx_submit = dwc_tx_submit;
+ desc->txd.flags = DMA_CTRL_ACK;
+ desc->txd.phys = phys;
+ return desc;
}
-/*
- * Move a descriptor, including any children, to the free list.
- * `desc' must not be on any lists.
- */
static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
{
- unsigned long flags;
+ struct dw_dma *dw = to_dw_dma(dwc->chan.device);
+ struct dw_desc *child, *_next;
- if (desc) {
- struct dw_desc *child;
+ if (unlikely(!desc))
+ return;
- spin_lock_irqsave(&dwc->lock, flags);
- list_for_each_entry(child, &desc->tx_list, desc_node)
- dev_vdbg(chan2dev(&dwc->chan),
- "moving child desc %p to freelist\n",
- child);
- list_splice_init(&desc->tx_list, &dwc->free_list);
- dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
- list_add(&desc->desc_node, &dwc->free_list);
- spin_unlock_irqrestore(&dwc->lock, flags);
+ list_for_each_entry_safe(child, _next, &desc->tx_list, desc_node) {
+ list_del(&child->desc_node);
+ dma_pool_free(dw->desc_pool, child, child->txd.phys);
+ dwc->descs_allocated--;
}
+
+ dma_pool_free(dw->desc_pool, desc, desc->txd.phys);
+ dwc->descs_allocated--;
}
static void dwc_initialize(struct dw_dma_chan *dwc)
{
struct dw_dma *dw = to_dw_dma(dwc->chan.device);
- struct dw_dma_slave *dws = dwc->chan.private;
u32 cfghi = DWC_CFGH_FIFO_MODE;
u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority);
- if (dwc->initialized == true)
+ if (test_bit(DW_DMA_IS_INITIALIZED, &dwc->flags))
return;
- if (dws) {
- /*
- * We need controller-specific data to set up slave
- * transfers.
- */
- BUG_ON(!dws->dma_dev || dws->dma_dev != dw->dma.dev);
-
- cfghi |= DWC_CFGH_DST_PER(dws->dst_id);
- cfghi |= DWC_CFGH_SRC_PER(dws->src_id);
- } else {
- cfghi |= DWC_CFGH_DST_PER(dwc->dst_id);
- cfghi |= DWC_CFGH_SRC_PER(dwc->src_id);
- }
+ cfghi |= DWC_CFGH_DST_PER(dwc->dst_id);
+ cfghi |= DWC_CFGH_SRC_PER(dwc->src_id);
channel_writel(dwc, CFG_LO, cfglo);
channel_writel(dwc, CFG_HI, cfghi);
@@ -158,26 +157,11 @@ static void dwc_initialize(struct dw_dma_chan *dwc)
channel_set_bit(dw, MASK.XFER, dwc->mask);
channel_set_bit(dw, MASK.ERROR, dwc->mask);
- dwc->initialized = true;
+ set_bit(DW_DMA_IS_INITIALIZED, &dwc->flags);
}
/*----------------------------------------------------------------------*/
-static inline unsigned int dwc_fast_ffs(unsigned long long v)
-{
- /*
- * We can be a lot more clever here, but this should take care
- * of the most common optimization.
- */
- if (!(v & 7))
- return 3;
- else if (!(v & 3))
- return 2;
- else if (!(v & 1))
- return 1;
- return 0;
-}
-
static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc)
{
dev_err(chan2dev(&dwc->chan),
@@ -209,12 +193,12 @@ static inline void dwc_do_single_block(struct dw_dma_chan *dwc,
* Software emulation of LLP mode relies on interrupts to continue
* multi block transfer.
*/
- ctllo = desc->lli.ctllo | DWC_CTLL_INT_EN;
+ ctllo = lli_read(desc, ctllo) | DWC_CTLL_INT_EN;
- channel_writel(dwc, SAR, desc->lli.sar);
- channel_writel(dwc, DAR, desc->lli.dar);
+ channel_writel(dwc, SAR, lli_read(desc, sar));
+ channel_writel(dwc, DAR, lli_read(desc, dar));
channel_writel(dwc, CTL_LO, ctllo);
- channel_writel(dwc, CTL_HI, desc->lli.ctlhi);
+ channel_writel(dwc, CTL_HI, lli_read(desc, ctlhi));
channel_set_bit(dw, CH_EN, dwc->mask);
/* Move pointer to next descriptor */
@@ -225,6 +209,7 @@ static inline void dwc_do_single_block(struct dw_dma_chan *dwc,
static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
{
struct dw_dma *dw = to_dw_dma(dwc->chan.device);
+ u8 lms = DWC_LLP_LMS(dwc->m_master);
unsigned long was_soft_llp;
/* ASSERT: channel is idle */
@@ -249,7 +234,7 @@ static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
dwc_initialize(dwc);
- dwc->residue = first->total_len;
+ first->residue = first->total_len;
dwc->tx_node_active = &first->tx_list;
/* Submit first block */
@@ -260,9 +245,8 @@ static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
dwc_initialize(dwc);
- channel_writel(dwc, LLP, first->txd.phys);
- channel_writel(dwc, CTL_LO,
- DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
+ channel_writel(dwc, LLP, first->txd.phys | lms);
+ channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
channel_writel(dwc, CTL_HI, 0);
channel_set_bit(dw, CH_EN, dwc->mask);
}
@@ -305,11 +289,7 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
list_for_each_entry(child, &desc->tx_list, desc_node)
async_tx_ack(&child->txd);
async_tx_ack(&desc->txd);
-
- list_splice_init(&desc->tx_list, &dwc->free_list);
- list_move(&desc->desc_node, &dwc->free_list);
-
- dma_descriptor_unmap(txd);
+ dwc_desc_put(dwc, desc);
spin_unlock_irqrestore(&dwc->lock, flags);
if (callback)
@@ -380,11 +360,11 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
head = &desc->tx_list;
if (active != head) {
- /* Update desc to reflect last sent one */
- if (active != head->next)
- desc = to_dw_desc(active->prev);
-
- dwc->residue -= desc->len;
+ /* Update residue to reflect last sent descriptor */
+ if (active == head->next)
+ desc->residue -= desc->len;
+ else
+ desc->residue -= to_dw_desc(active->prev)->len;
child = to_dw_desc(active);
@@ -399,8 +379,6 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
}
- dwc->residue = 0;
-
spin_unlock_irqrestore(&dwc->lock, flags);
dwc_complete_all(dw, dwc);
@@ -408,7 +386,6 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
}
if (list_empty(&dwc->active_list)) {
- dwc->residue = 0;
spin_unlock_irqrestore(&dwc->lock, flags);
return;
}
@@ -423,31 +400,31 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
/* Initial residue value */
- dwc->residue = desc->total_len;
+ desc->residue = desc->total_len;
/* Check first descriptors addr */
- if (desc->txd.phys == llp) {
+ if (desc->txd.phys == DWC_LLP_LOC(llp)) {
spin_unlock_irqrestore(&dwc->lock, flags);
return;
}
/* Check first descriptors llp */
- if (desc->lli.llp == llp) {
+ if (lli_read(desc, llp) == llp) {
/* This one is currently in progress */
- dwc->residue -= dwc_get_sent(dwc);
+ desc->residue -= dwc_get_sent(dwc);
spin_unlock_irqrestore(&dwc->lock, flags);
return;
}
- dwc->residue -= desc->len;
+ desc->residue -= desc->len;
list_for_each_entry(child, &desc->tx_list, desc_node) {
- if (child->lli.llp == llp) {
+ if (lli_read(child, llp) == llp) {
/* Currently in progress */
- dwc->residue -= dwc_get_sent(dwc);
+ desc->residue -= dwc_get_sent(dwc);
spin_unlock_irqrestore(&dwc->lock, flags);
return;
}
- dwc->residue -= child->len;
+ desc->residue -= child->len;
}
/*
@@ -469,10 +446,14 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
spin_unlock_irqrestore(&dwc->lock, flags);
}
-static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_lli *lli)
+static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_desc *desc)
{
dev_crit(chan2dev(&dwc->chan), " desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
- lli->sar, lli->dar, lli->llp, lli->ctlhi, lli->ctllo);
+ lli_read(desc, sar),
+ lli_read(desc, dar),
+ lli_read(desc, llp),
+ lli_read(desc, ctlhi),
+ lli_read(desc, ctllo));
}
static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
@@ -508,9 +489,9 @@ static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
*/
dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n"
" cookie: %d\n", bad_desc->txd.cookie);
- dwc_dump_lli(dwc, &bad_desc->lli);
+ dwc_dump_lli(dwc, bad_desc);
list_for_each_entry(child, &bad_desc->tx_list, desc_node)
- dwc_dump_lli(dwc, &child->lli);
+ dwc_dump_lli(dwc, child);
spin_unlock_irqrestore(&dwc->lock, flags);
@@ -561,7 +542,7 @@ static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
*/
if (unlikely(status_err & dwc->mask) ||
unlikely(status_xfer & dwc->mask)) {
- int i;
+ unsigned int i;
dev_err(chan2dev(&dwc->chan),
"cyclic DMA unexpected %s interrupt, stopping DMA transfer\n",
@@ -583,7 +564,7 @@ static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
dma_writel(dw, CLEAR.XFER, dwc->mask);
for (i = 0; i < dwc->cdesc->periods; i++)
- dwc_dump_lli(dwc, &dwc->cdesc->desc[i]->lli);
+ dwc_dump_lli(dwc, dwc->cdesc->desc[i]);
spin_unlock_irqrestore(&dwc->lock, flags);
}
@@ -601,7 +582,7 @@ static void dw_dma_tasklet(unsigned long data)
u32 status_block;
u32 status_xfer;
u32 status_err;
- int i;
+ unsigned int i;
status_block = dma_readl(dw, RAW.BLOCK);
status_xfer = dma_readl(dw, RAW.XFER);
@@ -670,30 +651,6 @@ static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
/*----------------------------------------------------------------------*/
-static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
-{
- struct dw_desc *desc = txd_to_dw_desc(tx);
- struct dw_dma_chan *dwc = to_dw_dma_chan(tx->chan);
- dma_cookie_t cookie;
- unsigned long flags;
-
- spin_lock_irqsave(&dwc->lock, flags);
- cookie = dma_cookie_assign(tx);
-
- /*
- * REVISIT: We should attempt to chain as many descriptors as
- * possible, perhaps even appending to those already submitted
- * for DMA. But this is hard to do in a race-free manner.
- */
-
- dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n", __func__, desc->txd.cookie);
- list_add_tail(&desc->desc_node, &dwc->queue);
-
- spin_unlock_irqrestore(&dwc->lock, flags);
-
- return cookie;
-}
-
static struct dma_async_tx_descriptor *
dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
size_t len, unsigned long flags)
@@ -705,10 +662,12 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
struct dw_desc *prev;
size_t xfer_count;
size_t offset;
+ u8 m_master = dwc->m_master;
unsigned int src_width;
unsigned int dst_width;
- unsigned int data_width;
+ unsigned int data_width = dw->pdata->data_width[m_master];
u32 ctllo;
+ u8 lms = DWC_LLP_LMS(m_master);
dev_vdbg(chan2dev(chan),
"%s: d%pad s%pad l0x%zx f0x%lx\n", __func__,
@@ -721,11 +680,7 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
dwc->direction = DMA_MEM_TO_MEM;
- data_width = min_t(unsigned int, dw->data_width[dwc->src_master],
- dw->data_width[dwc->dst_master]);
-
- src_width = dst_width = min_t(unsigned int, data_width,
- dwc_fast_ffs(src | dest | len));
+ src_width = dst_width = __ffs(data_width | src | dest | len);
ctllo = DWC_DEFAULT_CTLLO(chan)
| DWC_CTLL_DST_WIDTH(dst_width)
@@ -743,27 +698,27 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
if (!desc)
goto err_desc_get;
- desc->lli.sar = src + offset;
- desc->lli.dar = dest + offset;
- desc->lli.ctllo = ctllo;
- desc->lli.ctlhi = xfer_count;
+ lli_write(desc, sar, src + offset);
+ lli_write(desc, dar, dest + offset);
+ lli_write(desc, ctllo, ctllo);
+ lli_write(desc, ctlhi, xfer_count);
desc->len = xfer_count << src_width;
if (!first) {
first = desc;
} else {
- prev->lli.llp = desc->txd.phys;
- list_add_tail(&desc->desc_node,
- &first->tx_list);
+ lli_write(prev, llp, desc->txd.phys | lms);
+ list_add_tail(&desc->desc_node, &first->tx_list);
}
prev = desc;
}
if (flags & DMA_PREP_INTERRUPT)
/* Trigger interrupt after last block */
- prev->lli.ctllo |= DWC_CTLL_INT_EN;
+ lli_set(prev, ctllo, DWC_CTLL_INT_EN);
prev->lli.llp = 0;
+ lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
first->txd.flags = flags;
first->total_len = len;
@@ -785,10 +740,12 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
struct dw_desc *prev;
struct dw_desc *first;
u32 ctllo;
+ u8 m_master = dwc->m_master;
+ u8 lms = DWC_LLP_LMS(m_master);
dma_addr_t reg;
unsigned int reg_width;
unsigned int mem_width;
- unsigned int data_width;
+ unsigned int data_width = dw->pdata->data_width[m_master];
unsigned int i;
struct scatterlist *sg;
size_t total_len = 0;
@@ -814,8 +771,6 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
DWC_CTLL_FC(DW_DMA_FC_D_M2P);
- data_width = dw->data_width[dwc->src_master];
-
for_each_sg(sgl, sg, sg_len, i) {
struct dw_desc *desc;
u32 len, dlen, mem;
@@ -823,17 +778,16 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
mem = sg_dma_address(sg);
len = sg_dma_len(sg);
- mem_width = min_t(unsigned int,
- data_width, dwc_fast_ffs(mem | len));
+ mem_width = __ffs(data_width | mem | len);
slave_sg_todev_fill_desc:
desc = dwc_desc_get(dwc);
if (!desc)
goto err_desc_get;
- desc->lli.sar = mem;
- desc->lli.dar = reg;
- desc->lli.ctllo = ctllo | DWC_CTLL_SRC_WIDTH(mem_width);
+ lli_write(desc, sar, mem);
+ lli_write(desc, dar, reg);
+ lli_write(desc, ctllo, ctllo | DWC_CTLL_SRC_WIDTH(mem_width));
if ((len >> mem_width) > dwc->block_size) {
dlen = dwc->block_size << mem_width;
mem += dlen;
@@ -843,15 +797,14 @@ slave_sg_todev_fill_desc:
len = 0;
}
- desc->lli.ctlhi = dlen >> mem_width;
+ lli_write(desc, ctlhi, dlen >> mem_width);
desc->len = dlen;
if (!first) {
first = desc;
} else {
- prev->lli.llp = desc->txd.phys;
- list_add_tail(&desc->desc_node,
- &first->tx_list);
+ lli_write(prev, llp, desc->txd.phys | lms);
+ list_add_tail(&desc->desc_node, &first->tx_list);
}
prev = desc;
total_len += dlen;
@@ -871,8 +824,6 @@ slave_sg_todev_fill_desc:
ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
DWC_CTLL_FC(DW_DMA_FC_D_P2M);
- data_width = dw->data_width[dwc->dst_master];
-
for_each_sg(sgl, sg, sg_len, i) {
struct dw_desc *desc;
u32 len, dlen, mem;
@@ -880,17 +831,16 @@ slave_sg_todev_fill_desc:
mem = sg_dma_address(sg);
len = sg_dma_len(sg);
- mem_width = min_t(unsigned int,
- data_width, dwc_fast_ffs(mem | len));
+ mem_width = __ffs(data_width | mem | len);
slave_sg_fromdev_fill_desc:
desc = dwc_desc_get(dwc);
if (!desc)
goto err_desc_get;
- desc->lli.sar = reg;
- desc->lli.dar = mem;
- desc->lli.ctllo = ctllo | DWC_CTLL_DST_WIDTH(mem_width);
+ lli_write(desc, sar, reg);
+ lli_write(desc, dar, mem);
+ lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width));
if ((len >> reg_width) > dwc->block_size) {
dlen = dwc->block_size << reg_width;
mem += dlen;
@@ -899,15 +849,14 @@ slave_sg_fromdev_fill_desc:
dlen = len;
len = 0;
}
- desc->lli.ctlhi = dlen >> reg_width;
+ lli_write(desc, ctlhi, dlen >> reg_width);
desc->len = dlen;
if (!first) {
first = desc;
} else {
- prev->lli.llp = desc->txd.phys;
- list_add_tail(&desc->desc_node,
- &first->tx_list);
+ lli_write(prev, llp, desc->txd.phys | lms);
+ list_add_tail(&desc->desc_node, &first->tx_list);
}
prev = desc;
total_len += dlen;
@@ -922,9 +871,10 @@ slave_sg_fromdev_fill_desc:
if (flags & DMA_PREP_INTERRUPT)
/* Trigger interrupt after last block */
- prev->lli.ctllo |= DWC_CTLL_INT_EN;
+ lli_set(prev, ctllo, DWC_CTLL_INT_EN);
prev->lli.llp = 0;
+ lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
first->total_len = total_len;
return &first->txd;
@@ -941,7 +891,7 @@ bool dw_dma_filter(struct dma_chan *chan, void *param)
struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
struct dw_dma_slave *dws = param;
- if (!dws || dws->dma_dev != chan->device->dev)
+ if (dws->dma_dev != chan->device->dev)
return false;
/* We have to copy data since dws can be temporary storage */
@@ -949,8 +899,8 @@ bool dw_dma_filter(struct dma_chan *chan, void *param)
dwc->src_id = dws->src_id;
dwc->dst_id = dws->dst_id;
- dwc->src_master = dws->src_master;
- dwc->dst_master = dws->dst_master;
+ dwc->m_master = dws->m_master;
+ dwc->p_master = dws->p_master;
return true;
}
@@ -1003,7 +953,7 @@ static int dwc_pause(struct dma_chan *chan)
while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY) && count--)
udelay(2);
- dwc->paused = true;
+ set_bit(DW_DMA_IS_PAUSED, &dwc->flags);
spin_unlock_irqrestore(&dwc->lock, flags);
@@ -1016,7 +966,7 @@ static inline void dwc_chan_resume(struct dw_dma_chan *dwc)
channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP);
- dwc->paused = false;
+ clear_bit(DW_DMA_IS_PAUSED, &dwc->flags);
}
static int dwc_resume(struct dma_chan *chan)
@@ -1024,12 +974,10 @@ static int dwc_resume(struct dma_chan *chan)
struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
unsigned long flags;
- if (!dwc->paused)
- return 0;
-
spin_lock_irqsave(&dwc->lock, flags);
- dwc_chan_resume(dwc);
+ if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags))
+ dwc_chan_resume(dwc);
spin_unlock_irqrestore(&dwc->lock, flags);
@@ -1065,16 +1013,37 @@ static int dwc_terminate_all(struct dma_chan *chan)
return 0;
}
-static inline u32 dwc_get_residue(struct dw_dma_chan *dwc)
+static struct dw_desc *dwc_find_desc(struct dw_dma_chan *dwc, dma_cookie_t c)
+{
+ struct dw_desc *desc;
+
+ list_for_each_entry(desc, &dwc->active_list, desc_node)
+ if (desc->txd.cookie == c)
+ return desc;
+
+ return NULL;
+}
+
+static u32 dwc_get_residue(struct dw_dma_chan *dwc, dma_cookie_t cookie)
{
+ struct dw_desc *desc;
unsigned long flags;
u32 residue;
spin_lock_irqsave(&dwc->lock, flags);
- residue = dwc->residue;
- if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags) && residue)
- residue -= dwc_get_sent(dwc);
+ desc = dwc_find_desc(dwc, cookie);
+ if (desc) {
+ if (desc == dwc_first_active(dwc)) {
+ residue = desc->residue;
+ if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags) && residue)
+ residue -= dwc_get_sent(dwc);
+ } else {
+ residue = desc->total_len;
+ }
+ } else {
+ residue = 0;
+ }
spin_unlock_irqrestore(&dwc->lock, flags);
return residue;
@@ -1095,10 +1064,12 @@ dwc_tx_status(struct dma_chan *chan,
dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
ret = dma_cookie_status(chan, cookie, txstate);
- if (ret != DMA_COMPLETE)
- dma_set_residue(txstate, dwc_get_residue(dwc));
+ if (ret == DMA_COMPLETE)
+ return ret;
- if (dwc->paused && ret == DMA_IN_PROGRESS)
+ dma_set_residue(txstate, dwc_get_residue(dwc, cookie));
+
+ if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags) && ret == DMA_IN_PROGRESS)
return DMA_PAUSED;
return ret;
@@ -1119,7 +1090,7 @@ static void dwc_issue_pending(struct dma_chan *chan)
static void dw_dma_off(struct dw_dma *dw)
{
- int i;
+ unsigned int i;
dma_writel(dw, CFG, 0);
@@ -1133,7 +1104,7 @@ static void dw_dma_off(struct dw_dma *dw)
cpu_relax();
for (i = 0; i < dw->dma.chancnt; i++)
- dw->chan[i].initialized = false;
+ clear_bit(DW_DMA_IS_INITIALIZED, &dw->chan[i].flags);
}
static void dw_dma_on(struct dw_dma *dw)
@@ -1145,9 +1116,6 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
{
struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
struct dw_dma *dw = to_dw_dma(chan->device);
- struct dw_desc *desc;
- int i;
- unsigned long flags;
dev_vdbg(chan2dev(chan), "%s\n", __func__);
@@ -1165,53 +1133,26 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
* doesn't mean what you think it means), and status writeback.
*/
+ /*
+ * We need controller-specific data to set up slave transfers.
+ */
+ if (chan->private && !dw_dma_filter(chan, chan->private)) {
+ dev_warn(chan2dev(chan), "Wrong controller-specific data\n");
+ return -EINVAL;
+ }
+
/* Enable controller here if needed */
if (!dw->in_use)
dw_dma_on(dw);
dw->in_use |= dwc->mask;
- spin_lock_irqsave(&dwc->lock, flags);
- i = dwc->descs_allocated;
- while (dwc->descs_allocated < NR_DESCS_PER_CHANNEL) {
- dma_addr_t phys;
-
- spin_unlock_irqrestore(&dwc->lock, flags);
-
- desc = dma_pool_alloc(dw->desc_pool, GFP_ATOMIC, &phys);
- if (!desc)
- goto err_desc_alloc;
-
- memset(desc, 0, sizeof(struct dw_desc));
-
- INIT_LIST_HEAD(&desc->tx_list);
- dma_async_tx_descriptor_init(&desc->txd, chan);
- desc->txd.tx_submit = dwc_tx_submit;
- desc->txd.flags = DMA_CTRL_ACK;
- desc->txd.phys = phys;
-
- dwc_desc_put(dwc, desc);
-
- spin_lock_irqsave(&dwc->lock, flags);
- i = ++dwc->descs_allocated;
- }
-
- spin_unlock_irqrestore(&dwc->lock, flags);
-
- dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);
-
- return i;
-
-err_desc_alloc:
- dev_info(chan2dev(chan), "only allocated %d descriptors\n", i);
-
- return i;
+ return 0;
}
static void dwc_free_chan_resources(struct dma_chan *chan)
{
struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
struct dw_dma *dw = to_dw_dma(chan->device);
- struct dw_desc *desc, *_desc;
unsigned long flags;
LIST_HEAD(list);
@@ -1224,9 +1165,15 @@ static void dwc_free_chan_resources(struct dma_chan *chan)
BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);
spin_lock_irqsave(&dwc->lock, flags);
- list_splice_init(&dwc->free_list, &list);
- dwc->descs_allocated = 0;
- dwc->initialized = false;
+
+ /* Clear custom channel configuration */
+ dwc->src_id = 0;
+ dwc->dst_id = 0;
+
+ dwc->m_master = 0;
+ dwc->p_master = 0;
+
+ clear_bit(DW_DMA_IS_INITIALIZED, &dwc->flags);
/* Disable interrupts */
channel_clear_bit(dw, MASK.XFER, dwc->mask);
@@ -1240,11 +1187,6 @@ static void dwc_free_chan_resources(struct dma_chan *chan)
if (!dw->in_use)
dw_dma_off(dw);
- list_for_each_entry_safe(desc, _desc, &list, desc_node) {
- dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc);
- dma_pool_free(dw->desc_pool, desc, desc->txd.phys);
- }
-
dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
}
@@ -1322,6 +1264,7 @@ struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
struct dw_cyclic_desc *retval = NULL;
struct dw_desc *desc;
struct dw_desc *last = NULL;
+ u8 lms = DWC_LLP_LMS(dwc->m_master);
unsigned long was_cyclic;
unsigned int reg_width;
unsigned int periods;
@@ -1375,9 +1318,6 @@ struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
retval = ERR_PTR(-ENOMEM);
- if (periods > NR_DESCS_PER_CHANNEL)
- goto out_err;
-
cdesc = kzalloc(sizeof(struct dw_cyclic_desc), GFP_KERNEL);
if (!cdesc)
goto out_err;
@@ -1393,50 +1333,50 @@ struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
switch (direction) {
case DMA_MEM_TO_DEV:
- desc->lli.dar = sconfig->dst_addr;
- desc->lli.sar = buf_addr + (period_len * i);
- desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
- | DWC_CTLL_DST_WIDTH(reg_width)
- | DWC_CTLL_SRC_WIDTH(reg_width)
- | DWC_CTLL_DST_FIX
- | DWC_CTLL_SRC_INC
- | DWC_CTLL_INT_EN);
-
- desc->lli.ctllo |= sconfig->device_fc ?
- DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
- DWC_CTLL_FC(DW_DMA_FC_D_M2P);
+ lli_write(desc, dar, sconfig->dst_addr);
+ lli_write(desc, sar, buf_addr + period_len * i);
+ lli_write(desc, ctllo, (DWC_DEFAULT_CTLLO(chan)
+ | DWC_CTLL_DST_WIDTH(reg_width)
+ | DWC_CTLL_SRC_WIDTH(reg_width)
+ | DWC_CTLL_DST_FIX
+ | DWC_CTLL_SRC_INC
+ | DWC_CTLL_INT_EN));
+
+ lli_set(desc, ctllo, sconfig->device_fc ?
+ DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
+ DWC_CTLL_FC(DW_DMA_FC_D_M2P));
break;
case DMA_DEV_TO_MEM:
- desc->lli.dar = buf_addr + (period_len * i);
- desc->lli.sar = sconfig->src_addr;
- desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
- | DWC_CTLL_SRC_WIDTH(reg_width)
- | DWC_CTLL_DST_WIDTH(reg_width)
- | DWC_CTLL_DST_INC
- | DWC_CTLL_SRC_FIX
- | DWC_CTLL_INT_EN);
-
- desc->lli.ctllo |= sconfig->device_fc ?
- DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
- DWC_CTLL_FC(DW_DMA_FC_D_P2M);
+ lli_write(desc, dar, buf_addr + period_len * i);
+ lli_write(desc, sar, sconfig->src_addr);
+ lli_write(desc, ctllo, (DWC_DEFAULT_CTLLO(chan)
+ | DWC_CTLL_SRC_WIDTH(reg_width)
+ | DWC_CTLL_DST_WIDTH(reg_width)
+ | DWC_CTLL_DST_INC
+ | DWC_CTLL_SRC_FIX
+ | DWC_CTLL_INT_EN));
+
+ lli_set(desc, ctllo, sconfig->device_fc ?
+ DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
+ DWC_CTLL_FC(DW_DMA_FC_D_P2M));
break;
default:
break;
}
- desc->lli.ctlhi = (period_len >> reg_width);
+ lli_write(desc, ctlhi, period_len >> reg_width);
cdesc->desc[i] = desc;
if (last)
- last->lli.llp = desc->txd.phys;
+ lli_write(last, llp, desc->txd.phys | lms);
last = desc;
}
/* Let's make a cyclic list */
- last->lli.llp = cdesc->desc[0]->txd.phys;
+ lli_write(last, llp, cdesc->desc[0]->txd.phys | lms);
dev_dbg(chan2dev(&dwc->chan),
"cyclic prepared buf %pad len %zu period %zu periods %d\n",
@@ -1467,7 +1407,7 @@ void dw_dma_cyclic_free(struct dma_chan *chan)
struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
struct dw_dma *dw = to_dw_dma(dwc->chan.device);
struct dw_cyclic_desc *cdesc = dwc->cdesc;
- int i;
+ unsigned int i;
unsigned long flags;
dev_dbg(chan2dev(&dwc->chan), "%s\n", __func__);
@@ -1491,32 +1431,38 @@ void dw_dma_cyclic_free(struct dma_chan *chan)
kfree(cdesc->desc);
kfree(cdesc);
+ dwc->cdesc = NULL;
+
clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
}
EXPORT_SYMBOL(dw_dma_cyclic_free);
/*----------------------------------------------------------------------*/
-int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata)
+int dw_dma_probe(struct dw_dma_chip *chip)
{
+ struct dw_dma_platform_data *pdata;
struct dw_dma *dw;
bool autocfg = false;
unsigned int dw_params;
- unsigned int max_blk_size = 0;
+ unsigned int i;
int err;
- int i;
dw = devm_kzalloc(chip->dev, sizeof(*dw), GFP_KERNEL);
if (!dw)
return -ENOMEM;
+ dw->pdata = devm_kzalloc(chip->dev, sizeof(*dw->pdata), GFP_KERNEL);
+ if (!dw->pdata)
+ return -ENOMEM;
+
dw->regs = chip->regs;
chip->dw = dw;
pm_runtime_get_sync(chip->dev);
- if (!pdata) {
- dw_params = dma_read_byaddr(chip->regs, DW_PARAMS);
+ if (!chip->pdata) {
+ dw_params = dma_readl(dw, DW_PARAMS);
dev_dbg(chip->dev, "DW_PARAMS: 0x%08x\n", dw_params);
autocfg = dw_params >> DW_PARAMS_EN & 1;
@@ -1525,29 +1471,31 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata)
goto err_pdata;
}
- pdata = devm_kzalloc(chip->dev, sizeof(*pdata), GFP_KERNEL);
- if (!pdata) {
- err = -ENOMEM;
- goto err_pdata;
- }
+ /* Reassign the platform data pointer */
+ pdata = dw->pdata;
/* Get hardware configuration parameters */
pdata->nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 7) + 1;
pdata->nr_masters = (dw_params >> DW_PARAMS_NR_MASTER & 3) + 1;
for (i = 0; i < pdata->nr_masters; i++) {
pdata->data_width[i] =
- (dw_params >> DW_PARAMS_DATA_WIDTH(i) & 3) + 2;
+ 4 << (dw_params >> DW_PARAMS_DATA_WIDTH(i) & 3);
}
- max_blk_size = dma_readl(dw, MAX_BLK_SIZE);
+ pdata->block_size = dma_readl(dw, MAX_BLK_SIZE);
/* Fill platform data with the default values */
pdata->is_private = true;
pdata->is_memcpy = true;
pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING;
pdata->chan_priority = CHAN_PRIORITY_ASCENDING;
- } else if (pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) {
+ } else if (chip->pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) {
err = -EINVAL;
goto err_pdata;
+ } else {
+ memcpy(dw->pdata, chip->pdata, sizeof(*dw->pdata));
+
+ /* Reassign the platform data pointer */
+ pdata = dw->pdata;
}
dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan),
@@ -1557,11 +1505,6 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata)
goto err_pdata;
}
- /* Get hardware configuration parameters */
- dw->nr_masters = pdata->nr_masters;
- for (i = 0; i < dw->nr_masters; i++)
- dw->data_width[i] = pdata->data_width[i];
-
/* Calculate all channel mask before DMA setup */
dw->all_chan_mask = (1 << pdata->nr_channels) - 1;
@@ -1608,7 +1551,6 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata)
INIT_LIST_HEAD(&dwc->active_list);
INIT_LIST_HEAD(&dwc->queue);
- INIT_LIST_HEAD(&dwc->free_list);
channel_clear_bit(dw, CH_EN, dwc->mask);
@@ -1616,11 +1558,9 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata)
/* Hardware configuration */
if (autocfg) {
- unsigned int dwc_params;
unsigned int r = DW_DMA_MAX_NR_CHANNELS - i - 1;
- void __iomem *addr = chip->regs + r * sizeof(u32);
-
- dwc_params = dma_read_byaddr(addr, DWC_PARAMS);
+ void __iomem *addr = &__dw_regs(dw)->DWC_PARAMS[r];
+ unsigned int dwc_params = dma_readl_native(addr);
dev_dbg(chip->dev, "DWC_PARAMS[%d]: 0x%08x\n", i,
dwc_params);
@@ -1631,16 +1571,15 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata)
* up to 0x0a for 4095.
*/
dwc->block_size =
- (4 << ((max_blk_size >> 4 * i) & 0xf)) - 1;
+ (4 << ((pdata->block_size >> 4 * i) & 0xf)) - 1;
dwc->nollp =
(dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0;
} else {
dwc->block_size = pdata->block_size;
/* Check if channel supports multi block transfer */
- channel_writel(dwc, LLP, 0xfffffffc);
- dwc->nollp =
- (channel_readl(dwc, LLP) & 0xfffffffc) == 0;
+ channel_writel(dwc, LLP, DWC_LLP_LOC(0xffffffff));
+ dwc->nollp = DWC_LLP_LOC(channel_readl(dwc, LLP)) == 0;
channel_writel(dwc, LLP, 0);
}
}
diff --git a/drivers/dma/dw/pci.c b/drivers/dma/dw/pci.c
index 358f9689a3f5..0ae6c3b1d34e 100644
--- a/drivers/dma/dw/pci.c
+++ b/drivers/dma/dw/pci.c
@@ -17,8 +17,8 @@
static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid)
{
+ const struct dw_dma_platform_data *pdata = (void *)pid->driver_data;
struct dw_dma_chip *chip;
- struct dw_dma_platform_data *pdata = (void *)pid->driver_data;
int ret;
ret = pcim_enable_device(pdev);
@@ -49,8 +49,9 @@ static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid)
chip->dev = &pdev->dev;
chip->regs = pcim_iomap_table(pdev)[0];
chip->irq = pdev->irq;
+ chip->pdata = pdata;
- ret = dw_dma_probe(chip, pdata);
+ ret = dw_dma_probe(chip);
if (ret)
return ret;
diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c
index 26edbe3a27ac..5bda0eb9f393 100644
--- a/drivers/dma/dw/platform.c
+++ b/drivers/dma/dw/platform.c
@@ -42,13 +42,13 @@ static struct dma_chan *dw_dma_of_xlate(struct of_phandle_args *dma_spec,
slave.src_id = dma_spec->args[0];
slave.dst_id = dma_spec->args[0];
- slave.src_master = dma_spec->args[1];
- slave.dst_master = dma_spec->args[2];
+ slave.m_master = dma_spec->args[1];
+ slave.p_master = dma_spec->args[2];
if (WARN_ON(slave.src_id >= DW_DMA_MAX_NR_REQUESTS ||
slave.dst_id >= DW_DMA_MAX_NR_REQUESTS ||
- slave.src_master >= dw->nr_masters ||
- slave.dst_master >= dw->nr_masters))
+ slave.m_master >= dw->pdata->nr_masters ||
+ slave.p_master >= dw->pdata->nr_masters))
return NULL;
dma_cap_zero(cap);
@@ -66,8 +66,8 @@ static bool dw_dma_acpi_filter(struct dma_chan *chan, void *param)
.dma_dev = dma_spec->dev,
.src_id = dma_spec->slave_id,
.dst_id = dma_spec->slave_id,
- .src_master = 1,
- .dst_master = 0,
+ .m_master = 0,
+ .p_master = 1,
};
return dw_dma_filter(chan, &slave);
@@ -103,6 +103,7 @@ dw_dma_parse_dt(struct platform_device *pdev)
struct device_node *np = pdev->dev.of_node;
struct dw_dma_platform_data *pdata;
u32 tmp, arr[DW_DMA_MAX_NR_MASTERS];
+ u32 nr_masters;
u32 nr_channels;
if (!np) {
@@ -110,6 +111,11 @@ dw_dma_parse_dt(struct platform_device *pdev)
return NULL;
}
+ if (of_property_read_u32(np, "dma-masters", &nr_masters))
+ return NULL;
+ if (nr_masters < 1 || nr_masters > DW_DMA_MAX_NR_MASTERS)
+ return NULL;
+
if (of_property_read_u32(np, "dma-channels", &nr_channels))
return NULL;
@@ -117,6 +123,7 @@ dw_dma_parse_dt(struct platform_device *pdev)
if (!pdata)
return NULL;
+ pdata->nr_masters = nr_masters;
pdata->nr_channels = nr_channels;
if (of_property_read_bool(np, "is_private"))
@@ -131,17 +138,13 @@ dw_dma_parse_dt(struct platform_device *pdev)
if (!of_property_read_u32(np, "block_size", &tmp))
pdata->block_size = tmp;
- if (!of_property_read_u32(np, "dma-masters", &tmp)) {
- if (tmp > DW_DMA_MAX_NR_MASTERS)
- return NULL;
-
- pdata->nr_masters = tmp;
- }
-
- if (!of_property_read_u32_array(np, "data_width", arr,
- pdata->nr_masters))
- for (tmp = 0; tmp < pdata->nr_masters; tmp++)
+ if (!of_property_read_u32_array(np, "data-width", arr, nr_masters)) {
+ for (tmp = 0; tmp < nr_masters; tmp++)
pdata->data_width[tmp] = arr[tmp];
+ } else if (!of_property_read_u32_array(np, "data_width", arr, nr_masters)) {
+ for (tmp = 0; tmp < nr_masters; tmp++)
+ pdata->data_width[tmp] = BIT(arr[tmp] & 0x07);
+ }
return pdata;
}
@@ -158,7 +161,7 @@ static int dw_probe(struct platform_device *pdev)
struct dw_dma_chip *chip;
struct device *dev = &pdev->dev;
struct resource *mem;
- struct dw_dma_platform_data *pdata;
+ const struct dw_dma_platform_data *pdata;
int err;
chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
@@ -183,6 +186,7 @@ static int dw_probe(struct platform_device *pdev)
pdata = dw_dma_parse_dt(pdev);
chip->dev = dev;
+ chip->pdata = pdata;
chip->clk = devm_clk_get(chip->dev, "hclk");
if (IS_ERR(chip->clk))
@@ -193,7 +197,7 @@ static int dw_probe(struct platform_device *pdev)
pm_runtime_enable(&pdev->dev);
- err = dw_dma_probe(chip, pdata);
+ err = dw_dma_probe(chip);
if (err)
goto err_dw_dma_probe;
diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h
index 0a50c18d85b8..4b7bd7834046 100644
--- a/drivers/dma/dw/regs.h
+++ b/drivers/dma/dw/regs.h
@@ -114,10 +114,6 @@ struct dw_dma_regs {
#define dma_writel_native writel
#endif
-/* To access the registers in early stage of probe */
-#define dma_read_byaddr(addr, name) \
- dma_readl_native((addr) + offsetof(struct dw_dma_regs, name))
-
/* Bitfields in DW_PARAMS */
#define DW_PARAMS_NR_CHAN 8 /* number of channels */
#define DW_PARAMS_NR_MASTER 11 /* number of AHB masters */
@@ -143,6 +139,10 @@ enum dw_dma_msize {
DW_DMA_MSIZE_256,
};
+/* Bitfields in LLP */
+#define DWC_LLP_LMS(x) ((x) & 3) /* list master select */
+#define DWC_LLP_LOC(x) ((x) & ~3) /* next lli */
+
/* Bitfields in CTL_LO */
#define DWC_CTLL_INT_EN (1 << 0) /* irqs enabled? */
#define DWC_CTLL_DST_WIDTH(n) ((n)<<1) /* bytes per element */
@@ -216,6 +216,8 @@ enum dw_dma_msize {
enum dw_dmac_flags {
DW_DMA_IS_CYCLIC = 0,
DW_DMA_IS_SOFT_LLP = 1,
+ DW_DMA_IS_PAUSED = 2,
+ DW_DMA_IS_INITIALIZED = 3,
};
struct dw_dma_chan {
@@ -224,8 +226,6 @@ struct dw_dma_chan {
u8 mask;
u8 priority;
enum dma_transfer_direction direction;
- bool paused;
- bool initialized;
/* software emulation of the LLP transfers */
struct list_head *tx_node_active;
@@ -236,8 +236,6 @@ struct dw_dma_chan {
unsigned long flags;
struct list_head active_list;
struct list_head queue;
- struct list_head free_list;
- u32 residue;
struct dw_cyclic_desc *cdesc;
unsigned int descs_allocated;
@@ -249,8 +247,8 @@ struct dw_dma_chan {
/* custom slave configuration */
u8 src_id;
u8 dst_id;
- u8 src_master;
- u8 dst_master;
+ u8 m_master;
+ u8 p_master;
/* configuration passed via .device_config */
struct dma_slave_config dma_sconfig;
@@ -283,9 +281,8 @@ struct dw_dma {
u8 all_chan_mask;
u8 in_use;
- /* hardware configuration */
- unsigned char nr_masters;
- unsigned char data_width[DW_DMA_MAX_NR_MASTERS];
+ /* platform data */
+ struct dw_dma_platform_data *pdata;
};
static inline struct dw_dma_regs __iomem *__dw_regs(struct dw_dma *dw)
@@ -308,32 +305,51 @@ static inline struct dw_dma *to_dw_dma(struct dma_device *ddev)
return container_of(ddev, struct dw_dma, dma);
}
+#ifdef CONFIG_DW_DMAC_BIG_ENDIAN_IO
+typedef __be32 __dw32;
+#else
+typedef __le32 __dw32;
+#endif
+
/* LLI == Linked List Item; a.k.a. DMA block descriptor */
struct dw_lli {
/* values that are not changed by hardware */
- u32 sar;
- u32 dar;
- u32 llp; /* chain to next lli */
- u32 ctllo;
+ __dw32 sar;
+ __dw32 dar;
+ __dw32 llp; /* chain to next lli */
+ __dw32 ctllo;
/* values that may get written back: */
- u32 ctlhi;
+ __dw32 ctlhi;
/* sstat and dstat can snapshot peripheral register state.
* silicon config may discard either or both...
*/
- u32 sstat;
- u32 dstat;
+ __dw32 sstat;
+ __dw32 dstat;
};
struct dw_desc {
/* FIRST values the hardware uses */
struct dw_lli lli;
+#ifdef CONFIG_DW_DMAC_BIG_ENDIAN_IO
+#define lli_set(d, reg, v) ((d)->lli.reg |= cpu_to_be32(v))
+#define lli_clear(d, reg, v) ((d)->lli.reg &= ~cpu_to_be32(v))
+#define lli_read(d, reg) be32_to_cpu((d)->lli.reg)
+#define lli_write(d, reg, v) ((d)->lli.reg = cpu_to_be32(v))
+#else
+#define lli_set(d, reg, v) ((d)->lli.reg |= cpu_to_le32(v))
+#define lli_clear(d, reg, v) ((d)->lli.reg &= ~cpu_to_le32(v))
+#define lli_read(d, reg) le32_to_cpu((d)->lli.reg)
+#define lli_write(d, reg, v) ((d)->lli.reg = cpu_to_le32(v))
+#endif
+
/* THEN values for driver housekeeping */
struct list_head desc_node;
struct list_head tx_list;
struct dma_async_tx_descriptor txd;
size_t len;
size_t total_len;
+ u32 residue;
};
#define to_dw_desc(h) list_entry(h, struct dw_desc, desc_node)
diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c
index ee3463e774f8..694c44e487ed 100644
--- a/drivers/dma/edma.c
+++ b/drivers/dma/edma.c
@@ -1518,8 +1518,17 @@ static irqreturn_t dma_ccerr_handler(int irq, void *data)
dev_vdbg(ecc->dev, "dma_ccerr_handler\n");
- if (!edma_error_pending(ecc))
+ if (!edma_error_pending(ecc)) {
+ /*
+ * The registers indicate no pending error event but the irq
+ * handler has been called.
+ * Ask eDMA to re-evaluate the error registers.
+ */
+ dev_err(ecc->dev, "%s: Error interrupt without error event!\n",
+ __func__);
+ edma_write(ecc, EDMA_EEVAL, 1);
return IRQ_NONE;
+ }
while (1) {
/* Event missed register(s) */
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index aac85c30c2cf..a8828ed639b3 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -462,13 +462,12 @@ static struct fsl_desc_sw *fsl_dma_alloc_descriptor(struct fsldma_chan *chan)
struct fsl_desc_sw *desc;
dma_addr_t pdesc;
- desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
+ desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
if (!desc) {
chan_dbg(chan, "out of memory for link descriptor\n");
return NULL;
}
- memset(desc, 0, sizeof(*desc));
INIT_LIST_HEAD(&desc->tx_list);
dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
desc->async_tx.tx_submit = fsl_dma_tx_submit;
diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c
index eef145edb936..59d1e7c6fd0f 100644
--- a/drivers/dma/hsu/hsu.c
+++ b/drivers/dma/hsu/hsu.c
@@ -77,8 +77,8 @@ static void hsu_dma_chan_start(struct hsu_dma_chan *hsuc)
hsu_chan_writel(hsuc, HSU_CH_MTSR, mtsr);
/* Set descriptors */
- count = (desc->nents - desc->active) % HSU_DMA_CHAN_NR_DESC;
- for (i = 0; i < count; i++) {
+ count = desc->nents - desc->active;
+ for (i = 0; i < count && i < HSU_DMA_CHAN_NR_DESC; i++) {
hsu_chan_writel(hsuc, HSU_CH_DxSAR(i), desc->sg[i].addr);
hsu_chan_writel(hsuc, HSU_CH_DxTSR(i), desc->sg[i].len);
@@ -160,7 +160,7 @@ irqreturn_t hsu_dma_irq(struct hsu_dma_chip *chip, unsigned short nr)
return IRQ_NONE;
/* Timeout IRQ, need wait some time, see Errata 2 */
- if (hsuc->direction == DMA_DEV_TO_MEM && (sr & HSU_CH_SR_DESCTO_ANY))
+ if (sr & HSU_CH_SR_DESCTO_ANY)
udelay(2);
sr &= ~HSU_CH_SR_DESCTO_ANY;
@@ -417,6 +417,8 @@ int hsu_dma_probe(struct hsu_dma_chip *chip)
hsu->dma.dev = chip->dev;
+ dma_set_max_seg_size(hsu->dma.dev, HSU_CH_DxTSR_MASK);
+
ret = dma_async_device_register(&hsu->dma);
if (ret)
return ret;
diff --git a/drivers/dma/hsu/hsu.h b/drivers/dma/hsu/hsu.h
index 578a8ee8cd05..50a9d1bda253 100644
--- a/drivers/dma/hsu/hsu.h
+++ b/drivers/dma/hsu/hsu.h
@@ -55,6 +55,10 @@
#define HSU_CH_DCR_CHEI BIT(23)
#define HSU_CH_DCR_CHTOI(x) BIT(24 + (x))
+/* Bits in HSU_CH_DxTSR */
+#define HSU_CH_DxTSR_MASK GENMASK(15, 0)
+#define HSU_CH_DxTSR_TSR(x) ((x) & HSU_CH_DxTSR_MASK)
+
struct hsu_dma_sg {
dma_addr_t addr;
unsigned int len;
diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c
index efdee1a69fc4..d406056e8892 100644
--- a/drivers/dma/ioat/init.c
+++ b/drivers/dma/ioat/init.c
@@ -690,12 +690,11 @@ static int ioat_alloc_chan_resources(struct dma_chan *c)
/* allocate a completion writeback area */
/* doing 2 32bit writes to mmio since 1 64b write doesn't work */
ioat_chan->completion =
- dma_pool_alloc(ioat_chan->ioat_dma->completion_pool,
- GFP_KERNEL, &ioat_chan->completion_dma);
+ dma_pool_zalloc(ioat_chan->ioat_dma->completion_pool,
+ GFP_KERNEL, &ioat_chan->completion_dma);
if (!ioat_chan->completion)
return -ENOMEM;
- memset(ioat_chan->completion, 0, sizeof(*ioat_chan->completion));
writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF,
ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
writel(((u64)ioat_chan->completion_dma) >> 32,
@@ -1074,6 +1073,7 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
struct ioatdma_chan *ioat_chan;
bool is_raid_device = false;
int err;
+ u16 val16;
dma = &ioat_dma->dma_dev;
dma->device_prep_dma_memcpy = ioat_dma_prep_memcpy_lock;
@@ -1173,6 +1173,17 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
if (dca)
ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base);
+ /* disable relaxed ordering */
+ err = pcie_capability_read_word(pdev, IOAT_DEVCTRL_OFFSET, &val16);
+ if (err)
+ return err;
+
+ /* clear relaxed ordering enable */
+ val16 &= ~IOAT_DEVCTRL_ROE;
+ err = pcie_capability_write_word(pdev, IOAT_DEVCTRL_OFFSET, val16);
+ if (err)
+ return err;
+
return 0;
}
diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h
index 4994a3623aee..70534981a49b 100644
--- a/drivers/dma/ioat/registers.h
+++ b/drivers/dma/ioat/registers.h
@@ -26,6 +26,13 @@
#define IOAT_PCI_CHANERR_INT_OFFSET 0x180
#define IOAT_PCI_CHANERRMASK_INT_OFFSET 0x184
+/* PCIe config registers */
+
+/* EXPCAPID + N */
+#define IOAT_DEVCTRL_OFFSET 0x8
+/* relaxed ordering enable */
+#define IOAT_DEVCTRL_ROE 0x10
+
/* MMIO Device Registers */
#define IOAT_CHANCNT_OFFSET 0x00 /* 8-bit */
diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c
index e39457f13d4d..56f1fd68b620 100644
--- a/drivers/dma/mmp_pdma.c
+++ b/drivers/dma/mmp_pdma.c
@@ -364,13 +364,12 @@ mmp_pdma_alloc_descriptor(struct mmp_pdma_chan *chan)
struct mmp_pdma_desc_sw *desc;
dma_addr_t pdesc;
- desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
+ desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
if (!desc) {
dev_err(chan->dev, "out of memory for link descriptor\n");
return NULL;
}
- memset(desc, 0, sizeof(*desc));
INIT_LIST_HEAD(&desc->tx_list);
dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan);
/* each desc has submit */
diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c
index aae76fb39adc..ccadafa51d5e 100644
--- a/drivers/dma/mpc512x_dma.c
+++ b/drivers/dma/mpc512x_dma.c
@@ -3,6 +3,7 @@
* Copyright (C) Semihalf 2009
* Copyright (C) Ilya Yanok, Emcraft Systems 2010
* Copyright (C) Alexander Popov, Promcontroller 2014
+ * Copyright (C) Mario Six, Guntermann & Drunck GmbH, 2016
*
* Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description
* (defines, structures and comments) was taken from MPC5121 DMA driver
@@ -26,18 +27,19 @@
*/
/*
- * MPC512x and MPC8308 DMA driver. It supports
- * memory to memory data transfers (tested using dmatest module) and
- * data transfers between memory and peripheral I/O memory
- * by means of slave scatter/gather with these limitations:
- * - chunked transfers (described by s/g lists with more than one item)
- * are refused as long as proper support for scatter/gather is missing;
- * - transfers on MPC8308 always start from software as this SoC appears
- * not to have external request lines for peripheral flow control;
- * - only peripheral devices with 4-byte FIFO access register are supported;
- * - minimal memory <-> I/O memory transfer chunk is 4 bytes and consequently
- * source and destination addresses must be 4-byte aligned
- * and transfer size must be aligned on (4 * maxburst) boundary;
+ * MPC512x and MPC8308 DMA driver. It supports memory to memory data transfers
+ * (tested using dmatest module) and data transfers between memory and
+ * peripheral I/O memory by means of slave scatter/gather with these
+ * limitations:
+ * - chunked transfers (described by s/g lists with more than one item) are
+ * refused as long as proper support for scatter/gather is missing
+ * - transfers on MPC8308 always start from software as this SoC does not have
+ * external request lines for peripheral flow control
+ * - memory <-> I/O memory transfer chunks of sizes of 1, 2, 4, 16 (for
+ * MPC512x), and 32 bytes are supported, and, consequently, source
+ * addresses and destination addresses must be aligned accordingly;
+ * furthermore, for MPC512x SoCs, the transfer size must be aligned on
+ * (chunk size * maxburst)
*/
#include <linux/module.h>
@@ -213,8 +215,10 @@ struct mpc_dma_chan {
/* Settings for access to peripheral FIFO */
dma_addr_t src_per_paddr;
u32 src_tcd_nunits;
+ u8 swidth;
dma_addr_t dst_per_paddr;
u32 dst_tcd_nunits;
+ u8 dwidth;
/* Lock for this structure */
spinlock_t lock;
@@ -247,6 +251,7 @@ static inline struct mpc_dma_chan *dma_chan_to_mpc_dma_chan(struct dma_chan *c)
static inline struct mpc_dma *dma_chan_to_mpc_dma(struct dma_chan *c)
{
struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(c);
+
return container_of(mchan, struct mpc_dma, channels[c->chan_id]);
}
@@ -254,9 +259,9 @@ static inline struct mpc_dma *dma_chan_to_mpc_dma(struct dma_chan *c)
* Execute all queued DMA descriptors.
*
* Following requirements must be met while calling mpc_dma_execute():
- * a) mchan->lock is acquired,
- * b) mchan->active list is empty,
- * c) mchan->queued list contains at least one entry.
+ * a) mchan->lock is acquired,
+ * b) mchan->active list is empty,
+ * c) mchan->queued list contains at least one entry.
*/
static void mpc_dma_execute(struct mpc_dma_chan *mchan)
{
@@ -446,20 +451,15 @@ static void mpc_dma_tasklet(unsigned long data)
if (es & MPC_DMA_DMAES_SAE)
dev_err(mdma->dma.dev, "- Source Address Error\n");
if (es & MPC_DMA_DMAES_SOE)
- dev_err(mdma->dma.dev, "- Source Offset"
- " Configuration Error\n");
+ dev_err(mdma->dma.dev, "- Source Offset Configuration Error\n");
if (es & MPC_DMA_DMAES_DAE)
- dev_err(mdma->dma.dev, "- Destination Address"
- " Error\n");
+ dev_err(mdma->dma.dev, "- Destination Address Error\n");
if (es & MPC_DMA_DMAES_DOE)
- dev_err(mdma->dma.dev, "- Destination Offset"
- " Configuration Error\n");
+ dev_err(mdma->dma.dev, "- Destination Offset Configuration Error\n");
if (es & MPC_DMA_DMAES_NCE)
- dev_err(mdma->dma.dev, "- NBytes/Citter"
- " Configuration Error\n");
+ dev_err(mdma->dma.dev, "- NBytes/Citter Configuration Error\n");
if (es & MPC_DMA_DMAES_SGE)
- dev_err(mdma->dma.dev, "- Scatter/Gather"
- " Configuration Error\n");
+ dev_err(mdma->dma.dev, "- Scatter/Gather Configuration Error\n");
if (es & MPC_DMA_DMAES_SBE)
dev_err(mdma->dma.dev, "- Source Bus Error\n");
if (es & MPC_DMA_DMAES_DBE)
@@ -518,8 +518,8 @@ static int mpc_dma_alloc_chan_resources(struct dma_chan *chan)
for (i = 0; i < MPC_DMA_DESCRIPTORS; i++) {
mdesc = kzalloc(sizeof(struct mpc_dma_desc), GFP_KERNEL);
if (!mdesc) {
- dev_notice(mdma->dma.dev, "Memory allocation error. "
- "Allocated only %u descriptors\n", i);
+ dev_notice(mdma->dma.dev,
+ "Memory allocation error. Allocated only %u descriptors\n", i);
break;
}
@@ -684,6 +684,15 @@ mpc_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
return &mdesc->desc;
}
+inline u8 buswidth_to_dmatsize(u8 buswidth)
+{
+ u8 res;
+
+ for (res = 0; buswidth > 1; buswidth /= 2)
+ res++;
+ return res;
+}
+
static struct dma_async_tx_descriptor *
mpc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_transfer_direction direction,
@@ -742,39 +751,54 @@ mpc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
memset(tcd, 0, sizeof(struct mpc_dma_tcd));
- if (!IS_ALIGNED(sg_dma_address(sg), 4))
- goto err_prep;
-
if (direction == DMA_DEV_TO_MEM) {
tcd->saddr = per_paddr;
tcd->daddr = sg_dma_address(sg);
+
+ if (!IS_ALIGNED(sg_dma_address(sg), mchan->dwidth))
+ goto err_prep;
+
tcd->soff = 0;
- tcd->doff = 4;
+ tcd->doff = mchan->dwidth;
} else {
tcd->saddr = sg_dma_address(sg);
tcd->daddr = per_paddr;
- tcd->soff = 4;
+
+ if (!IS_ALIGNED(sg_dma_address(sg), mchan->swidth))
+ goto err_prep;
+
+ tcd->soff = mchan->swidth;
tcd->doff = 0;
}
- tcd->ssize = MPC_DMA_TSIZE_4;
- tcd->dsize = MPC_DMA_TSIZE_4;
+ tcd->ssize = buswidth_to_dmatsize(mchan->swidth);
+ tcd->dsize = buswidth_to_dmatsize(mchan->dwidth);
- len = sg_dma_len(sg);
- tcd->nbytes = tcd_nunits * 4;
- if (!IS_ALIGNED(len, tcd->nbytes))
- goto err_prep;
+ if (mdma->is_mpc8308) {
+ tcd->nbytes = sg_dma_len(sg);
+ if (!IS_ALIGNED(tcd->nbytes, mchan->swidth))
+ goto err_prep;
- iter = len / tcd->nbytes;
- if (iter >= 1 << 15) {
- /* len is too big */
- goto err_prep;
+ /* No major loops for MPC8303 */
+ tcd->biter = 1;
+ tcd->citer = 1;
+ } else {
+ len = sg_dma_len(sg);
+ tcd->nbytes = tcd_nunits * tcd->ssize;
+ if (!IS_ALIGNED(len, tcd->nbytes))
+ goto err_prep;
+
+ iter = len / tcd->nbytes;
+ if (iter >= 1 << 15) {
+ /* len is too big */
+ goto err_prep;
+ }
+ /* citer_linkch contains the high bits of iter */
+ tcd->biter = iter & 0x1ff;
+ tcd->biter_linkch = iter >> 9;
+ tcd->citer = tcd->biter;
+ tcd->citer_linkch = tcd->biter_linkch;
}
- /* citer_linkch contains the high bits of iter */
- tcd->biter = iter & 0x1ff;
- tcd->biter_linkch = iter >> 9;
- tcd->citer = tcd->biter;
- tcd->citer_linkch = tcd->biter_linkch;
tcd->e_sg = 0;
tcd->d_req = 1;
@@ -796,40 +820,62 @@ err_prep:
return NULL;
}
+inline bool is_buswidth_valid(u8 buswidth, bool is_mpc8308)
+{
+ switch (buswidth) {
+ case 16:
+ if (is_mpc8308)
+ return false;
+ case 1:
+ case 2:
+ case 4:
+ case 32:
+ break;
+ default:
+ return false;
+ }
+
+ return true;
+}
+
static int mpc_dma_device_config(struct dma_chan *chan,
struct dma_slave_config *cfg)
{
struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
+ struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan);
unsigned long flags;
/*
* Software constraints:
- * - only transfers between a peripheral device and
- * memory are supported;
- * - only peripheral devices with 4-byte FIFO access register
- * are supported;
- * - minimal transfer chunk is 4 bytes and consequently
- * source and destination addresses must be 4-byte aligned
- * and transfer size must be aligned on (4 * maxburst)
- * boundary;
- * - during the transfer RAM address is being incremented by
- * the size of minimal transfer chunk;
- * - peripheral port's address is constant during the transfer.
+ * - only transfers between a peripheral device and memory are
+ * supported
+ * - transfer chunk sizes of 1, 2, 4, 16 (for MPC512x), and 32 bytes
+ * are supported, and, consequently, source addresses and
+ * destination addresses; must be aligned accordingly; furthermore,
+ * for MPC512x SoCs, the transfer size must be aligned on (chunk
+ * size * maxburst)
+ * - during the transfer, the RAM address is incremented by the size
+ * of transfer chunk
+ * - the peripheral port's address is constant during the transfer.
*/
- if (cfg->src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES ||
- cfg->dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES ||
- !IS_ALIGNED(cfg->src_addr, 4) ||
- !IS_ALIGNED(cfg->dst_addr, 4)) {
+ if (!IS_ALIGNED(cfg->src_addr, cfg->src_addr_width) ||
+ !IS_ALIGNED(cfg->dst_addr, cfg->dst_addr_width)) {
return -EINVAL;
}
+ if (!is_buswidth_valid(cfg->src_addr_width, mdma->is_mpc8308) ||
+ !is_buswidth_valid(cfg->dst_addr_width, mdma->is_mpc8308))
+ return -EINVAL;
+
spin_lock_irqsave(&mchan->lock, flags);
mchan->src_per_paddr = cfg->src_addr;
mchan->src_tcd_nunits = cfg->src_maxburst;
+ mchan->swidth = cfg->src_addr_width;
mchan->dst_per_paddr = cfg->dst_addr;
mchan->dst_tcd_nunits = cfg->dst_maxburst;
+ mchan->dwidth = cfg->dst_addr_width;
/* Apply defaults */
if (mchan->src_tcd_nunits == 0)
@@ -875,7 +921,6 @@ static int mpc_dma_probe(struct platform_device *op)
mdma = devm_kzalloc(dev, sizeof(struct mpc_dma), GFP_KERNEL);
if (!mdma) {
- dev_err(dev, "Memory exhausted!\n");
retval = -ENOMEM;
goto err;
}
@@ -999,7 +1044,8 @@ static int mpc_dma_probe(struct platform_device *op)
out_be32(&mdma->regs->dmaerrl, 0xFFFF);
} else {
out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_EDCG |
- MPC_DMA_DMACR_ERGA | MPC_DMA_DMACR_ERCA);
+ MPC_DMA_DMACR_ERGA |
+ MPC_DMA_DMACR_ERCA);
/* Disable hardware DMA requests */
out_be32(&mdma->regs->dmaerqh, 0);
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index 3922a5d56806..25d1dadcddd1 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -31,6 +31,12 @@
#include "dmaengine.h"
#include "mv_xor.h"
+enum mv_xor_type {
+ XOR_ORION,
+ XOR_ARMADA_38X,
+ XOR_ARMADA_37XX,
+};
+
enum mv_xor_mode {
XOR_MODE_IN_REG,
XOR_MODE_IN_DESC,
@@ -477,7 +483,7 @@ mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);
dev_dbg(mv_chan_to_devp(mv_chan),
- "%s src_cnt: %d len: %u dest %pad flags: %ld\n",
+ "%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
__func__, src_cnt, len, &dest, flags);
sw_desc = mv_chan_alloc_slot(mv_chan);
@@ -933,7 +939,7 @@ static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)
static struct mv_xor_chan *
mv_xor_channel_add(struct mv_xor_device *xordev,
struct platform_device *pdev,
- int idx, dma_cap_mask_t cap_mask, int irq, int op_in_desc)
+ int idx, dma_cap_mask_t cap_mask, int irq)
{
int ret = 0;
struct mv_xor_chan *mv_chan;
@@ -945,7 +951,10 @@ mv_xor_channel_add(struct mv_xor_device *xordev,
mv_chan->idx = idx;
mv_chan->irq = irq;
- mv_chan->op_in_desc = op_in_desc;
+ if (xordev->xor_type == XOR_ORION)
+ mv_chan->op_in_desc = XOR_MODE_IN_REG;
+ else
+ mv_chan->op_in_desc = XOR_MODE_IN_DESC;
dma_dev = &mv_chan->dmadev;
@@ -1085,6 +1094,33 @@ mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
writel(0, base + WINDOW_OVERRIDE_CTRL(1));
}
+static void
+mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev)
+{
+ void __iomem *base = xordev->xor_high_base;
+ u32 win_enable = 0;
+ int i;
+
+ for (i = 0; i < 8; i++) {
+ writel(0, base + WINDOW_BASE(i));
+ writel(0, base + WINDOW_SIZE(i));
+ if (i < 4)
+ writel(0, base + WINDOW_REMAP_HIGH(i));
+ }
+ /*
+ * For Armada3700 open default 4GB Mbus window. The dram
+ * related configuration are done at AXIS level.
+ */
+ writel(0xffff0000, base + WINDOW_SIZE(0));
+ win_enable |= 1;
+ win_enable |= 3 << 16;
+
+ writel(win_enable, base + WINDOW_BAR_ENABLE(0));
+ writel(win_enable, base + WINDOW_BAR_ENABLE(1));
+ writel(0, base + WINDOW_OVERRIDE_CTRL(0));
+ writel(0, base + WINDOW_OVERRIDE_CTRL(1));
+}
+
/*
* Since this XOR driver is basically used only for RAID5, we don't
* need to care about synchronizing ->suspend with DMA activity,
@@ -1129,6 +1165,11 @@ static int mv_xor_resume(struct platform_device *dev)
XOR_INTR_MASK(mv_chan));
}
+ if (xordev->xor_type == XOR_ARMADA_37XX) {
+ mv_xor_conf_mbus_windows_a3700(xordev);
+ return 0;
+ }
+
dram = mv_mbus_dram_info();
if (dram)
mv_xor_conf_mbus_windows(xordev, dram);
@@ -1137,8 +1178,9 @@ static int mv_xor_resume(struct platform_device *dev)
}
static const struct of_device_id mv_xor_dt_ids[] = {
- { .compatible = "marvell,orion-xor", .data = (void *)XOR_MODE_IN_REG },
- { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_MODE_IN_DESC },
+ { .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION },
+ { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X },
+ { .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX },
{},
};
@@ -1152,7 +1194,6 @@ static int mv_xor_probe(struct platform_device *pdev)
struct resource *res;
unsigned int max_engines, max_channels;
int i, ret;
- int op_in_desc;
dev_notice(&pdev->dev, "Marvell shared XOR driver\n");
@@ -1180,12 +1221,30 @@ static int mv_xor_probe(struct platform_device *pdev)
platform_set_drvdata(pdev, xordev);
+
+ /*
+ * We need to know which type of XOR device we use before
+ * setting up. In non-dt case it can only be the legacy one.
+ */
+ xordev->xor_type = XOR_ORION;
+ if (pdev->dev.of_node) {
+ const struct of_device_id *of_id =
+ of_match_device(mv_xor_dt_ids,
+ &pdev->dev);
+
+ xordev->xor_type = (uintptr_t)of_id->data;
+ }
+
/*
* (Re-)program MBUS remapping windows if we are asked to.
*/
- dram = mv_mbus_dram_info();
- if (dram)
- mv_xor_conf_mbus_windows(xordev, dram);
+ if (xordev->xor_type == XOR_ARMADA_37XX) {
+ mv_xor_conf_mbus_windows_a3700(xordev);
+ } else {
+ dram = mv_mbus_dram_info();
+ if (dram)
+ mv_xor_conf_mbus_windows(xordev, dram);
+ }
/* Not all platforms can gate the clock, so it is not
* an error if the clock does not exists.
@@ -1199,12 +1258,16 @@ static int mv_xor_probe(struct platform_device *pdev)
* order for async_tx to perform well. So we limit the number
* of engines and channels so that we take into account this
* constraint. Note that we also want to use channels from
- * separate engines when possible.
+ * separate engines when possible. For dual-CPU Armada 3700
+ * SoC with single XOR engine allow using its both channels.
*/
max_engines = num_present_cpus();
- max_channels = min_t(unsigned int,
- MV_XOR_MAX_CHANNELS,
- DIV_ROUND_UP(num_present_cpus(), 2));
+ if (xordev->xor_type == XOR_ARMADA_37XX)
+ max_channels = num_present_cpus();
+ else
+ max_channels = min_t(unsigned int,
+ MV_XOR_MAX_CHANNELS,
+ DIV_ROUND_UP(num_present_cpus(), 2));
if (mv_xor_engine_count >= max_engines)
return 0;
@@ -1212,15 +1275,11 @@ static int mv_xor_probe(struct platform_device *pdev)
if (pdev->dev.of_node) {
struct device_node *np;
int i = 0;
- const struct of_device_id *of_id =
- of_match_device(mv_xor_dt_ids,
- &pdev->dev);
for_each_child_of_node(pdev->dev.of_node, np) {
struct mv_xor_chan *chan;
dma_cap_mask_t cap_mask;
int irq;
- op_in_desc = (int)of_id->data;
if (i >= max_channels)
continue;
@@ -1237,7 +1296,7 @@ static int mv_xor_probe(struct platform_device *pdev)
}
chan = mv_xor_channel_add(xordev, pdev, i,
- cap_mask, irq, op_in_desc);
+ cap_mask, irq);
if (IS_ERR(chan)) {
ret = PTR_ERR(chan);
irq_dispose_mapping(irq);
@@ -1266,8 +1325,7 @@ static int mv_xor_probe(struct platform_device *pdev)
}
chan = mv_xor_channel_add(xordev, pdev, i,
- cd->cap_mask, irq,
- XOR_MODE_IN_REG);
+ cd->cap_mask, irq);
if (IS_ERR(chan)) {
ret = PTR_ERR(chan);
goto err_channel_add;
diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h
index c19fe30e5ae9..bf56e082e7cd 100644
--- a/drivers/dma/mv_xor.h
+++ b/drivers/dma/mv_xor.h
@@ -85,6 +85,7 @@ struct mv_xor_device {
void __iomem *xor_high_base;
struct clk *clk;
struct mv_xor_chan *channels[MV_XOR_MAX_CHANNELS];
+ int xor_type;
};
/**
diff --git a/drivers/dma/of-dma.c b/drivers/dma/of-dma.c
index 1e1f2986eba8..faae0bfe1109 100644
--- a/drivers/dma/of-dma.c
+++ b/drivers/dma/of-dma.c
@@ -240,8 +240,9 @@ struct dma_chan *of_dma_request_slave_channel(struct device_node *np,
struct of_phandle_args dma_spec;
struct of_dma *ofdma;
struct dma_chan *chan;
- int count, i;
+ int count, i, start;
int ret_no_channel = -ENODEV;
+ static atomic_t last_index;
if (!np || !name) {
pr_err("%s: not enough information provided\n", __func__);
@@ -259,8 +260,15 @@ struct dma_chan *of_dma_request_slave_channel(struct device_node *np,
return ERR_PTR(-ENODEV);
}
+ /*
+ * approximate an average distribution across multiple
+ * entries with the same name
+ */
+ start = atomic_inc_return(&last_index);
for (i = 0; i < count; i++) {
- if (of_dma_match_channel(np, name, i, &dma_spec))
+ if (of_dma_match_channel(np, name,
+ (i + start) % count,
+ &dma_spec))
continue;
mutex_lock(&of_dma_lock);
generated by cgit v1.2.3 (git 2.39.0) at 2025-02-22 20:46:14 +0300