diff options
Diffstat (limited to 'drivers/dma')
| -rw-r--r-- | drivers/dma/dmaengine.c | 205 | ||||
| -rw-r--r-- | drivers/dma/dmatest.c | 2 | ||||
| -rw-r--r-- | drivers/dma/dw_dmac.c | 2 |
3 files changed, 129 insertions, 80 deletions
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index b9008932a8f3..d4d925912c47 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -74,6 +74,7 @@ static DEFINE_MUTEX(dma_list_mutex); static LIST_HEAD(dma_device_list); static LIST_HEAD(dma_client_list); +static long dmaengine_ref_count; /* --- sysfs implementation --- */ @@ -105,19 +106,8 @@ static ssize_t show_bytes_transferred(struct device *dev, struct device_attribut static ssize_t show_in_use(struct device *dev, struct device_attribute *attr, char *buf) { struct dma_chan *chan = to_dma_chan(dev); - int in_use = 0; - - if (unlikely(chan->slow_ref) && - atomic_read(&chan->refcount.refcount) > 1) - in_use = 1; - else { - if (local_read(&(per_cpu_ptr(chan->local, - get_cpu())->refcount)) > 0) - in_use = 1; - put_cpu(); - } - return sprintf(buf, "%d\n", in_use); + return sprintf(buf, "%d\n", chan->client_count); } static struct device_attribute dma_attrs[] = { @@ -155,6 +145,78 @@ __dma_chan_satisfies_mask(struct dma_chan *chan, dma_cap_mask_t *want) return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END); } +static struct module *dma_chan_to_owner(struct dma_chan *chan) +{ + return chan->device->dev->driver->owner; +} + +/** + * balance_ref_count - catch up the channel reference count + * @chan - channel to balance ->client_count versus dmaengine_ref_count + * + * balance_ref_count must be called under dma_list_mutex + */ +static void balance_ref_count(struct dma_chan *chan) +{ + struct module *owner = dma_chan_to_owner(chan); + + while (chan->client_count < dmaengine_ref_count) { + __module_get(owner); + chan->client_count++; + } +} + +/** + * dma_chan_get - try to grab a dma channel's parent driver module + * @chan - channel to grab + * + * Must be called under dma_list_mutex + */ +static int dma_chan_get(struct dma_chan *chan) +{ + int err = -ENODEV; + struct module *owner = dma_chan_to_owner(chan); + + if (chan->client_count) { + __module_get(owner); + err = 0; + } else if (try_module_get(owner)) + err = 0; + + if (err == 0) + chan->client_count++; + + /* allocate upon first client reference */ + if (chan->client_count == 1 && err == 0) { + int desc_cnt = chan->device->device_alloc_chan_resources(chan, NULL); + + if (desc_cnt < 0) { + err = desc_cnt; + chan->client_count = 0; + module_put(owner); + } else + balance_ref_count(chan); + } + + return err; +} + +/** + * dma_chan_put - drop a reference to a dma channel's parent driver module + * @chan - channel to release + * + * Must be called under dma_list_mutex + */ +static void dma_chan_put(struct dma_chan *chan) +{ + if (!chan->client_count) + return; /* this channel failed alloc_chan_resources */ + chan->client_count--; + module_put(dma_chan_to_owner(chan)); + if (chan->client_count == 0) + chan->device->device_free_chan_resources(chan); +} + /** * dma_client_chan_alloc - try to allocate channels to a client * @client: &dma_client @@ -165,7 +227,6 @@ static void dma_client_chan_alloc(struct dma_client *client) { struct dma_device *device; struct dma_chan *chan; - int desc; /* allocated descriptor count */ enum dma_state_client ack; /* Find a channel */ @@ -178,23 +239,16 @@ static void dma_client_chan_alloc(struct dma_client *client) list_for_each_entry(chan, &device->channels, device_node) { if (!dma_chan_satisfies_mask(chan, client->cap_mask)) continue; + if (!chan->client_count) + continue; + ack = client->event_callback(client, chan, + DMA_RESOURCE_AVAILABLE); - desc = chan->device->device_alloc_chan_resources( - chan, client); - if (desc >= 0) { - ack = client->event_callback(client, - chan, - DMA_RESOURCE_AVAILABLE); - - /* we are done once this client rejects - * an available resource - */ - if (ack == DMA_ACK) { - dma_chan_get(chan); - chan->client_count++; - } else if (ack == DMA_NAK) - return; - } + /* we are done once this client rejects + * an available resource + */ + if (ack == DMA_NAK) + return; } } } @@ -224,7 +278,6 @@ EXPORT_SYMBOL(dma_sync_wait); void dma_chan_cleanup(struct kref *kref) { struct dma_chan *chan = container_of(kref, struct dma_chan, refcount); - chan->device->device_free_chan_resources(chan); kref_put(&chan->device->refcount, dma_async_device_cleanup); } EXPORT_SYMBOL(dma_chan_cleanup); @@ -232,18 +285,12 @@ EXPORT_SYMBOL(dma_chan_cleanup); static void dma_chan_free_rcu(struct rcu_head *rcu) { struct dma_chan *chan = container_of(rcu, struct dma_chan, rcu); - int bias = 0x7FFFFFFF; - int i; - for_each_possible_cpu(i) - bias -= local_read(&per_cpu_ptr(chan->local, i)->refcount); - atomic_sub(bias, &chan->refcount.refcount); + kref_put(&chan->refcount, dma_chan_cleanup); } static void dma_chan_release(struct dma_chan *chan) { - atomic_add(0x7FFFFFFF, &chan->refcount.refcount); - chan->slow_ref = 1; call_rcu(&chan->rcu, dma_chan_free_rcu); } @@ -263,43 +310,36 @@ static void dma_clients_notify_available(void) } /** - * dma_chans_notify_available - tell the clients that a channel is going away - * @chan: channel on its way out - */ -static void dma_clients_notify_removed(struct dma_chan *chan) -{ - struct dma_client *client; - enum dma_state_client ack; - - mutex_lock(&dma_list_mutex); - - list_for_each_entry(client, &dma_client_list, global_node) { - ack = client->event_callback(client, chan, - DMA_RESOURCE_REMOVED); - - /* client was holding resources for this channel so - * free it - */ - if (ack == DMA_ACK) { - dma_chan_put(chan); - chan->client_count--; - } - } - - mutex_unlock(&dma_list_mutex); -} - -/** * dma_async_client_register - register a &dma_client * @client: ptr to a client structure with valid 'event_callback' and 'cap_mask' */ void dma_async_client_register(struct dma_client *client) { + struct dma_device *device, *_d; + struct dma_chan *chan; + int err; + /* validate client data */ BUG_ON(dma_has_cap(DMA_SLAVE, client->cap_mask) && !client->slave); mutex_lock(&dma_list_mutex); + dmaengine_ref_count++; + + /* try to grab channels */ + list_for_each_entry_safe(device, _d, &dma_device_list, global_node) + list_for_each_entry(chan, &device->channels, device_node) { + err = dma_chan_get(chan); + if (err == -ENODEV) { + /* module removed before we could use it */ + list_del_init(&device->global_node); + break; + } else if (err) + pr_err("dmaengine: failed to get %s: (%d)\n", + dev_name(&chan->dev), err); + } + + list_add_tail(&client->global_node, &dma_client_list); mutex_unlock(&dma_list_mutex); } @@ -315,23 +355,17 @@ void dma_async_client_unregister(struct dma_client *client) { struct dma_device *device; struct dma_chan *chan; - enum dma_state_client ack; if (!client) return; mutex_lock(&dma_list_mutex); - /* free all channels the client is holding */ + dmaengine_ref_count--; + BUG_ON(dmaengine_ref_count < 0); + /* drop channel references */ list_for_each_entry(device, &dma_device_list, global_node) - list_for_each_entry(chan, &device->channels, device_node) { - ack = client->event_callback(client, chan, - DMA_RESOURCE_REMOVED); - - if (ack == DMA_ACK) { - dma_chan_put(chan); - chan->client_count--; - } - } + list_for_each_entry(chan, &device->channels, device_node) + dma_chan_put(chan); list_del(&client->global_node); mutex_unlock(&dma_list_mutex); @@ -423,6 +457,21 @@ int dma_async_device_register(struct dma_device *device) } mutex_lock(&dma_list_mutex); + if (dmaengine_ref_count) + list_for_each_entry(chan, &device->channels, device_node) { + /* if clients are already waiting for channels we need + * to take references on their behalf + */ + if (dma_chan_get(chan) == -ENODEV) { + /* note we can only get here for the first + * channel as the remaining channels are + * guaranteed to get a reference + */ + rc = -ENODEV; + mutex_unlock(&dma_list_mutex); + goto err_out; + } + } list_add_tail(&device->global_node, &dma_device_list); mutex_unlock(&dma_list_mutex); @@ -456,7 +505,7 @@ static void dma_async_device_cleanup(struct kref *kref) } /** - * dma_async_device_unregister - unregisters DMA devices + * dma_async_device_unregister - unregister a DMA device * @device: &dma_device */ void dma_async_device_unregister(struct dma_device *device) @@ -468,7 +517,9 @@ void dma_async_device_unregister(struct dma_device *device) mutex_unlock(&dma_list_mutex); list_for_each_entry(chan, &device->channels, device_node) { - dma_clients_notify_removed(chan); + WARN_ONCE(chan->client_count, + "%s called while %d clients hold a reference\n", + __func__, chan->client_count); device_unregister(&chan->dev); dma_chan_release(chan); } diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index ed9636bfb54a..db4050884713 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -215,7 +215,6 @@ static int dmatest_func(void *data) smp_rmb(); chan = thread->chan; - dma_chan_get(chan); while (!kthread_should_stop()) { total_tests++; @@ -293,7 +292,6 @@ static int dmatest_func(void *data) } ret = 0; - dma_chan_put(chan); kfree(thread->dstbuf); err_dstbuf: kfree(thread->srcbuf); diff --git a/drivers/dma/dw_dmac.c b/drivers/dma/dw_dmac.c index 0778d99aea7c..377dafa37a20 100644 --- a/drivers/dma/dw_dmac.c +++ b/drivers/dma/dw_dmac.c @@ -773,7 +773,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan, dev_vdbg(&chan->dev, "alloc_chan_resources\n"); /* Channels doing slave DMA can only handle one client. */ - if (dwc->dws || client->slave) { + if (dwc->dws || (client && client->slave)) { if (chan->client_count) return -EBUSY; } |