summaryrefslogtreecommitdiff
path: root/drivers/s390/block/scm_blk.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/s390/block/scm_blk.c')
-rw-r--r--drivers/s390/block/scm_blk.c262
1 files changed, 118 insertions, 144 deletions
diff --git a/drivers/s390/block/scm_blk.c b/drivers/s390/block/scm_blk.c
index 3c2c84b72877..42018a20f2b7 100644
--- a/drivers/s390/block/scm_blk.c
+++ b/drivers/s390/block/scm_blk.c
@@ -13,6 +13,7 @@
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/blkdev.h>
+#include <linux/blk-mq.h>
#include <linux/genhd.h>
#include <linux/slab.h>
#include <linux/list.h>
@@ -42,7 +43,6 @@ static void __scm_free_rq(struct scm_request *scmrq)
struct aob_rq_header *aobrq = to_aobrq(scmrq);
free_page((unsigned long) scmrq->aob);
- __scm_free_rq_cluster(scmrq);
kfree(scmrq->request);
kfree(aobrq);
}
@@ -82,9 +82,6 @@ static int __scm_alloc_rq(void)
if (!scmrq->request)
goto free;
- if (__scm_alloc_rq_cluster(scmrq))
- goto free;
-
INIT_LIST_HEAD(&scmrq->list);
spin_lock_irq(&list_lock);
list_add(&scmrq->list, &inactive_requests);
@@ -114,13 +111,13 @@ static struct scm_request *scm_request_fetch(void)
{
struct scm_request *scmrq = NULL;
- spin_lock(&list_lock);
+ spin_lock_irq(&list_lock);
if (list_empty(&inactive_requests))
goto out;
scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
list_del(&scmrq->list);
out:
- spin_unlock(&list_lock);
+ spin_unlock_irq(&list_lock);
return scmrq;
}
@@ -234,130 +231,123 @@ static inline void scm_request_init(struct scm_blk_dev *bdev,
scmrq->error = BLK_STS_OK;
/* We don't use all msbs - place aidaws at the end of the aob page. */
scmrq->next_aidaw = (void *) &aob->msb[nr_requests_per_io];
- scm_request_cluster_init(scmrq);
}
-static void scm_ensure_queue_restart(struct scm_blk_dev *bdev)
-{
- if (atomic_read(&bdev->queued_reqs)) {
- /* Queue restart is triggered by the next interrupt. */
- return;
- }
- blk_delay_queue(bdev->rq, SCM_QUEUE_DELAY);
-}
-
-void scm_request_requeue(struct scm_request *scmrq)
+static void scm_request_requeue(struct scm_request *scmrq)
{
struct scm_blk_dev *bdev = scmrq->bdev;
int i;
- scm_release_cluster(scmrq);
for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++)
- blk_requeue_request(bdev->rq, scmrq->request[i]);
+ blk_mq_requeue_request(scmrq->request[i], false);
atomic_dec(&bdev->queued_reqs);
scm_request_done(scmrq);
- scm_ensure_queue_restart(bdev);
+ blk_mq_kick_requeue_list(bdev->rq);
}
-void scm_request_finish(struct scm_request *scmrq)
+static void scm_request_finish(struct scm_request *scmrq)
{
struct scm_blk_dev *bdev = scmrq->bdev;
int i;
- scm_release_cluster(scmrq);
- for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++)
- blk_end_request_all(scmrq->request[i], scmrq->error);
+ for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {
+ if (scmrq->error)
+ blk_mq_end_request(scmrq->request[i], scmrq->error);
+ else
+ blk_mq_complete_request(scmrq->request[i]);
+ }
atomic_dec(&bdev->queued_reqs);
scm_request_done(scmrq);
}
-static int scm_request_start(struct scm_request *scmrq)
+static void scm_request_start(struct scm_request *scmrq)
{
struct scm_blk_dev *bdev = scmrq->bdev;
- int ret;
atomic_inc(&bdev->queued_reqs);
- if (!scmrq->aob->request.msb_count) {
- scm_request_requeue(scmrq);
- return -EINVAL;
- }
-
- ret = eadm_start_aob(scmrq->aob);
- if (ret) {
+ if (eadm_start_aob(scmrq->aob)) {
SCM_LOG(5, "no subchannel");
scm_request_requeue(scmrq);
}
- return ret;
}
-static void scm_blk_request(struct request_queue *rq)
+struct scm_queue {
+ struct scm_request *scmrq;
+ spinlock_t lock;
+};
+
+static int scm_blk_request(struct blk_mq_hw_ctx *hctx,
+ const struct blk_mq_queue_data *qd)
{
- struct scm_device *scmdev = rq->queuedata;
+ struct scm_device *scmdev = hctx->queue->queuedata;
struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
- struct scm_request *scmrq = NULL;
- struct request *req;
+ struct scm_queue *sq = hctx->driver_data;
+ struct request *req = qd->rq;
+ struct scm_request *scmrq;
- while ((req = blk_peek_request(rq))) {
- if (!scm_permit_request(bdev, req))
- goto out;
+ spin_lock(&sq->lock);
+ if (!scm_permit_request(bdev, req)) {
+ spin_unlock(&sq->lock);
+ return BLK_MQ_RQ_QUEUE_BUSY;
+ }
+ scmrq = sq->scmrq;
+ if (!scmrq) {
+ scmrq = scm_request_fetch();
if (!scmrq) {
- scmrq = scm_request_fetch();
- if (!scmrq) {
- SCM_LOG(5, "no request");
- goto out;
- }
- scm_request_init(bdev, scmrq);
+ SCM_LOG(5, "no request");
+ spin_unlock(&sq->lock);
+ return BLK_MQ_RQ_QUEUE_BUSY;
}
- scm_request_set(scmrq, req);
+ scm_request_init(bdev, scmrq);
+ sq->scmrq = scmrq;
+ }
+ scm_request_set(scmrq, req);
- if (!scm_reserve_cluster(scmrq)) {
- SCM_LOG(5, "cluster busy");
- scm_request_set(scmrq, NULL);
- if (scmrq->aob->request.msb_count)
- goto out;
+ if (scm_request_prepare(scmrq)) {
+ SCM_LOG(5, "aidaw alloc failed");
+ scm_request_set(scmrq, NULL);
- scm_request_done(scmrq);
- return;
- }
+ if (scmrq->aob->request.msb_count)
+ scm_request_start(scmrq);
- if (scm_need_cluster_request(scmrq)) {
- if (scmrq->aob->request.msb_count) {
- /* Start cluster requests separately. */
- scm_request_set(scmrq, NULL);
- if (scm_request_start(scmrq))
- return;
- } else {
- atomic_inc(&bdev->queued_reqs);
- blk_start_request(req);
- scm_initiate_cluster_request(scmrq);
- }
- scmrq = NULL;
- continue;
- }
+ sq->scmrq = NULL;
+ spin_unlock(&sq->lock);
+ return BLK_MQ_RQ_QUEUE_BUSY;
+ }
+ blk_mq_start_request(req);
- if (scm_request_prepare(scmrq)) {
- SCM_LOG(5, "aidaw alloc failed");
- scm_request_set(scmrq, NULL);
- goto out;
- }
- blk_start_request(req);
+ if (qd->last || scmrq->aob->request.msb_count == nr_requests_per_io) {
+ scm_request_start(scmrq);
+ sq->scmrq = NULL;
+ }
+ spin_unlock(&sq->lock);
+ return BLK_MQ_RQ_QUEUE_OK;
+}
- if (scmrq->aob->request.msb_count < nr_requests_per_io)
- continue;
+static int scm_blk_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
+ unsigned int idx)
+{
+ struct scm_queue *qd = kzalloc(sizeof(*qd), GFP_KERNEL);
- if (scm_request_start(scmrq))
- return;
+ if (!qd)
+ return -ENOMEM;
- scmrq = NULL;
- }
-out:
- if (scmrq)
- scm_request_start(scmrq);
- else
- scm_ensure_queue_restart(bdev);
+ spin_lock_init(&qd->lock);
+ hctx->driver_data = qd;
+
+ return 0;
+}
+
+static void scm_blk_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int idx)
+{
+ struct scm_queue *qd = hctx->driver_data;
+
+ WARN_ON(qd->scmrq);
+ kfree(hctx->driver_data);
+ hctx->driver_data = NULL;
}
static void __scmrq_log_error(struct scm_request *scmrq)
@@ -377,21 +367,6 @@ static void __scmrq_log_error(struct scm_request *scmrq)
scmrq->error);
}
-void scm_blk_irq(struct scm_device *scmdev, void *data, blk_status_t error)
-{
- struct scm_request *scmrq = data;
- struct scm_blk_dev *bdev = scmrq->bdev;
-
- scmrq->error = error;
- if (error)
- __scmrq_log_error(scmrq);
-
- spin_lock(&bdev->lock);
- list_add_tail(&scmrq->list, &bdev->finished_requests);
- spin_unlock(&bdev->lock);
- tasklet_hi_schedule(&bdev->tasklet);
-}
-
static void scm_blk_handle_error(struct scm_request *scmrq)
{
struct scm_blk_dev *bdev = scmrq->bdev;
@@ -419,54 +394,46 @@ restart:
return;
requeue:
- spin_lock_irqsave(&bdev->rq_lock, flags);
scm_request_requeue(scmrq);
- spin_unlock_irqrestore(&bdev->rq_lock, flags);
}
-static void scm_blk_tasklet(struct scm_blk_dev *bdev)
+void scm_blk_irq(struct scm_device *scmdev, void *data, blk_status_t error)
{
- struct scm_request *scmrq;
- unsigned long flags;
-
- spin_lock_irqsave(&bdev->lock, flags);
- while (!list_empty(&bdev->finished_requests)) {
- scmrq = list_first_entry(&bdev->finished_requests,
- struct scm_request, list);
- list_del(&scmrq->list);
- spin_unlock_irqrestore(&bdev->lock, flags);
+ struct scm_request *scmrq = data;
- if (scmrq->error && scmrq->retries-- > 0) {
+ scmrq->error = error;
+ if (error) {
+ __scmrq_log_error(scmrq);
+ if (scmrq->retries-- > 0) {
scm_blk_handle_error(scmrq);
-
- /* Request restarted or requeued, handle next. */
- spin_lock_irqsave(&bdev->lock, flags);
- continue;
+ return;
}
+ }
- if (scm_test_cluster_request(scmrq)) {
- scm_cluster_request_irq(scmrq);
- spin_lock_irqsave(&bdev->lock, flags);
- continue;
- }
+ scm_request_finish(scmrq);
+}
- scm_request_finish(scmrq);
- spin_lock_irqsave(&bdev->lock, flags);
- }
- spin_unlock_irqrestore(&bdev->lock, flags);
- /* Look out for more requests. */
- blk_run_queue(bdev->rq);
+static void scm_blk_request_done(struct request *req)
+{
+ blk_mq_end_request(req, 0);
}
static const struct block_device_operations scm_blk_devops = {
.owner = THIS_MODULE,
};
+static const struct blk_mq_ops scm_mq_ops = {
+ .queue_rq = scm_blk_request,
+ .complete = scm_blk_request_done,
+ .init_hctx = scm_blk_init_hctx,
+ .exit_hctx = scm_blk_exit_hctx,
+};
+
int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
{
- struct request_queue *rq;
- int len, ret = -ENOMEM;
unsigned int devindex, nr_max_blk;
+ struct request_queue *rq;
+ int len, ret;
devindex = atomic_inc_return(&nr_devices) - 1;
/* scma..scmz + scmaa..scmzz */
@@ -477,18 +444,23 @@ int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
bdev->scmdev = scmdev;
bdev->state = SCM_OPER;
- spin_lock_init(&bdev->rq_lock);
spin_lock_init(&bdev->lock);
- INIT_LIST_HEAD(&bdev->finished_requests);
atomic_set(&bdev->queued_reqs, 0);
- tasklet_init(&bdev->tasklet,
- (void (*)(unsigned long)) scm_blk_tasklet,
- (unsigned long) bdev);
- rq = blk_init_queue(scm_blk_request, &bdev->rq_lock);
- if (!rq)
+ bdev->tag_set.ops = &scm_mq_ops;
+ bdev->tag_set.nr_hw_queues = nr_requests;
+ bdev->tag_set.queue_depth = nr_requests_per_io * nr_requests;
+ bdev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
+
+ ret = blk_mq_alloc_tag_set(&bdev->tag_set);
+ if (ret)
goto out;
+ rq = blk_mq_init_queue(&bdev->tag_set);
+ if (IS_ERR(rq)) {
+ ret = PTR_ERR(rq);
+ goto out_tag;
+ }
bdev->rq = rq;
nr_max_blk = min(scmdev->nr_max_block,
(unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));
@@ -498,12 +470,12 @@ int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
blk_queue_max_segments(rq, nr_max_blk);
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, rq);
- scm_blk_dev_cluster_setup(bdev);
bdev->gendisk = alloc_disk(SCM_NR_PARTS);
- if (!bdev->gendisk)
+ if (!bdev->gendisk) {
+ ret = -ENOMEM;
goto out_queue;
-
+ }
rq->queuedata = scmdev;
bdev->gendisk->private_data = scmdev;
bdev->gendisk->fops = &scm_blk_devops;
@@ -528,6 +500,8 @@ int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
out_queue:
blk_cleanup_queue(rq);
+out_tag:
+ blk_mq_free_tag_set(&bdev->tag_set);
out:
atomic_dec(&nr_devices);
return ret;
@@ -535,9 +509,9 @@ out:
void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
{
- tasklet_kill(&bdev->tasklet);
del_gendisk(bdev->gendisk);
blk_cleanup_queue(bdev->gendisk->queue);
+ blk_mq_free_tag_set(&bdev->tag_set);
put_disk(bdev->gendisk);
}
@@ -558,7 +532,7 @@ static bool __init scm_blk_params_valid(void)
if (!nr_requests_per_io || nr_requests_per_io > 64)
return false;
- return scm_cluster_size_valid();
+ return true;
}
static int __init scm_blk_init(void)