diff options
| -rw-r--r-- | drivers/block/nvme.c | 167 |
1 files changed, 81 insertions, 86 deletions
diff --git a/drivers/block/nvme.c b/drivers/block/nvme.c index a17f80fa3881..4724655a6ebf 100644 --- a/drivers/block/nvme.c +++ b/drivers/block/nvme.c @@ -135,8 +135,12 @@ static inline void _nvme_check_size(void) BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); } +typedef void (*nvme_completion_fn)(struct nvme_queue *, void *, + struct nvme_completion *); + struct nvme_cmd_info { - unsigned long ctx; + nvme_completion_fn fn; + void *ctx; unsigned long timeout; }; @@ -149,7 +153,7 @@ static struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq) * alloc_cmdid() - Allocate a Command ID * @nvmeq: The queue that will be used for this command * @ctx: A pointer that will be passed to the handler - * @handler: The ID of the handler to call + * @handler: The function to call on completion * * Allocate a Command ID for a queue. The data passed in will * be passed to the completion handler. This is implemented by using @@ -160,28 +164,27 @@ static struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq) * May be called with local interrupts disabled and the q_lock held, * or with interrupts enabled and no locks held. */ -static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, int handler, - unsigned timeout) +static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, + nvme_completion_fn handler, unsigned timeout) { int depth = nvmeq->q_depth - 1; struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); int cmdid; - BUG_ON((unsigned long)ctx & 3); - do { cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth); if (cmdid >= depth) return -EBUSY; } while (test_and_set_bit(cmdid, nvmeq->cmdid_data)); - info[cmdid].ctx = (unsigned long)ctx | handler; + info[cmdid].fn = handler; + info[cmdid].ctx = ctx; info[cmdid].timeout = jiffies + timeout; return cmdid; } static int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, - int handler, unsigned timeout) + nvme_completion_fn handler, unsigned timeout) { int cmdid; wait_event_killable(nvmeq->sq_full, @@ -189,47 +192,69 @@ static int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx, return (cmdid < 0) ? -EINTR : cmdid; } -/* - * If you need more than four handlers, you'll need to change how - * alloc_cmdid and nvme_process_cq work. Consider using a special - * CMD_CTX value instead, if that works for your situation. - */ -enum { - sync_completion_id = 0, - bio_completion_id, -}; - -/* Special values must be a multiple of 4, and less than 0x1000 */ -#define CMD_CTX_BASE (POISON_POINTER_DELTA + sync_completion_id) +/* Special values must be less than 0x1000 */ +#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA) #define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE) #define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE) #define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE) #define CMD_CTX_FLUSH (0x318 + CMD_CTX_BASE) +static void special_completion(struct nvme_queue *nvmeq, void *ctx, + struct nvme_completion *cqe) +{ + if (ctx == CMD_CTX_CANCELLED) + return; + if (ctx == CMD_CTX_FLUSH) + return; + if (ctx == CMD_CTX_COMPLETED) { + dev_warn(nvmeq->q_dmadev, + "completed id %d twice on queue %d\n", + cqe->command_id, le16_to_cpup(&cqe->sq_id)); + return; + } + if (ctx == CMD_CTX_INVALID) { + dev_warn(nvmeq->q_dmadev, + "invalid id %d completed on queue %d\n", + cqe->command_id, le16_to_cpup(&cqe->sq_id)); + return; + } + + dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx); +} + /* * Called with local interrupts disabled and the q_lock held. May not sleep. */ -static unsigned long free_cmdid(struct nvme_queue *nvmeq, int cmdid) +static void *free_cmdid(struct nvme_queue *nvmeq, int cmdid, + nvme_completion_fn *fn) { - unsigned long data; + void *ctx; struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); - if (cmdid >= nvmeq->q_depth) + if (cmdid >= nvmeq->q_depth) { + *fn = special_completion; return CMD_CTX_INVALID; - data = info[cmdid].ctx; + } + *fn = info[cmdid].fn; + ctx = info[cmdid].ctx; + info[cmdid].fn = special_completion; info[cmdid].ctx = CMD_CTX_COMPLETED; clear_bit(cmdid, nvmeq->cmdid_data); wake_up(&nvmeq->sq_full); - return data; + return ctx; } -static unsigned long cancel_cmdid(struct nvme_queue *nvmeq, int cmdid) +static void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid, + nvme_completion_fn *fn) { - unsigned long data; + void *ctx; struct nvme_cmd_info *info = nvme_cmd_info(nvmeq); - data = info[cmdid].ctx; + if (fn) + *fn = info[cmdid].fn; + ctx = info[cmdid].ctx; + info[cmdid].fn = special_completion; info[cmdid].ctx = CMD_CTX_CANCELLED; - return data; + return ctx; } static struct nvme_queue *get_nvmeq(struct nvme_ns *ns) @@ -485,7 +510,7 @@ static int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns, static int nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns) { int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH, - sync_completion_id, IO_TIMEOUT); + special_completion, IO_TIMEOUT); if (unlikely(cmdid < 0)) return cmdid; @@ -518,7 +543,7 @@ static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, nbio->bio = bio; result = -EBUSY; - cmdid = alloc_cmdid(nvmeq, nbio, bio_completion_id, IO_TIMEOUT); + cmdid = alloc_cmdid(nvmeq, nbio, bio_completion, IO_TIMEOUT); if (unlikely(cmdid < 0)) goto free_nbio; @@ -599,45 +624,6 @@ static int nvme_make_request(struct request_queue *q, struct bio *bio) return 0; } -struct sync_cmd_info { - struct task_struct *task; - u32 result; - int status; -}; - -static void sync_completion(struct nvme_queue *nvmeq, void *ctx, - struct nvme_completion *cqe) -{ - struct sync_cmd_info *cmdinfo = ctx; - if (unlikely((unsigned long)cmdinfo == CMD_CTX_CANCELLED)) - return; - if ((unsigned long)cmdinfo == CMD_CTX_FLUSH) - return; - if (unlikely((unsigned long)cmdinfo == CMD_CTX_COMPLETED)) { - dev_warn(nvmeq->q_dmadev, - "completed id %d twice on queue %d\n", - cqe->command_id, le16_to_cpup(&cqe->sq_id)); - return; - } - if (unlikely((unsigned long)cmdinfo == CMD_CTX_INVALID)) { - dev_warn(nvmeq->q_dmadev, - "invalid id %d completed on queue %d\n", - cqe->command_id, le16_to_cpup(&cqe->sq_id)); - return; - } - cmdinfo->result = le32_to_cpup(&cqe->result); - cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; - wake_up_process(cmdinfo->task); -} - -typedef void (*completion_fn)(struct nvme_queue *, void *, - struct nvme_completion *); - -static const completion_fn nvme_completions[4] = { - [sync_completion_id] = sync_completion, - [bio_completion_id] = bio_completion, -}; - static irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq) { u16 head, phase; @@ -646,9 +632,8 @@ static irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq) phase = nvmeq->cq_phase; for (;;) { - unsigned long data; - void *ptr; - unsigned char handler; + void *ctx; + nvme_completion_fn fn; struct nvme_completion cqe = nvmeq->cqes[head]; if ((le16_to_cpu(cqe.status) & 1) != phase) break; @@ -658,10 +643,8 @@ static irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq) phase = !phase; } - data = free_cmdid(nvmeq, cqe.command_id); - handler = data & 3; - ptr = (void *)(data & ~3UL); - nvme_completions[handler](nvmeq, ptr, &cqe); + ctx = free_cmdid(nvmeq, cqe.command_id, &fn); + fn(nvmeq, ctx, &cqe); } /* If the controller ignores the cq head doorbell and continuously @@ -702,10 +685,25 @@ static irqreturn_t nvme_irq_check(int irq, void *data) static void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid) { spin_lock_irq(&nvmeq->q_lock); - cancel_cmdid(nvmeq, cmdid); + cancel_cmdid(nvmeq, cmdid, NULL); spin_unlock_irq(&nvmeq->q_lock); } +struct sync_cmd_info { + struct task_struct *task; + u32 result; + int status; +}; + +static void sync_completion(struct nvme_queue *nvmeq, void *ctx, + struct nvme_completion *cqe) +{ + struct sync_cmd_info *cmdinfo = ctx; + cmdinfo->result = le32_to_cpup(&cqe->result); + cmdinfo->status = le16_to_cpup(&cqe->status) >> 1; + wake_up_process(cmdinfo->task); +} + /* * Returns 0 on success. If the result is negative, it's a Linux error code; * if the result is positive, it's an NVM Express status code @@ -719,7 +717,7 @@ static int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, cmdinfo.task = current; cmdinfo.status = -EINTR; - cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion_id, + cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion, timeout); if (cmdid < 0) return cmdid; @@ -1201,18 +1199,15 @@ static void nvme_timeout_ios(struct nvme_queue *nvmeq) int cmdid; for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) { - unsigned long data; - void *ptr; - unsigned char handler; + void *ctx; + nvme_completion_fn fn; static struct nvme_completion cqe = { .status = cpu_to_le16(NVME_SC_ABORT_REQ) << 1, }; if (!time_after(now, info[cmdid].timeout)) continue; dev_warn(nvmeq->q_dmadev, "Timing out I/O %d\n", cmdid); - data = cancel_cmdid(nvmeq, cmdid); - handler = data & 3; - ptr = (void *)(data & ~3UL); - nvme_completions[handler](nvmeq, ptr, &cqe); + ctx = cancel_cmdid(nvmeq, cmdid, &fn); + fn(nvmeq, ctx, &cqe); } } |