From 4c9dacb82d5aa36aa2568df60d897f2eb3d8819b Mon Sep 17 00:00:00 2001 From: Wenwei Tao Date: Thu, 3 Mar 2016 15:06:37 +0100 Subject: lightnvm: specify target's logical address area MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can create more than one target on a lightnvm device by specifying its begin lun and end lun. But only specify the physical address area is not enough, we need to get the corresponding non- intersection logical address area division from the backend device's logcial address space. Otherwise the targets on the device might use the same logical addresses cause incorrect information in the device's l2p table. Signed-off-by: Wenwei Tao Signed-off-by: Matias Bjørling Signed-off-by: Jens Axboe --- include/linux/lightnvm.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include') diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h index c3c43184a787..b466bd9f2cf8 100644 --- a/include/linux/lightnvm.h +++ b/include/linux/lightnvm.h @@ -355,6 +355,7 @@ struct nvm_dev { char name[DISK_NAME_LEN]; struct mutex mlock; + spinlock_t lock; }; static inline struct ppa_addr generic_to_dev_addr(struct nvm_dev *dev, @@ -467,6 +468,9 @@ typedef int (nvmm_erase_blk_fn)(struct nvm_dev *, struct nvm_block *, typedef struct nvm_lun *(nvmm_get_lun_fn)(struct nvm_dev *, int); typedef void (nvmm_lun_info_print_fn)(struct nvm_dev *); +typedef int (nvmm_get_area_fn)(struct nvm_dev *, sector_t *, sector_t); +typedef void (nvmm_put_area_fn)(struct nvm_dev *, sector_t); + struct nvmm_type { const char *name; unsigned int version[3]; @@ -491,6 +495,10 @@ struct nvmm_type { /* Statistics */ nvmm_lun_info_print_fn *lun_info_print; + + nvmm_get_area_fn *get_area; + nvmm_put_area_fn *put_area; + struct list_head list; }; -- cgit v1.2.3 From da1e284919b0b99c5bf0618b6c98cbaf2c17e62e Mon Sep 17 00:00:00 2001 From: Wenwei Tao Date: Thu, 3 Mar 2016 15:06:38 +0100 Subject: lightnvm: add a bitmap of luns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a bitmap of luns to indicate the status of luns: inuse/available. When create targets do the necessary check to avoid allocating luns that are already allocated. Signed-off-by: Wenwei Tao Freed dev->lun_map if nvm_core_init later failed in the init process. Signed-off-by: Matias Bjørling Signed-off-by: Jens Axboe --- drivers/lightnvm/core.c | 6 ++++ drivers/lightnvm/gennvm.c | 18 ++++++++++++ drivers/lightnvm/rrpc.c | 74 ++++++++++++++++++++++++++++------------------- include/linux/lightnvm.h | 5 ++++ 4 files changed, 74 insertions(+), 29 deletions(-) (limited to 'include') diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c index 2925fd0b82bb..0dc9a80adb94 100644 --- a/drivers/lightnvm/core.c +++ b/drivers/lightnvm/core.c @@ -464,6 +464,10 @@ static int nvm_core_init(struct nvm_dev *dev) dev->nr_luns = dev->luns_per_chnl * dev->nr_chnls; dev->total_secs = dev->nr_luns * dev->sec_per_lun; + dev->lun_map = kcalloc(BITS_TO_LONGS(dev->nr_luns), + sizeof(unsigned long), GFP_KERNEL); + if (!dev->lun_map) + return -ENOMEM; INIT_LIST_HEAD(&dev->online_targets); mutex_init(&dev->mlock); spin_lock_init(&dev->lock); @@ -586,6 +590,7 @@ int nvm_register(struct request_queue *q, char *disk_name, return 0; err_init: + kfree(dev->lun_map); kfree(dev); return ret; } @@ -608,6 +613,7 @@ void nvm_unregister(char *disk_name) up_write(&nvm_lock); nvm_exit(dev); + kfree(dev->lun_map); kfree(dev); } EXPORT_SYMBOL(nvm_unregister); diff --git a/drivers/lightnvm/gennvm.c b/drivers/lightnvm/gennvm.c index d460b37bb016..b97801c00099 100644 --- a/drivers/lightnvm/gennvm.c +++ b/drivers/lightnvm/gennvm.c @@ -192,6 +192,9 @@ static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private) lun_id = div_u64(pba, dev->sec_per_lun); lun = &gn->luns[lun_id]; + if (!test_bit(lun_id, dev->lun_map)) + __set_bit(lun_id, dev->lun_map); + /* Calculate block offset into lun */ pba = pba - (dev->sec_per_lun * lun_id); blk = &lun->vlun.blocks[div_u64(pba, dev->sec_per_blk)]; @@ -482,10 +485,23 @@ static int gennvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk, return nvm_erase_ppa(dev, &addr, 1); } +static int gennvm_reserve_lun(struct nvm_dev *dev, int lunid) +{ + return test_and_set_bit(lunid, dev->lun_map); +} + +static void gennvm_release_lun(struct nvm_dev *dev, int lunid) +{ + WARN_ON(!test_and_clear_bit(lunid, dev->lun_map)); +} + static struct nvm_lun *gennvm_get_lun(struct nvm_dev *dev, int lunid) { struct gen_nvm *gn = dev->mp; + if (unlikely(lunid >= dev->nr_luns)) + return NULL; + return &gn->luns[lunid].vlun; } @@ -527,6 +543,8 @@ static struct nvmm_type gennvm = { .erase_blk = gennvm_erase_blk, .get_lun = gennvm_get_lun, + .reserve_lun = gennvm_reserve_lun, + .release_lun = gennvm_release_lun, .lun_info_print = gennvm_lun_info_print, .get_area = gennvm_get_area, diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c index c1e3c83f06b3..3ab6495c3fd8 100644 --- a/drivers/lightnvm/rrpc.c +++ b/drivers/lightnvm/rrpc.c @@ -965,25 +965,11 @@ static void rrpc_requeue(struct work_struct *work) static void rrpc_gc_free(struct rrpc *rrpc) { - struct rrpc_lun *rlun; - int i; - if (rrpc->krqd_wq) destroy_workqueue(rrpc->krqd_wq); if (rrpc->kgc_wq) destroy_workqueue(rrpc->kgc_wq); - - if (!rrpc->luns) - return; - - for (i = 0; i < rrpc->nr_luns; i++) { - rlun = &rrpc->luns[i]; - - if (!rlun->blocks) - break; - vfree(rlun->blocks); - } } static int rrpc_gc_init(struct rrpc *rrpc) @@ -1143,6 +1129,23 @@ static void rrpc_core_free(struct rrpc *rrpc) static void rrpc_luns_free(struct rrpc *rrpc) { + struct nvm_dev *dev = rrpc->dev; + struct nvm_lun *lun; + struct rrpc_lun *rlun; + int i; + + if (!rrpc->luns) + return; + + for (i = 0; i < rrpc->nr_luns; i++) { + rlun = &rrpc->luns[i]; + lun = rlun->parent; + if (!lun) + break; + dev->mt->release_lun(dev, lun->id); + vfree(rlun->blocks); + } + kfree(rrpc->luns); } @@ -1150,7 +1153,7 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end) { struct nvm_dev *dev = rrpc->dev; struct rrpc_lun *rlun; - int i, j; + int i, j, ret = -EINVAL; if (dev->sec_per_blk > MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) { pr_err("rrpc: number of pages per block too high."); @@ -1166,25 +1169,26 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end) /* 1:1 mapping */ for (i = 0; i < rrpc->nr_luns; i++) { - struct nvm_lun *lun = dev->mt->get_lun(dev, lun_begin + i); - - rlun = &rrpc->luns[i]; - rlun->rrpc = rrpc; - rlun->parent = lun; - INIT_LIST_HEAD(&rlun->prio_list); - INIT_LIST_HEAD(&rlun->open_list); - INIT_LIST_HEAD(&rlun->closed_list); + int lunid = lun_begin + i; + struct nvm_lun *lun; - INIT_WORK(&rlun->ws_gc, rrpc_lun_gc); - spin_lock_init(&rlun->lock); + if (dev->mt->reserve_lun(dev, lunid)) { + pr_err("rrpc: lun %u is already allocated\n", lunid); + goto err; + } - rrpc->total_blocks += dev->blks_per_lun; - rrpc->nr_sects += dev->sec_per_lun; + lun = dev->mt->get_lun(dev, lunid); + if (!lun) + goto err; + rlun = &rrpc->luns[i]; + rlun->parent = lun; rlun->blocks = vzalloc(sizeof(struct rrpc_block) * rrpc->dev->blks_per_lun); - if (!rlun->blocks) + if (!rlun->blocks) { + ret = -ENOMEM; goto err; + } for (j = 0; j < rrpc->dev->blks_per_lun; j++) { struct rrpc_block *rblk = &rlun->blocks[j]; @@ -1195,11 +1199,23 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end) INIT_LIST_HEAD(&rblk->prio); spin_lock_init(&rblk->lock); } + + rlun->rrpc = rrpc; + INIT_LIST_HEAD(&rlun->prio_list); + INIT_LIST_HEAD(&rlun->open_list); + INIT_LIST_HEAD(&rlun->closed_list); + + INIT_WORK(&rlun->ws_gc, rrpc_lun_gc); + spin_lock_init(&rlun->lock); + + rrpc->total_blocks += dev->blks_per_lun; + rrpc->nr_sects += dev->sec_per_lun; + } return 0; err: - return -ENOMEM; + return ret; } /* returns 0 on success and stores the beginning address in *begin */ diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h index b466bd9f2cf8..0ee2c2c78ffd 100644 --- a/include/linux/lightnvm.h +++ b/include/linux/lightnvm.h @@ -346,6 +346,7 @@ struct nvm_dev { int nr_luns; unsigned max_pages_per_blk; + unsigned long *lun_map; void *ppalist_pool; struct nvm_id identity; @@ -466,6 +467,8 @@ typedef int (nvmm_submit_io_fn)(struct nvm_dev *, struct nvm_rq *); typedef int (nvmm_erase_blk_fn)(struct nvm_dev *, struct nvm_block *, unsigned long); typedef struct nvm_lun *(nvmm_get_lun_fn)(struct nvm_dev *, int); +typedef int (nvmm_reserve_lun)(struct nvm_dev *, int); +typedef void (nvmm_release_lun)(struct nvm_dev *, int); typedef void (nvmm_lun_info_print_fn)(struct nvm_dev *); typedef int (nvmm_get_area_fn)(struct nvm_dev *, sector_t *, sector_t); @@ -492,6 +495,8 @@ struct nvmm_type { /* Configuration management */ nvmm_get_lun_fn *get_lun; + nvmm_reserve_lun *reserve_lun; + nvmm_release_lun *release_lun; /* Statistics */ nvmm_lun_info_print_fn *lun_info_print; -- cgit v1.2.3 From 9f867268436d799549909437e627e7cf279e1127 Mon Sep 17 00:00:00 2001 From: Matias Bjorling Date: Thu, 3 Mar 2016 15:06:39 +0100 Subject: nvme: lightnvm: return ppa completion status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PPAs sent to device is separately acknowledge in a 64bit status variable. The status is stored in DW0 and DW1 of the completion queue entry. Store this status inside the nvm_rq for further processing. This can later be used to implement retry techniques for failed writes and reads. Reviewed-by: Christoph Hellwig Signed-off-by: Matias Bjørling Signed-off-by: Jens Axboe --- drivers/nvme/host/lightnvm.c | 17 +++++++++++++++-- include/linux/lightnvm.h | 1 + 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c index 42a01a931989..9461dd639acd 100644 --- a/drivers/nvme/host/lightnvm.c +++ b/drivers/nvme/host/lightnvm.c @@ -146,6 +146,14 @@ struct nvme_nvm_command { }; }; +struct nvme_nvm_completion { + __le64 result; /* Used by LightNVM to return ppa completions */ + __le16 sq_head; /* how much of this queue may be reclaimed */ + __le16 sq_id; /* submission queue that generated this entry */ + __u16 command_id; /* of the command which completed */ + __le16 status; /* did the command fail, and if so, why? */ +}; + #define NVME_NVM_LP_MLC_PAIRS 886 struct nvme_nvm_lp_mlc { __u16 num_pairs; @@ -507,6 +515,10 @@ static inline void nvme_nvm_rqtocmd(struct request *rq, struct nvm_rq *rqd, static void nvme_nvm_end_io(struct request *rq, int error) { struct nvm_rq *rqd = rq->end_io_data; + struct nvme_nvm_completion *cqe = rq->special; + + if (cqe) + rqd->ppa_status = le64_to_cpu(cqe->result); nvm_end_io(rqd, error); @@ -526,7 +538,8 @@ static int nvme_nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) if (IS_ERR(rq)) return -ENOMEM; - cmd = kzalloc(sizeof(struct nvme_nvm_command), GFP_KERNEL); + cmd = kzalloc(sizeof(struct nvme_nvm_command) + + sizeof(struct nvme_nvm_completion), GFP_KERNEL); if (!cmd) { blk_mq_free_request(rq); return -ENOMEM; @@ -545,7 +558,7 @@ static int nvme_nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) rq->cmd = (unsigned char *)cmd; rq->cmd_len = sizeof(struct nvme_nvm_command); - rq->special = (void *)0; + rq->special = cmd + 1; rq->end_io_data = rqd; diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h index 0ee2c2c78ffd..cdcb2ccbefa8 100644 --- a/include/linux/lightnvm.h +++ b/include/linux/lightnvm.h @@ -242,6 +242,7 @@ struct nvm_rq { uint16_t nr_pages; uint16_t flags; + u64 ppa_status; /* ppa media status */ int error; }; -- cgit v1.2.3 From 897bb0c7f1ea82d7cc882b19790b5e1df00ffc29 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sat, 19 Mar 2016 11:30:33 +0100 Subject: blk-mq: Use proper cpumask iterator queue_for_each_ctx() iterates over per_cpu variables under the assumption that the possible cpu mask cannot have holes. That's wrong as all cpumasks can have holes. In case there are holes the iteration ends up accessing uninitialized memory and crashing as a result. Replace the macro by a proper for_each_possible_cpu() loop and drop the unused macro blk_ctx_sum() which references queue_for_each_ctx(). Reported-by: Xiong Zhou Signed-off-by: Thomas Gleixner Signed-off-by: Jens Axboe --- block/blk-mq-sysfs.c | 6 ++++-- block/blk-mq.c | 3 ++- include/linux/blk-mq.h | 14 -------------- 3 files changed, 6 insertions(+), 17 deletions(-) (limited to 'include') diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c index 431fdda21737..4ea4dd8a1eed 100644 --- a/block/blk-mq-sysfs.c +++ b/block/blk-mq-sysfs.c @@ -416,12 +416,14 @@ void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx) static void blk_mq_sysfs_init(struct request_queue *q) { struct blk_mq_ctx *ctx; - int i; + int cpu; kobject_init(&q->mq_kobj, &blk_mq_ktype); - queue_for_each_ctx(q, ctx, i) + for_each_possible_cpu(cpu) { + ctx = per_cpu_ptr(q->queue_ctx, cpu); kobject_init(&ctx->kobj, &blk_mq_ctx_ktype); + } } int blk_mq_register_disk(struct gendisk *disk) diff --git a/block/blk-mq.c b/block/blk-mq.c index 050f7a13021b..1699baf39b78 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1798,11 +1798,12 @@ static void blk_mq_map_swqueue(struct request_queue *q, /* * Map software to hardware queues */ - queue_for_each_ctx(q, ctx, i) { + for_each_possible_cpu(i) { /* If the cpu isn't online, the cpu is mapped to first hctx */ if (!cpumask_test_cpu(i, online_mask)) continue; + ctx = per_cpu_ptr(q->queue_ctx, i); hctx = q->mq_ops->map_queue(q, i); cpumask_set_cpu(i, hctx->cpumask); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 15a73d49fd1d..9ac9799b702b 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -263,22 +263,8 @@ static inline void *blk_mq_rq_to_pdu(struct request *rq) for ((i) = 0; (i) < (q)->nr_hw_queues && \ ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++) -#define queue_for_each_ctx(q, ctx, i) \ - for ((i) = 0; (i) < (q)->nr_queues && \ - ({ ctx = per_cpu_ptr((q)->queue_ctx, (i)); 1; }); (i)++) - #define hctx_for_each_ctx(hctx, ctx, i) \ for ((i) = 0; (i) < (hctx)->nr_ctx && \ ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) -#define blk_ctx_sum(q, sum) \ -({ \ - struct blk_mq_ctx *__x; \ - unsigned int __ret = 0, __i; \ - \ - queue_for_each_ctx((q), __x, __i) \ - __ret += sum; \ - __ret; \ -}) - #endif -- cgit v1.2.3