diff options
Diffstat (limited to 'drivers/lightnvm')
| -rw-r--r-- | drivers/lightnvm/Makefile | 2 | ||||
| -rw-r--r-- | drivers/lightnvm/core.c | 387 | ||||
| -rw-r--r-- | drivers/lightnvm/gennvm.c | 645 | ||||
| -rw-r--r-- | drivers/lightnvm/gennvm.h | 34 | ||||
| -rw-r--r-- | drivers/lightnvm/lightnvm.h | 35 | ||||
| -rw-r--r-- | drivers/lightnvm/rrpc.c | 514 | ||||
| -rw-r--r-- | drivers/lightnvm/rrpc.h | 65 | ||||
| -rw-r--r-- | drivers/lightnvm/sysblk.c | 98 | ||||
| -rw-r--r-- | drivers/lightnvm/sysfs.c | 198 |
9 files changed, 932 insertions, 1046 deletions
diff --git a/drivers/lightnvm/Makefile b/drivers/lightnvm/Makefile index 1f6b6521016a..a7a0a22cf1a5 100644 --- a/drivers/lightnvm/Makefile +++ b/drivers/lightnvm/Makefile @@ -2,6 +2,6 @@ # Makefile for Open-Channel SSDs. # -obj-$(CONFIG_NVM) := core.o sysblk.o sysfs.o +obj-$(CONFIG_NVM) := core.o sysblk.o obj-$(CONFIG_NVM_GENNVM) += gennvm.o obj-$(CONFIG_NVM_RRPC) += rrpc.o diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c index 1cac0f8bc0dc..02240a0b39c9 100644 --- a/drivers/lightnvm/core.c +++ b/drivers/lightnvm/core.c @@ -22,13 +22,11 @@ #include <linux/types.h> #include <linux/sem.h> #include <linux/bitmap.h> -#include <linux/module.h> +#include <linux/moduleparam.h> #include <linux/miscdevice.h> #include <linux/lightnvm.h> #include <linux/sched/sysctl.h> -#include "lightnvm.h" - static LIST_HEAD(nvm_tgt_types); static DECLARE_RWSEM(nvm_tgtt_lock); static LIST_HEAD(nvm_mgrs); @@ -88,8 +86,7 @@ void *nvm_dev_dma_alloc(struct nvm_dev *dev, gfp_t mem_flags, } EXPORT_SYMBOL(nvm_dev_dma_alloc); -void nvm_dev_dma_free(struct nvm_dev *dev, void *addr, - dma_addr_t dma_handler) +void nvm_dev_dma_free(struct nvm_dev *dev, void *addr, dma_addr_t dma_handler) { dev->ops->dev_dma_free(dev->dma_pool, addr, dma_handler); } @@ -178,38 +175,133 @@ static struct nvm_dev *nvm_find_nvm_dev(const char *name) return NULL; } -struct nvm_block *nvm_get_blk(struct nvm_dev *dev, struct nvm_lun *lun, - unsigned long flags) +static void nvm_tgt_generic_to_addr_mode(struct nvm_tgt_dev *tgt_dev, + struct nvm_rq *rqd) { - return dev->mt->get_blk(dev, lun, flags); + struct nvm_dev *dev = tgt_dev->parent; + int i; + + if (rqd->nr_ppas > 1) { + for (i = 0; i < rqd->nr_ppas; i++) { + rqd->ppa_list[i] = dev->mt->trans_ppa(tgt_dev, + rqd->ppa_list[i], TRANS_TGT_TO_DEV); + rqd->ppa_list[i] = generic_to_dev_addr(dev, + rqd->ppa_list[i]); + } + } else { + rqd->ppa_addr = dev->mt->trans_ppa(tgt_dev, rqd->ppa_addr, + TRANS_TGT_TO_DEV); + rqd->ppa_addr = generic_to_dev_addr(dev, rqd->ppa_addr); + } +} + +int nvm_set_bb_tbl(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas, + int type) +{ + struct nvm_rq rqd; + int ret; + + if (nr_ppas > dev->ops->max_phys_sect) { + pr_err("nvm: unable to update all sysblocks atomically\n"); + return -EINVAL; + } + + memset(&rqd, 0, sizeof(struct nvm_rq)); + + nvm_set_rqd_ppalist(dev, &rqd, ppas, nr_ppas, 1); + nvm_generic_to_addr_mode(dev, &rqd); + + ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type); + nvm_free_rqd_ppalist(dev, &rqd); + if (ret) { + pr_err("nvm: sysblk failed bb mark\n"); + return -EINVAL; + } + + return 0; } -EXPORT_SYMBOL(nvm_get_blk); +EXPORT_SYMBOL(nvm_set_bb_tbl); -/* Assumes that all valid pages have already been moved on release to bm */ -void nvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk) +int nvm_set_tgt_bb_tbl(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppas, + int nr_ppas, int type) { - return dev->mt->put_blk(dev, blk); + struct nvm_dev *dev = tgt_dev->parent; + struct nvm_rq rqd; + int ret; + + if (nr_ppas > dev->ops->max_phys_sect) { + pr_err("nvm: unable to update all blocks atomically\n"); + return -EINVAL; + } + + memset(&rqd, 0, sizeof(struct nvm_rq)); + + nvm_set_rqd_ppalist(dev, &rqd, ppas, nr_ppas, 1); + nvm_tgt_generic_to_addr_mode(tgt_dev, &rqd); + + ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type); + nvm_free_rqd_ppalist(dev, &rqd); + if (ret) { + pr_err("nvm: sysblk failed bb mark\n"); + return -EINVAL; + } + + return 0; } -EXPORT_SYMBOL(nvm_put_blk); +EXPORT_SYMBOL(nvm_set_tgt_bb_tbl); -void nvm_mark_blk(struct nvm_dev *dev, struct ppa_addr ppa, int type) +int nvm_max_phys_sects(struct nvm_tgt_dev *tgt_dev) { - return dev->mt->mark_blk(dev, ppa, type); + struct nvm_dev *dev = tgt_dev->parent; + + return dev->ops->max_phys_sect; } -EXPORT_SYMBOL(nvm_mark_blk); +EXPORT_SYMBOL(nvm_max_phys_sects); -int nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) +int nvm_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd) { - return dev->mt->submit_io(dev, rqd); + struct nvm_dev *dev = tgt_dev->parent; + + return dev->mt->submit_io(tgt_dev, rqd); } EXPORT_SYMBOL(nvm_submit_io); -int nvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk) +int nvm_erase_blk(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p, int flags) { - return dev->mt->erase_blk(dev, blk, 0); + struct nvm_dev *dev = tgt_dev->parent; + + return dev->mt->erase_blk(tgt_dev, p, flags); } EXPORT_SYMBOL(nvm_erase_blk); +int nvm_get_l2p_tbl(struct nvm_tgt_dev *tgt_dev, u64 slba, u32 nlb, + nvm_l2p_update_fn *update_l2p, void *priv) +{ + struct nvm_dev *dev = tgt_dev->parent; + + if (!dev->ops->get_l2p_tbl) + return 0; + + return dev->ops->get_l2p_tbl(dev, slba, nlb, update_l2p, priv); +} +EXPORT_SYMBOL(nvm_get_l2p_tbl); + +int nvm_get_area(struct nvm_tgt_dev *tgt_dev, sector_t *lba, sector_t len) +{ + struct nvm_dev *dev = tgt_dev->parent; + + return dev->mt->get_area(dev, lba, len); +} +EXPORT_SYMBOL(nvm_get_area); + +void nvm_put_area(struct nvm_tgt_dev *tgt_dev, sector_t lba) +{ + struct nvm_dev *dev = tgt_dev->parent; + + dev->mt->put_area(dev, lba); +} +EXPORT_SYMBOL(nvm_put_area); + void nvm_addr_to_generic_mode(struct nvm_dev *dev, struct nvm_rq *rqd) { int i; @@ -241,10 +333,11 @@ EXPORT_SYMBOL(nvm_generic_to_addr_mode); int nvm_set_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd, const struct ppa_addr *ppas, int nr_ppas, int vblk) { + struct nvm_geo *geo = &dev->geo; int i, plane_cnt, pl_idx; struct ppa_addr ppa; - if ((!vblk || dev->plane_mode == NVM_PLANE_SINGLE) && nr_ppas == 1) { + if ((!vblk || geo->plane_mode == NVM_PLANE_SINGLE) && nr_ppas == 1) { rqd->nr_ppas = nr_ppas; rqd->ppa_addr = ppas[0]; @@ -262,7 +355,7 @@ int nvm_set_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd, for (i = 0; i < nr_ppas; i++) rqd->ppa_list[i] = ppas[i]; } else { - plane_cnt = dev->plane_mode; + plane_cnt = geo->plane_mode; rqd->nr_ppas *= plane_cnt; for (i = 0; i < nr_ppas; i++) { @@ -287,7 +380,8 @@ void nvm_free_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd) } EXPORT_SYMBOL(nvm_free_rqd_ppalist); -int nvm_erase_ppa(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas) +int nvm_erase_ppa(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas, + int flags) { struct nvm_rq rqd; int ret; @@ -303,6 +397,8 @@ int nvm_erase_ppa(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas) nvm_generic_to_addr_mode(dev, &rqd); + rqd.flags = flags; + ret = dev->ops->erase_block(dev, &rqd); nvm_free_rqd_ppalist(dev, &rqd); @@ -341,7 +437,7 @@ static int __nvm_submit_ppa(struct nvm_dev *dev, struct nvm_rq *rqd, int opcode, nvm_generic_to_addr_mode(dev, rqd); - rqd->dev = dev; + rqd->dev = NULL; rqd->opcode = opcode; rqd->flags = flags; rqd->bio = bio; @@ -437,17 +533,18 @@ EXPORT_SYMBOL(nvm_submit_ppa); */ int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks) { + struct nvm_geo *geo = &dev->geo; int blk, offset, pl, blktype; - if (nr_blks != dev->blks_per_lun * dev->plane_mode) + if (nr_blks != geo->blks_per_lun * geo->plane_mode) return -EINVAL; - for (blk = 0; blk < dev->blks_per_lun; blk++) { - offset = blk * dev->plane_mode; + for (blk = 0; blk < geo->blks_per_lun; blk++) { + offset = blk * geo->plane_mode; blktype = blks[offset]; /* Bad blocks on any planes take precedence over other types */ - for (pl = 0; pl < dev->plane_mode; pl++) { + for (pl = 0; pl < geo->plane_mode; pl++) { if (blks[offset + pl] & (NVM_BLK_T_BAD|NVM_BLK_T_GRWN_BAD)) { blktype = blks[offset + pl]; @@ -458,7 +555,7 @@ int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks) blks[blk] = blktype; } - return dev->blks_per_lun; + return geo->blks_per_lun; } EXPORT_SYMBOL(nvm_bb_tbl_fold); @@ -470,11 +567,22 @@ int nvm_get_bb_tbl(struct nvm_dev *dev, struct ppa_addr ppa, u8 *blks) } EXPORT_SYMBOL(nvm_get_bb_tbl); +int nvm_get_tgt_bb_tbl(struct nvm_tgt_dev *tgt_dev, struct ppa_addr ppa, + u8 *blks) +{ + struct nvm_dev *dev = tgt_dev->parent; + + ppa = dev->mt->trans_ppa(tgt_dev, ppa, TRANS_TGT_TO_DEV); + return nvm_get_bb_tbl(dev, ppa, blks); +} +EXPORT_SYMBOL(nvm_get_tgt_bb_tbl); + static int nvm_init_slc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp) { + struct nvm_geo *geo = &dev->geo; int i; - dev->lps_per_blk = dev->pgs_per_blk; + dev->lps_per_blk = geo->pgs_per_blk; dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL); if (!dev->lptbl) return -ENOMEM; @@ -520,29 +628,32 @@ static int nvm_core_init(struct nvm_dev *dev) { struct nvm_id *id = &dev->identity; struct nvm_id_group *grp = &id->groups[0]; + struct nvm_geo *geo = &dev->geo; int ret; - /* device values */ - dev->nr_chnls = grp->num_ch; - dev->luns_per_chnl = grp->num_lun; - dev->pgs_per_blk = grp->num_pg; - dev->blks_per_lun = grp->num_blk; - dev->nr_planes = grp->num_pln; - dev->fpg_size = grp->fpg_sz; - dev->pfpg_size = grp->fpg_sz * grp->num_pln; - dev->sec_size = grp->csecs; - dev->oob_size = grp->sos; - dev->sec_per_pg = grp->fpg_sz / grp->csecs; - dev->mccap = grp->mccap; - memcpy(&dev->ppaf, &id->ppaf, sizeof(struct nvm_addr_format)); - - dev->plane_mode = NVM_PLANE_SINGLE; - dev->max_rq_size = dev->ops->max_phys_sect * dev->sec_size; + /* Whole device values */ + geo->nr_chnls = grp->num_ch; + geo->luns_per_chnl = grp->num_lun; + + /* Generic device values */ + geo->pgs_per_blk = grp->num_pg; + geo->blks_per_lun = grp->num_blk; + geo->nr_planes = grp->num_pln; + geo->fpg_size = grp->fpg_sz; + geo->pfpg_size = grp->fpg_sz * grp->num_pln; + geo->sec_size = grp->csecs; + geo->oob_size = grp->sos; + geo->sec_per_pg = grp->fpg_sz / grp->csecs; + geo->mccap = grp->mccap; + memcpy(&geo->ppaf, &id->ppaf, sizeof(struct nvm_addr_format)); + + geo->plane_mode = NVM_PLANE_SINGLE; + geo->max_rq_size = dev->ops->max_phys_sect * geo->sec_size; if (grp->mpos & 0x020202) - dev->plane_mode = NVM_PLANE_DOUBLE; + geo->plane_mode = NVM_PLANE_DOUBLE; if (grp->mpos & 0x040404) - dev->plane_mode = NVM_PLANE_QUAD; + geo->plane_mode = NVM_PLANE_QUAD; if (grp->mtype != 0) { pr_err("nvm: memory type not supported\n"); @@ -550,13 +661,13 @@ static int nvm_core_init(struct nvm_dev *dev) } /* calculated values */ - dev->sec_per_pl = dev->sec_per_pg * dev->nr_planes; - dev->sec_per_blk = dev->sec_per_pl * dev->pgs_per_blk; - dev->sec_per_lun = dev->sec_per_blk * dev->blks_per_lun; - dev->nr_luns = dev->luns_per_chnl * dev->nr_chnls; + geo->sec_per_pl = geo->sec_per_pg * geo->nr_planes; + geo->sec_per_blk = geo->sec_per_pl * geo->pgs_per_blk; + geo->sec_per_lun = geo->sec_per_blk * geo->blks_per_lun; + geo->nr_luns = geo->luns_per_chnl * geo->nr_chnls; - dev->total_secs = dev->nr_luns * dev->sec_per_lun; - dev->lun_map = kcalloc(BITS_TO_LONGS(dev->nr_luns), + dev->total_secs = geo->nr_luns * geo->sec_per_lun; + dev->lun_map = kcalloc(BITS_TO_LONGS(geo->nr_luns), sizeof(unsigned long), GFP_KERNEL); if (!dev->lun_map) return -ENOMEM; @@ -583,7 +694,7 @@ static int nvm_core_init(struct nvm_dev *dev) mutex_init(&dev->mlock); spin_lock_init(&dev->lock); - blk_queue_logical_block_size(dev->q, dev->sec_size); + blk_queue_logical_block_size(dev->q, geo->sec_size); return 0; err_fmtype: @@ -617,6 +728,7 @@ void nvm_free(struct nvm_dev *dev) static int nvm_init(struct nvm_dev *dev) { + struct nvm_geo *geo = &dev->geo; int ret = -EINVAL; if (!dev->q || !dev->ops) @@ -648,20 +760,15 @@ static int nvm_init(struct nvm_dev *dev) } pr_info("nvm: registered %s [%u/%u/%u/%u/%u/%u]\n", - dev->name, dev->sec_per_pg, dev->nr_planes, - dev->pgs_per_blk, dev->blks_per_lun, dev->nr_luns, - dev->nr_chnls); + dev->name, geo->sec_per_pg, geo->nr_planes, + geo->pgs_per_blk, geo->blks_per_lun, + geo->nr_luns, geo->nr_chnls); return 0; err: pr_err("nvm: failed to initialize nvm\n"); return ret; } -static void nvm_exit(struct nvm_dev *dev) -{ - nvm_sysfs_unregister_dev(dev); -} - struct nvm_dev *nvm_alloc_dev(int node) { return kzalloc_node(sizeof(struct nvm_dev), GFP_KERNEL, node); @@ -691,10 +798,6 @@ int nvm_register(struct nvm_dev *dev) } } - ret = nvm_sysfs_register_dev(dev); - if (ret) - goto err_ppalist; - if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT) { ret = nvm_get_sysblock(dev, &dev->sb); if (!ret) @@ -711,8 +814,6 @@ int nvm_register(struct nvm_dev *dev) up_write(&nvm_lock); return 0; -err_ppalist: - dev->ops->destroy_dma_pool(dev->dma_pool); err_init: kfree(dev->lun_map); return ret; @@ -725,7 +826,7 @@ void nvm_unregister(struct nvm_dev *dev) list_del(&dev->devices); up_write(&nvm_lock); - nvm_exit(dev); + nvm_free(dev); } EXPORT_SYMBOL(nvm_unregister); @@ -754,149 +855,15 @@ static int __nvm_configure_create(struct nvm_ioctl_create *create) } s = &create->conf.s; - if (s->lun_begin > s->lun_end || s->lun_end > dev->nr_luns) { + if (s->lun_begin > s->lun_end || s->lun_end > dev->geo.nr_luns) { pr_err("nvm: lun out of bound (%u:%u > %u)\n", - s->lun_begin, s->lun_end, dev->nr_luns); + s->lun_begin, s->lun_end, dev->geo.nr_luns); return -EINVAL; } return dev->mt->create_tgt(dev, create); } -#ifdef CONFIG_NVM_DEBUG -static int nvm_configure_show(const char *val) -{ - struct nvm_dev *dev; - char opcode, devname[DISK_NAME_LEN]; - int ret; - - ret = sscanf(val, "%c %32s", &opcode, devname); - if (ret != 2) { - pr_err("nvm: invalid command. Use \"opcode devicename\".\n"); - return -EINVAL; - } - - down_write(&nvm_lock); - dev = nvm_find_nvm_dev(devname); - up_write(&nvm_lock); - if (!dev) { - pr_err("nvm: device not found\n"); - return -EINVAL; - } - - if (!dev->mt) - return 0; - - dev->mt->lun_info_print(dev); - - return 0; -} - -static int nvm_configure_remove(const char *val) -{ - struct nvm_ioctl_remove remove; - struct nvm_dev *dev; - char opcode; - int ret = 0; - - ret = sscanf(val, "%c %256s", &opcode, remove.tgtname); - if (ret != 2) { - pr_err("nvm: invalid command. Use \"d targetname\".\n"); - return -EINVAL; - } - - remove.flags = 0; - - list_for_each_entry(dev, &nvm_devices, devices) { - ret = dev->mt->remove_tgt(dev, &remove); - if (!ret) - break; - } - - return ret; -} - -static int nvm_configure_create(const char *val) -{ - struct nvm_ioctl_create create; - char opcode; - int lun_begin, lun_end, ret; - - ret = sscanf(val, "%c %256s %256s %48s %u:%u", &opcode, create.dev, - create.tgtname, create.tgttype, - &lun_begin, &lun_end); - if (ret != 6) { - pr_err("nvm: invalid command. Use \"opcode device name tgttype lun_begin:lun_end\".\n"); - return -EINVAL; - } - - create.flags = 0; - create.conf.type = NVM_CONFIG_TYPE_SIMPLE; - create.conf.s.lun_begin = lun_begin; - create.conf.s.lun_end = lun_end; - - return __nvm_configure_create(&create); -} - - -/* Exposes administrative interface through /sys/module/lnvm/configure_by_str */ -static int nvm_configure_by_str_event(const char *val, - const struct kernel_param *kp) -{ - char opcode; - int ret; - - ret = sscanf(val, "%c", &opcode); - if (ret != 1) { - pr_err("nvm: string must have the format of \"cmd ...\"\n"); - return -EINVAL; - } - - switch (opcode) { - case 'a': - return nvm_configure_create(val); - case 'd': - return nvm_configure_remove(val); - case 's': - return nvm_configure_show(val); - default: - pr_err("nvm: invalid command\n"); - return -EINVAL; - } - - return 0; -} - -static int nvm_configure_get(char *buf, const struct kernel_param *kp) -{ - int sz; - struct nvm_dev *dev; - - sz = sprintf(buf, "available devices:\n"); - down_write(&nvm_lock); - list_for_each_entry(dev, &nvm_devices, devices) { - if (sz > 4095 - DISK_NAME_LEN - 2) - break; - sz += sprintf(buf + sz, " %32s\n", dev->name); - } - up_write(&nvm_lock); - - return sz; -} - -static const struct kernel_param_ops nvm_configure_by_str_event_param_ops = { - .set = nvm_configure_by_str_event, - .get = nvm_configure_get, -}; - -#undef MODULE_PARAM_PREFIX -#define MODULE_PARAM_PREFIX "lnvm." - -module_param_cb(configure_debug, &nvm_configure_by_str_event_param_ops, NULL, - 0644); - -#endif /* CONFIG_NVM_DEBUG */ - static long nvm_ioctl_info(struct file *file, void __user *arg) { struct nvm_ioctl_info *info; @@ -1162,10 +1129,4 @@ static struct miscdevice _nvm_misc = { .nodename = "lightnvm/control", .fops = &_ctl_fops, }; -module_misc_device(_nvm_misc); - -MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR); - -MODULE_AUTHOR("Matias Bjorling <m@bjorling.me>"); -MODULE_LICENSE("GPL v2"); -MODULE_VERSION("0.1"); +builtin_misc_device(_nvm_misc); diff --git a/drivers/lightnvm/gennvm.c b/drivers/lightnvm/gennvm.c index b74174c6d021..ca7880082d80 100644 --- a/drivers/lightnvm/gennvm.c +++ b/drivers/lightnvm/gennvm.c @@ -35,6 +35,165 @@ static const struct block_device_operations gen_fops = { .owner = THIS_MODULE, }; +static int gen_reserve_luns(struct nvm_dev *dev, struct nvm_target *t, + int lun_begin, int lun_end) +{ + int i; + + for (i = lun_begin; i <= lun_end; i++) { + if (test_and_set_bit(i, dev->lun_map)) { + pr_err("nvm: lun %d already allocated\n", i); + goto err; + } + } + + return 0; + +err: + while (--i > lun_begin) + clear_bit(i, dev->lun_map); + + return -EBUSY; +} + +static void gen_release_luns_err(struct nvm_dev *dev, int lun_begin, + int lun_end) +{ + int i; + + for (i = lun_begin; i <= lun_end; i++) + WARN_ON(!test_and_clear_bit(i, dev->lun_map)); +} + +static void gen_remove_tgt_dev(struct nvm_tgt_dev *tgt_dev) +{ + struct nvm_dev *dev = tgt_dev->parent; + struct gen_dev_map *dev_map = tgt_dev->map; + int i, j; + + for (i = 0; i < dev_map->nr_chnls; i++) { + struct gen_ch_map *ch_map = &dev_map->chnls[i]; + int *lun_offs = ch_map->lun_offs; + int ch = i + ch_map->ch_off; + + for (j = 0; j < ch_map->nr_luns; j++) { + int lun = j + lun_offs[j]; + int lunid = (ch * dev->geo.luns_per_chnl) + lun; + + WARN_ON(!test_and_clear_bit(lunid, dev->lun_map)); + } + + kfree(ch_map->lun_offs); + } + + kfree(dev_map->chnls); + kfree(dev_map); + kfree(tgt_dev->luns); + kfree(tgt_dev); +} + +static struct nvm_tgt_dev *gen_create_tgt_dev(struct nvm_dev *dev, + int lun_begin, int lun_end) +{ + struct nvm_tgt_dev *tgt_dev = NULL; + struct gen_dev_map *dev_rmap = dev->rmap; + struct gen_dev_map *dev_map; + struct ppa_addr *luns; + int nr_luns = lun_end - lun_begin + 1; + int luns_left = nr_luns; + int nr_chnls = nr_luns / dev->geo.luns_per_chnl; + int nr_chnls_mod = nr_luns % dev->geo.luns_per_chnl; + int bch = lun_begin / dev->geo.luns_per_chnl; + int blun = lun_begin % dev->geo.luns_per_chnl; + int lunid = 0; + int lun_balanced = 1; + int prev_nr_luns; + int i, j; + + nr_chnls = nr_luns / dev->geo.luns_per_chnl; + nr_chnls = (nr_chnls_mod == 0) ? nr_chnls : nr_chnls + 1; + + dev_map = kmalloc(sizeof(struct gen_dev_map), GFP_KERNEL); + if (!dev_map) + goto err_dev; + + dev_map->chnls = kcalloc(nr_chnls, sizeof(struct gen_ch_map), + GFP_KERNEL); + if (!dev_map->chnls) + goto err_chnls; + + luns = kcalloc(nr_luns, sizeof(struct ppa_addr), GFP_KERNEL); + if (!luns) + goto err_luns; + + prev_nr_luns = (luns_left > dev->geo.luns_per_chnl) ? + dev->geo.luns_per_chnl : luns_left; + for (i = 0; i < nr_chnls; i++) { + struct gen_ch_map *ch_rmap = &dev_rmap->chnls[i + bch]; + int *lun_roffs = ch_rmap->lun_offs; + struct gen_ch_map *ch_map = &dev_map->chnls[i]; + int *lun_offs; + int luns_in_chnl = (luns_left > dev->geo.luns_per_chnl) ? + dev->geo.luns_per_chnl : luns_left; + + if (lun_balanced && prev_nr_luns != luns_in_chnl) + lun_balanced = 0; + + ch_map->ch_off = ch_rmap->ch_off = bch; + ch_map->nr_luns = luns_in_chnl; + + lun_offs = kcalloc(luns_in_chnl, sizeof(int), GFP_KERNEL); + if (!lun_offs) + goto err_ch; + + for (j = 0; j < luns_in_chnl; j++) { + luns[lunid].ppa = 0; + luns[lunid].g.ch = i; + luns[lunid++].g.lun = j; + + lun_offs[j] = blun; + lun_roffs[j + blun] = blun; + } + + ch_map->lun_offs = lun_offs; + + /* when starting a new channel, lun offset is reset */ + blun = 0; + luns_left -= luns_in_chnl; + } + + dev_map->nr_chnls = nr_chnls; + + tgt_dev = kmalloc(sizeof(struct nvm_tgt_dev), GFP_KERNEL); + if (!tgt_dev) + goto err_ch; + + memcpy(&tgt_dev->geo, &dev->geo, sizeof(struct nvm_geo)); + /* Target device only owns a portion of the physical device */ + tgt_dev->geo.nr_chnls = nr_chnls; + tgt_dev->geo.nr_luns = nr_luns; + tgt_dev->geo.luns_per_chnl = (lun_balanced) ? prev_nr_luns : -1; + tgt_dev->total_secs = nr_luns * tgt_dev->geo.sec_per_lun; + tgt_dev->q = dev->q; + tgt_dev->map = dev_map; + tgt_dev->luns = luns; + memcpy(&tgt_dev->identity, &dev->identity, sizeof(struct nvm_id)); + + tgt_dev->parent = dev; + + return tgt_dev; +err_ch: + while (--i > 0) + kfree(dev_map->chnls[i].lun_offs); + kfree(luns); +err_luns: + kfree(dev_map->chnls); +err_chnls: + kfree(dev_map); +err_dev: + return tgt_dev; +} + static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create) { struct gen_dev *gn = dev->mp; @@ -43,6 +202,7 @@ static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create) struct gendisk *tdisk; struct nvm_tgt_type *tt; struct nvm_target *t; + struct nvm_tgt_dev *tgt_dev; void *targetdata; tt = nvm_find_target_type(create->tgttype, 1); @@ -64,9 +224,18 @@ static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create) if (!t) return -ENOMEM; + if (gen_reserve_luns(dev, t, s->lun_begin, s->lun_end)) + goto err_t; + + tgt_dev = gen_create_tgt_dev(dev, s->lun_begin, s->lun_end); + if (!tgt_dev) { + pr_err("nvm: could not create target device\n"); + goto err_reserve; + } + tqueue = blk_alloc_queue_node(GFP_KERNEL, dev->q->node); if (!tqueue) - goto err_t; + goto err_dev; blk_queue_make_request(tqueue, tt->make_rq); tdisk = alloc_disk(0); @@ -80,7 +249,7 @@ static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create) tdisk->fops = &gen_fops; tdisk->queue = tqueue; - targetdata = tt->init(dev, tdisk, s->lun_begin, s->lun_end); + targetdata = tt->init(tgt_dev, tdisk); if (IS_ERR(targetdata)) goto err_init; @@ -94,7 +263,7 @@ static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create) t->type = tt; t->disk = tdisk; - t->dev = dev; + t->dev = tgt_dev; mutex_lock(&gn->lock); list_add_tail(&t->list, &gn->targets); @@ -105,6 +274,10 @@ err_init: put_disk(tdisk); err_queue: blk_cleanup_queue(tqueue); +err_dev: + kfree(tgt_dev); +err_reserve: + gen_release_luns_err(dev, s->lun_begin, s->lun_end); err_t: kfree(t); return -ENOMEM; @@ -122,6 +295,7 @@ static void __gen_remove_target(struct nvm_target *t) if (tt->exit) tt->exit(tdisk->private_data); + gen_remove_tgt_dev(t->dev); put_disk(tdisk); list_del(&t->list); @@ -160,10 +334,11 @@ static int gen_remove_tgt(struct nvm_dev *dev, struct nvm_ioctl_remove *remove) static int gen_get_area(struct nvm_dev *dev, sector_t *lba, sector_t len) { + struct nvm_geo *geo = &dev->geo; struct gen_dev *gn = dev->mp; struct gen_area *area, *prev, *next; sector_t begin = 0; - sector_t max_sectors = (dev->sec_size * dev->total_secs) >> 9; + sector_t max_sectors = (geo->sec_size * dev->total_secs) >> 9; if (len > max_sectors) return -EINVAL; @@ -220,240 +395,74 @@ static void gen_put_area(struct nvm_dev *dev, sector_t begin) spin_unlock(&dev->lock); } -static void gen_blocks_free(struct nvm_dev *dev) -{ - struct gen_dev *gn = dev->mp; - struct gen_lun *lun; - int i; - - gen_for_each_lun(gn, lun, i) { - if (!lun->vlun.blocks) - break; - vfree(lun->vlun.blocks); - } -} - -static void gen_luns_free(struct nvm_dev *dev) -{ - struct gen_dev *gn = dev->mp; - - kfree(gn->luns); -} - -static int gen_luns_init(struct nvm_dev *dev, struct gen_dev *gn) -{ - struct gen_lun *lun; - int i; - - gn->luns = kcalloc(dev->nr_luns, sizeof(struct gen_lun), GFP_KERNEL); - if (!gn->luns) - return -ENOMEM; - - gen_for_each_lun(gn, lun, i) { - spin_lock_init(&lun->vlun.lock); - INIT_LIST_HEAD(&lun->free_list); - INIT_LIST_HEAD(&lun->used_list); - INIT_LIST_HEAD(&lun->bb_list); - - lun->reserved_blocks = 2; /* for GC only */ - lun->vlun.id = i; - lun->vlun.lun_id = i % dev->luns_per_chnl; - lun->vlun.chnl_id = i / dev->luns_per_chnl; - lun->vlun.nr_free_blocks = dev->blks_per_lun; - } - return 0; -} - -static int gen_block_bb(struct gen_dev *gn, struct ppa_addr ppa, - u8 *blks, int nr_blks) -{ - struct nvm_dev *dev = gn->dev; - struct gen_lun *lun; - struct nvm_block *blk; - int i; - - nr_blks = nvm_bb_tbl_fold(dev, blks, nr_blks); - if (nr_blks < 0) - return nr_blks; - - lun = &gn->luns[(dev->luns_per_chnl * ppa.g.ch) + ppa.g.lun]; - - for (i = 0; i < nr_blks; i++) { - if (blks[i] == 0) - continue; - - blk = &lun->vlun.blocks[i]; - list_move_tail(&blk->list, &lun->bb_list); - lun->vlun.nr_free_blocks--; - } - - return 0; -} - -static int gen_block_map(u64 slba, u32 nlb, __le64 *entries, void *private) -{ - struct nvm_dev *dev = private; - struct gen_dev *gn = dev->mp; - u64 elba = slba + nlb; - struct gen_lun *lun; - struct nvm_block *blk; - u64 i; - int lun_id; - - if (unlikely(elba > dev->total_secs)) { - pr_err("gen: L2P data from device is out of bounds!\n"); - return -EINVAL; - } - - for (i = 0; i < nlb; i++) { - u64 pba = le64_to_cpu(entries[i]); - - if (unlikely(pba >= dev->total_secs && pba != U64_MAX)) { - pr_err("gen: L2P data entry is out of bounds!\n"); - return -EINVAL; - } - - /* Address zero is a special one. The first page on a disk is - * protected. It often holds internal device boot - * information. - */ - if (!pba) - continue; - - /* resolve block from physical address */ - lun_id = div_u64(pba, dev->sec_per_lun); - lun = &gn->luns[lun_id]; - - /* Calculate block offset into lun */ - pba = pba - (dev->sec_per_lun * lun_id); - blk = &lun->vlun.blocks[div_u64(pba, dev->sec_per_blk)]; - - if (!blk->state) { - /* at this point, we don't know anything about the - * block. It's up to the FTL on top to re-etablish the - * block state. The block is assumed to be open. - */ - list_move_tail(&blk->list, &lun->used_list); - blk->state = NVM_BLK_ST_TGT; - lun->vlun.nr_free_blocks--; - } - } - - return 0; -} - -static int gen_blocks_init(struct nvm_dev *dev, struct gen_dev *gn) -{ - struct gen_lun *lun; - struct nvm_block *block; - sector_t lun_iter, blk_iter, cur_block_id = 0; - int ret, nr_blks; - u8 *blks; - - nr_blks = dev->blks_per_lun * dev->plane_mode; - blks = kmalloc(nr_blks, GFP_KERNEL); - if (!blks) - return -ENOMEM; - - gen_for_each_lun(gn, lun, lun_iter) { - lun->vlun.blocks = vzalloc(sizeof(struct nvm_block) * - dev->blks_per_lun); - if (!lun->vlun.blocks) { - kfree(blks); - return -ENOMEM; - } - - for (blk_iter = 0; blk_iter < dev->blks_per_lun; blk_iter++) { - block = &lun->vlun.blocks[blk_iter]; - - INIT_LIST_HEAD(&block->list); - - block->lun = &lun->vlun; - block->id = cur_block_id++; - - /* First block is reserved for device */ - if (unlikely(lun_iter == 0 && blk_iter == 0)) { - lun->vlun.nr_free_blocks--; - continue; - } - - list_add_tail(&block->list, &lun->free_list); - } - - if (dev->ops->get_bb_tbl) { - struct ppa_addr ppa; - - ppa.ppa = 0; - ppa.g.ch = lun->vlun.chnl_id; - ppa.g.lun = lun->vlun.lun_id; - - ret = nvm_get_bb_tbl(dev, ppa, blks); - if (ret) - pr_err("gen: could not get BB table\n"); - - ret = gen_block_bb(gn, ppa, blks, nr_blks); - if (ret) - pr_err("gen: BB table map failed\n"); - } - } - - if ((dev->identity.dom & NVM_RSP_L2P) && dev->ops->get_l2p_tbl) { - ret = dev->ops->get_l2p_tbl(dev, 0, dev->total_secs, - gen_block_map, dev); - if (ret) { - pr_err("gen: could not read L2P table.\n"); - pr_warn("gen: default block initialization"); - } - } - - kfree(blks); - return 0; -} - static void gen_free(struct nvm_dev *dev) { - gen_blocks_free(dev); - gen_luns_free(dev); kfree(dev->mp); + kfree(dev->rmap); dev->mp = NULL; } static int gen_register(struct nvm_dev *dev) { struct gen_dev *gn; - int ret; + struct gen_dev_map *dev_rmap; + int i, j; if (!try_module_get(THIS_MODULE)) return -ENODEV; gn = kzalloc(sizeof(struct gen_dev), GFP_KERNEL); if (!gn) - return -ENOMEM; + goto err_gn; + + dev_rmap = kmalloc(sizeof(struct gen_dev_map), GFP_KERNEL); + if (!dev_rmap) + goto err_rmap; + + dev_rmap->chnls = kcalloc(dev->geo.nr_chnls, sizeof(struct gen_ch_map), + GFP_KERNEL); + if (!dev_rmap->chnls) + goto err_chnls; + + for (i = 0; i < dev->geo.nr_chnls; i++) { + struct gen_ch_map *ch_rmap; + int *lun_roffs; + int luns_in_chnl = dev->geo.luns_per_chnl; + + ch_rmap = &dev_rmap->chnls[i]; + + ch_rmap->ch_off = -1; + ch_rmap->nr_luns = luns_in_chnl; + + lun_roffs = kcalloc(luns_in_chnl, sizeof(int), GFP_KERNEL); + if (!lun_roffs) + goto err_ch; + + for (j = 0; j < luns_in_chnl; j++) + lun_roffs[j] = -1; + + ch_rmap->lun_offs = lun_roffs; + } gn->dev = dev; - gn->nr_luns = dev->nr_luns; + gn->nr_luns = dev->geo.nr_luns; INIT_LIST_HEAD(&gn->area_list); mutex_init(&gn->lock); INIT_LIST_HEAD(&gn->targets); dev->mp = gn; - - ret = gen_luns_init(dev, gn); - if (ret) { - pr_err("gen: could not initialize luns\n"); - goto err; - } - - ret = gen_blocks_init(dev, gn); - if (ret) { - pr_err("gen: could not initialize blocks\n"); - goto err; - } + dev->rmap = dev_rmap; return 1; -err: +err_ch: + while (--i >= 0) + kfree(dev_rmap->chnls[i].lun_offs); +err_chnls: + kfree(dev_rmap); +err_rmap: gen_free(dev); +err_gn: module_put(THIS_MODULE); - return ret; + return -ENOMEM; } static void gen_unregister(struct nvm_dev *dev) @@ -463,7 +472,7 @@ static void gen_unregister(struct nvm_dev *dev) mutex_lock(&gn->lock); list_for_each_entry_safe(t, tmp, &gn->targets, list) { - if (t->dev != dev) + if (t->dev->parent != dev) continue; __gen_remove_target(t); } @@ -473,168 +482,142 @@ static void gen_unregister(struct nvm_dev *dev) module_put(THIS_MODULE); } -static struct nvm_block *gen_get_blk(struct nvm_dev *dev, - struct nvm_lun *vlun, unsigned long flags) +static int gen_map_to_dev(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p) { - struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun); - struct nvm_block *blk = NULL; - int is_gc = flags & NVM_IOTYPE_GC; - - spin_lock(&vlun->lock); - if (list_empty(&lun->free_list)) { - pr_err_ratelimited("gen: lun %u have no free pages available", - lun->vlun.id); - goto out; + struct gen_dev_map *dev_map = tgt_dev->map; + struct gen_ch_map *ch_map = &dev_map->chnls[p->g.ch]; + int lun_off = ch_map->lun_offs[p->g.lun]; + struct nvm_dev *dev = tgt_dev->parent; + struct gen_dev_map *dev_rmap = dev->rmap; + struct gen_ch_map *ch_rmap; + int lun_roff; + + p->g.ch += ch_map->ch_off; + p->g.lun += lun_off; + + ch_rmap = &dev_rmap->chnls[p->g.ch]; + lun_roff = ch_rmap->lun_offs[p->g.lun]; + + if (unlikely(ch_rmap->ch_off < 0 || lun_roff < 0)) { + pr_err("nvm: corrupted device partition table\n"); + return -EINVAL; } - if (!is_gc && lun->vlun.nr_free_blocks < lun->reserved_blocks) - goto out; - - blk = list_first_entry(&lun->free_list, struct nvm_block, list); - - list_move_tail(&blk->list, &lun->used_list); - blk->state = NVM_BLK_ST_TGT; - lun->vlun.nr_free_blocks--; -out: - spin_unlock(&vlun->lock); - return blk; -} - -static void gen_put_blk(struct nvm_dev *dev, struct nvm_block *blk) -{ - struct nvm_lun *vlun = blk->lun; - struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun); - - spin_lock(&vlun->lock); - if (blk->state & NVM_BLK_ST_TGT) { - list_move_tail(&blk->list, &lun->free_list); - lun->vlun.nr_free_blocks++; - blk->state = NVM_BLK_ST_FREE; - } else if (blk->state & NVM_BLK_ST_BAD) { - list_move_tail(&blk->list, &lun->bb_list); - blk->state = NVM_BLK_ST_BAD; - } else { - WARN_ON_ONCE(1); - pr_err("gen: erroneous block type (%lu -> %u)\n", - blk->id, blk->state); - list_move_tail(&blk->list, &lun->bb_list); - } - spin_unlock(&vlun->lock); + return 0; } -static void gen_mark_blk(struct nvm_dev *dev, struct ppa_addr ppa, int type) +static int gen_map_to_tgt(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p) { - struct gen_dev *gn = dev->mp; - struct gen_lun *lun; - struct nvm_block *blk; - - pr_debug("gen: ppa (ch: %u lun: %u blk: %u pg: %u) -> %u\n", - ppa.g.ch, ppa.g.lun, ppa.g.blk, ppa.g.pg, type); - - if (unlikely(ppa.g.ch > dev->nr_chnls || - ppa.g.lun > dev->luns_per_chnl || - ppa.g.blk > dev->blks_per_lun)) { - WARN_ON_ONCE(1); - pr_err("gen: ppa broken (ch: %u > %u lun: %u > %u blk: %u > %u", - ppa.g.ch, dev->nr_chnls, - ppa.g.lun, dev->luns_per_chnl, - ppa.g.blk, dev->blks_per_lun); - return; - } + struct nvm_dev *dev = tgt_dev->parent; + struct gen_dev_map *dev_rmap = dev->rmap; + struct gen_ch_map *ch_rmap = &dev_rmap->chnls[p->g.ch]; + int lun_roff = ch_rmap->lun_offs[p->g.lun]; - lun = &gn->luns[(dev->luns_per_chnl * ppa.g.ch) + ppa.g.lun]; - blk = &lun->vlun.blocks[ppa.g.blk]; + p->g.ch -= ch_rmap->ch_off; + p->g.lun -= lun_roff; - /* will be moved to bb list on put_blk from target */ - blk->state = type; + return 0; } -/* - * mark block bad in gen. It is expected that the target recovers separately - */ -static void gen_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd) +static int gen_trans_rq(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd, + int flag) { - int bit = -1; - int max_secs = dev->ops->max_phys_sect; - void *comp_bits = &rqd->ppa_status; + gen_trans_fn *f; + int i; + int ret = 0; - nvm_addr_to_generic_mode(dev, rqd); + f = (flag == TRANS_TGT_TO_DEV) ? gen_map_to_dev : gen_map_to_tgt; - /* look up blocks and mark them as bad */ - if (rqd->nr_ppas == 1) { - gen_mark_blk(dev, rqd->ppa_addr, NVM_BLK_ST_BAD); - return; + if (rqd->nr_ppas == 1) + return f(tgt_dev, &rqd->ppa_addr); + + for (i = 0; i < rqd->nr_ppas; i++) { + ret = f(tgt_dev, &rqd->ppa_list[i]); + if (ret) + goto out; } - while ((bit = find_next_bit(comp_bits, max_secs, bit + 1)) < max_secs) - gen_mark_blk(dev, rqd->ppa_list[bit], NVM_BLK_ST_BAD); +out: + return ret; } static void gen_end_io(struct nvm_rq *rqd) { + struct nvm_tgt_dev *tgt_dev = rqd->dev; struct nvm_tgt_instance *ins = rqd->ins; - if (rqd->error == NVM_RSP_ERR_FAILWRITE) - gen_mark_blk_bad(rqd->dev, rqd); + /* Convert address space */ + if (tgt_dev) + gen_trans_rq(tgt_dev, rqd, TRANS_DEV_TO_TGT); ins->tt->end_io(rqd); } -static int gen_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) +static int gen_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd) { + struct nvm_dev *dev = tgt_dev->parent; + if (!dev->ops->submit_io) return -ENODEV; /* Convert address space */ + gen_trans_rq(tgt_dev, rqd, TRANS_TGT_TO_DEV); nvm_generic_to_addr_mode(dev, rqd); - rqd->dev = dev; + rqd->dev = tgt_dev; rqd->end_io = gen_end_io; return dev->ops->submit_io(dev, rqd); } -static int gen_erase_blk(struct nvm_dev *dev, struct nvm_block *blk, - unsigned long flags) +static int gen_erase_blk(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p, + int flags) { - struct ppa_addr addr = block_to_ppa(dev, blk); + /* Convert address space */ + gen_map_to_dev(tgt_dev, p); - return nvm_erase_ppa(dev, &addr, 1); + return nvm_erase_ppa(tgt_dev->parent, p, 1, flags); } -static int gen_reserve_lun(struct nvm_dev *dev, int lunid) +static struct ppa_addr gen_trans_ppa(struct nvm_tgt_dev *tgt_dev, + struct ppa_addr p, int direction) { - return test_and_set_bit(lunid, dev->lun_map); -} + gen_trans_fn *f; + struct ppa_addr ppa = p; -static void gen_release_lun(struct nvm_dev *dev, int lunid) -{ - WARN_ON(!test_and_clear_bit(lunid, dev->lun_map)); + f = (direction == TRANS_TGT_TO_DEV) ? gen_map_to_dev : gen_map_to_tgt; + f(tgt_dev, &ppa); + + return ppa; } -static struct nvm_lun *gen_get_lun(struct nvm_dev *dev, int lunid) +static void gen_part_to_tgt(struct nvm_dev *dev, sector_t *entries, + int len) { - struct gen_dev *gn = dev->mp; - - if (unlikely(lunid >= dev->nr_luns)) - return NULL; + struct nvm_geo *geo = &dev->geo; + struct gen_dev_map *dev_rmap = dev->rmap; + u64 i; - return &gn->luns[lunid].vlun; -} + for (i = 0; i < len; i++) { + struct gen_ch_map *ch_rmap; + int *lun_roffs; + struct ppa_addr gaddr; + u64 pba = le64_to_cpu(entries[i]); + int off; + u64 diff; -static void gen_lun_info_print(struct nvm_dev *dev) -{ - struct gen_dev *gn = dev->mp; - struct gen_lun *lun; - unsigned int i; + if (!pba) + continue; + gaddr = linear_to_generic_addr(geo, pba); + ch_rmap = &dev_rmap->chnls[gaddr.g.ch]; + lun_roffs = ch_rmap->lun_offs; - gen_for_each_lun(gn, lun, i) { - spin_lock(&lun->vlun.lock); + off = gaddr.g.ch * geo->luns_per_chnl + gaddr.g.lun; - pr_info("%s: lun%8u\t%u\n", dev->name, i, - lun->vlun.nr_free_blocks); + diff = ((ch_rmap->ch_off * geo->luns_per_chnl) + + (lun_roffs[gaddr.g.lun])) * geo->sec_per_lun; - spin_unlock(&lun->vlun.lock); + entries[i] -= cpu_to_le64(diff); } } @@ -648,22 +631,14 @@ static struct nvmm_type gen = { .create_tgt = gen_create_tgt, .remove_tgt = gen_remove_tgt, - .get_blk = gen_get_blk, - .put_blk = gen_put_blk, - .submit_io = gen_submit_io, .erase_blk = gen_erase_blk, - .mark_blk = gen_mark_blk, - - .get_lun = gen_get_lun, - .reserve_lun = gen_reserve_lun, - .release_lun = gen_release_lun, - .lun_info_print = gen_lun_info_print, - .get_area = gen_get_area, .put_area = gen_put_area, + .trans_ppa = gen_trans_ppa, + .part_to_tgt = gen_part_to_tgt, }; static int __init gen_module_init(void) diff --git a/drivers/lightnvm/gennvm.h b/drivers/lightnvm/gennvm.h index 8ecfa817d21d..6a4b3f368848 100644 --- a/drivers/lightnvm/gennvm.h +++ b/drivers/lightnvm/gennvm.h @@ -20,37 +20,41 @@ #include <linux/lightnvm.h> -struct gen_lun { - struct nvm_lun vlun; - - int reserved_blocks; - /* lun block lists */ - struct list_head used_list; /* In-use blocks */ - struct list_head free_list; /* Not used blocks i.e. released - * and ready for use - */ - struct list_head bb_list; /* Bad blocks. Mutually exclusive with - * free_list and used_list - */ -}; - struct gen_dev { struct nvm_dev *dev; int nr_luns; - struct gen_lun *luns; struct list_head area_list; struct mutex lock; struct list_head targets; }; +/* Map between virtual and physical channel and lun */ +struct gen_ch_map { + int ch_off; + int nr_luns; + int *lun_offs; +}; + +struct gen_dev_map { + struct gen_ch_map *chnls; + int nr_chnls; +}; + struct gen_area { struct list_head list; sector_t begin; sector_t end; /* end is excluded */ }; +static inline void *ch_map_to_lun_offs(struct gen_ch_map *ch_map) +{ + return ch_map + 1; +} + +typedef int (gen_trans_fn)(struct nvm_tgt_dev *, struct ppa_addr *); + #define gen_for_each_lun(bm, lun, i) \ for ((i) = 0, lun = &(bm)->luns[0]; \ (i) < (bm)->nr_luns; (i)++, lun = &(bm)->luns[(i)]) diff --git a/drivers/lightnvm/lightnvm.h b/drivers/lightnvm/lightnvm.h deleted file mode 100644 index 305c181509a6..000000000000 --- a/drivers/lightnvm/lightnvm.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2016 CNEX Labs. All rights reserved. - * Initial release: Matias Bjorling <matias@cnexlabs.com> - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version - * 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; see the file COPYING. If not, write to - * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, - * USA. - * - */ - -#ifndef LIGHTNVM_H -#define LIGHTNVM_H - -#include <linux/lightnvm.h> - -/* core -> sysfs.c */ -int __must_check nvm_sysfs_register_dev(struct nvm_dev *); -void nvm_sysfs_unregister_dev(struct nvm_dev *); -int nvm_sysfs_register(void); -void nvm_sysfs_unregister(void); - -/* sysfs > core */ -void nvm_free(struct nvm_dev *); - -#endif diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c index 37fcaadbf80c..9fb7de395915 100644 --- a/drivers/lightnvm/rrpc.c +++ b/drivers/lightnvm/rrpc.c @@ -28,6 +28,7 @@ static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio, static void rrpc_page_invalidate(struct rrpc *rrpc, struct rrpc_addr *a) { + struct nvm_tgt_dev *dev = rrpc->dev; struct rrpc_block *rblk = a->rblk; unsigned int pg_offset; @@ -38,13 +39,13 @@ static void rrpc_page_invalidate(struct rrpc *rrpc, struct rrpc_addr *a) spin_lock(&rblk->lock); - div_u64_rem(a->addr, rrpc->dev->sec_per_blk, &pg_offset); + div_u64_rem(a->addr, dev->geo.sec_per_blk, &pg_offset); WARN_ON(test_and_set_bit(pg_offset, rblk->invalid_pages)); rblk->nr_invalid_pages++; spin_unlock(&rblk->lock); - rrpc->rev_trans_map[a->addr - rrpc->poffset].addr = ADDR_EMPTY; + rrpc->rev_trans_map[a->addr].addr = ADDR_EMPTY; } static void rrpc_invalidate_range(struct rrpc *rrpc, sector_t slba, @@ -116,62 +117,35 @@ static void rrpc_discard(struct rrpc *rrpc, struct bio *bio) static int block_is_full(struct rrpc *rrpc, struct rrpc_block *rblk) { - return (rblk->next_page == rrpc->dev->sec_per_blk); + struct nvm_tgt_dev *dev = rrpc->dev; + + return (rblk->next_page == dev->geo.sec_per_blk); } /* Calculate relative addr for the given block, considering instantiated LUNs */ static u64 block_to_rel_addr(struct rrpc *rrpc, struct rrpc_block *rblk) { - struct nvm_block *blk = rblk->parent; - int lun_blk = blk->id % (rrpc->dev->blks_per_lun * rrpc->nr_luns); - - return lun_blk * rrpc->dev->sec_per_blk; -} - -/* Calculate global addr for the given block */ -static u64 block_to_addr(struct rrpc *rrpc, struct rrpc_block *rblk) -{ - struct nvm_block *blk = rblk->parent; - - return blk->id * rrpc->dev->sec_per_blk; -} - -static struct ppa_addr linear_to_generic_addr(struct nvm_dev *dev, - struct ppa_addr r) -{ - struct ppa_addr l; - int secs, pgs, blks, luns; - sector_t ppa = r.ppa; - - l.ppa = 0; - - div_u64_rem(ppa, dev->sec_per_pg, &secs); - l.g.sec = secs; - - sector_div(ppa, dev->sec_per_pg); - div_u64_rem(ppa, dev->pgs_per_blk, &pgs); - l.g.pg = pgs; - - sector_div(ppa, dev->pgs_per_blk); - div_u64_rem(ppa, dev->blks_per_lun, &blks); - l.g.blk = blks; - - sector_div(ppa, dev->blks_per_lun); - div_u64_rem(ppa, dev->luns_per_chnl, &luns); - l.g.lun = luns; - - sector_div(ppa, dev->luns_per_chnl); - l.g.ch = ppa; + struct nvm_tgt_dev *dev = rrpc->dev; + struct rrpc_lun *rlun = rblk->rlun; - return l; + return rlun->id * dev->geo.sec_per_blk; } -static struct ppa_addr rrpc_ppa_to_gaddr(struct nvm_dev *dev, u64 addr) +static struct ppa_addr rrpc_ppa_to_gaddr(struct nvm_tgt_dev *dev, + struct rrpc_addr *gp) { + struct rrpc_block *rblk = gp->rblk; + struct rrpc_lun *rlun = rblk->rlun; + u64 addr = gp->addr; struct ppa_addr paddr; paddr.ppa = addr; - return linear_to_generic_addr(dev, paddr); + paddr = rrpc_linear_to_generic_addr(&dev->geo, paddr); + paddr.g.ch = rlun->bppa.g.ch; + paddr.g.lun = rlun->bppa.g.lun; + paddr.g.blk = rblk->id; + + return paddr; } /* requires lun->lock taken */ @@ -188,21 +162,47 @@ static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *new_rblk, *cur_rblk = new_rblk; } +static struct rrpc_block *__rrpc_get_blk(struct rrpc *rrpc, + struct rrpc_lun *rlun) +{ + struct rrpc_block *rblk = NULL; + + if (list_empty(&rlun->free_list)) + goto out; + + rblk = list_first_entry(&rlun->free_list, struct rrpc_block, list); + + list_move_tail(&rblk->list, &rlun->used_list); + rblk->state = NVM_BLK_ST_TGT; + rlun->nr_free_blocks--; + +out: + return rblk; +} + static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun, unsigned long flags) { - struct nvm_block *blk; + struct nvm_tgt_dev *dev = rrpc->dev; struct rrpc_block *rblk; + int is_gc = flags & NVM_IOTYPE_GC; + + spin_lock(&rlun->lock); + if (!is_gc && rlun->nr_free_blocks < rlun->reserved_blocks) { + pr_err("nvm: rrpc: cannot give block to non GC request\n"); + spin_unlock(&rlun->lock); + return NULL; + } - blk = nvm_get_blk(rrpc->dev, rlun->parent, flags); - if (!blk) { - pr_err("nvm: rrpc: cannot get new block from media manager\n"); + rblk = __rrpc_get_blk(rrpc, rlun); + if (!rblk) { + pr_err("nvm: rrpc: cannot get new block\n"); + spin_unlock(&rlun->lock); return NULL; } + spin_unlock(&rlun->lock); - rblk = rrpc_get_rblk(rlun, blk->id); - blk->priv = rblk; - bitmap_zero(rblk->invalid_pages, rrpc->dev->sec_per_blk); + bitmap_zero(rblk->invalid_pages, dev->geo.sec_per_blk); rblk->next_page = 0; rblk->nr_invalid_pages = 0; atomic_set(&rblk->data_cmnt_size, 0); @@ -212,7 +212,24 @@ static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun, static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk) { - nvm_put_blk(rrpc->dev, rblk->parent); + struct rrpc_lun *rlun = rblk->rlun; + + spin_lock(&rlun->lock); + if (rblk->state & NVM_BLK_ST_TGT) { + list_move_tail(&rblk->list, &rlun->free_list); + rlun->nr_free_blocks++; + rblk->state = NVM_BLK_ST_FREE; + } else if (rblk->state & NVM_BLK_ST_BAD) { + list_move_tail(&rblk->list, &rlun->bb_list); + rblk->state = NVM_BLK_ST_BAD; + } else { + WARN_ON_ONCE(1); + pr_err("rrpc: erroneous type (ch:%d,lun:%d,blk%d-> %u)\n", + rlun->bppa.g.ch, rlun->bppa.g.lun, + rblk->id, rblk->state); + list_move_tail(&rblk->list, &rlun->bb_list); + } + spin_unlock(&rlun->lock); } static void rrpc_put_blks(struct rrpc *rrpc) @@ -280,13 +297,14 @@ static void rrpc_end_sync_bio(struct bio *bio) */ static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk) { - struct request_queue *q = rrpc->dev->q; + struct nvm_tgt_dev *dev = rrpc->dev; + struct request_queue *q = dev->q; struct rrpc_rev_addr *rev; struct nvm_rq *rqd; struct bio *bio; struct page *page; int slot; - int nr_sec_per_blk = rrpc->dev->sec_per_blk; + int nr_sec_per_blk = dev->geo.sec_per_blk; u64 phys_addr; DECLARE_COMPLETION_ONSTACK(wait); @@ -309,12 +327,12 @@ static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk) nr_sec_per_blk)) < nr_sec_per_blk) { /* Lock laddr */ - phys_addr = rblk->parent->id * nr_sec_per_blk + slot; + phys_addr = rrpc_blk_to_ppa(rrpc, rblk) + slot; try: spin_lock(&rrpc->rev_lock); /* Get logical address from physical to logical table */ - rev = &rrpc->rev_trans_map[phys_addr - rrpc->poffset]; + rev = &rrpc->rev_trans_map[phys_addr]; /* already updated by previous regular write */ if (rev->addr == ADDR_EMPTY) { spin_unlock(&rrpc->rev_lock); @@ -396,15 +414,23 @@ static void rrpc_block_gc(struct work_struct *work) struct rrpc *rrpc = gcb->rrpc; struct rrpc_block *rblk = gcb->rblk; struct rrpc_lun *rlun = rblk->rlun; - struct nvm_dev *dev = rrpc->dev; + struct nvm_tgt_dev *dev = rrpc->dev; + struct ppa_addr ppa; mempool_free(gcb, rrpc->gcb_pool); - pr_debug("nvm: block '%lu' being reclaimed\n", rblk->parent->id); + pr_debug("nvm: block 'ch:%d,lun:%d,blk:%d' being reclaimed\n", + rlun->bppa.g.ch, rlun->bppa.g.lun, + rblk->id); if (rrpc_move_valid_pages(rrpc, rblk)) goto put_back; - if (nvm_erase_blk(dev, rblk->parent)) + ppa.ppa = 0; + ppa.g.ch = rlun->bppa.g.ch; + ppa.g.lun = rlun->bppa.g.lun; + ppa.g.blk = rblk->id; + + if (nvm_erase_blk(dev, &ppa, 0)) goto put_back; rrpc_put_blk(rrpc, rblk); @@ -420,7 +446,7 @@ put_back: /* the block with highest number of invalid pages, will be in the beginning * of the list */ -static struct rrpc_block *rblock_max_invalid(struct rrpc_block *ra, +static struct rrpc_block *rblk_max_invalid(struct rrpc_block *ra, struct rrpc_block *rb) { if (ra->nr_invalid_pages == rb->nr_invalid_pages) @@ -435,13 +461,13 @@ static struct rrpc_block *rblock_max_invalid(struct rrpc_block *ra, static struct rrpc_block *block_prio_find_max(struct rrpc_lun *rlun) { struct list_head *prio_list = &rlun->prio_list; - struct rrpc_block *rblock, *max; + struct rrpc_block *rblk, *max; BUG_ON(list_empty(prio_list)); max = list_first_entry(prio_list, struct rrpc_block, prio); - list_for_each_entry(rblock, prio_list, prio) - max = rblock_max_invalid(max, rblock); + list_for_each_entry(rblk, prio_list, prio) + max = rblk_max_invalid(max, rblk); return max; } @@ -450,36 +476,37 @@ static void rrpc_lun_gc(struct work_struct *work) { struct rrpc_lun *rlun = container_of(work, struct rrpc_lun, ws_gc); struct rrpc *rrpc = rlun->rrpc; - struct nvm_lun *lun = rlun->parent; + struct nvm_tgt_dev *dev = rrpc->dev; struct rrpc_block_gc *gcb; unsigned int nr_blocks_need; - nr_blocks_need = rrpc->dev->blks_per_lun / GC_LIMIT_INVERSE; + nr_blocks_need = dev->geo.blks_per_lun / GC_LIMIT_INVERSE; if (nr_blocks_need < rrpc->nr_luns) nr_blocks_need = rrpc->nr_luns; spin_lock(&rlun->lock); - while (nr_blocks_need > lun->nr_free_blocks && + while (nr_blocks_need > rlun->nr_free_blocks && !list_empty(&rlun->prio_list)) { - struct rrpc_block *rblock = block_prio_find_max(rlun); - struct nvm_block *block = rblock->parent; + struct rrpc_block *rblk = block_prio_find_max(rlun); - if (!rblock->nr_invalid_pages) + if (!rblk->nr_invalid_pages) break; gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC); if (!gcb) break; - list_del_init(&rblock->prio); + list_del_init(&rblk->prio); - BUG_ON(!block_is_full(rrpc, rblock)); + WARN_ON(!block_is_full(rrpc, rblk)); - pr_debug("rrpc: selected block '%lu' for GC\n", block->id); + pr_debug("rrpc: selected block 'ch:%d,lun:%d,blk:%d' for GC\n", + rlun->bppa.g.ch, rlun->bppa.g.lun, + rblk->id); gcb->rrpc = rrpc; - gcb->rblk = rblock; + gcb->rblk = rblk; INIT_WORK(&gcb->ws_gc, rrpc_block_gc); queue_work(rrpc->kgc_wq, &gcb->ws_gc); @@ -504,8 +531,9 @@ static void rrpc_gc_queue(struct work_struct *work) spin_unlock(&rlun->lock); mempool_free(gcb, rrpc->gcb_pool); - pr_debug("nvm: block '%lu' is full, allow GC (sched)\n", - rblk->parent->id); + pr_debug("nvm: block 'ch:%d,lun:%d,blk:%d' full, allow GC (sched)\n", + rlun->bppa.g.ch, rlun->bppa.g.lun, + rblk->id); } static const struct block_device_operations rrpc_fops = { @@ -529,8 +557,7 @@ static struct rrpc_lun *rrpc_get_lun_rr(struct rrpc *rrpc, int is_gc) * estimate. */ rrpc_for_each_lun(rrpc, rlun, i) { - if (rlun->parent->nr_free_blocks > - max_free->parent->nr_free_blocks) + if (rlun->nr_free_blocks > max_free->nr_free_blocks) max_free = rlun; } @@ -553,7 +580,7 @@ static struct rrpc_addr *rrpc_update_map(struct rrpc *rrpc, sector_t laddr, gp->addr = paddr; gp->rblk = rblk; - rev = &rrpc->rev_trans_map[gp->addr - rrpc->poffset]; + rev = &rrpc->rev_trans_map[gp->addr]; rev->addr = laddr; spin_unlock(&rrpc->rev_lock); @@ -568,7 +595,7 @@ static u64 rrpc_alloc_addr(struct rrpc *rrpc, struct rrpc_block *rblk) if (block_is_full(rrpc, rblk)) goto out; - addr = block_to_addr(rrpc, rblk) + rblk->next_page; + addr = rblk->next_page; rblk->next_page++; out: @@ -582,20 +609,22 @@ out: * Returns rrpc_addr with the physical address and block. Returns NULL if no * blocks in the next rlun are available. */ -static struct rrpc_addr *rrpc_map_page(struct rrpc *rrpc, sector_t laddr, +static struct ppa_addr rrpc_map_page(struct rrpc *rrpc, sector_t laddr, int is_gc) { + struct nvm_tgt_dev *tgt_dev = rrpc->dev; struct rrpc_lun *rlun; struct rrpc_block *rblk, **cur_rblk; - struct nvm_lun *lun; + struct rrpc_addr *p; + struct ppa_addr ppa; u64 paddr; int gc_force = 0; + ppa.ppa = ADDR_EMPTY; rlun = rrpc_get_lun_rr(rrpc, is_gc); - lun = rlun->parent; - if (!is_gc && lun->nr_free_blocks < rrpc->nr_luns * 4) - return NULL; + if (!is_gc && rlun->nr_free_blocks < rrpc->nr_luns * 4) + return ppa; /* * page allocation steps: @@ -652,10 +681,15 @@ new_blk: } pr_err("rrpc: failed to allocate new block\n"); - return NULL; + return ppa; done: spin_unlock(&rlun->lock); - return rrpc_update_map(rrpc, laddr, rblk, paddr); + p = rrpc_update_map(rrpc, laddr, rblk, paddr); + if (!p) + return ppa; + + /* return global address */ + return rrpc_ppa_to_gaddr(tgt_dev, p); } static void rrpc_run_gc(struct rrpc *rrpc, struct rrpc_block *rblk) @@ -675,21 +709,70 @@ static void rrpc_run_gc(struct rrpc *rrpc, struct rrpc_block *rblk) queue_work(rrpc->kgc_wq, &gcb->ws_gc); } +static struct rrpc_lun *rrpc_ppa_to_lun(struct rrpc *rrpc, struct ppa_addr p) +{ + struct rrpc_lun *rlun = NULL; + int i; + + for (i = 0; i < rrpc->nr_luns; i++) { + if (rrpc->luns[i].bppa.g.ch == p.g.ch && + rrpc->luns[i].bppa.g.lun == p.g.lun) { + rlun = &rrpc->luns[i]; + break; + } + } + + return rlun; +} + +static void __rrpc_mark_bad_block(struct rrpc *rrpc, struct ppa_addr ppa) +{ + struct nvm_tgt_dev *dev = rrpc->dev; + struct rrpc_lun *rlun; + struct rrpc_block *rblk; + + rlun = rrpc_ppa_to_lun(rrpc, ppa); + rblk = &rlun->blocks[ppa.g.blk]; + rblk->state = NVM_BLK_ST_BAD; + + nvm_set_tgt_bb_tbl(dev, &ppa, 1, NVM_BLK_T_GRWN_BAD); +} + +static void rrpc_mark_bad_block(struct rrpc *rrpc, struct nvm_rq *rqd) +{ + void *comp_bits = &rqd->ppa_status; + struct ppa_addr ppa, prev_ppa; + int nr_ppas = rqd->nr_ppas; + int bit; + + if (rqd->nr_ppas == 1) + __rrpc_mark_bad_block(rrpc, rqd->ppa_addr); + + ppa_set_empty(&prev_ppa); + bit = -1; + while ((bit = find_next_bit(comp_bits, nr_ppas, bit + 1)) < nr_ppas) { + ppa = rqd->ppa_list[bit]; + if (ppa_cmp_blk(ppa, prev_ppa)) + continue; + + __rrpc_mark_bad_block(rrpc, ppa); + } +} + static void rrpc_end_io_write(struct rrpc *rrpc, struct rrpc_rq *rrqd, sector_t laddr, uint8_t npages) { + struct nvm_tgt_dev *dev = rrpc->dev; struct rrpc_addr *p; struct rrpc_block *rblk; - struct nvm_lun *lun; int cmnt_size, i; for (i = 0; i < npages; i++) { p = &rrpc->trans_map[laddr + i]; rblk = p->rblk; - lun = rblk->parent->lun; cmnt_size = atomic_inc_return(&rblk->data_cmnt_size); - if (unlikely(cmnt_size == rrpc->dev->sec_per_blk)) + if (unlikely(cmnt_size == dev->geo.sec_per_blk)) rrpc_run_gc(rrpc, rblk); } } @@ -697,12 +780,17 @@ static void rrpc_end_io_write(struct rrpc *rrpc, struct rrpc_rq *rrqd, static void rrpc_end_io(struct nvm_rq *rqd) { struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance); + struct nvm_tgt_dev *dev = rrpc->dev; struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); uint8_t npages = rqd->nr_ppas; sector_t laddr = rrpc_get_laddr(rqd->bio) - npages; - if (bio_data_dir(rqd->bio) == WRITE) + if (bio_data_dir(rqd->bio) == WRITE) { + if (rqd->error == NVM_RSP_ERR_FAILWRITE) + rrpc_mark_bad_block(rrpc, rqd); + rrpc_end_io_write(rrpc, rrqd, laddr, npages); + } bio_put(rqd->bio); @@ -712,7 +800,7 @@ static void rrpc_end_io(struct nvm_rq *rqd) rrpc_unlock_rq(rrpc, rqd); if (npages > 1) - nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); + nvm_dev_dma_free(dev->parent, rqd->ppa_list, rqd->dma_ppa_list); mempool_free(rqd, rrpc->rq_pool); } @@ -720,6 +808,7 @@ static void rrpc_end_io(struct nvm_rq *rqd) static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, unsigned long flags, int npages) { + struct nvm_tgt_dev *dev = rrpc->dev; struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd); struct rrpc_addr *gp; sector_t laddr = rrpc_get_laddr(bio); @@ -727,7 +816,7 @@ static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio, int i; if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) { - nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); + nvm_dev_dma_free(dev->parent, rqd->ppa_list, rqd->dma_ppa_list); return NVM_IO_REQUEUE; } @@ -737,12 +826,11 @@ static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio, gp = &rrpc->trans_map[laddr + i]; if (gp->rblk) { - rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev, - gp->addr); + rqd->ppa_list[i] = rrpc_ppa_to_gaddr(dev, gp); } else { BUG_ON(is_gc); rrpc_unlock_laddr(rrpc, r); - nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, + nvm_dev_dma_free(dev->parent, rqd->ppa_list, rqd->dma_ppa_list); return NVM_IO_DONE; } @@ -756,7 +844,6 @@ static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio, static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, unsigned long flags) { - struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); int is_gc = flags & NVM_IOTYPE_GC; sector_t laddr = rrpc_get_laddr(bio); struct rrpc_addr *gp; @@ -768,7 +855,7 @@ static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, gp = &rrpc->trans_map[laddr]; if (gp->rblk) { - rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr); + rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp); } else { BUG_ON(is_gc); rrpc_unlock_rq(rrpc, rqd); @@ -776,7 +863,6 @@ static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, } rqd->opcode = NVM_OP_HBREAD; - rrqd->addr = gp; return NVM_IO_OK; } @@ -784,31 +870,31 @@ static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, unsigned long flags, int npages) { + struct nvm_tgt_dev *dev = rrpc->dev; struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd); - struct rrpc_addr *p; + struct ppa_addr p; sector_t laddr = rrpc_get_laddr(bio); int is_gc = flags & NVM_IOTYPE_GC; int i; if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) { - nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); + nvm_dev_dma_free(dev->parent, rqd->ppa_list, rqd->dma_ppa_list); return NVM_IO_REQUEUE; } for (i = 0; i < npages; i++) { /* We assume that mapping occurs at 4KB granularity */ p = rrpc_map_page(rrpc, laddr + i, is_gc); - if (!p) { + if (p.ppa == ADDR_EMPTY) { BUG_ON(is_gc); rrpc_unlock_laddr(rrpc, r); - nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, + nvm_dev_dma_free(dev->parent, rqd->ppa_list, rqd->dma_ppa_list); rrpc_gc_kick(rrpc); return NVM_IO_REQUEUE; } - rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev, - p->addr); + rqd->ppa_list[i] = p; } rqd->opcode = NVM_OP_HBWRITE; @@ -819,8 +905,7 @@ static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio, static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, unsigned long flags) { - struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); - struct rrpc_addr *p; + struct ppa_addr p; int is_gc = flags & NVM_IOTYPE_GC; sector_t laddr = rrpc_get_laddr(bio); @@ -828,16 +913,15 @@ static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio, return NVM_IO_REQUEUE; p = rrpc_map_page(rrpc, laddr, is_gc); - if (!p) { + if (p.ppa == ADDR_EMPTY) { BUG_ON(is_gc); rrpc_unlock_rq(rrpc, rqd); rrpc_gc_kick(rrpc); return NVM_IO_REQUEUE; } - rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, p->addr); + rqd->ppa_addr = p; rqd->opcode = NVM_OP_HBWRITE; - rrqd->addr = p; return NVM_IO_OK; } @@ -845,8 +929,10 @@ static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio, static int rrpc_setup_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, unsigned long flags, uint8_t npages) { + struct nvm_tgt_dev *dev = rrpc->dev; + if (npages > 1) { - rqd->ppa_list = nvm_dev_dma_alloc(rrpc->dev, GFP_KERNEL, + rqd->ppa_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, &rqd->dma_ppa_list); if (!rqd->ppa_list) { pr_err("rrpc: not able to allocate ppa list\n"); @@ -869,14 +955,15 @@ static int rrpc_setup_rq(struct rrpc *rrpc, struct bio *bio, static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, unsigned long flags) { - int err; + struct nvm_tgt_dev *dev = rrpc->dev; struct rrpc_rq *rrq = nvm_rq_to_pdu(rqd); uint8_t nr_pages = rrpc_get_pages(bio); int bio_size = bio_sectors(bio) << 9; + int err; - if (bio_size < rrpc->dev->sec_size) + if (bio_size < dev->geo.sec_size) return NVM_IO_ERR; - else if (bio_size > rrpc->dev->max_rq_size) + else if (bio_size > dev->geo.max_rq_size) return NVM_IO_ERR; err = rrpc_setup_rq(rrpc, bio, rqd, flags, nr_pages); @@ -889,15 +976,15 @@ static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio, rqd->nr_ppas = nr_pages; rrq->flags = flags; - err = nvm_submit_io(rrpc->dev, rqd); + err = nvm_submit_io(dev, rqd); if (err) { pr_err("rrpc: I/O submission failed: %d\n", err); bio_put(bio); if (!(flags & NVM_IOTYPE_GC)) { rrpc_unlock_rq(rrpc, rqd); if (rqd->nr_ppas > 1) - nvm_dev_dma_free(rrpc->dev, - rqd->ppa_list, rqd->dma_ppa_list); + nvm_dev_dma_free(dev->parent, rqd->ppa_list, + rqd->dma_ppa_list); } return NVM_IO_ERR; } @@ -911,6 +998,8 @@ static blk_qc_t rrpc_make_rq(struct request_queue *q, struct bio *bio) struct nvm_rq *rqd; int err; + blk_queue_split(q, &bio, q->bio_split); + if (bio_op(bio) == REQ_OP_DISCARD) { rrpc_discard(rrpc, bio); return BLK_QC_T_NONE; @@ -997,25 +1086,24 @@ static void rrpc_map_free(struct rrpc *rrpc) static int rrpc_l2p_update(u64 slba, u32 nlb, __le64 *entries, void *private) { struct rrpc *rrpc = (struct rrpc *)private; - struct nvm_dev *dev = rrpc->dev; + struct nvm_tgt_dev *dev = rrpc->dev; struct rrpc_addr *addr = rrpc->trans_map + slba; struct rrpc_rev_addr *raddr = rrpc->rev_trans_map; - u64 elba = slba + nlb; + struct rrpc_lun *rlun; + struct rrpc_block *rblk; u64 i; - if (unlikely(elba > dev->total_secs)) { - pr_err("nvm: L2P data from device is out of bounds!\n"); - return -EINVAL; - } - for (i = 0; i < nlb; i++) { + struct ppa_addr gaddr; u64 pba = le64_to_cpu(entries[i]); unsigned int mod; + /* LNVM treats address-spaces as silos, LBA and PBA are * equally large and zero-indexed. */ if (unlikely(pba >= dev->total_secs && pba != U64_MAX)) { pr_err("nvm: L2P data entry is out of bounds!\n"); + pr_err("nvm: Maybe loaded an old target L2P\n"); return -EINVAL; } @@ -1028,7 +1116,27 @@ static int rrpc_l2p_update(u64 slba, u32 nlb, __le64 *entries, void *private) div_u64_rem(pba, rrpc->nr_sects, &mod); + gaddr = rrpc_recov_addr(dev, pba); + rlun = rrpc_ppa_to_lun(rrpc, gaddr); + if (!rlun) { + pr_err("rrpc: l2p corruption on lba %llu\n", + slba + i); + return -EINVAL; + } + + rblk = &rlun->blocks[gaddr.g.blk]; + if (!rblk->state) { + /* at this point, we don't know anything about the + * block. It's up to the FTL on top to re-etablish the + * block state. The block is assumed to be open. + */ + list_move_tail(&rblk->list, &rlun->used_list); + rblk->state = NVM_BLK_ST_TGT; + rlun->nr_free_blocks--; + } + addr[i].addr = pba; + addr[i].rblk = rblk; raddr[mod].addr = slba + i; } @@ -1037,7 +1145,7 @@ static int rrpc_l2p_update(u64 slba, u32 nlb, __le64 *entries, void *private) static int rrpc_map_init(struct rrpc *rrpc) { - struct nvm_dev *dev = rrpc->dev; + struct nvm_tgt_dev *dev = rrpc->dev; sector_t i; int ret; @@ -1058,12 +1166,9 @@ static int rrpc_map_init(struct rrpc *rrpc) r->addr = ADDR_EMPTY; } - if (!dev->ops->get_l2p_tbl) - return 0; - /* Bring up the mapping table from device */ - ret = dev->ops->get_l2p_tbl(dev, rrpc->soffset, rrpc->nr_sects, - rrpc_l2p_update, rrpc); + ret = nvm_get_l2p_tbl(dev, rrpc->soffset, rrpc->nr_sects, + rrpc_l2p_update, rrpc); if (ret) { pr_err("nvm: rrpc: could not read L2P table.\n"); return -EINVAL; @@ -1102,7 +1207,7 @@ static int rrpc_core_init(struct rrpc *rrpc) if (!rrpc->page_pool) return -ENOMEM; - rrpc->gcb_pool = mempool_create_slab_pool(rrpc->dev->nr_luns, + rrpc->gcb_pool = mempool_create_slab_pool(rrpc->dev->geo.nr_luns, rrpc_gcb_cache); if (!rrpc->gcb_pool) return -ENOMEM; @@ -1126,8 +1231,6 @@ static void rrpc_core_free(struct rrpc *rrpc) static void rrpc_luns_free(struct rrpc *rrpc) { - struct nvm_dev *dev = rrpc->dev; - struct nvm_lun *lun; struct rrpc_lun *rlun; int i; @@ -1136,23 +1239,74 @@ static void rrpc_luns_free(struct rrpc *rrpc) for (i = 0; i < rrpc->nr_luns; i++) { rlun = &rrpc->luns[i]; - lun = rlun->parent; - if (!lun) - break; - dev->mt->release_lun(dev, lun->id); vfree(rlun->blocks); } kfree(rrpc->luns); } -static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end) +static int rrpc_bb_discovery(struct nvm_tgt_dev *dev, struct rrpc_lun *rlun) { - struct nvm_dev *dev = rrpc->dev; + struct nvm_geo *geo = &dev->geo; + struct rrpc_block *rblk; + struct ppa_addr ppa; + u8 *blks; + int nr_blks; + int i; + int ret; + + if (!dev->parent->ops->get_bb_tbl) + return 0; + + nr_blks = geo->blks_per_lun * geo->plane_mode; + blks = kmalloc(nr_blks, GFP_KERNEL); + if (!blks) + return -ENOMEM; + + ppa.ppa = 0; + ppa.g.ch = rlun->bppa.g.ch; + ppa.g.lun = rlun->bppa.g.lun; + + ret = nvm_get_tgt_bb_tbl(dev, ppa, blks); + if (ret) { + pr_err("rrpc: could not get BB table\n"); + goto out; + } + + nr_blks = nvm_bb_tbl_fold(dev->parent, blks, nr_blks); + if (nr_blks < 0) + return nr_blks; + + for (i = 0; i < nr_blks; i++) { + if (blks[i] == NVM_BLK_T_FREE) + continue; + + rblk = &rlun->blocks[i]; + list_move_tail(&rblk->list, &rlun->bb_list); + rblk->state = NVM_BLK_ST_BAD; + rlun->nr_free_blocks--; + } + +out: + kfree(blks); + return ret; +} + +static void rrpc_set_lun_ppa(struct rrpc_lun *rlun, struct ppa_addr ppa) +{ + rlun->bppa.ppa = 0; + rlun->bppa.g.ch = ppa.g.ch; + rlun->bppa.g.lun = ppa.g.lun; +} + +static int rrpc_luns_init(struct rrpc *rrpc, struct ppa_addr *luns) +{ + struct nvm_tgt_dev *dev = rrpc->dev; + struct nvm_geo *geo = &dev->geo; struct rrpc_lun *rlun; int i, j, ret = -EINVAL; - if (dev->sec_per_blk > MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) { + if (geo->sec_per_blk > MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) { pr_err("rrpc: number of pages per block too high."); return -EINVAL; } @@ -1166,43 +1320,46 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end) /* 1:1 mapping */ for (i = 0; i < rrpc->nr_luns; i++) { - int lunid = lun_begin + i; - struct nvm_lun *lun; - - if (dev->mt->reserve_lun(dev, lunid)) { - pr_err("rrpc: lun %u is already allocated\n", lunid); - goto err; - } - - lun = dev->mt->get_lun(dev, lunid); - if (!lun) - goto err; - rlun = &rrpc->luns[i]; - rlun->parent = lun; + rlun->id = i; + rrpc_set_lun_ppa(rlun, luns[i]); rlun->blocks = vzalloc(sizeof(struct rrpc_block) * - rrpc->dev->blks_per_lun); + geo->blks_per_lun); if (!rlun->blocks) { ret = -ENOMEM; goto err; } - for (j = 0; j < rrpc->dev->blks_per_lun; j++) { + INIT_LIST_HEAD(&rlun->free_list); + INIT_LIST_HEAD(&rlun->used_list); + INIT_LIST_HEAD(&rlun->bb_list); + + for (j = 0; j < geo->blks_per_lun; j++) { struct rrpc_block *rblk = &rlun->blocks[j]; - struct nvm_block *blk = &lun->blocks[j]; - rblk->parent = blk; + rblk->id = j; rblk->rlun = rlun; + rblk->state = NVM_BLK_T_FREE; INIT_LIST_HEAD(&rblk->prio); + INIT_LIST_HEAD(&rblk->list); spin_lock_init(&rblk->lock); + + list_add_tail(&rblk->list, &rlun->free_list); } rlun->rrpc = rrpc; + rlun->nr_free_blocks = geo->blks_per_lun; + rlun->reserved_blocks = 2; /* for GC only */ + INIT_LIST_HEAD(&rlun->prio_list); INIT_LIST_HEAD(&rlun->wblk_list); INIT_WORK(&rlun->ws_gc, rrpc_lun_gc); spin_lock_init(&rlun->lock); + + if (rrpc_bb_discovery(dev, rlun)) + goto err; + } return 0; @@ -1213,27 +1370,25 @@ err: /* returns 0 on success and stores the beginning address in *begin */ static int rrpc_area_init(struct rrpc *rrpc, sector_t *begin) { - struct nvm_dev *dev = rrpc->dev; - struct nvmm_type *mt = dev->mt; - sector_t size = rrpc->nr_sects * dev->sec_size; + struct nvm_tgt_dev *dev = rrpc->dev; + sector_t size = rrpc->nr_sects * dev->geo.sec_size; int ret; size >>= 9; - ret = mt->get_area(dev, begin, size); + ret = nvm_get_area(dev, begin, size); if (!ret) - *begin >>= (ilog2(dev->sec_size) - 9); + *begin >>= (ilog2(dev->geo.sec_size) - 9); return ret; } static void rrpc_area_free(struct rrpc *rrpc) { - struct nvm_dev *dev = rrpc->dev; - struct nvmm_type *mt = dev->mt; - sector_t begin = rrpc->soffset << (ilog2(dev->sec_size) - 9); + struct nvm_tgt_dev *dev = rrpc->dev; + sector_t begin = rrpc->soffset << (ilog2(dev->geo.sec_size) - 9); - mt->put_area(dev, begin); + nvm_put_area(dev, begin); } static void rrpc_free(struct rrpc *rrpc) @@ -1262,11 +1417,11 @@ static void rrpc_exit(void *private) static sector_t rrpc_capacity(void *private) { struct rrpc *rrpc = private; - struct nvm_dev *dev = rrpc->dev; + struct nvm_tgt_dev *dev = rrpc->dev; sector_t reserved, provisioned; /* cur, gc, and two emergency blocks for each lun */ - reserved = rrpc->nr_luns * dev->sec_per_blk * 4; + reserved = rrpc->nr_luns * dev->geo.sec_per_blk * 4; provisioned = rrpc->nr_sects - reserved; if (reserved > rrpc->nr_sects) { @@ -1285,13 +1440,13 @@ static sector_t rrpc_capacity(void *private) */ static void rrpc_block_map_update(struct rrpc *rrpc, struct rrpc_block *rblk) { - struct nvm_dev *dev = rrpc->dev; + struct nvm_tgt_dev *dev = rrpc->dev; int offset; struct rrpc_addr *laddr; u64 bpaddr, paddr, pladdr; bpaddr = block_to_rel_addr(rrpc, rblk); - for (offset = 0; offset < dev->sec_per_blk; offset++) { + for (offset = 0; offset < dev->geo.sec_per_blk; offset++) { paddr = bpaddr + offset; pladdr = rrpc->rev_trans_map[paddr].addr; @@ -1311,6 +1466,7 @@ static void rrpc_block_map_update(struct rrpc *rrpc, struct rrpc_block *rblk) static int rrpc_blocks_init(struct rrpc *rrpc) { + struct nvm_tgt_dev *dev = rrpc->dev; struct rrpc_lun *rlun; struct rrpc_block *rblk; int lun_iter, blk_iter; @@ -1318,7 +1474,7 @@ static int rrpc_blocks_init(struct rrpc *rrpc) for (lun_iter = 0; lun_iter < rrpc->nr_luns; lun_iter++) { rlun = &rrpc->luns[lun_iter]; - for (blk_iter = 0; blk_iter < rrpc->dev->blks_per_lun; + for (blk_iter = 0; blk_iter < dev->geo.blks_per_lun; blk_iter++) { rblk = &rlun->blocks[blk_iter]; rrpc_block_map_update(rrpc, rblk); @@ -1357,11 +1513,11 @@ err: static struct nvm_tgt_type tt_rrpc; -static void *rrpc_init(struct nvm_dev *dev, struct gendisk *tdisk, - int lun_begin, int lun_end) +static void *rrpc_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk) { struct request_queue *bqueue = dev->q; struct request_queue *tqueue = tdisk->queue; + struct nvm_geo *geo = &dev->geo; struct rrpc *rrpc; sector_t soffset; int ret; @@ -1384,9 +1540,8 @@ static void *rrpc_init(struct nvm_dev *dev, struct gendisk *tdisk, spin_lock_init(&rrpc->bio_lock); INIT_WORK(&rrpc->ws_requeue, rrpc_requeue); - rrpc->nr_luns = lun_end - lun_begin + 1; - rrpc->total_blocks = (unsigned long)dev->blks_per_lun * rrpc->nr_luns; - rrpc->nr_sects = (unsigned long long)dev->sec_per_lun * rrpc->nr_luns; + rrpc->nr_luns = geo->nr_luns; + rrpc->nr_sects = (unsigned long long)geo->sec_per_lun * rrpc->nr_luns; /* simple round-robin strategy */ atomic_set(&rrpc->next_lun, -1); @@ -1398,15 +1553,12 @@ static void *rrpc_init(struct nvm_dev *dev, struct gendisk *tdisk, } rrpc->soffset = soffset; - ret = rrpc_luns_init(rrpc, lun_begin, lun_end); + ret = rrpc_luns_init(rrpc, dev->luns); if (ret) { pr_err("nvm: rrpc: could not initialize luns\n"); goto err; } - rrpc->poffset = dev->sec_per_lun * lun_begin; - rrpc->lun_offset = lun_begin; - ret = rrpc_core_init(rrpc); if (ret) { pr_err("nvm: rrpc: could not initialize core\n"); diff --git a/drivers/lightnvm/rrpc.h b/drivers/lightnvm/rrpc.h index 5e87d52cb983..94e4d73116b2 100644 --- a/drivers/lightnvm/rrpc.h +++ b/drivers/lightnvm/rrpc.h @@ -48,14 +48,15 @@ struct rrpc_inflight_rq { struct rrpc_rq { struct rrpc_inflight_rq inflight_rq; - struct rrpc_addr *addr; unsigned long flags; }; struct rrpc_block { - struct nvm_block *parent; + int id; /* id inside of LUN */ struct rrpc_lun *rlun; - struct list_head prio; + + struct list_head prio; /* LUN CG list */ + struct list_head list; /* LUN free, used, bb list */ #define MAX_INVALID_PAGES_STORAGE 8 /* Bitmap for invalid page intries */ @@ -65,21 +66,38 @@ struct rrpc_block { /* number of pages that are invalid, wrt host page size */ unsigned int nr_invalid_pages; + int state; + spinlock_t lock; atomic_t data_cmnt_size; /* data pages committed to stable storage */ }; struct rrpc_lun { struct rrpc *rrpc; - struct nvm_lun *parent; + + int id; + struct ppa_addr bppa; + struct rrpc_block *cur, *gc_cur; struct rrpc_block *blocks; /* Reference to block allocation */ struct list_head prio_list; /* Blocks that may be GC'ed */ struct list_head wblk_list; /* Queued blocks to be written to */ + /* lun block lists */ + struct list_head used_list; /* In-use blocks */ + struct list_head free_list; /* Not used blocks i.e. released + * and ready for use + */ + struct list_head bb_list; /* Bad blocks. Mutually exclusive with + * free_list and used_list + */ + unsigned int nr_free_blocks; /* Number of unused blocks */ + struct work_struct ws_gc; + int reserved_blocks; + spinlock_t lock; }; @@ -87,19 +105,16 @@ struct rrpc { /* instance must be kept in top to resolve rrpc in unprep */ struct nvm_tgt_instance instance; - struct nvm_dev *dev; + struct nvm_tgt_dev *dev; struct gendisk *disk; sector_t soffset; /* logical sector offset */ - u64 poffset; /* physical page offset */ - int lun_offset; int nr_luns; struct rrpc_lun *luns; /* calculated values */ unsigned long long nr_sects; - unsigned long total_blocks; /* Write strategy variables. Move these into each for structure for each * strategy @@ -150,13 +165,37 @@ struct rrpc_rev_addr { u64 addr; }; -static inline struct rrpc_block *rrpc_get_rblk(struct rrpc_lun *rlun, - int blk_id) +static inline struct ppa_addr rrpc_linear_to_generic_addr(struct nvm_geo *geo, + struct ppa_addr r) +{ + struct ppa_addr l; + int secs, pgs; + sector_t ppa = r.ppa; + + l.ppa = 0; + + div_u64_rem(ppa, geo->sec_per_pg, &secs); + l.g.sec = secs; + + sector_div(ppa, geo->sec_per_pg); + div_u64_rem(ppa, geo->pgs_per_blk, &pgs); + l.g.pg = pgs; + + return l; +} + +static inline struct ppa_addr rrpc_recov_addr(struct nvm_tgt_dev *dev, u64 pba) +{ + return linear_to_generic_addr(&dev->geo, pba); +} + +static inline u64 rrpc_blk_to_ppa(struct rrpc *rrpc, struct rrpc_block *rblk) { - struct rrpc *rrpc = rlun->rrpc; - int lun_blk = blk_id % rrpc->dev->blks_per_lun; + struct nvm_tgt_dev *dev = rrpc->dev; + struct nvm_geo *geo = &dev->geo; + struct rrpc_lun *rlun = rblk->rlun; - return &rlun->blocks[lun_blk]; + return (rlun->id * geo->sec_per_lun) + (rblk->id * geo->sec_per_blk); } static inline sector_t rrpc_get_laddr(struct bio *bio) diff --git a/drivers/lightnvm/sysblk.c b/drivers/lightnvm/sysblk.c index a75bd28aaca3..12002bf4efc2 100644 --- a/drivers/lightnvm/sysblk.c +++ b/drivers/lightnvm/sysblk.c @@ -62,7 +62,8 @@ static void nvm_cpu_to_sysblk(struct nvm_system_block *sb, static int nvm_setup_sysblks(struct nvm_dev *dev, struct ppa_addr *sysblk_ppas) { - int nr_rows = min_t(int, MAX_SYSBLKS, dev->nr_chnls); + struct nvm_geo *geo = &dev->geo; + int nr_rows = min_t(int, MAX_SYSBLKS, geo->nr_chnls); int i; for (i = 0; i < nr_rows; i++) @@ -71,7 +72,7 @@ static int nvm_setup_sysblks(struct nvm_dev *dev, struct ppa_addr *sysblk_ppas) /* if possible, place sysblk at first channel, middle channel and last * channel of the device. If not, create only one or two sys blocks */ - switch (dev->nr_chnls) { + switch (geo->nr_chnls) { case 2: sysblk_ppas[1].g.ch = 1; /* fall-through */ @@ -80,8 +81,8 @@ static int nvm_setup_sysblks(struct nvm_dev *dev, struct ppa_addr *sysblk_ppas) break; default: sysblk_ppas[0].g.ch = 0; - sysblk_ppas[1].g.ch = dev->nr_chnls / 2; - sysblk_ppas[2].g.ch = dev->nr_chnls - 1; + sysblk_ppas[1].g.ch = geo->nr_chnls / 2; + sysblk_ppas[2].g.ch = geo->nr_chnls - 1; break; } @@ -162,11 +163,12 @@ static int sysblk_get_host_blks(struct nvm_dev *dev, struct ppa_addr ppa, static int nvm_get_all_sysblks(struct nvm_dev *dev, struct sysblk_scan *s, struct ppa_addr *ppas, int get_free) { + struct nvm_geo *geo = &dev->geo; int i, nr_blks, ret = 0; u8 *blks; s->nr_ppas = 0; - nr_blks = dev->blks_per_lun * dev->plane_mode; + nr_blks = geo->blks_per_lun * geo->plane_mode; blks = kmalloc(nr_blks, GFP_KERNEL); if (!blks) @@ -210,13 +212,14 @@ err_get: static int nvm_scan_block(struct nvm_dev *dev, struct ppa_addr *ppa, struct nvm_system_block *sblk) { + struct nvm_geo *geo = &dev->geo; struct nvm_system_block *cur; int pg, ret, found = 0; /* the full buffer for a flash page is allocated. Only the first of it * contains the system block information */ - cur = kmalloc(dev->pfpg_size, GFP_KERNEL); + cur = kmalloc(geo->pfpg_size, GFP_KERNEL); if (!cur) return -ENOMEM; @@ -225,7 +228,7 @@ static int nvm_scan_block(struct nvm_dev *dev, struct ppa_addr *ppa, ppa->g.pg = ppa_to_slc(dev, pg); ret = nvm_submit_ppa(dev, ppa, 1, NVM_OP_PREAD, NVM_IO_SLC_MODE, - cur, dev->pfpg_size); + cur, geo->pfpg_size); if (ret) { if (ret == NVM_RSP_ERR_EMPTYPAGE) { pr_debug("nvm: sysblk scan empty ppa (%u %u %u %u)\n", @@ -267,34 +270,16 @@ static int nvm_scan_block(struct nvm_dev *dev, struct ppa_addr *ppa, return found; } -static int nvm_set_bb_tbl(struct nvm_dev *dev, struct sysblk_scan *s, int type) +static int nvm_sysblk_set_bb_tbl(struct nvm_dev *dev, struct sysblk_scan *s, + int type) { - struct nvm_rq rqd; - int ret; - - if (s->nr_ppas > dev->ops->max_phys_sect) { - pr_err("nvm: unable to update all sysblocks atomically\n"); - return -EINVAL; - } - - memset(&rqd, 0, sizeof(struct nvm_rq)); - - nvm_set_rqd_ppalist(dev, &rqd, s->ppas, s->nr_ppas, 1); - nvm_generic_to_addr_mode(dev, &rqd); - - ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type); - nvm_free_rqd_ppalist(dev, &rqd); - if (ret) { - pr_err("nvm: sysblk failed bb mark\n"); - return -EINVAL; - } - - return 0; + return nvm_set_bb_tbl(dev, s->ppas, s->nr_ppas, type); } static int nvm_write_and_verify(struct nvm_dev *dev, struct nvm_sb_info *info, struct sysblk_scan *s) { + struct nvm_geo *geo = &dev->geo; struct nvm_system_block nvmsb; void *buf; int i, sect, ret = 0; @@ -302,12 +287,12 @@ static int nvm_write_and_verify(struct nvm_dev *dev, struct nvm_sb_info *info, nvm_cpu_to_sysblk(&nvmsb, info); - buf = kzalloc(dev->pfpg_size, GFP_KERNEL); + buf = kzalloc(geo->pfpg_size, GFP_KERNEL); if (!buf) return -ENOMEM; memcpy(buf, &nvmsb, sizeof(struct nvm_system_block)); - ppas = kcalloc(dev->sec_per_pg, sizeof(struct ppa_addr), GFP_KERNEL); + ppas = kcalloc(geo->sec_per_pg, sizeof(struct ppa_addr), GFP_KERNEL); if (!ppas) { ret = -ENOMEM; goto err; @@ -324,15 +309,15 @@ static int nvm_write_and_verify(struct nvm_dev *dev, struct nvm_sb_info *info, ppas[0].g.pg); /* Expand to all sectors within a flash page */ - if (dev->sec_per_pg > 1) { - for (sect = 1; sect < dev->sec_per_pg; sect++) { + if (geo->sec_per_pg > 1) { + for (sect = 1; sect < geo->sec_per_pg; sect++) { ppas[sect].ppa = ppas[0].ppa; ppas[sect].g.sec = sect; } } - ret = nvm_submit_ppa(dev, ppas, dev->sec_per_pg, NVM_OP_PWRITE, - NVM_IO_SLC_MODE, buf, dev->pfpg_size); + ret = nvm_submit_ppa(dev, ppas, geo->sec_per_pg, NVM_OP_PWRITE, + NVM_IO_SLC_MODE, buf, geo->pfpg_size); if (ret) { pr_err("nvm: sysblk failed program (%u %u %u)\n", ppas[0].g.ch, @@ -341,8 +326,8 @@ static int nvm_write_and_verify(struct nvm_dev *dev, struct nvm_sb_info *info, break; } - ret = nvm_submit_ppa(dev, ppas, dev->sec_per_pg, NVM_OP_PREAD, - NVM_IO_SLC_MODE, buf, dev->pfpg_size); + ret = nvm_submit_ppa(dev, ppas, geo->sec_per_pg, NVM_OP_PREAD, + NVM_IO_SLC_MODE, buf, geo->pfpg_size); if (ret) { pr_err("nvm: sysblk failed read (%u %u %u)\n", ppas[0].g.ch, @@ -379,7 +364,7 @@ static int nvm_prepare_new_sysblks(struct nvm_dev *dev, struct sysblk_scan *s) ppa = &s->ppas[scan_ppa_idx(i, nxt_blk)]; ppa->g.pg = ppa_to_slc(dev, 0); - ret = nvm_erase_ppa(dev, ppa, 1); + ret = nvm_erase_ppa(dev, ppa, 1, 0); if (ret) return ret; @@ -546,6 +531,7 @@ err_sysblk: int nvm_init_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info) { + struct nvm_geo *geo = &dev->geo; struct ppa_addr sysblk_ppas[MAX_SYSBLKS]; struct sysblk_scan s; int ret; @@ -560,7 +546,7 @@ int nvm_init_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info) if (!dev->ops->get_bb_tbl || !dev->ops->set_bb_tbl) return -EINVAL; - if (!(dev->mccap & NVM_ID_CAP_SLC) || !dev->lps_per_blk) { + if (!(geo->mccap & NVM_ID_CAP_SLC) || !dev->lps_per_blk) { pr_err("nvm: memory does not support SLC access\n"); return -EINVAL; } @@ -573,7 +559,7 @@ int nvm_init_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info) if (ret) goto err_mark; - ret = nvm_set_bb_tbl(dev, &s, NVM_BLK_T_HOST); + ret = nvm_sysblk_set_bb_tbl(dev, &s, NVM_BLK_T_HOST); if (ret) goto err_mark; @@ -590,11 +576,11 @@ static int factory_nblks(int nblks) return (nblks + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1); } -static unsigned int factory_blk_offset(struct nvm_dev *dev, struct ppa_addr ppa) +static unsigned int factory_blk_offset(struct nvm_geo *geo, struct ppa_addr ppa) { - int nblks = factory_nblks(dev->blks_per_lun); + int nblks = factory_nblks(geo->blks_per_lun); - return ((ppa.g.ch * dev->luns_per_chnl * nblks) + (ppa.g.lun * nblks)) / + return ((ppa.g.ch * geo->luns_per_chnl * nblks) + (ppa.g.lun * nblks)) / BITS_PER_LONG; } @@ -608,7 +594,7 @@ static int nvm_factory_blks(struct nvm_dev *dev, struct ppa_addr ppa, if (nr_blks < 0) return nr_blks; - lunoff = factory_blk_offset(dev, ppa); + lunoff = factory_blk_offset(&dev->geo, ppa); /* non-set bits correspond to the block must be erased */ for (i = 0; i < nr_blks; i++) { @@ -637,19 +623,19 @@ static int nvm_factory_blks(struct nvm_dev *dev, struct ppa_addr ppa, static int nvm_fact_get_blks(struct nvm_dev *dev, struct ppa_addr *erase_list, int max_ppas, unsigned long *blk_bitmap) { + struct nvm_geo *geo = &dev->geo; struct ppa_addr ppa; int ch, lun, blkid, idx, done = 0, ppa_cnt = 0; unsigned long *offset; while (!done) { done = 1; - nvm_for_each_lun_ppa(dev, ppa, ch, lun) { - idx = factory_blk_offset(dev, ppa); + nvm_for_each_lun_ppa(geo, ppa, ch, lun) { + idx = factory_blk_offset(geo, ppa); offset = &blk_bitmap[idx]; - blkid = find_first_zero_bit(offset, - dev->blks_per_lun); - if (blkid >= dev->blks_per_lun) + blkid = find_first_zero_bit(offset, geo->blks_per_lun); + if (blkid >= geo->blks_per_lun) continue; set_bit(blkid, offset); @@ -674,16 +660,17 @@ static int nvm_fact_get_blks(struct nvm_dev *dev, struct ppa_addr *erase_list, static int nvm_fact_select_blks(struct nvm_dev *dev, unsigned long *blk_bitmap, int flags) { + struct nvm_geo *geo = &dev->geo; struct ppa_addr ppa; int ch, lun, nr_blks, ret = 0; u8 *blks; - nr_blks = dev->blks_per_lun * dev->plane_mode; + nr_blks = geo->blks_per_lun * geo->plane_mode; blks = kmalloc(nr_blks, GFP_KERNEL); if (!blks) return -ENOMEM; - nvm_for_each_lun_ppa(dev, ppa, ch, lun) { + nvm_for_each_lun_ppa(geo, ppa, ch, lun) { ret = nvm_get_bb_tbl(dev, ppa, blks); if (ret) pr_err("nvm: failed bb tbl for ch%u lun%u\n", @@ -701,14 +688,15 @@ static int nvm_fact_select_blks(struct nvm_dev *dev, unsigned long *blk_bitmap, int nvm_dev_factory(struct nvm_dev *dev, int flags) { + struct nvm_geo *geo = &dev->geo; struct ppa_addr *ppas; int ppa_cnt, ret = -ENOMEM; - int max_ppas = dev->ops->max_phys_sect / dev->nr_planes; + int max_ppas = dev->ops->max_phys_sect / geo->nr_planes; struct ppa_addr sysblk_ppas[MAX_SYSBLKS]; struct sysblk_scan s; unsigned long *blk_bitmap; - blk_bitmap = kzalloc(factory_nblks(dev->blks_per_lun) * dev->nr_luns, + blk_bitmap = kzalloc(factory_nblks(geo->blks_per_lun) * geo->nr_luns, GFP_KERNEL); if (!blk_bitmap) return ret; @@ -725,7 +713,7 @@ int nvm_dev_factory(struct nvm_dev *dev, int flags) /* continue to erase until list of blks until empty */ while ((ppa_cnt = nvm_fact_get_blks(dev, ppas, max_ppas, blk_bitmap)) > 0) - nvm_erase_ppa(dev, ppas, ppa_cnt); + nvm_erase_ppa(dev, ppas, ppa_cnt, 0); /* mark host reserved blocks free */ if (flags & NVM_FACTORY_RESET_HOST_BLKS) { @@ -733,7 +721,7 @@ int nvm_dev_factory(struct nvm_dev *dev, int flags) mutex_lock(&dev->mlock); ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, 0); if (!ret) - ret = nvm_set_bb_tbl(dev, &s, NVM_BLK_T_FREE); + ret = nvm_sysblk_set_bb_tbl(dev, &s, NVM_BLK_T_FREE); mutex_unlock(&dev->mlock); } err_ppas: diff --git a/drivers/lightnvm/sysfs.c b/drivers/lightnvm/sysfs.c deleted file mode 100644 index 0338c27ab95a..000000000000 --- a/drivers/lightnvm/sysfs.c +++ /dev/null @@ -1,198 +0,0 @@ -#include <linux/kernel.h> -#include <linux/lightnvm.h> -#include <linux/miscdevice.h> -#include <linux/kobject.h> -#include <linux/blk-mq.h> - -#include "lightnvm.h" - -static ssize_t nvm_dev_attr_show(struct device *dev, - struct device_attribute *dattr, char *page) -{ - struct nvm_dev *ndev = container_of(dev, struct nvm_dev, dev); - struct nvm_id *id = &ndev->identity; - struct nvm_id_group *grp = &id->groups[0]; - struct attribute *attr = &dattr->attr; - - if (strcmp(attr->name, "version") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", id->ver_id); - } else if (strcmp(attr->name, "vendor_opcode") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", id->vmnt); - } else if (strcmp(attr->name, "capabilities") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", id->cap); - } else if (strcmp(attr->name, "device_mode") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", id->dom); - } else if (strcmp(attr->name, "media_manager") == 0) { - if (!ndev->mt) - return scnprintf(page, PAGE_SIZE, "%s\n", "none"); - return scnprintf(page, PAGE_SIZE, "%s\n", ndev->mt->name); - } else if (strcmp(attr->name, "ppa_format") == 0) { - return scnprintf(page, PAGE_SIZE, - "0x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n", - id->ppaf.ch_offset, id->ppaf.ch_len, - id->ppaf.lun_offset, id->ppaf.lun_len, - id->ppaf.pln_offset, id->ppaf.pln_len, - id->ppaf.blk_offset, id->ppaf.blk_len, - id->ppaf.pg_offset, id->ppaf.pg_len, - id->ppaf.sect_offset, id->ppaf.sect_len); - } else if (strcmp(attr->name, "media_type") == 0) { /* u8 */ - return scnprintf(page, PAGE_SIZE, "%u\n", grp->mtype); - } else if (strcmp(attr->name, "flash_media_type") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->fmtype); - } else if (strcmp(attr->name, "num_channels") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_ch); - } else if (strcmp(attr->name, "num_luns") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_lun); - } else if (strcmp(attr->name, "num_planes") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_pln); - } else if (strcmp(attr->name, "num_blocks") == 0) { /* u16 */ - return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_blk); - } else if (strcmp(attr->name, "num_pages") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_pg); - } else if (strcmp(attr->name, "page_size") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->fpg_sz); - } else if (strcmp(attr->name, "hw_sector_size") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->csecs); - } else if (strcmp(attr->name, "oob_sector_size") == 0) {/* u32 */ - return scnprintf(page, PAGE_SIZE, "%u\n", grp->sos); - } else if (strcmp(attr->name, "read_typ") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->trdt); - } else if (strcmp(attr->name, "read_max") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->trdm); - } else if (strcmp(attr->name, "prog_typ") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->tprt); - } else if (strcmp(attr->name, "prog_max") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->tprm); - } else if (strcmp(attr->name, "erase_typ") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->tbet); - } else if (strcmp(attr->name, "erase_max") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", grp->tbem); - } else if (strcmp(attr->name, "multiplane_modes") == 0) { - return scnprintf(page, PAGE_SIZE, "0x%08x\n", grp->mpos); - } else if (strcmp(attr->name, "media_capabilities") == 0) { - return scnprintf(page, PAGE_SIZE, "0x%08x\n", grp->mccap); - } else if (strcmp(attr->name, "max_phys_secs") == 0) { - return scnprintf(page, PAGE_SIZE, "%u\n", - ndev->ops->max_phys_sect); - } else { - return scnprintf(page, - PAGE_SIZE, - "Unhandled attr(%s) in `nvm_dev_attr_show`\n", - attr->name); - } -} - -#define NVM_DEV_ATTR_RO(_name) \ - DEVICE_ATTR(_name, S_IRUGO, nvm_dev_attr_show, NULL) - -static NVM_DEV_ATTR_RO(version); -static NVM_DEV_ATTR_RO(vendor_opcode); -static NVM_DEV_ATTR_RO(capabilities); -static NVM_DEV_ATTR_RO(device_mode); -static NVM_DEV_ATTR_RO(ppa_format); -static NVM_DEV_ATTR_RO(media_manager); - -static NVM_DEV_ATTR_RO(media_type); -static NVM_DEV_ATTR_RO(flash_media_type); -static NVM_DEV_ATTR_RO(num_channels); -static NVM_DEV_ATTR_RO(num_luns); -static NVM_DEV_ATTR_RO(num_planes); -static NVM_DEV_ATTR_RO(num_blocks); -static NVM_DEV_ATTR_RO(num_pages); -static NVM_DEV_ATTR_RO(page_size); -static NVM_DEV_ATTR_RO(hw_sector_size); -static NVM_DEV_ATTR_RO(oob_sector_size); -static NVM_DEV_ATTR_RO(read_typ); -static NVM_DEV_ATTR_RO(read_max); -static NVM_DEV_ATTR_RO(prog_typ); -static NVM_DEV_ATTR_RO(prog_max); -static NVM_DEV_ATTR_RO(erase_typ); -static NVM_DEV_ATTR_RO(erase_max); -static NVM_DEV_ATTR_RO(multiplane_modes); -static NVM_DEV_ATTR_RO(media_capabilities); -static NVM_DEV_ATTR_RO(max_phys_secs); - -#define NVM_DEV_ATTR(_name) (dev_attr_##_name##) - -static struct attribute *nvm_dev_attrs[] = { - &dev_attr_version.attr, - &dev_attr_vendor_opcode.attr, - &dev_attr_capabilities.attr, - &dev_attr_device_mode.attr, - &dev_attr_media_manager.attr, - - &dev_attr_ppa_format.attr, - &dev_attr_media_type.attr, - &dev_attr_flash_media_type.attr, - &dev_attr_num_channels.attr, - &dev_attr_num_luns.attr, - &dev_attr_num_planes.attr, - &dev_attr_num_blocks.attr, - &dev_attr_num_pages.attr, - &dev_attr_page_size.attr, - &dev_attr_hw_sector_size.attr, - &dev_attr_oob_sector_size.attr, - &dev_attr_read_typ.attr, - &dev_attr_read_max.attr, - &dev_attr_prog_typ.attr, - &dev_attr_prog_max.attr, - &dev_attr_erase_typ.attr, - &dev_attr_erase_max.attr, - &dev_attr_multiplane_modes.attr, - &dev_attr_media_capabilities.attr, - &dev_attr_max_phys_secs.attr, - NULL, -}; - -static struct attribute_group nvm_dev_attr_group = { - .name = "lightnvm", - .attrs = nvm_dev_attrs, -}; - -static const struct attribute_group *nvm_dev_attr_groups[] = { - &nvm_dev_attr_group, - NULL, -}; - -static void nvm_dev_release(struct device *device) -{ - struct nvm_dev *dev = container_of(device, struct nvm_dev, dev); - struct request_queue *q = dev->q; - - pr_debug("nvm/sysfs: `nvm_dev_release`\n"); - - blk_mq_unregister_dev(device, q); - - nvm_free(dev); -} - -static struct device_type nvm_type = { - .name = "lightnvm", - .groups = nvm_dev_attr_groups, - .release = nvm_dev_release, -}; - -int nvm_sysfs_register_dev(struct nvm_dev *dev) -{ - int ret; - - if (!dev->parent_dev) - return 0; - - dev->dev.parent = dev->parent_dev; - dev_set_name(&dev->dev, "%s", dev->name); - dev->dev.type = &nvm_type; - device_initialize(&dev->dev); - ret = device_add(&dev->dev); - - if (!ret) - blk_mq_register_dev(&dev->dev, dev->q); - - return ret; -} - -void nvm_sysfs_unregister_dev(struct nvm_dev *dev) -{ - if (dev && dev->parent_dev) - kobject_put(&dev->dev.kobj); -} |