summaryrefslogtreecommitdiff
path: root/drivers
diff options
context:
space:
mode:
Diffstat (limited to 'drivers')
-rw-r--r--drivers/dax/device.c75
-rw-r--r--drivers/nvdimm/pmem.c26
-rw-r--r--drivers/nvdimm/pmem.h13
3 files changed, 87 insertions, 27 deletions
diff --git a/drivers/dax/device.c b/drivers/dax/device.c
index 0a2acd7993f0..6fd46083e629 100644
--- a/drivers/dax/device.c
+++ b/drivers/dax/device.c
@@ -248,13 +248,12 @@ __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
return -1;
}
-static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
+static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,
+ struct vm_fault *vmf, pfn_t *pfn)
{
struct device *dev = &dev_dax->dev;
struct dax_region *dax_region;
- int rc = VM_FAULT_SIGBUS;
phys_addr_t phys;
- pfn_t pfn;
unsigned int fault_size = PAGE_SIZE;
if (check_vma(dev_dax, vmf->vma, __func__))
@@ -276,26 +275,19 @@ static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
return VM_FAULT_SIGBUS;
}
- pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
-
- rc = vm_insert_mixed(vmf->vma, vmf->address, pfn);
-
- if (rc == -ENOMEM)
- return VM_FAULT_OOM;
- if (rc < 0 && rc != -EBUSY)
- return VM_FAULT_SIGBUS;
+ *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
- return VM_FAULT_NOPAGE;
+ return vmf_insert_mixed(vmf->vma, vmf->address, *pfn);
}
-static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
+static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,
+ struct vm_fault *vmf, pfn_t *pfn)
{
unsigned long pmd_addr = vmf->address & PMD_MASK;
struct device *dev = &dev_dax->dev;
struct dax_region *dax_region;
phys_addr_t phys;
pgoff_t pgoff;
- pfn_t pfn;
unsigned int fault_size = PMD_SIZE;
if (check_vma(dev_dax, vmf->vma, __func__))
@@ -331,21 +323,21 @@ static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
return VM_FAULT_SIGBUS;
}
- pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
+ *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
- return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, pfn,
+ return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, *pfn,
vmf->flags & FAULT_FLAG_WRITE);
}
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
-static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
+static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
+ struct vm_fault *vmf, pfn_t *pfn)
{
unsigned long pud_addr = vmf->address & PUD_MASK;
struct device *dev = &dev_dax->dev;
struct dax_region *dax_region;
phys_addr_t phys;
pgoff_t pgoff;
- pfn_t pfn;
unsigned int fault_size = PUD_SIZE;
@@ -382,23 +374,26 @@ static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
return VM_FAULT_SIGBUS;
}
- pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
+ *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
- return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, pfn,
+ return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, *pfn,
vmf->flags & FAULT_FLAG_WRITE);
}
#else
-static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
+static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
+ struct vm_fault *vmf, pfn_t *pfn)
{
return VM_FAULT_FALLBACK;
}
#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
-static int dev_dax_huge_fault(struct vm_fault *vmf,
+static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf,
enum page_entry_size pe_size)
{
- int rc, id;
struct file *filp = vmf->vma->vm_file;
+ unsigned long fault_size;
+ int rc, id;
+ pfn_t pfn;
struct dev_dax *dev_dax = filp->private_data;
dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm,
@@ -408,23 +403,49 @@ static int dev_dax_huge_fault(struct vm_fault *vmf,
id = dax_read_lock();
switch (pe_size) {
case PE_SIZE_PTE:
- rc = __dev_dax_pte_fault(dev_dax, vmf);
+ fault_size = PAGE_SIZE;
+ rc = __dev_dax_pte_fault(dev_dax, vmf, &pfn);
break;
case PE_SIZE_PMD:
- rc = __dev_dax_pmd_fault(dev_dax, vmf);
+ fault_size = PMD_SIZE;
+ rc = __dev_dax_pmd_fault(dev_dax, vmf, &pfn);
break;
case PE_SIZE_PUD:
- rc = __dev_dax_pud_fault(dev_dax, vmf);
+ fault_size = PUD_SIZE;
+ rc = __dev_dax_pud_fault(dev_dax, vmf, &pfn);
break;
default:
rc = VM_FAULT_SIGBUS;
}
+
+ if (rc == VM_FAULT_NOPAGE) {
+ unsigned long i;
+ pgoff_t pgoff;
+
+ /*
+ * In the device-dax case the only possibility for a
+ * VM_FAULT_NOPAGE result is when device-dax capacity is
+ * mapped. No need to consider the zero page, or racing
+ * conflicting mappings.
+ */
+ pgoff = linear_page_index(vmf->vma, vmf->address
+ & ~(fault_size - 1));
+ for (i = 0; i < fault_size / PAGE_SIZE; i++) {
+ struct page *page;
+
+ page = pfn_to_page(pfn_t_to_pfn(pfn) + i);
+ if (page->mapping)
+ continue;
+ page->mapping = filp->f_mapping;
+ page->index = pgoff + i;
+ }
+ }
dax_read_unlock(id);
return rc;
}
-static int dev_dax_fault(struct vm_fault *vmf)
+static vm_fault_t dev_dax_fault(struct vm_fault *vmf)
{
return dev_dax_huge_fault(vmf, PE_SIZE_PTE);
}
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index c23649867696..6071e2942053 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -20,6 +20,7 @@
#include <linux/hdreg.h>
#include <linux/init.h>
#include <linux/platform_device.h>
+#include <linux/set_memory.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/badblocks.h>
@@ -51,6 +52,30 @@ static struct nd_region *to_region(struct pmem_device *pmem)
return to_nd_region(to_dev(pmem)->parent);
}
+static void hwpoison_clear(struct pmem_device *pmem,
+ phys_addr_t phys, unsigned int len)
+{
+ unsigned long pfn_start, pfn_end, pfn;
+
+ /* only pmem in the linear map supports HWPoison */
+ if (is_vmalloc_addr(pmem->virt_addr))
+ return;
+
+ pfn_start = PHYS_PFN(phys);
+ pfn_end = pfn_start + PHYS_PFN(len);
+ for (pfn = pfn_start; pfn < pfn_end; pfn++) {
+ struct page *page = pfn_to_page(pfn);
+
+ /*
+ * Note, no need to hold a get_dev_pagemap() reference
+ * here since we're in the driver I/O path and
+ * outstanding I/O requests pin the dev_pagemap.
+ */
+ if (test_and_clear_pmem_poison(page))
+ clear_mce_nospec(pfn);
+ }
+}
+
static blk_status_t pmem_clear_poison(struct pmem_device *pmem,
phys_addr_t offset, unsigned int len)
{
@@ -65,6 +90,7 @@ static blk_status_t pmem_clear_poison(struct pmem_device *pmem,
if (cleared < len)
rc = BLK_STS_IOERR;
if (cleared > 0 && cleared / 512) {
+ hwpoison_clear(pmem, pmem->phys_addr + offset, cleared);
cleared /= 512;
dev_dbg(dev, "%#llx clear %ld sector%s\n",
(unsigned long long) sector, cleared,
diff --git a/drivers/nvdimm/pmem.h b/drivers/nvdimm/pmem.h
index a64ebc78b5df..59cfe13ea8a8 100644
--- a/drivers/nvdimm/pmem.h
+++ b/drivers/nvdimm/pmem.h
@@ -1,6 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NVDIMM_PMEM_H__
#define __NVDIMM_PMEM_H__
+#include <linux/page-flags.h>
#include <linux/badblocks.h>
#include <linux/types.h>
#include <linux/pfn_t.h>
@@ -27,4 +28,16 @@ struct pmem_device {
long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,
long nr_pages, void **kaddr, pfn_t *pfn);
+
+#ifdef CONFIG_MEMORY_FAILURE
+static inline bool test_and_clear_pmem_poison(struct page *page)
+{
+ return TestClearPageHWPoison(page);
+}
+#else
+static inline bool test_and_clear_pmem_poison(struct page *page)
+{
+ return false;
+}
+#endif
#endif /* __NVDIMM_PMEM_H__ */
generated by cgit v1.2.3 (git 2.39.0) at 2024-11-21 14:30:20 +0300