From 5d3c8b21e22712137db6bbd246d1bdcbe0a09914 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Wed, 13 Feb 2008 16:20:35 +0100 Subject: x86: CPA: fix gbpages support in try_preserve_large_page [ mingo@elte.hu: while gbpages cannot be enabled on mainline currently, keep the code uptodate and this fix is easy enough. ] Use correct page sizes and masks for GB pages in try_preserve_large_page() This prevents a boot hang on a GB capable system with CONFIG_DIRECT_GBPAGES enabled. Signed-off-by: Andi Kleen Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/mm/pageattr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/x86/mm/pageattr.c') diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index 440210a2277d..bd61ed13f9cf 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -275,8 +275,8 @@ try_preserve_large_page(pte_t *kpte, unsigned long address, break; #ifdef CONFIG_X86_64 case PG_LEVEL_1G: - psize = PMD_PAGE_SIZE; - pmask = PMD_PAGE_MASK; + psize = PUD_PAGE_SIZE; + pmask = PUD_PAGE_MASK; break; #endif default: -- cgit v1.2.3 From 69b1415e934fc1a796a817e32d84162ae65962f5 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 13 Feb 2008 11:04:50 +0100 Subject: x86: cpa: ensure page alignment the cpa API is page aligned - warn about any weird alignments. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/mm/pageattr.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'arch/x86/mm/pageattr.c') diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index bd61ed13f9cf..5d2259468d3c 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -688,6 +688,15 @@ static int change_page_attr_set_clr(unsigned long addr, int numpages, if (!pgprot_val(mask_set) && !pgprot_val(mask_clr)) return 0; + /* Ensure we are PAGE_SIZE aligned */ + if (addr & ~PAGE_MASK) { + addr &= PAGE_MASK; + /* + * People should not be passing in unaligned addresses: + */ + WARN_ON_ONCE(1); + } + cpa.vaddr = addr; cpa.numpages = numpages; cpa.mask_set = mask_set; -- cgit v1.2.3 From f8d8406bcb58ff70e97b71c35ff5be90c54fc3d0 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 13 Feb 2008 14:09:53 +0100 Subject: x86: cpa, fix out of date comment Signed-off-by: Ingo Molnar --- arch/x86/mm/pageattr.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'arch/x86/mm/pageattr.c') diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index 5d2259468d3c..4119379f80ff 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -870,8 +870,12 @@ void kernel_map_pages(struct page *page, int numpages, int enable) return; /* - * The return value is ignored - the calls cannot fail, - * large pages are disabled at boot time: + * The return value is ignored as the calls cannot fail. + * Large pages are kept enabled at boot time, and are + * split up quickly with DEBUG_PAGEALLOC. If a splitup + * fails here (due to temporary memory shortage) no damage + * is done because we just keep the largepage intact up + * to the next attempt when it will likely be split up: */ if (enable) __set_pages_p(page, numpages); -- cgit v1.2.3 From c31c7d4844ea4817692ae16bf70f9c96c05a50eb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 18 Feb 2008 20:54:14 +0100 Subject: x86: CPA, fix alias checks c_p_a() did not discover all aliases correctly. (such as when called on vmalloc()-ed areas or ioremap()-ed areas) Push the alias checks to the lower, physical level and consistently discover all aliases that might exist: the low direct mappings and the high linear kernel-text mappings (on 64-bit). Thanks to Andi Kleen for pointing out that this was buggy. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/mm/pageattr.c | 192 ++++++++++++++++++++++++------------------------- 1 file changed, 92 insertions(+), 100 deletions(-) (limited to 'arch/x86/mm/pageattr.c') diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index 4119379f80ff..7b4a17236972 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -16,6 +16,7 @@ #include #include #include +#include /* * The current flushing context - we pass it instead of 5 arguments: @@ -26,8 +27,23 @@ struct cpa_data { pgprot_t mask_clr; int numpages; int flushtlb; + unsigned long pfn; }; +#ifdef CONFIG_X86_64 + +static inline unsigned long highmap_start_pfn(void) +{ + return __pa(_text) >> PAGE_SHIFT; +} + +static inline unsigned long highmap_end_pfn(void) +{ + return __pa(round_up((unsigned long)_end, PMD_SIZE)) >> PAGE_SHIFT; +} + +#endif + static inline int within(unsigned long addr, unsigned long start, unsigned long end) { @@ -123,29 +139,14 @@ static void cpa_flush_range(unsigned long start, int numpages, int cache) } } -#define HIGH_MAP_START __START_KERNEL_map -#define HIGH_MAP_END (__START_KERNEL_map + KERNEL_TEXT_SIZE) - - -/* - * Converts a virtual address to a X86-64 highmap address - */ -static unsigned long virt_to_highmap(void *address) -{ -#ifdef CONFIG_X86_64 - return __pa((unsigned long)address) + HIGH_MAP_START - phys_base; -#else - return (unsigned long)address; -#endif -} - /* * Certain areas of memory on x86 require very specific protection flags, * for example the BIOS area or kernel text. Callers don't always get this * right (again, ioremap() on BIOS memory is not uncommon) so this function * checks and fixes these known static required protection bits. */ -static inline pgprot_t static_protections(pgprot_t prot, unsigned long address) +static inline pgprot_t static_protections(pgprot_t prot, unsigned long address, + unsigned long pfn) { pgprot_t forbidden = __pgprot(0); @@ -153,30 +154,23 @@ static inline pgprot_t static_protections(pgprot_t prot, unsigned long address) * The BIOS area between 640k and 1Mb needs to be executable for * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support. */ - if (within(__pa(address), BIOS_BEGIN, BIOS_END)) + if (within(pfn, BIOS_BEGIN >> PAGE_SHIFT, BIOS_END >> PAGE_SHIFT)) pgprot_val(forbidden) |= _PAGE_NX; /* * The kernel text needs to be executable for obvious reasons - * Does not cover __inittext since that is gone later on + * Does not cover __inittext since that is gone later on. On + * 64bit we do not enforce !NX on the low mapping */ if (within(address, (unsigned long)_text, (unsigned long)_etext)) pgprot_val(forbidden) |= _PAGE_NX; - /* - * Do the same for the x86-64 high kernel mapping - */ - if (within(address, virt_to_highmap(_text), virt_to_highmap(_etext))) - pgprot_val(forbidden) |= _PAGE_NX; - /* The .rodata section needs to be read-only */ - if (within(address, (unsigned long)__start_rodata, - (unsigned long)__end_rodata)) - pgprot_val(forbidden) |= _PAGE_RW; /* - * Do the same for the x86-64 high kernel mapping + * The .rodata section needs to be read-only. Using the pfn + * catches all aliases. */ - if (within(address, virt_to_highmap(__start_rodata), - virt_to_highmap(__end_rodata))) + if (within(pfn, __pa((unsigned long)__start_rodata) >> PAGE_SHIFT, + __pa((unsigned long)__end_rodata) >> PAGE_SHIFT)) pgprot_val(forbidden) |= _PAGE_RW; prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden)); @@ -253,7 +247,7 @@ static int try_preserve_large_page(pte_t *kpte, unsigned long address, struct cpa_data *cpa) { - unsigned long nextpage_addr, numpages, pmask, psize, flags, addr; + unsigned long nextpage_addr, numpages, pmask, psize, flags, addr, pfn; pte_t new_pte, old_pte, *tmp; pgprot_t old_prot, new_prot; int i, do_split = 1; @@ -301,7 +295,15 @@ try_preserve_large_page(pte_t *kpte, unsigned long address, pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr); pgprot_val(new_prot) |= pgprot_val(cpa->mask_set); - new_prot = static_protections(new_prot, address); + + /* + * old_pte points to the large page base address. So we need + * to add the offset of the virtual address: + */ + pfn = pte_pfn(old_pte) + ((address & (psize - 1)) >> PAGE_SHIFT); + cpa->pfn = pfn; + + new_prot = static_protections(new_prot, address, pfn); /* * We need to check the full range, whether @@ -309,8 +311,9 @@ try_preserve_large_page(pte_t *kpte, unsigned long address, * the pages in the range we try to preserve: */ addr = address + PAGE_SIZE; - for (i = 1; i < cpa->numpages; i++, addr += PAGE_SIZE) { - pgprot_t chk_prot = static_protections(new_prot, addr); + pfn++; + for (i = 1; i < cpa->numpages; i++, addr += PAGE_SIZE, pfn++) { + pgprot_t chk_prot = static_protections(new_prot, addr, pfn); if (pgprot_val(chk_prot) != pgprot_val(new_prot)) goto out_unlock; @@ -505,46 +508,51 @@ out_unlock: return 0; } -static int __change_page_attr(unsigned long address, struct cpa_data *cpa) +static int __change_page_attr(struct cpa_data *cpa, int primary) { + unsigned long address = cpa->vaddr; int do_split, err; unsigned int level; struct page *kpte_page; - pte_t *kpte; + pte_t *kpte, old_pte; repeat: kpte = lookup_address(address, &level); if (!kpte) + return primary ? -EINVAL : 0; + + old_pte = *kpte; + if (!pte_val(old_pte)) { + if (!primary) + return 0; + printk(KERN_WARNING "CPA: called for zero pte. " + "vaddr = %lx cpa->vaddr = %lx\n", address, + cpa->vaddr); + WARN_ON(1); return -EINVAL; + } kpte_page = virt_to_page(kpte); BUG_ON(PageLRU(kpte_page)); BUG_ON(PageCompound(kpte_page)); if (level == PG_LEVEL_4K) { - pte_t new_pte, old_pte = *kpte; + pte_t new_pte; pgprot_t new_prot = pte_pgprot(old_pte); - - if(!pte_val(old_pte)) { - printk(KERN_WARNING "CPA: called for zero pte. " - "vaddr = %lx cpa->vaddr = %lx\n", address, - cpa->vaddr); - WARN_ON(1); - return -EINVAL; - } + unsigned long pfn = pte_pfn(old_pte); pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr); pgprot_val(new_prot) |= pgprot_val(cpa->mask_set); - new_prot = static_protections(new_prot, address); + new_prot = static_protections(new_prot, address, pfn); /* * We need to keep the pfn from the existing PTE, * after all we're only going to change it's attributes * not the memory it points to */ - new_pte = pfn_pte(pte_pfn(old_pte), canon_pgprot(new_prot)); - + new_pte = pfn_pte(pfn, canon_pgprot(new_prot)); + cpa->pfn = pfn; /* * Do we really change anything ? */ @@ -581,67 +589,44 @@ repeat: return err; } -/** - * change_page_attr_addr - Change page table attributes in linear mapping - * @address: Virtual address in linear mapping. - * @prot: New page table attribute (PAGE_*) - * - * Change page attributes of a page in the direct mapping. This is a variant - * of change_page_attr() that also works on memory holes that do not have - * mem_map entry (pfn_valid() is false). - * - * See change_page_attr() documentation for more details. - * - * Modules and drivers should use the set_memory_* APIs instead. - */ -static int change_page_attr_addr(struct cpa_data *cpa) +static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias); + +static int cpa_process_alias(struct cpa_data *cpa) { - int err; - unsigned long address = cpa->vaddr; + struct cpa_data alias_cpa; + int ret; -#ifdef CONFIG_X86_64 - unsigned long phys_addr = __pa(address); + if (cpa->pfn > max_pfn_mapped) + return 0; - /* - * If we are inside the high mapped kernel range, then we - * fixup the low mapping first. __va() returns the virtual - * address in the linear mapping: - */ - if (within(address, HIGH_MAP_START, HIGH_MAP_END)) - address = (unsigned long) __va(phys_addr); -#endif + alias_cpa = *cpa; + alias_cpa.vaddr = (unsigned long) __va(cpa->pfn << PAGE_SHIFT); - err = __change_page_attr(address, cpa); - if (err) - return err; + ret = __change_page_attr_set_clr(&alias_cpa, 0); #ifdef CONFIG_X86_64 + if (ret) + return ret; /* * If the physical address is inside the kernel map, we need * to touch the high mapped kernel as well: */ - if (within(phys_addr, 0, KERNEL_TEXT_SIZE)) { - /* - * Calc the high mapping address. See __phys_addr() - * for the non obvious details. - * - * Note that NX and other required permissions are - * checked in static_protections(). - */ - address = phys_addr + HIGH_MAP_START - phys_base; + if (!within(cpa->pfn, highmap_start_pfn(), highmap_end_pfn())) + return 0; - /* - * Our high aliases are imprecise, because we check - * everything between 0 and KERNEL_TEXT_SIZE, so do - * not propagate lookup failures back to users: - */ - __change_page_attr(address, cpa); - } + alias_cpa = *cpa; + alias_cpa.vaddr = + (cpa->pfn << PAGE_SHIFT) + __START_KERNEL_map - phys_base; + + /* + * The high mapping range is imprecise, so ignore the return value. + */ + __change_page_attr_set_clr(&alias_cpa, 0); #endif - return err; + return ret; } -static int __change_page_attr_set_clr(struct cpa_data *cpa) +static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias) { int ret, numpages = cpa->numpages; @@ -651,10 +636,17 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa) * preservation check. */ cpa->numpages = numpages; - ret = change_page_attr_addr(cpa); + + ret = __change_page_attr(cpa, checkalias); if (ret) return ret; + if (checkalias) { + ret = cpa_process_alias(cpa); + if (ret) + return ret; + } + /* * Adjust the number of pages with the result of the * CPA operation. Either a large page has been @@ -703,7 +695,7 @@ static int change_page_attr_set_clr(unsigned long addr, int numpages, cpa.mask_clr = mask_clr; cpa.flushtlb = 0; - ret = __change_page_attr_set_clr(&cpa); + ret = __change_page_attr_set_clr(&cpa, 1); /* * Check whether we really changed something: @@ -841,7 +833,7 @@ static int __set_pages_p(struct page *page, int numpages) .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW), .mask_clr = __pgprot(0)}; - return __change_page_attr_set_clr(&cpa); + return __change_page_attr_set_clr(&cpa, 1); } static int __set_pages_np(struct page *page, int numpages) @@ -851,7 +843,7 @@ static int __set_pages_np(struct page *page, int numpages) .mask_set = __pgprot(0), .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW)}; - return __change_page_attr_set_clr(&cpa); + return __change_page_attr_set_clr(&cpa, 1); } void kernel_map_pages(struct page *page, int numpages, int enable) -- cgit v1.2.3 From af96e4438a4b34a257f5318a296e0b9e182e7ab9 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 15 Feb 2008 21:49:46 +0100 Subject: x86: CPA no alias checking for _NX NX settings are not required to be consistent across alias mappings. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/mm/pageattr.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'arch/x86/mm/pageattr.c') diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index 7b4a17236972..b8f53233151d 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -669,7 +669,7 @@ static int change_page_attr_set_clr(unsigned long addr, int numpages, pgprot_t mask_set, pgprot_t mask_clr) { struct cpa_data cpa; - int ret, cache; + int ret, cache, checkalias; /* * Check, if we are requested to change a not supported @@ -695,7 +695,10 @@ static int change_page_attr_set_clr(unsigned long addr, int numpages, cpa.mask_clr = mask_clr; cpa.flushtlb = 0; - ret = __change_page_attr_set_clr(&cpa, 1); + /* No alias checking for _NX bit modifications */ + checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX; + + ret = __change_page_attr_set_clr(&cpa, checkalias); /* * Check whether we really changed something: -- cgit v1.2.3 From f34b439f34c49d7de858234bab5e2dd03cfaf3c1 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 15 Feb 2008 22:17:57 +0100 Subject: x86: CPA: avoid double checking of alias ranges When the CPA code is called with an virtual address in the range of the direct mapping or the high alias then we do not need to run through the alias check for this range. Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/mm/pageattr.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'arch/x86/mm/pageattr.c') diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index b8f53233151d..3ee14996c829 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -594,19 +594,34 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias); static int cpa_process_alias(struct cpa_data *cpa) { struct cpa_data alias_cpa; - int ret; + int ret = 0; if (cpa->pfn > max_pfn_mapped) return 0; - alias_cpa = *cpa; - alias_cpa.vaddr = (unsigned long) __va(cpa->pfn << PAGE_SHIFT); + /* + * No need to redo, when the primary call touched the direct + * mapping already: + */ + if (!within(cpa->vaddr, PAGE_OFFSET, + PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) { - ret = __change_page_attr_set_clr(&alias_cpa, 0); + alias_cpa = *cpa; + alias_cpa.vaddr = (unsigned long) __va(cpa->pfn << PAGE_SHIFT); + + ret = __change_page_attr_set_clr(&alias_cpa, 0); + } #ifdef CONFIG_X86_64 if (ret) return ret; + /* + * No need to redo, when the primary call touched the high + * mapping already: + */ + if (within(cpa->vaddr, (unsigned long) _text, (unsigned long) _end)) + return 0; + /* * If the physical address is inside the kernel map, we need * to touch the high mapped kernel as well: -- cgit v1.2.3 From 8e31c2ac1101b09ad6c6d3539b032d312cc3302b Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Mon, 11 Feb 2008 10:50:21 +0100 Subject: x86: CPA: remove BUG_ON for LRU/Compound pages New implementation does not use lru for anything so there is no need to reject pages that are in the LRU. Similar for compound pages (which were checked because they also use page->lru) [ tglx@linutronix.de: removed unused variable ] Signed-off-by: Andi Kleen Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/mm/pageattr.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'arch/x86/mm/pageattr.c') diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index 3ee14996c829..e2a74ea11a53 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -513,7 +513,6 @@ static int __change_page_attr(struct cpa_data *cpa, int primary) unsigned long address = cpa->vaddr; int do_split, err; unsigned int level; - struct page *kpte_page; pte_t *kpte, old_pte; repeat: @@ -532,10 +531,6 @@ repeat: return -EINVAL; } - kpte_page = virt_to_page(kpte); - BUG_ON(PageLRU(kpte_page)); - BUG_ON(PageCompound(kpte_page)); - if (level == PG_LEVEL_4K) { pte_t new_pte; pgprot_t new_prot = pte_pgprot(old_pte); -- cgit v1.2.3 From 8a235efad548abd2ab5ebea45a9ffa750c814375 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 20 Feb 2008 01:47:44 +0100 Subject: Hibernation: Handle DEBUG_PAGEALLOC on x86 Make hibernation work with CONFIG_DEBUG_PAGEALLOC set on x86, by checking if the pages to be copied are marked as present in the kernel mapping and temporarily marking them as present if that's not the case. No functional modifications are introduced if CONFIG_DEBUG_PAGEALLOC is unset. Signed-off-by: Rafael J. Wysocki Signed-off-by: Len Brown --- arch/x86/mm/pageattr.c | 19 ++++++++++++++++++- include/linux/mm.h | 6 ++++++ kernel/power/snapshot.c | 42 +++++++++++++++++++++++++++++------------- 3 files changed, 53 insertions(+), 14 deletions(-) (limited to 'arch/x86/mm/pageattr.c') diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index e2a74ea11a53..464d8fc21ce6 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -899,7 +899,24 @@ void kernel_map_pages(struct page *page, int numpages, int enable) */ cpa_fill_pool(); } -#endif + +#ifdef CONFIG_HIBERNATION + +bool kernel_page_present(struct page *page) +{ + unsigned int level; + pte_t *pte; + + if (PageHighMem(page)) + return false; + + pte = lookup_address((unsigned long)page_address(page), &level); + return (pte_val(*pte) & _PAGE_PRESENT); +} + +#endif /* CONFIG_HIBERNATION */ + +#endif /* CONFIG_DEBUG_PAGEALLOC */ /* * The testcases use internal knowledge of the implementation that shouldn't diff --git a/include/linux/mm.h b/include/linux/mm.h index 26c7124b841a..3b3e1341163f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1171,12 +1171,18 @@ static inline void enable_debug_pagealloc(void) { debug_pagealloc_enabled = 1; } +#ifdef CONFIG_HIBERNATION +extern bool kernel_page_present(struct page *page); +#endif /* CONFIG_HIBERNATION */ #else static inline void kernel_map_pages(struct page *page, int numpages, int enable) {} static inline void enable_debug_pagealloc(void) { } +#ifdef CONFIG_HIBERNATION +static inline bool kernel_page_present(struct page *page) { return true; } +#endif /* CONFIG_HIBERNATION */ #endif extern struct vm_area_struct *get_gate_vma(struct task_struct *tsk); diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index 95250d7c8d91..72a020cabb4c 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -875,8 +875,8 @@ static inline void *saveable_highmem_page(unsigned long pfn) { return NULL; } #endif /* CONFIG_HIGHMEM */ /** - * saveable - Determine whether a non-highmem page should be included in - * the suspend image. + * saveable_page - Determine whether a non-highmem page should be included + * in the suspend image. * * We should save the page if it isn't Nosave, and is not in the range * of pages statically defined as 'unsaveable', and it isn't a part of @@ -897,7 +897,8 @@ static struct page *saveable_page(unsigned long pfn) if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page)) return NULL; - if (PageReserved(page) && pfn_is_nosave(pfn)) + if (PageReserved(page) + && (!kernel_page_present(page) || pfn_is_nosave(pfn))) return NULL; return page; @@ -938,6 +939,25 @@ static inline void do_copy_page(long *dst, long *src) *dst++ = *src++; } + +/** + * safe_copy_page - check if the page we are going to copy is marked as + * present in the kernel page tables (this always is the case if + * CONFIG_DEBUG_PAGEALLOC is not set and in that case + * kernel_page_present() always returns 'true'). + */ +static void safe_copy_page(void *dst, struct page *s_page) +{ + if (kernel_page_present(s_page)) { + do_copy_page(dst, page_address(s_page)); + } else { + kernel_map_pages(s_page, 1, 1); + do_copy_page(dst, page_address(s_page)); + kernel_map_pages(s_page, 1, 0); + } +} + + #ifdef CONFIG_HIGHMEM static inline struct page * page_is_saveable(struct zone *zone, unsigned long pfn) @@ -946,8 +966,7 @@ page_is_saveable(struct zone *zone, unsigned long pfn) saveable_highmem_page(pfn) : saveable_page(pfn); } -static inline void -copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) +static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) { struct page *s_page, *d_page; void *src, *dst; @@ -961,29 +980,26 @@ copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) kunmap_atomic(src, KM_USER0); kunmap_atomic(dst, KM_USER1); } else { - src = page_address(s_page); if (PageHighMem(d_page)) { /* Page pointed to by src may contain some kernel * data modified by kmap_atomic() */ - do_copy_page(buffer, src); + safe_copy_page(buffer, s_page); dst = kmap_atomic(pfn_to_page(dst_pfn), KM_USER0); memcpy(dst, buffer, PAGE_SIZE); kunmap_atomic(dst, KM_USER0); } else { - dst = page_address(d_page); - do_copy_page(dst, src); + safe_copy_page(page_address(d_page), s_page); } } } #else #define page_is_saveable(zone, pfn) saveable_page(pfn) -static inline void -copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) +static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) { - do_copy_page(page_address(pfn_to_page(dst_pfn)), - page_address(pfn_to_page(src_pfn))); + safe_copy_page(page_address(pfn_to_page(dst_pfn)), + pfn_to_page(src_pfn)); } #endif /* CONFIG_HIGHMEM */ -- cgit v1.2.3