From 410f14837a92177242a403c20d0a0906fe357019 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 14 Feb 2011 12:58:04 +0100 Subject: ARM: 6667/1: Fix early_pte_alloc() assumption about the Linux PTE With LPAE we no longer have software bits in a separate Linux PTE and the early_pte_alloc() function should pass PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE to early_alloc() to avoid allocating extra memory. Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 546e44734db0..7515d4718645 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -532,7 +532,7 @@ static void __init *early_alloc(unsigned long sz) static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot) { if (pmd_none(*pmd)) { - pte_t *pte = early_alloc(2 * PTRS_PER_PTE * sizeof(pte_t)); + pte_t *pte = early_alloc(PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE); __pmd_populate(pmd, __pa(pte), prot); } BUG_ON(pmd_bad(*pmd)); -- cgit v1.2.3 From 29a38193c15bd72ba96c57a805443ef46ea6a6ac Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 15 Feb 2011 14:31:37 +0100 Subject: ARM: 6674/1: LPAE: use long long format when printing physical addresses and ptes For the Kernel to support 2 level and 3 level page tables, physical addresses (and also page table entries) need to be 32 or 64-bits depending upon the configuration. This patch uses the %08llx conversion specifier for physical addresses and page table entries, ensuring that they are cast to (long long) so that common code can be used regardless of the datatype widths. Acked-by: Catalin Marinas Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/kernel/setup.c | 2 +- arch/arm/kernel/traps.c | 6 +++--- arch/arm/mm/fault.c | 10 ++++++---- arch/arm/mm/mmu.c | 26 +++++++++++++------------- 4 files changed, 23 insertions(+), 21 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 336f14e0e5c2..77af580953e6 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -448,7 +448,7 @@ static int __init arm_add_memory(unsigned long start, unsigned long size) if (meminfo.nr_banks >= NR_BANKS) { printk(KERN_CRIT "NR_BANKS too low, " - "ignoring memory at %#lx\n", start); + "ignoring memory at 0x%08llx\n", (long long)start); return -EINVAL; } diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 354cd4ce400a..10f5b8926ee6 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -710,17 +710,17 @@ EXPORT_SYMBOL(__readwrite_bug); void __pte_error(const char *file, int line, pte_t pte) { - printk("%s:%d: bad pte %08lx.\n", file, line, pte_val(pte)); + printk("%s:%d: bad pte %08llx.\n", file, line, (long long)pte_val(pte)); } void __pmd_error(const char *file, int line, pmd_t pmd) { - printk("%s:%d: bad pmd %08lx.\n", file, line, pmd_val(pmd)); + printk("%s:%d: bad pmd %08llx.\n", file, line, (long long)pmd_val(pmd)); } void __pgd_error(const char *file, int line, pgd_t pgd) { - printk("%s:%d: bad pgd %08lx.\n", file, line, pgd_val(pgd)); + printk("%s:%d: bad pgd %08llx.\n", file, line, (long long)pgd_val(pgd)); } asmlinkage void __div0(void) diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index f10f9bac2206..ef0e24f578ef 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -76,7 +76,8 @@ void show_pte(struct mm_struct *mm, unsigned long addr) printk(KERN_ALERT "pgd = %p\n", mm->pgd); pgd = pgd_offset(mm, addr); - printk(KERN_ALERT "[%08lx] *pgd=%08lx", addr, pgd_val(*pgd)); + printk(KERN_ALERT "[%08lx] *pgd=%08llx", + addr, (long long)pgd_val(*pgd)); do { pmd_t *pmd; @@ -92,7 +93,7 @@ void show_pte(struct mm_struct *mm, unsigned long addr) pmd = pmd_offset(pgd, addr); if (PTRS_PER_PMD != 1) - printk(", *pmd=%08lx", pmd_val(*pmd)); + printk(", *pmd=%08llx", (long long)pmd_val(*pmd)); if (pmd_none(*pmd)) break; @@ -107,8 +108,9 @@ void show_pte(struct mm_struct *mm, unsigned long addr) break; pte = pte_offset_map(pmd, addr); - printk(", *pte=%08lx", pte_val(*pte)); - printk(", *ppte=%08lx", pte_val(pte[PTE_HWTABLE_PTRS])); + printk(", *pte=%08llx", (long long)pte_val(*pte)); + printk(", *ppte=%08llx", + (long long)pte_val(pte[PTE_HWTABLE_PTRS])); pte_unmap(pte); } while(0); diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 7515d4718645..f512ad97dafe 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -597,7 +597,7 @@ static void __init create_36bit_mapping(struct map_desc *md, if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) { printk(KERN_ERR "MM: CPU does not support supersection " "mapping for 0x%08llx at 0x%08lx\n", - __pfn_to_phys((u64)md->pfn), addr); + (long long)__pfn_to_phys((u64)md->pfn), addr); return; } @@ -610,14 +610,14 @@ static void __init create_36bit_mapping(struct map_desc *md, if (type->domain) { printk(KERN_ERR "MM: invalid domain in supersection " "mapping for 0x%08llx at 0x%08lx\n", - __pfn_to_phys((u64)md->pfn), addr); + (long long)__pfn_to_phys((u64)md->pfn), addr); return; } if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) { - printk(KERN_ERR "MM: cannot create mapping for " - "0x%08llx at 0x%08lx invalid alignment\n", - __pfn_to_phys((u64)md->pfn), addr); + printk(KERN_ERR "MM: cannot create mapping for 0x%08llx" + " at 0x%08lx invalid alignment\n", + (long long)__pfn_to_phys((u64)md->pfn), addr); return; } @@ -656,17 +656,17 @@ static void __init create_mapping(struct map_desc *md) pgd_t *pgd; if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) { - printk(KERN_WARNING "BUG: not creating mapping for " - "0x%08llx at 0x%08lx in user region\n", - __pfn_to_phys((u64)md->pfn), md->virtual); + printk(KERN_WARNING "BUG: not creating mapping for 0x%08llx" + " at 0x%08lx in user region\n", + (long long)__pfn_to_phys((u64)md->pfn), md->virtual); return; } if ((md->type == MT_DEVICE || md->type == MT_ROM) && md->virtual >= PAGE_OFFSET && md->virtual < VMALLOC_END) { - printk(KERN_WARNING "BUG: mapping for 0x%08llx at 0x%08lx " - "overlaps vmalloc space\n", - __pfn_to_phys((u64)md->pfn), md->virtual); + printk(KERN_WARNING "BUG: mapping for 0x%08llx" + " at 0x%08lx overlaps vmalloc space\n", + (long long)__pfn_to_phys((u64)md->pfn), md->virtual); } type = &mem_types[md->type]; @@ -684,9 +684,9 @@ static void __init create_mapping(struct map_desc *md) length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK)); if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) { - printk(KERN_WARNING "BUG: map for 0x%08lx at 0x%08lx can not " + printk(KERN_WARNING "BUG: map for 0x%08llx at 0x%08lx can not " "be mapped using pages, ignoring.\n", - __pfn_to_phys(md->pfn), addr); + (long long)__pfn_to_phys(md->pfn), addr); return; } -- cgit v1.2.3 From cae6292b653f5e3308bf2787a54b7dcd2cc7e2b3 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 15 Feb 2011 12:42:57 +0100 Subject: ARM: 6672/1: LPAE: use phys_addr_t instead of unsigned long in mapping functions The unsigned long datatype is not sufficient for mapping physical addresses >= 4GB. This patch ensures that the phys_addr_t datatype is used to represent physical addresses when converting from a PFN. Acked-by: Catalin Marinas Signed-off-by: Will Deacon Signed-off-by: Russell King --- arch/arm/include/asm/pgtable.h | 2 +- arch/arm/mm/init.c | 6 +++--- arch/arm/mm/mmu.c | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index ebcb6432f45f..ca567914d303 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -351,7 +351,7 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd) #define pte_unmap(pte) __pte_unmap(pte) #define pte_pfn(pte) (pte_val(pte) >> PAGE_SHIFT) -#define pfn_pte(pfn,prot) __pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) +#define pfn_pte(pfn,prot) __pte(__pfn_to_phys(pfn) | pgprot_val(prot)) #define pte_page(pte) pfn_to_page(pte_pfn(pte)) #define mk_pte(page,prot) pfn_pte(page_to_pfn(page), prot) diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 5164069ced42..14a00a1ef52f 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -344,7 +344,7 @@ void __init bootmem_init(void) */ arm_bootmem_free(min, max_low, max_high); - high_memory = __va((max_low << PAGE_SHIFT) - 1) + 1; + high_memory = __va(((phys_addr_t)max_low << PAGE_SHIFT) - 1) + 1; /* * This doesn't seem to be used by the Linux memory manager any @@ -392,8 +392,8 @@ free_memmap(unsigned long start_pfn, unsigned long end_pfn) * Convert to physical addresses, and * round start upwards and end downwards. */ - pg = PAGE_ALIGN(__pa(start_pg)); - pgend = __pa(end_pg) & PAGE_MASK; + pg = (unsigned long)PAGE_ALIGN(__pa(start_pg)); + pgend = (unsigned long)__pa(end_pg) & PAGE_MASK; /* * If there are free pages between these, diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index f512ad97dafe..e09744e82d50 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -591,7 +591,7 @@ static void __init create_36bit_mapping(struct map_desc *md, pgd_t *pgd; addr = md->virtual; - phys = (unsigned long)__pfn_to_phys(md->pfn); + phys = __pfn_to_phys(md->pfn); length = PAGE_ALIGN(md->length); if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) { @@ -651,7 +651,8 @@ static void __init create_36bit_mapping(struct map_desc *md, */ static void __init create_mapping(struct map_desc *md) { - unsigned long phys, addr, length, end; + unsigned long addr, length, end; + phys_addr_t phys; const struct mem_type *type; pgd_t *pgd; @@ -680,7 +681,7 @@ static void __init create_mapping(struct map_desc *md) } addr = md->virtual & PAGE_MASK; - phys = (unsigned long)__pfn_to_phys(md->pfn); + phys = __pfn_to_phys(md->pfn); length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK)); if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) { -- cgit v1.2.3 From e33b9d0878ff6b8a543b3423c3019a5dbbf5689a Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 20 Feb 2011 11:47:41 +0000 Subject: ARM: Use long long format when printing meminfo physical addresses Acked-by: Will Deacon Signed-off-by: Russell King --- arch/arm/mm/mmu.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index e09744e82d50..176749010935 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -794,9 +794,10 @@ static void __init sanity_check_meminfo(void) */ if (__va(bank->start) >= vmalloc_min || __va(bank->start) < (void *)PAGE_OFFSET) { - printk(KERN_NOTICE "Ignoring RAM at %.8lx-%.8lx " + printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx " "(vmalloc region overlap).\n", - bank->start, bank->start + bank->size - 1); + (unsigned long long)bank->start, + (unsigned long long)bank->start + bank->size - 1); continue; } @@ -807,10 +808,11 @@ static void __init sanity_check_meminfo(void) if (__va(bank->start + bank->size) > vmalloc_min || __va(bank->start + bank->size) < __va(bank->start)) { unsigned long newsize = vmalloc_min - __va(bank->start); - printk(KERN_NOTICE "Truncating RAM at %.8lx-%.8lx " - "to -%.8lx (vmalloc region overlap).\n", - bank->start, bank->start + bank->size - 1, - bank->start + newsize - 1); + printk(KERN_NOTICE "Truncating RAM at %.8llx-%.8llx " + "to -%.8llx (vmalloc region overlap).\n", + (unsigned long long)bank->start, + (unsigned long long)bank->start + bank->size - 1, + (unsigned long long)bank->start + newsize - 1); bank->size = newsize; } #endif -- cgit v1.2.3 From 516295e5ab4bf986865cfff856d484ec678e3b0b Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 21 Nov 2010 16:27:49 +0000 Subject: ARM: pgtable: add pud-level code Add pud_offset() et.al. between the pgd and pmd code in preparation of using pgtable-nopud.h rather than 4level-fixup.h. This incorporates a fix from Jamie Iles for uaccess_with_memcpy.c. Signed-off-by: Russell King --- arch/arm/include/asm/pgtable.h | 1 + arch/arm/lib/uaccess_with_memcpy.c | 7 ++++++- arch/arm/mm/dma-mapping.c | 11 ++++++++++- arch/arm/mm/fault-armv.c | 7 ++++++- arch/arm/mm/fault.c | 29 +++++++++++++++++++++++++---- arch/arm/mm/idmap.c | 35 +++++++++++++++++++++++++++++------ arch/arm/mm/mm.h | 2 +- arch/arm/mm/mmu.c | 22 ++++++++++++++++++---- arch/arm/mm/pgd.c | 24 ++++++++++++++++++++---- 9 files changed, 116 insertions(+), 22 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index ca567914d303..5750704e0271 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -301,6 +301,7 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; #define pgd_present(pgd) (1) #define pgd_clear(pgdp) do { } while (0) #define set_pgd(pgd,pgdp) do { } while (0) +#define set_pud(pud,pudp) do { } while (0) /* Find an entry in the second-level page table.. */ diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c index e2d2f2cd0c4f..8b9b13649f81 100644 --- a/arch/arm/lib/uaccess_with_memcpy.c +++ b/arch/arm/lib/uaccess_with_memcpy.c @@ -27,13 +27,18 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp) pgd_t *pgd; pmd_t *pmd; pte_t *pte; + pud_t *pud; spinlock_t *ptl; pgd = pgd_offset(current->mm, addr); if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd))) return 0; - pmd = pmd_offset(pgd, addr); + pud = pud_offset(pgd, addr); + if (unlikely(pud_none(*pud) || pud_bad(*pud))) + return 0; + + pmd = pmd_offset(pud, addr); if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd))) return 0; diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index ac6a36142fcd..a9bdfcda23f4 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -148,6 +148,7 @@ static int __init consistent_init(void) { int ret = 0; pgd_t *pgd; + pud_t *pud; pmd_t *pmd; pte_t *pte; int i = 0; @@ -155,7 +156,15 @@ static int __init consistent_init(void) do { pgd = pgd_offset(&init_mm, base); - pmd = pmd_alloc(&init_mm, pgd, base); + + pud = pud_alloc(&init_mm, pgd, base); + if (!pud) { + printk(KERN_ERR "%s: no pud tables\n", __func__); + ret = -ENOMEM; + break; + } + + pmd = pmd_alloc(&init_mm, pud, base); if (!pmd) { printk(KERN_ERR "%s: no pmd tables\n", __func__); ret = -ENOMEM; diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 01210dba0221..7cab79179421 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -95,6 +95,7 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, { spinlock_t *ptl; pgd_t *pgd; + pud_t *pud; pmd_t *pmd; pte_t *pte; int ret; @@ -103,7 +104,11 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, if (pgd_none_or_clear_bad(pgd)) return 0; - pmd = pmd_offset(pgd, address); + pud = pud_offset(pgd, address); + if (pud_none_or_clear_bad(pud)) + return 0; + + pmd = pmd_offset(pud, address); if (pmd_none_or_clear_bad(pmd)) return 0; diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index ef0e24f578ef..bc0e1d88fd3b 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -80,6 +80,7 @@ void show_pte(struct mm_struct *mm, unsigned long addr) addr, (long long)pgd_val(*pgd)); do { + pud_t *pud; pmd_t *pmd; pte_t *pte; @@ -91,7 +92,19 @@ void show_pte(struct mm_struct *mm, unsigned long addr) break; } - pmd = pmd_offset(pgd, addr); + pud = pud_offset(pgd, addr); + if (PTRS_PER_PUD != 1) + printk(", *pud=%08lx", pud_val(*pud)); + + if (pud_none(*pud)) + break; + + if (pud_bad(*pud)) { + printk("(bad)"); + break; + } + + pmd = pmd_offset(pud, addr); if (PTRS_PER_PMD != 1) printk(", *pmd=%08llx", (long long)pmd_val(*pmd)); @@ -390,6 +403,7 @@ do_translation_fault(unsigned long addr, unsigned int fsr, { unsigned int index; pgd_t *pgd, *pgd_k; + pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; if (addr < TASK_SIZE) @@ -408,12 +422,19 @@ do_translation_fault(unsigned long addr, unsigned int fsr, if (pgd_none(*pgd_k)) goto bad_area; - if (!pgd_present(*pgd)) set_pgd(pgd, *pgd_k); - pmd_k = pmd_offset(pgd_k, addr); - pmd = pmd_offset(pgd, addr); + pud = pud_offset(pgd, addr); + pud_k = pud_offset(pgd_k, addr); + + if (pud_none(*pud_k)) + goto bad_area; + if (!pud_present(*pud)) + set_pud(pud, *pud_k); + + pmd = pmd_offset(pud, addr); + pmd_k = pmd_offset(pud_k, addr); /* * On ARM one Linux PGD entry contains two hardware entries (see page diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c index 57299446f787..2be9139a4ef3 100644 --- a/arch/arm/mm/idmap.c +++ b/arch/arm/mm/idmap.c @@ -4,10 +4,10 @@ #include #include -static void idmap_add_pmd(pgd_t *pgd, unsigned long addr, unsigned long end, +static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end, unsigned long prot) { - pmd_t *pmd = pmd_offset(pgd, addr); + pmd_t *pmd = pmd_offset(pud, addr); addr = (addr & PMD_MASK) | prot; pmd[0] = __pmd(addr); @@ -16,6 +16,18 @@ static void idmap_add_pmd(pgd_t *pgd, unsigned long addr, unsigned long end, flush_pmd_entry(pmd); } +static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end, + unsigned long prot) +{ + pud_t *pud = pud_offset(pgd, addr); + unsigned long next; + + do { + next = pud_addr_end(addr, end); + idmap_add_pmd(pud, addr, next, prot); + } while (pud++, addr = next, addr != end); +} + void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end) { unsigned long prot, next; @@ -27,17 +39,28 @@ void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end) pgd += pgd_index(addr); do { next = pgd_addr_end(addr, end); - idmap_add_pmd(pgd, addr, next, prot); + idmap_add_pud(pgd, addr, next, prot); } while (pgd++, addr = next, addr != end); } #ifdef CONFIG_SMP -static void idmap_del_pmd(pgd_t *pgd, unsigned long addr, unsigned long end) +static void idmap_del_pmd(pud_t *pud, unsigned long addr, unsigned long end) { - pmd_t *pmd = pmd_offset(pgd, addr); + pmd_t *pmd = pmd_offset(pud, addr); pmd_clear(pmd); } +static void idmap_del_pud(pgd_t *pgd, unsigned long addr, unsigned long end) +{ + pud_t *pud = pud_offset(pgd, addr); + unsigned long next; + + do { + next = pud_addr_end(addr, end); + idmap_del_pmd(pud, addr, next); + } while (pud++, addr = next, addr != end); +} + void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end) { unsigned long next; @@ -45,7 +68,7 @@ void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end) pgd += pgd_index(addr); do { next = pgd_addr_end(addr, end); - idmap_del_pmd(pgd, addr, next); + idmap_del_pud(pgd, addr, next); } while (pgd++, addr = next, addr != end); } #endif diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index 36960df5fb76..d2384106af9c 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -7,7 +7,7 @@ extern pmd_t *top_pmd; static inline pmd_t *pmd_off(pgd_t *pgd, unsigned long virt) { - return pmd_offset(pgd, virt); + return pmd_offset(pud_offset(pgd, virt), virt); } static inline pmd_t *pmd_off_k(unsigned long virt) diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 176749010935..82ef6966ae09 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -550,11 +550,11 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, } while (pte++, addr += PAGE_SIZE, addr != end); } -static void __init alloc_init_section(pgd_t *pgd, unsigned long addr, +static void __init alloc_init_section(pud_t *pud, unsigned long addr, unsigned long end, phys_addr_t phys, const struct mem_type *type) { - pmd_t *pmd = pmd_offset(pgd, addr); + pmd_t *pmd = pmd_offset(pud, addr); /* * Try a section mapping - end, addr and phys must all be aligned @@ -583,6 +583,19 @@ static void __init alloc_init_section(pgd_t *pgd, unsigned long addr, } } +static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, + unsigned long phys, const struct mem_type *type) +{ + pud_t *pud = pud_offset(pgd, addr); + unsigned long next; + + do { + next = pud_addr_end(addr, end); + alloc_init_section(pud, addr, next, phys, type); + phys += next - addr; + } while (pud++, addr = next, addr != end); +} + static void __init create_36bit_mapping(struct map_desc *md, const struct mem_type *type) { @@ -630,7 +643,8 @@ static void __init create_36bit_mapping(struct map_desc *md, pgd = pgd_offset_k(addr); end = addr + length; do { - pmd_t *pmd = pmd_offset(pgd, addr); + pud_t *pud = pud_offset(pgd, addr); + pmd_t *pmd = pmd_offset(pud, addr); int i; for (i = 0; i < 16; i++) @@ -696,7 +710,7 @@ static void __init create_mapping(struct map_desc *md) do { unsigned long next = pgd_addr_end(addr, end); - alloc_init_section(pgd, addr, next, phys, type); + alloc_init_pud(pgd, addr, next, phys, type); phys += next - addr; addr = next; diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index 93292a18cf77..f7fafb1741f4 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -23,6 +23,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *new_pgd, *init_pgd; + pud_t *new_pud, *init_pud; pmd_t *new_pmd, *init_pmd; pte_t *new_pte, *init_pte; @@ -46,7 +47,11 @@ pgd_t *pgd_alloc(struct mm_struct *mm) * On ARM, first page must always be allocated since it * contains the machine vectors. */ - new_pmd = pmd_alloc(mm, new_pgd, 0); + new_pud = pud_alloc(mm, new_pgd, 0); + if (!new_pud) + goto no_pud; + + new_pmd = pmd_alloc(mm, new_pud, 0); if (!new_pmd) goto no_pmd; @@ -54,7 +59,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm) if (!new_pte) goto no_pte; - init_pmd = pmd_offset(init_pgd, 0); + init_pud = pud_offset(init_pgd, 0); + init_pmd = pmd_offset(init_pud, 0); init_pte = pte_offset_map(init_pmd, 0); set_pte_ext(new_pte, *init_pte, 0); pte_unmap(init_pte); @@ -66,6 +72,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm) no_pte: pmd_free(mm, new_pmd); no_pmd: + pud_free(mm, new_pud); +no_pud: free_pages((unsigned long)new_pgd, 2); no_pgd: return NULL; @@ -74,6 +82,7 @@ no_pgd: void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) { pgd_t *pgd; + pud_t *pud; pmd_t *pmd; pgtable_t pte; @@ -84,7 +93,11 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) if (pgd_none_or_clear_bad(pgd)) goto no_pgd; - pmd = pmd_offset(pgd, 0); + pud = pud_offset(pgd, 0); + if (pud_none_or_clear_bad(pud)) + goto no_pud; + + pmd = pmd_offset(pud, 0); if (pmd_none_or_clear_bad(pmd)) goto no_pmd; @@ -92,8 +105,11 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) pmd_clear(pmd); pte_free(mm, pte); no_pmd: - pgd_clear(pgd); + pud_clear(pud); pmd_free(mm, pmd); +no_pud: + pgd_clear(pgd); + pud_free(mm, pud); no_pgd: free_pages((unsigned long) pgd_base, 2); } -- cgit v1.2.3