diff options
| -rw-r--r-- | arch/powerpc/include/asm/hugetlb.h | 30 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/mmu-hash64.h | 20 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/page.h | 13 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/pgalloc-64.h | 5 | ||||
| -rw-r--r-- | arch/powerpc/mm/hugetlbpage.c | 26 | ||||
| -rw-r--r-- | arch/powerpc/mm/init_64.c | 3 |
6 files changed, 75 insertions, 22 deletions
diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h index 62e11a32c4c2..4daf7e684f58 100644 --- a/arch/powerpc/include/asm/hugetlb.h +++ b/arch/powerpc/include/asm/hugetlb.h @@ -6,6 +6,33 @@ extern struct kmem_cache *hugepte_cache; +#ifdef CONFIG_PPC_BOOK3S_64 +/* + * This should work for other subarchs too. But right now we use the + * new format only for 64bit book3s + */ +static inline pte_t *hugepd_page(hugepd_t hpd) +{ + BUG_ON(!hugepd_ok(hpd)); + /* + * We have only four bits to encode, MMU page size + */ + BUILD_BUG_ON((MMU_PAGE_COUNT - 1) > 0xf); + return (pte_t *)(hpd.pd & ~HUGEPD_SHIFT_MASK); +} + +static inline unsigned int hugepd_mmu_psize(hugepd_t hpd) +{ + return (hpd.pd & HUGEPD_SHIFT_MASK) >> 2; +} + +static inline unsigned int hugepd_shift(hugepd_t hpd) +{ + return mmu_psize_to_shift(hugepd_mmu_psize(hpd)); +} + +#else + static inline pte_t *hugepd_page(hugepd_t hpd) { BUG_ON(!hugepd_ok(hpd)); @@ -17,6 +44,9 @@ static inline unsigned int hugepd_shift(hugepd_t hpd) return hpd.pd & HUGEPD_SHIFT_MASK; } +#endif /* CONFIG_PPC_BOOK3S_64 */ + + static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr, unsigned pdshift) { diff --git a/arch/powerpc/include/asm/mmu-hash64.h b/arch/powerpc/include/asm/mmu-hash64.h index b59e06f507ea..05895cff1345 100644 --- a/arch/powerpc/include/asm/mmu-hash64.h +++ b/arch/powerpc/include/asm/mmu-hash64.h @@ -21,6 +21,7 @@ * complete pgtable.h but only a portion of it. */ #include <asm/pgtable-ppc64.h> +#include <asm/bug.h> /* * Segment table @@ -159,6 +160,24 @@ struct mmu_psize_def unsigned long avpnm; /* bits to mask out in AVPN in the HPTE */ unsigned long sllp; /* SLB L||LP (exact mask to use in slbmte) */ }; +extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; + +static inline int shift_to_mmu_psize(unsigned int shift) +{ + int psize; + + for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) + if (mmu_psize_defs[psize].shift == shift) + return psize; + return -1; +} + +static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize) +{ + if (mmu_psize_defs[mmu_psize].shift) + return mmu_psize_defs[mmu_psize].shift; + BUG(); +} #endif /* __ASSEMBLY__ */ @@ -193,7 +212,6 @@ static inline int segment_shift(int ssize) /* * The current system page and segment sizes */ -extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; extern int mmu_linear_psize; extern int mmu_virtual_psize; extern int mmu_vmalloc_psize; diff --git a/arch/powerpc/include/asm/page.h b/arch/powerpc/include/asm/page.h index f072e974f8a2..652719ccd2e9 100644 --- a/arch/powerpc/include/asm/page.h +++ b/arch/powerpc/include/asm/page.h @@ -249,6 +249,7 @@ extern long long virt_phys_offset; #define is_kernel_addr(x) ((x) >= PAGE_OFFSET) #endif +#ifndef CONFIG_PPC_BOOK3S_64 /* * Use the top bit of the higher-level page table entries to indicate whether * the entries we point to contain hugepages. This works because we know that @@ -260,6 +261,7 @@ extern long long virt_phys_offset; #else #define PD_HUGE 0x80000000 #endif +#endif /* CONFIG_PPC_BOOK3S_64 */ /* * Some number of bits at the level of the page table that points to @@ -354,10 +356,21 @@ typedef unsigned long pgprot_t; typedef struct { signed long pd; } hugepd_t; #ifdef CONFIG_HUGETLB_PAGE +#ifdef CONFIG_PPC_BOOK3S_64 +static inline int hugepd_ok(hugepd_t hpd) +{ + /* + * hugepd pointer, bottom two bits == 00 and next 4 bits + * indicate size of table + */ + return (((hpd.pd & 0x3) == 0x0) && ((hpd.pd & HUGEPD_SHIFT_MASK) != 0)); +} +#else static inline int hugepd_ok(hugepd_t hpd) { return (hpd.pd > 0); } +#endif #define is_hugepd(pdep) (hugepd_ok(*((hugepd_t *)(pdep)))) #else /* CONFIG_HUGETLB_PAGE */ diff --git a/arch/powerpc/include/asm/pgalloc-64.h b/arch/powerpc/include/asm/pgalloc-64.h index 292725cec2e3..69e352a5252b 100644 --- a/arch/powerpc/include/asm/pgalloc-64.h +++ b/arch/powerpc/include/asm/pgalloc-64.h @@ -35,7 +35,10 @@ struct vmemmap_backing { #define MAX_PGTABLE_INDEX_SIZE 0xf extern struct kmem_cache *pgtable_cache[]; -#define PGT_CACHE(shift) (pgtable_cache[(shift)-1]) +#define PGT_CACHE(shift) ({ \ + BUG_ON(!(shift)); \ + pgtable_cache[(shift) - 1]; \ + }) static inline pgd_t *pgd_alloc(struct mm_struct *mm) { diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 5dc52d803ed8..b4e2f24a9b8f 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -48,23 +48,6 @@ static u64 gpage_freearray[MAX_NUMBER_GPAGES]; static unsigned nr_gpages; #endif -static inline int shift_to_mmu_psize(unsigned int shift) -{ - int psize; - - for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) - if (mmu_psize_defs[psize].shift == shift) - return psize; - return -1; -} - -static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize) -{ - if (mmu_psize_defs[mmu_psize].shift) - return mmu_psize_defs[mmu_psize].shift; - BUG(); -} - #define hugepd_none(hpd) ((hpd).pd == 0) pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift) @@ -145,6 +128,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, if (unlikely(!hugepd_none(*hpdp))) break; else + /* We use the old format for PPC_FSL_BOOK3E */ hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift; } /* If we bailed from the for loop early, an error occurred, clean up */ @@ -156,9 +140,15 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, #else if (!hugepd_none(*hpdp)) kmem_cache_free(cachep, new); - else + else { +#ifdef CONFIG_PPC_BOOK3S_64 + hpdp->pd = (unsigned long)new | + (shift_to_mmu_psize(pshift) << 2); +#else hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift; #endif + } +#endif spin_unlock(&mm->page_table_lock); return 0; } diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c index 7e2246fb2f31..a56de85ad3b7 100644 --- a/arch/powerpc/mm/init_64.c +++ b/arch/powerpc/mm/init_64.c @@ -129,8 +129,7 @@ void pgtable_cache_add(unsigned shift, void (*ctor)(void *)) align = max_t(unsigned long, align, minalign); name = kasprintf(GFP_KERNEL, "pgtable-2^%d", shift); new = kmem_cache_create(name, table_size, align, 0, ctor); - PGT_CACHE(shift) = new; - + pgtable_cache[shift - 1] = new; pr_debug("Allocated pgtable cache for order %d\n", shift); } |