From badea125d7cbd93f1678a95cf009b3bdfe6065cd Mon Sep 17 00:00:00 2001 From: David Mosberger-Tang Date: Mon, 25 Jul 2005 22:23:00 -0700 Subject: [IA64] Fix race in mm-context wrap-around logic. The patch below should fix a race which could cause stale TLB entries. Specifically, when 2 CPUs ended up racing for entrance to wrap_mmu_context(). The losing CPU would find that by the time it acquired ctx.lock, mm->context already had a valid value, but then it failed to (re-)check the delayed TLB flushing logic and hence could end up using a context number when there were still stale entries in its TLB. The fix is to check for delayed TLB flushes only after mm->context is valid (non-zero). The patch also makes GCC v4.x happier by defining a non-volatile variant of mm_context_t called nv_mm_context_t. Signed-off-by: David Mosberger-Tang Signed-off-by: Tony Luck --- include/asm-ia64/mmu_context.h | 54 +++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 22 deletions(-) (limited to 'include/asm-ia64/mmu_context.h') diff --git a/include/asm-ia64/mmu_context.h b/include/asm-ia64/mmu_context.h index e3e5fededb04..0680d163be97 100644 --- a/include/asm-ia64/mmu_context.h +++ b/include/asm-ia64/mmu_context.h @@ -55,34 +55,46 @@ static inline void delayed_tlb_flush (void) { extern void local_flush_tlb_all (void); + unsigned long flags; if (unlikely(__ia64_per_cpu_var(ia64_need_tlb_flush))) { - local_flush_tlb_all(); - __ia64_per_cpu_var(ia64_need_tlb_flush) = 0; + spin_lock_irqsave(&ia64_ctx.lock, flags); + { + if (__ia64_per_cpu_var(ia64_need_tlb_flush)) { + local_flush_tlb_all(); + __ia64_per_cpu_var(ia64_need_tlb_flush) = 0; + } + } + spin_unlock_irqrestore(&ia64_ctx.lock, flags); } } -static inline mm_context_t +static inline nv_mm_context_t get_mmu_context (struct mm_struct *mm) { unsigned long flags; - mm_context_t context = mm->context; - - if (context) - return context; - - spin_lock_irqsave(&ia64_ctx.lock, flags); - { - /* re-check, now that we've got the lock: */ - context = mm->context; - if (context == 0) { - cpus_clear(mm->cpu_vm_mask); - if (ia64_ctx.next >= ia64_ctx.limit) - wrap_mmu_context(mm); - mm->context = context = ia64_ctx.next++; + nv_mm_context_t context = mm->context; + + if (unlikely(!context)) { + spin_lock_irqsave(&ia64_ctx.lock, flags); + { + /* re-check, now that we've got the lock: */ + context = mm->context; + if (context == 0) { + cpus_clear(mm->cpu_vm_mask); + if (ia64_ctx.next >= ia64_ctx.limit) + wrap_mmu_context(mm); + mm->context = context = ia64_ctx.next++; + } } + spin_unlock_irqrestore(&ia64_ctx.lock, flags); } - spin_unlock_irqrestore(&ia64_ctx.lock, flags); + /* + * Ensure we're not starting to use "context" before any old + * uses of it are gone from our TLB. + */ + delayed_tlb_flush(); + return context; } @@ -104,7 +116,7 @@ destroy_context (struct mm_struct *mm) } static inline void -reload_context (mm_context_t context) +reload_context (nv_mm_context_t context) { unsigned long rid; unsigned long rid_incr = 0; @@ -138,7 +150,7 @@ reload_context (mm_context_t context) static inline void activate_context (struct mm_struct *mm) { - mm_context_t context; + nv_mm_context_t context; do { context = get_mmu_context(mm); @@ -157,8 +169,6 @@ activate_context (struct mm_struct *mm) static inline void activate_mm (struct mm_struct *prev, struct mm_struct *next) { - delayed_tlb_flush(); - /* * We may get interrupts here, but that's OK because interrupt handlers cannot * touch user-space. -- cgit v1.2.3 From 0a41e2501160587eb8f66cef3bdf1c6f2cb86997 Mon Sep 17 00:00:00 2001 From: Peter Chubb Date: Tue, 16 Aug 2005 19:54:00 -0700 Subject: [IA64] Rationalise Region Definitions Currently, region numbers are defined in several files, with several names. For example, we have REGION_KERNEL in asm/page.h and RGN_KERNEL in pgtable.h We also have address definitions that should depend on the RGN_XXX macros, but are currently just long constants. The following patch reorganises all the definitions so that they have the same form (RGN_XXX), are in one place, and that addresses that depend on RGN_XXX are derived from them. (This is a necessary but not sufficient patch to allow UML-like operation on IA64). Thanks to David Mosberger for catching the change I missed in mmu_context.h. Signed-off-by: Peter Chubb Signed-off-by: Tony Luck --- arch/ia64/kernel/sys_ia64.c | 2 +- arch/ia64/mm/hugetlbpage.c | 8 ++++---- include/asm-ia64/io.h | 2 +- include/asm-ia64/mmu_context.h | 7 ++++++- include/asm-ia64/page.h | 27 ++++++++++++++++++--------- include/asm-ia64/pgtable.h | 13 +++++-------- include/asm-ia64/system.h | 5 +++-- 7 files changed, 38 insertions(+), 26 deletions(-) (limited to 'include/asm-ia64/mmu_context.h') diff --git a/arch/ia64/kernel/sys_ia64.c b/arch/ia64/kernel/sys_ia64.c index 770fab37928e..f2dbcd1db0d4 100644 --- a/arch/ia64/kernel/sys_ia64.c +++ b/arch/ia64/kernel/sys_ia64.c @@ -35,7 +35,7 @@ arch_get_unmapped_area (struct file *filp, unsigned long addr, unsigned long len return -ENOMEM; #ifdef CONFIG_HUGETLB_PAGE - if (REGION_NUMBER(addr) == REGION_HPAGE) + if (REGION_NUMBER(addr) == RGN_HPAGE) addr = 0; #endif if (!addr) diff --git a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c index e0a776a3044c..2d13889d0a99 100644 --- a/arch/ia64/mm/hugetlbpage.c +++ b/arch/ia64/mm/hugetlbpage.c @@ -76,7 +76,7 @@ int is_aligned_hugepage_range(unsigned long addr, unsigned long len) return -EINVAL; if (addr & ~HPAGE_MASK) return -EINVAL; - if (REGION_NUMBER(addr) != REGION_HPAGE) + if (REGION_NUMBER(addr) != RGN_HPAGE) return -EINVAL; return 0; @@ -87,7 +87,7 @@ struct page *follow_huge_addr(struct mm_struct *mm, unsigned long addr, int writ struct page *page; pte_t *ptep; - if (REGION_NUMBER(addr) != REGION_HPAGE) + if (REGION_NUMBER(addr) != RGN_HPAGE) return ERR_PTR(-EINVAL); ptep = huge_pte_offset(mm, addr); @@ -142,8 +142,8 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, u return -ENOMEM; if (len & ~HPAGE_MASK) return -EINVAL; - /* This code assumes that REGION_HPAGE != 0. */ - if ((REGION_NUMBER(addr) != REGION_HPAGE) || (addr & (HPAGE_SIZE - 1))) + /* This code assumes that RGN_HPAGE != 0. */ + if ((REGION_NUMBER(addr) != RGN_HPAGE) || (addr & (HPAGE_SIZE - 1))) addr = HPAGE_REGION_BASE; else addr = ALIGN(addr, HPAGE_SIZE); diff --git a/include/asm-ia64/io.h b/include/asm-ia64/io.h index 54e7637a326c..3c28eeb0b009 100644 --- a/include/asm-ia64/io.h +++ b/include/asm-ia64/io.h @@ -23,7 +23,7 @@ #define __SLOW_DOWN_IO do { } while (0) #define SLOW_DOWN_IO do { } while (0) -#define __IA64_UNCACHED_OFFSET 0xc000000000000000UL /* region 6 */ +#define __IA64_UNCACHED_OFFSET RGN_BASE(RGN_UNCACHED) /* * The legacy I/O space defined by the ia64 architecture supports only 65536 ports, but diff --git a/include/asm-ia64/mmu_context.h b/include/asm-ia64/mmu_context.h index e3e5fededb04..ab60a6a26911 100644 --- a/include/asm-ia64/mmu_context.h +++ b/include/asm-ia64/mmu_context.h @@ -19,6 +19,7 @@ #define ia64_rid(ctx,addr) (((ctx) << 3) | (addr >> 61)) +# include # ifndef __ASSEMBLY__ #include @@ -110,7 +111,7 @@ reload_context (mm_context_t context) unsigned long rid_incr = 0; unsigned long rr0, rr1, rr2, rr3, rr4, old_rr4; - old_rr4 = ia64_get_rr(0x8000000000000000UL); + old_rr4 = ia64_get_rr(RGN_BASE(RGN_HPAGE)); rid = context << 3; /* make space for encoding the region number */ rid_incr = 1 << 8; @@ -122,6 +123,10 @@ reload_context (mm_context_t context) rr4 = rr0 + 4*rid_incr; #ifdef CONFIG_HUGETLB_PAGE rr4 = (rr4 & (~(0xfcUL))) | (old_rr4 & 0xfc); + +# if RGN_HPAGE != 4 +# error "reload_context assumes RGN_HPAGE is 4" +# endif #endif ia64_set_rr(0x0000000000000000UL, rr0); diff --git a/include/asm-ia64/page.h b/include/asm-ia64/page.h index 08894f73abf0..ec17f9e9da75 100644 --- a/include/asm-ia64/page.h +++ b/include/asm-ia64/page.h @@ -12,6 +12,19 @@ #include #include +/* + * The top three bits of an IA64 address are its Region Number. + * Different regions are assigned to different purposes. + */ +#define RGN_SHIFT (61) +#define RGN_BASE(r) (__IA64_UL_CONST(r)<> (HPAGE_SHIFT-PAGE_SHIFT))) # define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) # define is_hugepage_only_range(mm, addr, len) \ - (REGION_NUMBER(addr) == REGION_HPAGE && \ - REGION_NUMBER((addr)+(len)-1) == REGION_HPAGE) + (REGION_NUMBER(addr) == RGN_HPAGE && \ + REGION_NUMBER((addr)+(len)-1) == RGN_HPAGE) extern unsigned int hpage_shift; #endif @@ -197,7 +206,7 @@ get_order (unsigned long size) # define __pgprot(x) (x) #endif /* !STRICT_MM_TYPECHECKS */ -#define PAGE_OFFSET __IA64_UL_CONST(0xe000000000000000) +#define PAGE_OFFSET RGN_BASE(RGN_KERNEL) #define VM_DATA_DEFAULT_FLAGS (VM_READ | VM_WRITE | \ VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC | \ diff --git a/include/asm-ia64/pgtable.h b/include/asm-ia64/pgtable.h index 48586e08f432..2e34c06e6777 100644 --- a/include/asm-ia64/pgtable.h +++ b/include/asm-ia64/pgtable.h @@ -204,21 +204,18 @@ ia64_phys_addr_valid (unsigned long addr) #define set_pte(ptep, pteval) (*(ptep) = (pteval)) #define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) -#define RGN_SIZE (1UL << 61) -#define RGN_KERNEL 7 - -#define VMALLOC_START 0xa000000200000000UL +#define VMALLOC_START (RGN_BASE(RGN_GATE) + 0x200000000UL) #ifdef CONFIG_VIRTUAL_MEM_MAP -# define VMALLOC_END_INIT (0xa000000000000000UL + (1UL << (4*PAGE_SHIFT - 9))) +# define VMALLOC_END_INIT (RGN_BASE(RGN_GATE) + (1UL << (4*PAGE_SHIFT - 9))) # define VMALLOC_END vmalloc_end extern unsigned long vmalloc_end; #else -# define VMALLOC_END (0xa000000000000000UL + (1UL << (4*PAGE_SHIFT - 9))) +# define VMALLOC_END (RGN_BASE(RGN_GATE) + (1UL << (4*PAGE_SHIFT - 9))) #endif /* fs/proc/kcore.c */ -#define kc_vaddr_to_offset(v) ((v) - 0xa000000000000000UL) -#define kc_offset_to_vaddr(o) ((o) + 0xa000000000000000UL) +#define kc_vaddr_to_offset(v) ((v) - RGN_BASE(RGN_GATE)) +#define kc_offset_to_vaddr(o) ((o) + RGN_BASE(RGN_GATE)) /* * Conversion functions: convert page frame number (pfn) and a protection value to a page diff --git a/include/asm-ia64/system.h b/include/asm-ia64/system.h index cd2cf76b2db1..33256db4a7cf 100644 --- a/include/asm-ia64/system.h +++ b/include/asm-ia64/system.h @@ -19,12 +19,13 @@ #include #include -#define GATE_ADDR __IA64_UL_CONST(0xa000000000000000) +#define GATE_ADDR RGN_BASE(RGN_GATE) + /* * 0xa000000000000000+2*PERCPU_PAGE_SIZE * - 0xa000000000000000+3*PERCPU_PAGE_SIZE remain unmapped (guard page) */ -#define KERNEL_START __IA64_UL_CONST(0xa000000100000000) +#define KERNEL_START (GATE_ADDR+0x100000000) #define PERCPU_ADDR (-PERCPU_PAGE_SIZE) #ifndef __ASSEMBLY__ -- cgit v1.2.3