diff options
| -rw-r--r-- | arch/mips/include/asm/hazards.h | 371 | ||||
| -rw-r--r-- | arch/mips/include/asm/irqflags.h | 153 | ||||
| -rw-r--r-- | arch/mips/lib/mips-atomic.c | 149 |
3 files changed, 403 insertions, 270 deletions
diff --git a/arch/mips/include/asm/hazards.h b/arch/mips/include/asm/hazards.h index 44d6a5bde4a1..e3ee92d4dbe7 100644 --- a/arch/mips/include/asm/hazards.h +++ b/arch/mips/include/asm/hazards.h @@ -10,34 +10,13 @@ #ifndef _ASM_HAZARDS_H #define _ASM_HAZARDS_H -#ifdef __ASSEMBLY__ -#define ASMMACRO(name, code...) .macro name; code; .endm -#else - -#include <asm/cpu-features.h> - -#define ASMMACRO(name, code...) \ -__asm__(".macro " #name "; " #code "; .endm"); \ - \ -static inline void name(void) \ -{ \ - __asm__ __volatile__ (#name); \ -} - -/* - * MIPS R2 instruction hazard barrier. Needs to be called as a subroutine. - */ -extern void mips_ihb(void); - -#endif +#include <linux/stringify.h> -ASMMACRO(_ssnop, - sll $0, $0, 1 - ) +#define ___ssnop \ + sll $0, $0, 1 -ASMMACRO(_ehb, - sll $0, $0, 3 - ) +#define ___ehb \ + sll $0, $0, 3 /* * TLB hazards @@ -48,24 +27,24 @@ ASMMACRO(_ehb, * MIPSR2 defines ehb for hazard avoidance */ -ASMMACRO(mtc0_tlbw_hazard, - _ehb - ) -ASMMACRO(tlbw_use_hazard, - _ehb - ) -ASMMACRO(tlb_probe_hazard, - _ehb - ) -ASMMACRO(irq_enable_hazard, - _ehb - ) -ASMMACRO(irq_disable_hazard, - _ehb - ) -ASMMACRO(back_to_back_c0_hazard, - _ehb - ) +#define __mtc0_tlbw_hazard \ + ___ehb + +#define __tlbw_use_hazard \ + ___ehb + +#define __tlb_probe_hazard \ + ___ehb + +#define __irq_enable_hazard \ + ___ehb + +#define __irq_disable_hazard \ + ___ehb + +#define __back_to_back_c0_hazard \ + ___ehb + /* * gcc has a tradition of misscompiling the previous construct using the * address of a label as argument to inline assembler. Gas otoh has the @@ -94,24 +73,42 @@ do { \ * These are slightly complicated by the fact that we guarantee R1 kernels to * run fine on R2 processors. */ -ASMMACRO(mtc0_tlbw_hazard, - _ssnop; _ssnop; _ehb - ) -ASMMACRO(tlbw_use_hazard, - _ssnop; _ssnop; _ssnop; _ehb - ) -ASMMACRO(tlb_probe_hazard, - _ssnop; _ssnop; _ssnop; _ehb - ) -ASMMACRO(irq_enable_hazard, - _ssnop; _ssnop; _ssnop; _ehb - ) -ASMMACRO(irq_disable_hazard, - _ssnop; _ssnop; _ssnop; _ehb - ) -ASMMACRO(back_to_back_c0_hazard, - _ssnop; _ssnop; _ssnop; _ehb - ) + +#define __mtc0_tlbw_hazard \ + ___ssnop; \ + ___ssnop; \ + ___ehb + +#define __tlbw_use_hazard \ + ___ssnop; \ + ___ssnop; \ + ___ssnop; \ + ___ehb + +#define __tlb_probe_hazard \ + ___ssnop; \ + ___ssnop; \ + ___ssnop; \ + ___ehb + +#define __irq_enable_hazard \ + ___ssnop; \ + ___ssnop; \ + ___ssnop; \ + ___ehb + +#define __irq_disable_hazard \ + ___ssnop; \ + ___ssnop; \ + ___ssnop; \ + ___ehb + +#define __back_to_back_c0_hazard \ + ___ssnop; \ + ___ssnop; \ + ___ssnop; \ + ___ehb + /* * gcc has a tradition of misscompiling the previous construct using the * address of a label as argument to inline assembler. Gas otoh has the @@ -147,18 +144,18 @@ do { \ * R10000 rocks - all hazards handled in hardware, so this becomes a nobrainer. */ -ASMMACRO(mtc0_tlbw_hazard, - ) -ASMMACRO(tlbw_use_hazard, - ) -ASMMACRO(tlb_probe_hazard, - ) -ASMMACRO(irq_enable_hazard, - ) -ASMMACRO(irq_disable_hazard, - ) -ASMMACRO(back_to_back_c0_hazard, - ) +#define __mtc0_tlbw_hazard + +#define __tlbw_use_hazard + +#define __tlb_probe_hazard + +#define __irq_enable_hazard + +#define __irq_disable_hazard + +#define __back_to_back_c0_hazard + #define instruction_hazard() do { } while (0) #elif defined(CONFIG_CPU_SB1) @@ -166,19 +163,21 @@ ASMMACRO(back_to_back_c0_hazard, /* * Mostly like R4000 for historic reasons */ -ASMMACRO(mtc0_tlbw_hazard, - ) -ASMMACRO(tlbw_use_hazard, - ) -ASMMACRO(tlb_probe_hazard, - ) -ASMMACRO(irq_enable_hazard, - ) -ASMMACRO(irq_disable_hazard, - _ssnop; _ssnop; _ssnop - ) -ASMMACRO(back_to_back_c0_hazard, - ) +#define __mtc0_tlbw_hazard + +#define __tlbw_use_hazard + +#define __tlb_probe_hazard + +#define __irq_enable_hazard + +#define __irq_disable_hazard \ + ___ssnop; \ + ___ssnop; \ + ___ssnop + +#define __back_to_back_c0_hazard + #define instruction_hazard() do { } while (0) #else @@ -192,24 +191,35 @@ ASMMACRO(back_to_back_c0_hazard, * hazard so this is nice trick to have an optimal code for a range of * processors. */ -ASMMACRO(mtc0_tlbw_hazard, - nop; nop - ) -ASMMACRO(tlbw_use_hazard, - nop; nop; nop - ) -ASMMACRO(tlb_probe_hazard, - nop; nop; nop - ) -ASMMACRO(irq_enable_hazard, - _ssnop; _ssnop; _ssnop; - ) -ASMMACRO(irq_disable_hazard, - nop; nop; nop - ) -ASMMACRO(back_to_back_c0_hazard, - _ssnop; _ssnop; _ssnop; - ) +#define __mtc0_tlbw_hazard \ + nop; \ + nop + +#define __tlbw_use_hazard \ + nop; \ + nop; \ + nop + +#define __tlb_probe_hazard \ + nop; \ + nop; \ + nop + +#define __irq_enable_hazard \ + ___ssnop; \ + ___ssnop; \ + ___ssnop + +#define __irq_disable_hazard \ + nop; \ + nop; \ + nop + +#define __back_to_back_c0_hazard \ + ___ssnop; \ + ___ssnop; \ + ___ssnop + #define instruction_hazard() do { } while (0) #endif @@ -218,32 +228,137 @@ ASMMACRO(back_to_back_c0_hazard, /* FPU hazards */ #if defined(CONFIG_CPU_SB1) -ASMMACRO(enable_fpu_hazard, - .set push; - .set mips64; - .set noreorder; - _ssnop; - bnezl $0, .+4; - _ssnop; - .set pop -) -ASMMACRO(disable_fpu_hazard, -) + +#define __enable_fpu_hazard \ + .set push; \ + .set mips64; \ + .set noreorder; \ + ___ssnop; \ + bnezl $0, .+4; \ + ___ssnop; \ + .set pop + +#define __disable_fpu_hazard #elif defined(CONFIG_CPU_MIPSR2) -ASMMACRO(enable_fpu_hazard, - _ehb -) -ASMMACRO(disable_fpu_hazard, - _ehb -) + +#define __enable_fpu_hazard \ + ___ehb + +#define __disable_fpu_hazard \ + ___ehb + #else -ASMMACRO(enable_fpu_hazard, - nop; nop; nop; nop -) -ASMMACRO(disable_fpu_hazard, - _ehb -) + +#define __enable_fpu_hazard \ + nop; \ + nop; \ + nop; \ + nop + +#define __disable_fpu_hazard \ + ___ehb + #endif +#ifdef __ASSEMBLY__ + +#define _ssnop ___ssnop +#define _ehb ___ehb +#define mtc0_tlbw_hazard __mtc0_tlbw_hazard +#define tlbw_use_hazard __tlbw_use_hazard +#define tlb_probe_hazard __tlb_probe_hazard +#define irq_enable_hazard __irq_enable_hazard +#define irq_disable_hazard __irq_disable_hazard +#define back_to_back_c0_hazard __back_to_back_c0_hazard +#define enable_fpu_hazard __enable_fpu_hazard +#define disable_fpu_hazard __disable_fpu_hazard + +#else + +#define _ssnop() \ +do { \ + __asm__ __volatile__( \ + __stringify(___ssnop) \ + ); \ +} while (0) + +#define _ehb() \ +do { \ + __asm__ __volatile__( \ + __stringify(___ehb) \ + ); \ +} while (0) + + +#define mtc0_tlbw_hazard() \ +do { \ + __asm__ __volatile__( \ + __stringify(__mtc0_tlbw_hazard) \ + ); \ +} while (0) + + +#define tlbw_use_hazard() \ +do { \ + __asm__ __volatile__( \ + __stringify(__tlbw_use_hazard) \ + ); \ +} while (0) + + +#define tlb_probe_hazard() \ +do { \ + __asm__ __volatile__( \ + __stringify(__tlb_probe_hazard) \ + ); \ +} while (0) + + +#define irq_enable_hazard() \ +do { \ + __asm__ __volatile__( \ + __stringify(__irq_enable_hazard) \ + ); \ +} while (0) + + +#define irq_disable_hazard() \ +do { \ + __asm__ __volatile__( \ + __stringify(__irq_disable_hazard) \ + ); \ +} while (0) + + +#define back_to_back_c0_hazard() \ +do { \ + __asm__ __volatile__( \ + __stringify(__back_to_back_c0_hazard) \ + ); \ +} while (0) + + +#define enable_fpu_hazard() \ +do { \ + __asm__ __volatile__( \ + __stringify(__enable_fpu_hazard) \ + ); \ +} while (0) + + +#define disable_fpu_hazard() \ +do { \ + __asm__ __volatile__( \ + __stringify(__disable_fpu_hazard) \ + ); \ +} while (0) + +/* + * MIPS R2 instruction hazard barrier. Needs to be called as a subroutine. + */ +extern void mips_ihb(void); + +#endif /* __ASSEMBLY__ */ + #endif /* _ASM_HAZARDS_H */ diff --git a/arch/mips/include/asm/irqflags.h b/arch/mips/include/asm/irqflags.h index 9f3384c789d7..45c00951888b 100644 --- a/arch/mips/include/asm/irqflags.h +++ b/arch/mips/include/asm/irqflags.h @@ -14,53 +14,48 @@ #ifndef __ASSEMBLY__ #include <linux/compiler.h> +#include <linux/stringify.h> #include <asm/hazards.h> #if defined(CONFIG_CPU_MIPSR2) && !defined(CONFIG_MIPS_MT_SMTC) -__asm__( - " .macro arch_local_irq_disable\n" +static inline void arch_local_irq_disable(void) +{ + __asm__ __volatile__( " .set push \n" " .set noat \n" " di \n" - " irq_disable_hazard \n" + " " __stringify(__irq_disable_hazard) " \n" " .set pop \n" - " .endm \n"); - -static inline void arch_local_irq_disable(void) -{ - __asm__ __volatile__( - "arch_local_irq_disable" - : /* no outputs */ - : /* no inputs */ - : "memory"); + : /* no outputs */ + : /* no inputs */ + : "memory"); } +static inline unsigned long arch_local_irq_save(void) +{ + unsigned long flags; -__asm__( - " .macro arch_local_irq_save result \n" + asm __volatile__( " .set push \n" " .set reorder \n" " .set noat \n" - " di \\result \n" - " andi \\result, 1 \n" - " irq_disable_hazard \n" + " di %[flags] \n" + " andi %[flags], 1 \n" + " " __stringify(__irq_disable_hazard) " \n" " .set pop \n" - " .endm \n"); + : [flags] "=r" (flags) + : /* no inputs */ + : "memory"); -static inline unsigned long arch_local_irq_save(void) -{ - unsigned long flags; - asm volatile("arch_local_irq_save\t%0" - : "=r" (flags) - : /* no inputs */ - : "memory"); return flags; } +static inline void arch_local_irq_restore(unsigned long flags) +{ + unsigned long __tmp1; -__asm__( - " .macro arch_local_irq_restore flags \n" + __asm__ __volatile__( " .set push \n" " .set noreorder \n" " .set noat \n" @@ -69,7 +64,7 @@ __asm__( * Slow, but doesn't suffer from a relatively unlikely race * condition we're having since days 1. */ - " beqz \\flags, 1f \n" + " beqz %[flags], 1f \n" " di \n" " ei \n" "1: \n" @@ -78,33 +73,44 @@ __asm__( * Fast, dangerous. Life is fun, life is good. */ " mfc0 $1, $12 \n" - " ins $1, \\flags, 0, 1 \n" + " ins $1, %[flags], 0, 1 \n" " mtc0 $1, $12 \n" #endif - " irq_disable_hazard \n" + " " __stringify(__irq_disable_hazard) " \n" " .set pop \n" - " .endm \n"); - -static inline void arch_local_irq_restore(unsigned long flags) -{ - unsigned long __tmp1; - - __asm__ __volatile__( - "arch_local_irq_restore\t%0" - : "=r" (__tmp1) - : "0" (flags) - : "memory"); + : [flags] "=r" (__tmp1) + : "0" (flags) + : "memory"); } static inline void __arch_local_irq_restore(unsigned long flags) { - unsigned long __tmp1; - __asm__ __volatile__( - "arch_local_irq_restore\t%0" - : "=r" (__tmp1) - : "0" (flags) - : "memory"); + " .set push \n" + " .set noreorder \n" + " .set noat \n" +#if defined(CONFIG_IRQ_CPU) + /* + * Slow, but doesn't suffer from a relatively unlikely race + * condition we're having since days 1. + */ + " beqz %[flags], 1f \n" + " di \n" + " ei \n" + "1: \n" +#else + /* + * Fast, dangerous. Life is fun, life is good. + */ + " mfc0 $1, $12 \n" + " ins $1, %[flags], 0, 1 \n" + " mtc0 $1, $12 \n" +#endif + " " __stringify(__irq_disable_hazard) " \n" + " .set pop \n" + : [flags] "=r" (flags) + : "0" (flags) + : "memory"); } #else /* Functions that require preempt_{dis,en}able() are in mips-atomic.c */ @@ -115,8 +121,18 @@ void __arch_local_irq_restore(unsigned long flags); #endif /* if defined(CONFIG_CPU_MIPSR2) && !defined(CONFIG_MIPS_MT_SMTC) */ -__asm__( - " .macro arch_local_irq_enable \n" +extern void smtc_ipi_replay(void); + +static inline void arch_local_irq_enable(void) +{ +#ifdef CONFIG_MIPS_MT_SMTC + /* + * SMTC kernel needs to do a software replay of queued + * IPIs, at the cost of call overhead on each local_irq_enable() + */ + smtc_ipi_replay(); +#endif + __asm__ __volatile__( " .set push \n" " .set reorder \n" " .set noat \n" @@ -133,45 +149,28 @@ __asm__( " xori $1,0x1e \n" " mtc0 $1,$12 \n" #endif - " irq_enable_hazard \n" + " " __stringify(__irq_enable_hazard) " \n" " .set pop \n" - " .endm"); - -extern void smtc_ipi_replay(void); - -static inline void arch_local_irq_enable(void) -{ -#ifdef CONFIG_MIPS_MT_SMTC - /* - * SMTC kernel needs to do a software replay of queued - * IPIs, at the cost of call overhead on each local_irq_enable() - */ - smtc_ipi_replay(); -#endif - __asm__ __volatile__( - "arch_local_irq_enable" - : /* no outputs */ - : /* no inputs */ - : "memory"); + : /* no outputs */ + : /* no inputs */ + : "memory"); } +static inline unsigned long arch_local_save_flags(void) +{ + unsigned long flags; -__asm__( - " .macro arch_local_save_flags flags \n" + asm __volatile__( " .set push \n" " .set reorder \n" #ifdef CONFIG_MIPS_MT_SMTC - " mfc0 \\flags, $2, 1 \n" + " mfc0 %[flags], $2, 1 \n" #else - " mfc0 \\flags, $12 \n" + " mfc0 %[flags], $12 \n" #endif " .set pop \n" - " .endm \n"); + : [flags] "=r" (flags)); -static inline unsigned long arch_local_save_flags(void) -{ - unsigned long flags; - asm volatile("arch_local_save_flags %0" : "=r" (flags)); return flags; } diff --git a/arch/mips/lib/mips-atomic.c b/arch/mips/lib/mips-atomic.c index cd160be3ce4d..6807f7172eaf 100644 --- a/arch/mips/lib/mips-atomic.c +++ b/arch/mips/lib/mips-atomic.c @@ -13,6 +13,7 @@ #include <linux/compiler.h> #include <linux/preempt.h> #include <linux/export.h> +#include <linux/stringify.h> #if !defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_MIPS_MT_SMTC) @@ -34,8 +35,11 @@ * * Workaround: mask EXL bit of the result or place a nop before mfc0. */ -__asm__( - " .macro arch_local_irq_disable\n" +notrace void arch_local_irq_disable(void) +{ + preempt_disable(); + + __asm__ __volatile__( " .set push \n" " .set noat \n" #ifdef CONFIG_MIPS_MT_SMTC @@ -52,108 +56,98 @@ __asm__( " .set noreorder \n" " mtc0 $1,$12 \n" #endif - " irq_disable_hazard \n" + " " __stringify(__irq_disable_hazard) " \n" " .set pop \n" - " .endm \n"); + : /* no outputs */ + : /* no inputs */ + : "memory"); -notrace void arch_local_irq_disable(void) -{ - preempt_disable(); - __asm__ __volatile__( - "arch_local_irq_disable" - : /* no outputs */ - : /* no inputs */ - : "memory"); preempt_enable(); } EXPORT_SYMBOL(arch_local_irq_disable); -__asm__( - " .macro arch_local_irq_save result \n" +notrace unsigned long arch_local_irq_save(void) +{ + unsigned long flags; + + preempt_disable(); + + __asm__ __volatile__( " .set push \n" " .set reorder \n" " .set noat \n" #ifdef CONFIG_MIPS_MT_SMTC - " mfc0 \\result, $2, 1 \n" - " ori $1, \\result, 0x400 \n" + " mfc0 %[flags], $2, 1 \n" + " ori $1, %[flags], 0x400 \n" " .set noreorder \n" " mtc0 $1, $2, 1 \n" - " andi \\result, \\result, 0x400 \n" + " andi %[flags], %[flags], 0x400 \n" #elif defined(CONFIG_CPU_MIPSR2) /* see irqflags.h for inline function */ #else - " mfc0 \\result, $12 \n" - " ori $1, \\result, 0x1f \n" + " mfc0 %[flags], $12 \n" + " ori $1, %[flags], 0x1f \n" " xori $1, 0x1f \n" " .set noreorder \n" " mtc0 $1, $12 \n" #endif - " irq_disable_hazard \n" + " " __stringify(__irq_disable_hazard) " \n" " .set pop \n" - " .endm \n"); + : [flags] "=r" (flags) + : /* no inputs */ + : "memory"); -notrace unsigned long arch_local_irq_save(void) -{ - unsigned long flags; - preempt_disable(); - asm volatile("arch_local_irq_save\t%0" - : "=r" (flags) - : /* no inputs */ - : "memory"); preempt_enable(); + return flags; } EXPORT_SYMBOL(arch_local_irq_save); +notrace void arch_local_irq_restore(unsigned long flags) +{ + unsigned long __tmp1; + +#ifdef CONFIG_MIPS_MT_SMTC + /* + * SMTC kernel needs to do a software replay of queued + * IPIs, at the cost of branch and call overhead on each + * local_irq_restore() + */ + if (unlikely(!(flags & 0x0400))) + smtc_ipi_replay(); +#endif + preempt_disable(); -__asm__( - " .macro arch_local_irq_restore flags \n" + __asm__ __volatile__( " .set push \n" " .set noreorder \n" " .set noat \n" #ifdef CONFIG_MIPS_MT_SMTC - "mfc0 $1, $2, 1 \n" - "andi \\flags, 0x400 \n" - "ori $1, 0x400 \n" - "xori $1, 0x400 \n" - "or \\flags, $1 \n" - "mtc0 \\flags, $2, 1 \n" + " mfc0 $1, $2, 1 \n" + " andi %[flags], 0x400 \n" + " ori $1, 0x400 \n" + " xori $1, 0x400 \n" + " or %[flags], $1 \n" + " mtc0 %[flags], $2, 1 \n" #elif defined(CONFIG_CPU_MIPSR2) && defined(CONFIG_IRQ_CPU) /* see irqflags.h for inline function */ #elif defined(CONFIG_CPU_MIPSR2) /* see irqflags.h for inline function */ #else " mfc0 $1, $12 \n" - " andi \\flags, 1 \n" + " andi %[flags], 1 \n" " ori $1, 0x1f \n" " xori $1, 0x1f \n" - " or \\flags, $1 \n" - " mtc0 \\flags, $12 \n" + " or %[flags], $1 \n" + " mtc0 %[flags], $12 \n" #endif - " irq_disable_hazard \n" + " " __stringify(__irq_disable_hazard) " \n" " .set pop \n" - " .endm \n"); + : [flags] "=r" (__tmp1) + : "0" (flags) + : "memory"); -notrace void arch_local_irq_restore(unsigned long flags) -{ - unsigned long __tmp1; - -#ifdef CONFIG_MIPS_MT_SMTC - /* - * SMTC kernel needs to do a software replay of queued - * IPIs, at the cost of branch and call overhead on each - * local_irq_restore() - */ - if (unlikely(!(flags & 0x0400))) - smtc_ipi_replay(); -#endif - preempt_disable(); - __asm__ __volatile__( - "arch_local_irq_restore\t%0" - : "=r" (__tmp1) - : "0" (flags) - : "memory"); preempt_enable(); } EXPORT_SYMBOL(arch_local_irq_restore); @@ -164,11 +158,36 @@ notrace void __arch_local_irq_restore(unsigned long flags) unsigned long __tmp1; preempt_disable(); + __asm__ __volatile__( - "arch_local_irq_restore\t%0" - : "=r" (__tmp1) - : "0" (flags) - : "memory"); + " .set push \n" + " .set noreorder \n" + " .set noat \n" +#ifdef CONFIG_MIPS_MT_SMTC + " mfc0 $1, $2, 1 \n" + " andi %[flags], 0x400 \n" + " ori $1, 0x400 \n" + " xori $1, 0x400 \n" + " or %[flags], $1 \n" + " mtc0 %[flags], $2, 1 \n" +#elif defined(CONFIG_CPU_MIPSR2) && defined(CONFIG_IRQ_CPU) + /* see irqflags.h for inline function */ +#elif defined(CONFIG_CPU_MIPSR2) + /* see irqflags.h for inline function */ +#else + " mfc0 $1, $12 \n" + " andi %[flags], 1 \n" + " ori $1, 0x1f \n" + " xori $1, 0x1f \n" + " or %[flags], $1 \n" + " mtc0 %[flags], $12 \n" +#endif + " " __stringify(__irq_disable_hazard) " \n" + " .set pop \n" + : [flags] "=r" (__tmp1) + : "0" (flags) + : "memory"); + preempt_enable(); } EXPORT_SYMBOL(__arch_local_irq_restore); |