diff options
Diffstat (limited to 'arch/x86/lib')
| -rw-r--r-- | arch/x86/lib/Makefile | 1 | ||||
| -rw-r--r-- | arch/x86/lib/atomic64_386_32.S | 238 | ||||
| -rw-r--r-- | arch/x86/lib/clear_page_64.S | 2 | ||||
| -rw-r--r-- | arch/x86/lib/cmpxchg.c | 54 | ||||
| -rw-r--r-- | arch/x86/lib/copy_page_64.S | 2 | ||||
| -rw-r--r-- | arch/x86/lib/copy_user_64.S | 2 | ||||
| -rw-r--r-- | arch/x86/lib/memcpy_64.S | 2 | ||||
| -rw-r--r-- | arch/x86/lib/memset_64.S | 2 |
8 files changed, 190 insertions, 113 deletions
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index f871e04b6965..e10cf070ede0 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -30,6 +30,7 @@ ifeq ($(CONFIG_X86_32),y) lib-y += checksum_32.o lib-y += strstr_32.o lib-y += semaphore_32.o string_32.o + lib-y += cmpxchg.o ifneq ($(CONFIG_X86_CMPXCHG64),y) lib-y += cmpxchg8b_emu.o atomic64_386_32.o endif diff --git a/arch/x86/lib/atomic64_386_32.S b/arch/x86/lib/atomic64_386_32.S index 4a5979aa6883..2cda60a06e65 100644 --- a/arch/x86/lib/atomic64_386_32.S +++ b/arch/x86/lib/atomic64_386_32.S @@ -25,150 +25,172 @@ CFI_ADJUST_CFA_OFFSET -4 .endm -.macro BEGIN func reg -$v = \reg - -ENTRY(atomic64_\func\()_386) - CFI_STARTPROC - LOCK $v - -.macro RETURN - UNLOCK $v +#define BEGIN(op) \ +.macro endp; \ + CFI_ENDPROC; \ +ENDPROC(atomic64_##op##_386); \ +.purgem endp; \ +.endm; \ +ENTRY(atomic64_##op##_386); \ + CFI_STARTPROC; \ + LOCK v; + +#define ENDP endp + +#define RET \ + UNLOCK v; \ ret -.endm - -.macro END_ - CFI_ENDPROC -ENDPROC(atomic64_\func\()_386) -.purgem RETURN -.purgem END_ -.purgem END -.endm - -.macro END -RETURN -END_ -.endm -.endm -BEGIN read %ecx - movl ($v), %eax - movl 4($v), %edx -END - -BEGIN set %esi - movl %ebx, ($v) - movl %ecx, 4($v) -END - -BEGIN xchg %esi - movl ($v), %eax - movl 4($v), %edx - movl %ebx, ($v) - movl %ecx, 4($v) -END - -BEGIN add %ecx - addl %eax, ($v) - adcl %edx, 4($v) -END - -BEGIN add_return %ecx - addl ($v), %eax - adcl 4($v), %edx - movl %eax, ($v) - movl %edx, 4($v) -END - -BEGIN sub %ecx - subl %eax, ($v) - sbbl %edx, 4($v) -END - -BEGIN sub_return %ecx +#define RET_ENDP \ + RET; \ + ENDP + +#define v %ecx +BEGIN(read) + movl (v), %eax + movl 4(v), %edx +RET_ENDP +#undef v + +#define v %esi +BEGIN(set) + movl %ebx, (v) + movl %ecx, 4(v) +RET_ENDP +#undef v + +#define v %esi +BEGIN(xchg) + movl (v), %eax + movl 4(v), %edx + movl %ebx, (v) + movl %ecx, 4(v) +RET_ENDP +#undef v + +#define v %ecx +BEGIN(add) + addl %eax, (v) + adcl %edx, 4(v) +RET_ENDP +#undef v + +#define v %ecx +BEGIN(add_return) + addl (v), %eax + adcl 4(v), %edx + movl %eax, (v) + movl %edx, 4(v) +RET_ENDP +#undef v + +#define v %ecx +BEGIN(sub) + subl %eax, (v) + sbbl %edx, 4(v) +RET_ENDP +#undef v + +#define v %ecx +BEGIN(sub_return) negl %edx negl %eax sbbl $0, %edx - addl ($v), %eax - adcl 4($v), %edx - movl %eax, ($v) - movl %edx, 4($v) -END - -BEGIN inc %esi - addl $1, ($v) - adcl $0, 4($v) -END - -BEGIN inc_return %esi - movl ($v), %eax - movl 4($v), %edx + addl (v), %eax + adcl 4(v), %edx + movl %eax, (v) + movl %edx, 4(v) +RET_ENDP +#undef v + +#define v %esi +BEGIN(inc) + addl $1, (v) + adcl $0, 4(v) +RET_ENDP +#undef v + +#define v %esi +BEGIN(inc_return) + movl (v), %eax + movl 4(v), %edx addl $1, %eax adcl $0, %edx - movl %eax, ($v) - movl %edx, 4($v) -END - -BEGIN dec %esi - subl $1, ($v) - sbbl $0, 4($v) -END - -BEGIN dec_return %esi - movl ($v), %eax - movl 4($v), %edx + movl %eax, (v) + movl %edx, 4(v) +RET_ENDP +#undef v + +#define v %esi +BEGIN(dec) + subl $1, (v) + sbbl $0, 4(v) +RET_ENDP +#undef v + +#define v %esi +BEGIN(dec_return) + movl (v), %eax + movl 4(v), %edx subl $1, %eax sbbl $0, %edx - movl %eax, ($v) - movl %edx, 4($v) -END + movl %eax, (v) + movl %edx, 4(v) +RET_ENDP +#undef v -BEGIN add_unless %ecx +#define v %ecx +BEGIN(add_unless) addl %eax, %esi adcl %edx, %edi - addl ($v), %eax - adcl 4($v), %edx + addl (v), %eax + adcl 4(v), %edx cmpl %eax, %esi je 3f 1: - movl %eax, ($v) - movl %edx, 4($v) + movl %eax, (v) + movl %edx, 4(v) movl $1, %eax 2: -RETURN + RET 3: cmpl %edx, %edi jne 1b xorl %eax, %eax jmp 2b -END_ +ENDP +#undef v -BEGIN inc_not_zero %esi - movl ($v), %eax - movl 4($v), %edx +#define v %esi +BEGIN(inc_not_zero) + movl (v), %eax + movl 4(v), %edx testl %eax, %eax je 3f 1: addl $1, %eax adcl $0, %edx - movl %eax, ($v) - movl %edx, 4($v) + movl %eax, (v) + movl %edx, 4(v) movl $1, %eax 2: -RETURN + RET 3: testl %edx, %edx jne 1b jmp 2b -END_ +ENDP +#undef v -BEGIN dec_if_positive %esi - movl ($v), %eax - movl 4($v), %edx +#define v %esi +BEGIN(dec_if_positive) + movl (v), %eax + movl 4(v), %edx subl $1, %eax sbbl $0, %edx js 1f - movl %eax, ($v) - movl %edx, 4($v) + movl %eax, (v) + movl %edx, 4(v) 1: -END +RET_ENDP +#undef v diff --git a/arch/x86/lib/clear_page_64.S b/arch/x86/lib/clear_page_64.S index ebeafcce04a9..aa4326bfb24a 100644 --- a/arch/x86/lib/clear_page_64.S +++ b/arch/x86/lib/clear_page_64.S @@ -52,7 +52,7 @@ ENDPROC(clear_page) .align 8 .quad clear_page .quad 1b - .byte X86_FEATURE_REP_GOOD + .word X86_FEATURE_REP_GOOD .byte .Lclear_page_end - clear_page .byte 2b - 1b .previous diff --git a/arch/x86/lib/cmpxchg.c b/arch/x86/lib/cmpxchg.c new file mode 100644 index 000000000000..5d619f6df3ee --- /dev/null +++ b/arch/x86/lib/cmpxchg.c @@ -0,0 +1,54 @@ +/* + * cmpxchg*() fallbacks for CPU not supporting these instructions + */ + +#include <linux/kernel.h> +#include <linux/smp.h> +#include <linux/module.h> + +#ifndef CONFIG_X86_CMPXCHG +unsigned long cmpxchg_386_u8(volatile void *ptr, u8 old, u8 new) +{ + u8 prev; + unsigned long flags; + + /* Poor man's cmpxchg for 386. Unsuitable for SMP */ + local_irq_save(flags); + prev = *(u8 *)ptr; + if (prev == old) + *(u8 *)ptr = new; + local_irq_restore(flags); + return prev; +} +EXPORT_SYMBOL(cmpxchg_386_u8); + +unsigned long cmpxchg_386_u16(volatile void *ptr, u16 old, u16 new) +{ + u16 prev; + unsigned long flags; + + /* Poor man's cmpxchg for 386. Unsuitable for SMP */ + local_irq_save(flags); + prev = *(u16 *)ptr; + if (prev == old) + *(u16 *)ptr = new; + local_irq_restore(flags); + return prev; +} +EXPORT_SYMBOL(cmpxchg_386_u16); + +unsigned long cmpxchg_386_u32(volatile void *ptr, u32 old, u32 new) +{ + u32 prev; + unsigned long flags; + + /* Poor man's cmpxchg for 386. Unsuitable for SMP */ + local_irq_save(flags); + prev = *(u32 *)ptr; + if (prev == old) + *(u32 *)ptr = new; + local_irq_restore(flags); + return prev; +} +EXPORT_SYMBOL(cmpxchg_386_u32); +#endif diff --git a/arch/x86/lib/copy_page_64.S b/arch/x86/lib/copy_page_64.S index 727a5d46d2fc..6fec2d1cebe1 100644 --- a/arch/x86/lib/copy_page_64.S +++ b/arch/x86/lib/copy_page_64.S @@ -113,7 +113,7 @@ ENDPROC(copy_page) .align 8 .quad copy_page .quad 1b - .byte X86_FEATURE_REP_GOOD + .word X86_FEATURE_REP_GOOD .byte .Lcopy_page_end - copy_page .byte 2b - 1b .previous diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S index 71100c98e337..a460158b5ac5 100644 --- a/arch/x86/lib/copy_user_64.S +++ b/arch/x86/lib/copy_user_64.S @@ -29,7 +29,7 @@ .align 8 .quad 0b .quad 2b - .byte \feature /* when feature is set */ + .word \feature /* when feature is set */ .byte 5 .byte 5 .previous diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S index f82e884928af..bcbcd1e0f7d5 100644 --- a/arch/x86/lib/memcpy_64.S +++ b/arch/x86/lib/memcpy_64.S @@ -131,7 +131,7 @@ ENDPROC(__memcpy) .align 8 .quad memcpy .quad .Lmemcpy_c - .byte X86_FEATURE_REP_GOOD + .word X86_FEATURE_REP_GOOD /* * Replace only beginning, memcpy is used to apply alternatives, diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S index e88d3b81644a..09d344269652 100644 --- a/arch/x86/lib/memset_64.S +++ b/arch/x86/lib/memset_64.S @@ -121,7 +121,7 @@ ENDPROC(__memset) .align 8 .quad memset .quad .Lmemset_c - .byte X86_FEATURE_REP_GOOD + .word X86_FEATURE_REP_GOOD .byte .Lfinal - memset .byte .Lmemset_e - .Lmemset_c .previous |