From e9b9eb59ffcdee09ec96b040f85c919618f4043e Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sat, 27 Oct 2012 23:00:41 -0700 Subject: sparc64: Use pause instruction when available. In atomic backoff and cpu_relax(), use the pause instruction found on SPARC-T4 and later. It makes the cpu strand unselectable for the given number of cycles, unless an intervening disrupting trap occurs. Signed-off-by: David S. Miller --- arch/sparc/kernel/entry.h | 7 +++++++ arch/sparc/kernel/setup_64.c | 21 +++++++++++++++++++++ arch/sparc/kernel/vmlinux.lds.S | 5 +++++ 3 files changed, 33 insertions(+) (limited to 'arch/sparc/kernel') diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h index 0c218e4c0881..51742df63c75 100644 --- a/arch/sparc/kernel/entry.h +++ b/arch/sparc/kernel/entry.h @@ -59,6 +59,13 @@ struct popc_6insn_patch_entry { extern struct popc_6insn_patch_entry __popc_6insn_patch, __popc_6insn_patch_end; +struct pause_patch_entry { + unsigned int addr; + unsigned int insns[3]; +}; +extern struct pause_patch_entry __pause_patch, + __pause_patch_end; + extern void __init per_cpu_patch(void); extern void sun4v_patch_1insn_range(struct sun4v_1insn_patch_entry *, struct sun4v_1insn_patch_entry *); diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c index 0800e71d8a88..b45cff408de3 100644 --- a/arch/sparc/kernel/setup_64.c +++ b/arch/sparc/kernel/setup_64.c @@ -316,6 +316,25 @@ static void __init popc_patch(void) } } +static void __init pause_patch(void) +{ + struct pause_patch_entry *p; + + p = &__pause_patch; + while (p < &__pause_patch_end) { + unsigned long i, addr = p->addr; + + for (i = 0; i < 3; i++) { + *(unsigned int *) (addr + (i * 4)) = p->insns[i]; + wmb(); + __asm__ __volatile__("flush %0" + : : "r" (addr + (i * 4))); + } + + p++; + } +} + #ifdef CONFIG_SMP void __init boot_cpu_id_too_large(int cpu) { @@ -528,6 +547,8 @@ static void __init init_sparc64_elf_hwcap(void) if (sparc64_elf_hwcap & AV_SPARC_POPC) popc_patch(); + if (sparc64_elf_hwcap & AV_SPARC_PAUSE) + pause_patch(); } void __init setup_arch(char **cmdline_p) diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index 89c2c29f154b..847f9f793618 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -132,6 +132,11 @@ SECTIONS *(.popc_6insn_patch) __popc_6insn_patch_end = .; } + .pause_patch : { + __pause_patch = .; + *(.pause_patch) + __pause_patch_end = .; + } PERCPU_SECTION(SMP_CACHE_BYTES) . = ALIGN(PAGE_SIZE); -- cgit v1.2.3 From 187818cd6a5ab6343eac47e52da2f3e40c544b98 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sun, 28 Oct 2012 13:04:47 -0700 Subject: sparc64: Improvde documentation and readability of atomic backoff code. Document what's going on in asm/backoff.h with a large and descriptive comment. Refer to it above the cpu_relax() definition in asm/processor_64.h Rename the pause patching section to have "3insn" in it's name like the other patching sections do. Based upon feedback from Sam Ravnborg. Signed-off-by: David S. Miller --- arch/sparc/include/asm/backoff.h | 42 ++++++++++++++++++++++++++++++++++- arch/sparc/include/asm/processor_64.h | 7 +++++- arch/sparc/kernel/entry.h | 4 ++-- arch/sparc/kernel/setup_64.c | 4 ++-- arch/sparc/kernel/vmlinux.lds.S | 8 +++---- 5 files changed, 55 insertions(+), 10 deletions(-) (limited to 'arch/sparc/kernel') diff --git a/arch/sparc/include/asm/backoff.h b/arch/sparc/include/asm/backoff.h index 20f01df0871b..4e02086b839c 100644 --- a/arch/sparc/include/asm/backoff.h +++ b/arch/sparc/include/asm/backoff.h @@ -1,6 +1,46 @@ #ifndef _SPARC64_BACKOFF_H #define _SPARC64_BACKOFF_H +/* The macros in this file implement an exponential backoff facility + * for atomic operations. + * + * When multiple threads compete on an atomic operation, it is + * possible for one thread to be continually denied a successful + * completion of the compare-and-swap instruction. Heavily + * threaded cpu implementations like Niagara can compound this + * problem even further. + * + * When an atomic operation fails and needs to be retried, we spin a + * certain number of times. At each subsequent failure of the same + * operation we double the spin count, realizing an exponential + * backoff. + * + * When we spin, we try to use an operation that will cause the + * current cpu strand to block, and therefore make the core fully + * available to any other other runnable strands. There are two + * options, based upon cpu capabilities. + * + * On all cpus prior to SPARC-T4 we do three dummy reads of the + * condition code register. Each read blocks the strand for something + * between 40 and 50 cpu cycles. + * + * For SPARC-T4 and later we have a special "pause" instruction + * available. This is implemented using writes to register %asr27. + * The cpu will block the number of cycles written into the register, + * unless a disrupting trap happens first. SPARC-T4 specifically + * implements pause with a granularity of 8 cycles. Each strand has + * an internal pause counter which decrements every 8 cycles. So the + * chip shifts the %asr27 value down by 3 bits, and writes the result + * into the pause counter. If a value smaller than 8 is written, the + * chip blocks for 1 cycle. + * + * To achieve the same amount of backoff as the three %ccr reads give + * on earlier chips, we shift the backoff value up by 7 bits. (Three + * %ccr reads block for about 128 cycles, 1 << 7 == 128) We write the + * whole amount we want to block into the pause register, rather than + * loop writing 128 each time. + */ + #define BACKOFF_LIMIT (4 * 1024) #ifdef CONFIG_SMP @@ -16,7 +56,7 @@ 88: rd %ccr, %g0; \ rd %ccr, %g0; \ rd %ccr, %g0; \ - .section .pause_patch,"ax"; \ + .section .pause_3insn_patch,"ax";\ .word 88b; \ sllx tmp, 7, tmp; \ wr tmp, 0, %asr27; \ diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h index 9cdf52eec48a..721e25f0e2ea 100644 --- a/arch/sparc/include/asm/processor_64.h +++ b/arch/sparc/include/asm/processor_64.h @@ -196,11 +196,16 @@ extern unsigned long get_wchan(struct task_struct *task); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->tpc) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->u_regs[UREG_FP]) +/* Please see the commentary in asm/backoff.h for a description of + * what these instructions are doing and how they have been choosen. + * To make a long story short, we are trying to yield the current cpu + * strand during busy loops. + */ #define cpu_relax() asm volatile("\n99:\n\t" \ "rd %%ccr, %%g0\n\t" \ "rd %%ccr, %%g0\n\t" \ "rd %%ccr, %%g0\n\t" \ - ".section .pause_patch,\"ax\"\n\t"\ + ".section .pause_3insn_patch,\"ax\"\n\t"\ ".word 99b\n\t" \ "wr %%g0, 128, %%asr27\n\t" \ "nop\n\t" \ diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h index 51742df63c75..cc3c5cb47cda 100644 --- a/arch/sparc/kernel/entry.h +++ b/arch/sparc/kernel/entry.h @@ -63,8 +63,8 @@ struct pause_patch_entry { unsigned int addr; unsigned int insns[3]; }; -extern struct pause_patch_entry __pause_patch, - __pause_patch_end; +extern struct pause_patch_entry __pause_3insn_patch, + __pause_3insn_patch_end; extern void __init per_cpu_patch(void); extern void sun4v_patch_1insn_range(struct sun4v_1insn_patch_entry *, diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c index b45cff408de3..0eaf0059aaef 100644 --- a/arch/sparc/kernel/setup_64.c +++ b/arch/sparc/kernel/setup_64.c @@ -320,8 +320,8 @@ static void __init pause_patch(void) { struct pause_patch_entry *p; - p = &__pause_patch; - while (p < &__pause_patch_end) { + p = &__pause_3insn_patch; + while (p < &__pause_3insn_patch_end) { unsigned long i, addr = p->addr; for (i = 0; i < 3; i++) { diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index 847f9f793618..0bacceb19150 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -132,10 +132,10 @@ SECTIONS *(.popc_6insn_patch) __popc_6insn_patch_end = .; } - .pause_patch : { - __pause_patch = .; - *(.pause_patch) - __pause_patch_end = .; + .pause_3insn_patch : { + __pause_3insn_patch = .; + *(.pause_3insn_patch) + __pause_3insn_patch_end = .; } PERCPU_SECTION(SMP_CACHE_BYTES) -- cgit v1.2.3 From 1df35f80f9d3bba7cd434b64c9eaff8c9109abad Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sun, 28 Oct 2012 13:15:09 -0700 Subject: sparc: Wire up sys_kcmp. Signed-off-by: David S. Miller --- arch/sparc/include/uapi/asm/unistd.h | 3 ++- arch/sparc/kernel/systbls_32.S | 1 + arch/sparc/kernel/systbls_64.S | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) (limited to 'arch/sparc/kernel') diff --git a/arch/sparc/include/uapi/asm/unistd.h b/arch/sparc/include/uapi/asm/unistd.h index bed86a820d09..cac719d1bc5c 100644 --- a/arch/sparc/include/uapi/asm/unistd.h +++ b/arch/sparc/include/uapi/asm/unistd.h @@ -406,8 +406,9 @@ #define __NR_process_vm_readv 338 #define __NR_process_vm_writev 339 #define __NR_kern_features 340 +#define __NR_kcmp 341 -#define NR_syscalls 341 +#define NR_syscalls 342 /* Bitmask values returned from kern_features system call. */ #define KERN_FEATURE_MIXED_MODE_STACK 0x00000001 diff --git a/arch/sparc/kernel/systbls_32.S b/arch/sparc/kernel/systbls_32.S index 63402f9e9f51..5147f574f125 100644 --- a/arch/sparc/kernel/systbls_32.S +++ b/arch/sparc/kernel/systbls_32.S @@ -85,3 +85,4 @@ sys_call_table: /*325*/ .long sys_pwritev, sys_rt_tgsigqueueinfo, sys_perf_event_open, sys_recvmmsg, sys_fanotify_init /*330*/ .long sys_fanotify_mark, sys_prlimit64, sys_name_to_handle_at, sys_open_by_handle_at, sys_clock_adjtime /*335*/ .long sys_syncfs, sys_sendmmsg, sys_setns, sys_process_vm_readv, sys_process_vm_writev +/*340*/ .long sys_ni_syscall, sys_kcmp diff --git a/arch/sparc/kernel/systbls_64.S b/arch/sparc/kernel/systbls_64.S index 45ce6be088e4..1c9af9fa38e9 100644 --- a/arch/sparc/kernel/systbls_64.S +++ b/arch/sparc/kernel/systbls_64.S @@ -86,7 +86,7 @@ sys_call_table32: .word compat_sys_pwritev, compat_sys_rt_tgsigqueueinfo, sys_perf_event_open, compat_sys_recvmmsg, sys_fanotify_init /*330*/ .word sys32_fanotify_mark, sys_prlimit64, sys_name_to_handle_at, compat_sys_open_by_handle_at, compat_sys_clock_adjtime .word sys_syncfs, compat_sys_sendmmsg, sys_setns, compat_sys_process_vm_readv, compat_sys_process_vm_writev -/*340*/ .word sys_kern_features +/*340*/ .word sys_kern_features, sys_kcmp #endif /* CONFIG_COMPAT */ @@ -164,4 +164,4 @@ sys_call_table: .word sys_pwritev, sys_rt_tgsigqueueinfo, sys_perf_event_open, sys_recvmmsg, sys_fanotify_init /*330*/ .word sys_fanotify_mark, sys_prlimit64, sys_name_to_handle_at, sys_open_by_handle_at, sys_clock_adjtime .word sys_syncfs, sys_sendmmsg, sys_setns, sys_process_vm_readv, sys_process_vm_writev -/*340*/ .word sys_kern_features +/*340*/ .word sys_kern_features, sys_kcmp -- cgit v1.2.3 From 20424d85f8a07090fd32c6fad343f91b63c730b0 Mon Sep 17 00:00:00 2001 From: Andreas Larsson Date: Tue, 30 Oct 2012 00:09:46 +0000 Subject: sparc32, leon: Check for existent irq_map entry in leon_handle_ext_irq If an irq is being unlinked concurrently with leon_handle_ext_irq, irq_map[eirq] might be null in leon_handle_ext_irq. Make sure that this is not dereferenced. Signed-off-by: Andreas Larsson Acked-by: Sam Ravnborg Signed-off-by: David S. Miller --- arch/sparc/kernel/leon_kernel.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'arch/sparc/kernel') diff --git a/arch/sparc/kernel/leon_kernel.c b/arch/sparc/kernel/leon_kernel.c index f8b6eee40bde..87f60ee65433 100644 --- a/arch/sparc/kernel/leon_kernel.c +++ b/arch/sparc/kernel/leon_kernel.c @@ -56,11 +56,13 @@ static inline unsigned int leon_eirq_get(int cpu) static void leon_handle_ext_irq(unsigned int irq, struct irq_desc *desc) { unsigned int eirq; + struct irq_bucket *p; int cpu = sparc_leon3_cpuid(); eirq = leon_eirq_get(cpu); - if ((eirq & 0x10) && irq_map[eirq]->irq) /* bit4 tells if IRQ happened */ - generic_handle_irq(irq_map[eirq]->irq); + p = irq_map[eirq]; + if ((eirq & 0x10) && p && p->irq) /* bit4 tells if IRQ happened */ + generic_handle_irq(p->irq); } /* The extended IRQ controller has been found, this function registers it */ -- cgit v1.2.3