From d4541805e812abb5110d5de83246488fa0aa9a8e Mon Sep 17 00:00:00 2001 From: "H. Peter Anvin" Date: Fri, 20 Apr 2012 12:12:27 -0700 Subject: x86, extable: Use .pushsection ... .popsection for _ASM_EXTABLE() Instead of using .section ... .previous, use .pushsection ... .popsection; this is (hopefully) a bit more robust, especially in complex assembly code. Signed-off-by: H. Peter Anvin Cc: David Daney Link: http://lkml.kernel.org/r/CA%2B55aFyijf43qSu3N9nWHEBwaGbb7T2Oq9A=9EyR=Jtyqfq_cQ@mail.gmail.com --- arch/x86/include/asm/asm.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'arch/x86/include/asm/asm.h') diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index 9412d6558c88..ff3f6bffcbf9 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -42,17 +42,17 @@ /* Exception table entry */ #ifdef __ASSEMBLY__ -# define _ASM_EXTABLE(from,to) \ - __ASM_EX_SEC ; \ - _ASM_ALIGN ; \ - _ASM_PTR from , to ; \ - .previous +# define _ASM_EXTABLE(from,to) \ + .pushsection "__ex_table","a" ; \ + _ASM_ALIGN ; \ + _ASM_PTR from , to ; \ + .popsection #else -# define _ASM_EXTABLE(from,to) \ - __ASM_EX_SEC \ - _ASM_ALIGN "\n" \ - _ASM_PTR #from "," #to "\n" \ - " .previous\n" +# define _ASM_EXTABLE(from,to) \ + " .pushsection \"__ex_table\",\"a\"\n" \ + _ASM_ALIGN "\n" \ + _ASM_PTR #from "," #to "\n" \ + " .popsection\n" #endif #endif /* _ASM_X86_ASM_H */ -- cgit v1.2.3 From 447657e31235c692f579c639250317c7f565cd0d Mon Sep 17 00:00:00 2001 From: "H. Peter Anvin" Date: Fri, 20 Apr 2012 12:20:30 -0700 Subject: x86, extable: Remove the now-unused __ASM_EX_SEC macros Nothing should use them anymore; only _ASM_EXTABLE() should ever be used. Signed-off-by: H. Peter Anvin Cc: David Daney Link: http://lkml.kernel.org/r/CA%2B55aFyijf43qSu3N9nWHEBwaGbb7T2Oq9A=9EyR=Jtyqfq_cQ@mail.gmail.com --- arch/x86/include/asm/asm.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'arch/x86/include/asm/asm.h') diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index ff3f6bffcbf9..53dce41f2517 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -4,11 +4,9 @@ #ifdef __ASSEMBLY__ # define __ASM_FORM(x) x # define __ASM_FORM_COMMA(x) x, -# define __ASM_EX_SEC .section __ex_table, "a" #else # define __ASM_FORM(x) " " #x " " # define __ASM_FORM_COMMA(x) " " #x "," -# define __ASM_EX_SEC " .section __ex_table,\"a\"\n" #endif #ifdef CONFIG_X86_32 -- cgit v1.2.3 From 535c0c34698061544f81a51c65fc51f4eeeebff6 Mon Sep 17 00:00:00 2001 From: "H. Peter Anvin" Date: Fri, 20 Apr 2012 16:57:35 -0700 Subject: x86, extable: Add _ASM_EXTABLE_EX() macro Add _ASM_EXTABLE_EX() to generate the special extable entries that are associated with uaccess_err. This allows us to change the protocol associated with these special entries. Signed-off-by: H. Peter Anvin Cc: David Daney Link: http://lkml.kernel.org/r/CA%2B55aFyijf43qSu3N9nWHEBwaGbb7T2Oq9A=9EyR=Jtyqfq_cQ@mail.gmail.com --- arch/x86/include/asm/asm.h | 28 ++++++++++++++++++++-------- arch/x86/include/asm/uaccess.h | 8 ++++---- 2 files changed, 24 insertions(+), 12 deletions(-) (limited to 'arch/x86/include/asm/asm.h') diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index 53dce41f2517..0f15e8a4f565 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -40,16 +40,28 @@ /* Exception table entry */ #ifdef __ASSEMBLY__ -# define _ASM_EXTABLE(from,to) \ - .pushsection "__ex_table","a" ; \ - _ASM_ALIGN ; \ - _ASM_PTR from , to ; \ +# define _ASM_EXTABLE(from,to) \ + .pushsection "__ex_table","a" ; \ + _ASM_ALIGN ; \ + _ASM_PTR from , to ; \ + .popsection + +# define _ASM_EXTABLE_EX(from,to) \ + .pushsection "__ex_table","a" ; \ + _ASM_ALIGN ; \ + _ASM_PTR from , (to) - (from) ; \ .popsection #else -# define _ASM_EXTABLE(from,to) \ - " .pushsection \"__ex_table\",\"a\"\n" \ - _ASM_ALIGN "\n" \ - _ASM_PTR #from "," #to "\n" \ +# define _ASM_EXTABLE(from,to) \ + " .pushsection \"__ex_table\",\"a\"\n" \ + _ASM_ALIGN "\n" \ + _ASM_PTR #from "," #to "\n" \ + " .popsection\n" + +# define _ASM_EXTABLE_EX(from,to) \ + " .pushsection \"__ex_table\",\"a\"\n" \ + _ASM_ALIGN "\n" \ + _ASM_PTR #from ",(" #to ")-(" #from ")\n" \ " .popsection\n" #endif diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index e0544597cfe7..4ee59dd66f5d 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -202,8 +202,8 @@ extern int __get_user_bad(void); asm volatile("1: movl %%eax,0(%1)\n" \ "2: movl %%edx,4(%1)\n" \ "3:\n" \ - _ASM_EXTABLE(1b, 2b - 1b) \ - _ASM_EXTABLE(2b, 3b - 2b) \ + _ASM_EXTABLE_EX(1b, 2b) \ + _ASM_EXTABLE_EX(2b, 3b) \ : : "A" (x), "r" (addr)) #define __put_user_x8(x, ptr, __ret_pu) \ @@ -408,7 +408,7 @@ do { \ #define __get_user_asm_ex(x, addr, itype, rtype, ltype) \ asm volatile("1: mov"itype" %1,%"rtype"0\n" \ "2:\n" \ - _ASM_EXTABLE(1b, 2b - 1b) \ + _ASM_EXTABLE_EX(1b, 2b) \ : ltype(x) : "m" (__m(addr))) #define __put_user_nocheck(x, ptr, size) \ @@ -450,7 +450,7 @@ struct __large_struct { unsigned long buf[100]; }; #define __put_user_asm_ex(x, addr, itype, rtype, ltype) \ asm volatile("1: mov"itype" %"rtype"0,%1\n" \ "2:\n" \ - _ASM_EXTABLE(1b, 2b - 1b) \ + _ASM_EXTABLE_EX(1b, 2b) \ : : ltype(x), "m" (__m(addr))) /* -- cgit v1.2.3 From 706276543b699d80f546e45f8b12574e7b18d952 Mon Sep 17 00:00:00 2001 From: "H. Peter Anvin" Date: Fri, 20 Apr 2012 17:12:48 -0700 Subject: x86, extable: Switch to relative exception table entries Switch to using relative exception table entries on x86. On i386, this has the advantage that the exception table entries don't need to be relocated; on x86-64 this means the exception table entries take up only half the space. In either case, a 32-bit delta is sufficient, as the range of kernel code addresses is limited. Since part of the goal is to avoid needing to adjust the entries when the kernel is relocated, the old trick of using addresses in the NULL pointer range to indicate uaccess_err no longer works (and unlike RISC architectures we can't use a flag bit); instead use an delta just below +2G to indicate these special entries. The reach is still limited to a single instruction. Signed-off-by: H. Peter Anvin Cc: David Daney Link: http://lkml.kernel.org/r/CA%2B55aFyijf43qSu3N9nWHEBwaGbb7T2Oq9A=9EyR=Jtyqfq_cQ@mail.gmail.com --- arch/x86/include/asm/asm.h | 20 ++++--- arch/x86/include/asm/uaccess.h | 17 ++++-- arch/x86/mm/extable.c | 131 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 146 insertions(+), 22 deletions(-) (limited to 'arch/x86/include/asm/asm.h') diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index 0f15e8a4f565..1c2d247f65ce 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -42,26 +42,30 @@ #ifdef __ASSEMBLY__ # define _ASM_EXTABLE(from,to) \ .pushsection "__ex_table","a" ; \ - _ASM_ALIGN ; \ - _ASM_PTR from , to ; \ + .balign 8 ; \ + .long (from) - . ; \ + .long (to) - . ; \ .popsection # define _ASM_EXTABLE_EX(from,to) \ .pushsection "__ex_table","a" ; \ - _ASM_ALIGN ; \ - _ASM_PTR from , (to) - (from) ; \ + .balign 8 ; \ + .long (from) - . ; \ + .long (to) - . + 0x7ffffff0 ; \ .popsection #else # define _ASM_EXTABLE(from,to) \ " .pushsection \"__ex_table\",\"a\"\n" \ - _ASM_ALIGN "\n" \ - _ASM_PTR #from "," #to "\n" \ + " .balign 8\n" \ + " .long (" #from ") - .\n" \ + " .long (" #to ") - .\n" \ " .popsection\n" # define _ASM_EXTABLE_EX(from,to) \ " .pushsection \"__ex_table\",\"a\"\n" \ - _ASM_ALIGN "\n" \ - _ASM_PTR #from ",(" #to ")-(" #from ")\n" \ + " .balign 8\n" \ + " .long (" #from ") - .\n" \ + " .long (" #to ") - . + 0x7ffffff0\n" \ " .popsection\n" #endif diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 4ee59dd66f5d..851fe0dc13bc 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -79,11 +79,12 @@ #define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0)) /* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. + * The exception table consists of pairs of addresses relative to the + * exception table enty itself: the first is the address of an + * instruction that is allowed to fault, and the second is the address + * at which the program should continue. No registers are modified, + * so it is entirely up to the continuation code to figure out what to + * do. * * All the routines below use bits of fixup code that are out of line * with the main instruction path. This means when everything is well, @@ -92,10 +93,14 @@ */ struct exception_table_entry { - unsigned long insn, fixup; + int insn, fixup; }; +/* This is not the generic standard exception_table_entry format */ +#define ARCH_HAS_SORT_EXTABLE +#define ARCH_HAS_SEARCH_EXTABLE extern int fixup_exception(struct pt_regs *regs); +extern int early_fixup_exception(unsigned long *ip); /* * These are the main single-value transfer routines. They automatically diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 5555675dadb6..903ec1e9c326 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -1,11 +1,23 @@ #include #include +#include #include +static inline unsigned long +ex_insn_addr(const struct exception_table_entry *x) +{ + return (unsigned long)&x->insn + x->insn; +} +static inline unsigned long +ex_fixup_addr(const struct exception_table_entry *x) +{ + return (unsigned long)&x->fixup + x->fixup; +} int fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *fixup; + unsigned long new_ip; #ifdef CONFIG_PNPBIOS if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) { @@ -23,13 +35,14 @@ int fixup_exception(struct pt_regs *regs) fixup = search_exception_tables(regs->ip); if (fixup) { - /* If fixup is less than 16, it means uaccess error */ - if (fixup->fixup < 16) { + new_ip = ex_fixup_addr(fixup); + + if (fixup->fixup - fixup->insn >= 0x7ffffff0 - 4) { + /* Special hack for uaccess_err */ current_thread_info()->uaccess_err = 1; - regs->ip += fixup->fixup; - return 1; + new_ip -= 0x7ffffff0; } - regs->ip = fixup->fixup; + regs->ip = new_ip; return 1; } @@ -40,15 +53,117 @@ int fixup_exception(struct pt_regs *regs) int __init early_fixup_exception(unsigned long *ip) { const struct exception_table_entry *fixup; + unsigned long new_ip; fixup = search_exception_tables(*ip); if (fixup) { - if (fixup->fixup < 16) - return 0; /* Not supported during early boot */ + new_ip = ex_fixup_addr(fixup); + + if (fixup->fixup - fixup->insn >= 0x7ffffff0 - 4) { + /* uaccess handling not supported during early boot */ + return 0; + } - *ip = fixup->fixup; + *ip = new_ip; return 1; } return 0; } + +/* + * Search one exception table for an entry corresponding to the + * given instruction address, and return the address of the entry, + * or NULL if none is found. + * We use a binary search, and thus we assume that the table is + * already sorted. + */ +const struct exception_table_entry * +search_extable(const struct exception_table_entry *first, + const struct exception_table_entry *last, + unsigned long value) +{ + while (first <= last) { + const struct exception_table_entry *mid; + unsigned long addr; + + mid = ((last - first) >> 1) + first; + addr = ex_insn_addr(mid); + if (addr < value) + first = mid + 1; + else if (addr > value) + last = mid - 1; + else + return mid; + } + return NULL; +} + +/* + * The exception table needs to be sorted so that the binary + * search that we use to find entries in it works properly. + * This is used both for the kernel exception table and for + * the exception tables of modules that get loaded. + * + */ +static int cmp_ex(const void *a, const void *b) +{ + const struct exception_table_entry *x = a, *y = b; + + /* + * This value will always end up fittin in an int, because on + * both i386 and x86-64 the kernel symbol-reachable address + * space is < 2 GiB. + * + * This compare is only valid after normalization. + */ + return x->insn - y->insn; +} + +void sort_extable(struct exception_table_entry *start, + struct exception_table_entry *finish) +{ + struct exception_table_entry *p; + int i; + + /* Convert all entries to being relative to the start of the section */ + i = 0; + for (p = start; p < finish; p++) { + p->insn += i; + i += 4; + p->fixup += i; + i += 4; + } + + sort(start, finish - start, sizeof(struct exception_table_entry), + cmp_ex, NULL); + + /* Denormalize all entries */ + i = 0; + for (p = start; p < finish; p++) { + p->insn -= i; + i += 4; + p->fixup -= i; + i += 4; + } +} + +#ifdef CONFIG_MODULES +/* + * If the exception table is sorted, any referring to the module init + * will be at the beginning or the end. + */ +void trim_init_extable(struct module *m) +{ + /*trim the beginning*/ + while (m->num_exentries && + within_module_init(ex_insn_addr(&m->extable[0]), m)) { + m->extable++; + m->num_exentries--; + } + /*trim the end*/ + while (m->num_exentries && + within_module_init(ex_insn_addr(&m->extable[m->num_exentries-1]), m)) + m->num_exentries--; +} +#endif /* CONFIG_MODULES */ -- cgit v1.2.3