summaryrefslogtreecommitdiff
path: root/arch/s390/lib
diff options
context:
space:
mode:
Diffstat (limited to 'arch/s390/lib')
-rw-r--r--arch/s390/lib/Makefile4
-rw-r--r--arch/s390/lib/delay.c1
-rw-r--r--arch/s390/lib/expoline.S12
-rw-r--r--arch/s390/lib/test_modules.c32
-rw-r--r--arch/s390/lib/test_modules.h53
-rw-r--r--arch/s390/lib/test_modules_helpers.c13
-rw-r--r--arch/s390/lib/test_unwind.c274
-rw-r--r--arch/s390/lib/uaccess.c200
-rw-r--r--arch/s390/lib/xor.c21
9 files changed, 368 insertions, 242 deletions
diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile
index 707cd4622c13..5d415b3db6d1 100644
--- a/arch/s390/lib/Makefile
+++ b/arch/s390/lib/Makefile
@@ -7,6 +7,7 @@ lib-y += delay.o string.o uaccess.o find.o spinlock.o
obj-y += mem.o xor.o
lib-$(CONFIG_KPROBES) += probes.o
lib-$(CONFIG_UPROBES) += probes.o
+obj-$(CONFIG_EXPOLINE_EXTERN) += expoline.o
obj-$(CONFIG_S390_KPROBES_SANITY_TEST) += test_kprobes_s390.o
test_kprobes_s390-objs += test_kprobes_asm.o test_kprobes.o
@@ -17,4 +18,7 @@ KASAN_SANITIZE_uaccess.o := n
obj-$(CONFIG_S390_UNWIND_SELFTEST) += test_unwind.o
CFLAGS_test_unwind.o += -fno-optimize-sibling-calls
+obj-$(CONFIG_S390_MODULES_SANITY_TEST) += test_modules.o
+obj-$(CONFIG_S390_MODULES_SANITY_TEST_HELPERS) += test_modules_helpers.o
+
lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c
index bccbf394ae7e..f7f5adea8940 100644
--- a/arch/s390/lib/delay.c
+++ b/arch/s390/lib/delay.c
@@ -4,7 +4,6 @@
*
* Copyright IBM Corp. 1999, 2008
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
- * Heiko Carstens <heiko.carstens@de.ibm.com>,
*/
#include <linux/processor.h>
diff --git a/arch/s390/lib/expoline.S b/arch/s390/lib/expoline.S
new file mode 100644
index 000000000000..92ed8409a7a4
--- /dev/null
+++ b/arch/s390/lib/expoline.S
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <asm/nospec-insn.h>
+#include <linux/linkage.h>
+
+.macro GEN_ALL_BR_THUNK_EXTERN
+ .irp r1,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+ GEN_BR_THUNK_EXTERN %r\r1
+ .endr
+.endm
+
+GEN_ALL_BR_THUNK_EXTERN
diff --git a/arch/s390/lib/test_modules.c b/arch/s390/lib/test_modules.c
new file mode 100644
index 000000000000..9894009fc1f2
--- /dev/null
+++ b/arch/s390/lib/test_modules.c
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <kunit/test.h>
+#include <linux/module.h>
+
+#include "test_modules.h"
+
+/*
+ * Test that modules with many relocations are loaded properly.
+ */
+static void test_modules_many_vmlinux_relocs(struct kunit *test)
+{
+ int result = 0;
+
+#define CALL_RETURN(i) result += test_modules_return_ ## i()
+ REPEAT_10000(CALL_RETURN);
+ KUNIT_ASSERT_EQ(test, result, 49995000);
+}
+
+static struct kunit_case modules_testcases[] = {
+ KUNIT_CASE(test_modules_many_vmlinux_relocs),
+ {}
+};
+
+static struct kunit_suite modules_test_suite = {
+ .name = "modules_test_s390",
+ .test_cases = modules_testcases,
+};
+
+kunit_test_suites(&modules_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/arch/s390/lib/test_modules.h b/arch/s390/lib/test_modules.h
new file mode 100644
index 000000000000..6371fcf17684
--- /dev/null
+++ b/arch/s390/lib/test_modules.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef TEST_MODULES_H
+#define TEST_MODULES_H
+
+#define __REPEAT_10000_3(f, x) \
+ f(x ## 0); \
+ f(x ## 1); \
+ f(x ## 2); \
+ f(x ## 3); \
+ f(x ## 4); \
+ f(x ## 5); \
+ f(x ## 6); \
+ f(x ## 7); \
+ f(x ## 8); \
+ f(x ## 9)
+#define __REPEAT_10000_2(f, x) \
+ __REPEAT_10000_3(f, x ## 0); \
+ __REPEAT_10000_3(f, x ## 1); \
+ __REPEAT_10000_3(f, x ## 2); \
+ __REPEAT_10000_3(f, x ## 3); \
+ __REPEAT_10000_3(f, x ## 4); \
+ __REPEAT_10000_3(f, x ## 5); \
+ __REPEAT_10000_3(f, x ## 6); \
+ __REPEAT_10000_3(f, x ## 7); \
+ __REPEAT_10000_3(f, x ## 8); \
+ __REPEAT_10000_3(f, x ## 9)
+#define __REPEAT_10000_1(f, x) \
+ __REPEAT_10000_2(f, x ## 0); \
+ __REPEAT_10000_2(f, x ## 1); \
+ __REPEAT_10000_2(f, x ## 2); \
+ __REPEAT_10000_2(f, x ## 3); \
+ __REPEAT_10000_2(f, x ## 4); \
+ __REPEAT_10000_2(f, x ## 5); \
+ __REPEAT_10000_2(f, x ## 6); \
+ __REPEAT_10000_2(f, x ## 7); \
+ __REPEAT_10000_2(f, x ## 8); \
+ __REPEAT_10000_2(f, x ## 9)
+#define REPEAT_10000(f) \
+ __REPEAT_10000_1(f, 0); \
+ __REPEAT_10000_1(f, 1); \
+ __REPEAT_10000_1(f, 2); \
+ __REPEAT_10000_1(f, 3); \
+ __REPEAT_10000_1(f, 4); \
+ __REPEAT_10000_1(f, 5); \
+ __REPEAT_10000_1(f, 6); \
+ __REPEAT_10000_1(f, 7); \
+ __REPEAT_10000_1(f, 8); \
+ __REPEAT_10000_1(f, 9)
+
+#define DECLARE_RETURN(i) int test_modules_return_ ## i(void)
+REPEAT_10000(DECLARE_RETURN);
+
+#endif
diff --git a/arch/s390/lib/test_modules_helpers.c b/arch/s390/lib/test_modules_helpers.c
new file mode 100644
index 000000000000..1670349a03eb
--- /dev/null
+++ b/arch/s390/lib/test_modules_helpers.c
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/export.h>
+
+#include "test_modules.h"
+
+#define DEFINE_RETURN(i) \
+ int test_modules_return_ ## i(void) \
+ { \
+ return 1 ## i - 10000; \
+ } \
+ EXPORT_SYMBOL_GPL(test_modules_return_ ## i)
+REPEAT_10000(DEFINE_RETURN);
diff --git a/arch/s390/lib/test_unwind.c b/arch/s390/lib/test_unwind.c
index bc7973359ae2..c01f02887de4 100644
--- a/arch/s390/lib/test_unwind.c
+++ b/arch/s390/lib/test_unwind.c
@@ -8,6 +8,7 @@
#include <linux/completion.h>
#include <linux/kallsyms.h>
#include <linux/kthread.h>
+#include <linux/ftrace.h>
#include <linux/module.h>
#include <linux/timer.h>
#include <linux/slab.h>
@@ -16,10 +17,14 @@
#include <linux/wait.h>
#include <asm/irq.h>
-struct kunit *current_test;
+static struct kunit *current_test;
#define BT_BUF_SIZE (PAGE_SIZE * 4)
+static bool force_bt;
+module_param_named(backtrace, force_bt, bool, 0444);
+MODULE_PARM_DESC(backtrace, "print backtraces for all tests");
+
/*
* To avoid printk line limit split backtrace by lines
*/
@@ -98,7 +103,7 @@ static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs,
kunit_err(current_test, "Maximum number of frames exceeded\n");
ret = -EINVAL;
}
- if (ret)
+ if (ret || force_bt)
print_backtrace(bt);
kfree(bt);
return ret;
@@ -124,7 +129,10 @@ static struct unwindme *unwindme;
#define UWM_CALLER 0x8 /* Unwind starting from caller. */
#define UWM_SWITCH_STACK 0x10 /* Use call_on_stack. */
#define UWM_IRQ 0x20 /* Unwind from irq context. */
-#define UWM_PGM 0x40 /* Unwind from program check handler. */
+#define UWM_PGM 0x40 /* Unwind from program check handler */
+#define UWM_KPROBE_ON_FTRACE 0x80 /* Unwind from kprobe handler called via ftrace. */
+#define UWM_FTRACE 0x100 /* Unwind from ftrace handler. */
+#define UWM_KRETPROBE 0x200 /* Unwind kretprobe handlers. */
static __always_inline unsigned long get_psw_addr(void)
{
@@ -136,8 +144,56 @@ static __always_inline unsigned long get_psw_addr(void)
return psw_addr;
}
-#ifdef CONFIG_KPROBES
-static int pgm_pre_handler(struct kprobe *p, struct pt_regs *regs)
+static int kretprobe_ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs)
+{
+ struct unwindme *u = unwindme;
+
+ u->ret = test_unwind(NULL, (u->flags & UWM_REGS) ? regs : NULL,
+ (u->flags & UWM_SP) ? u->sp : 0);
+
+ return 0;
+}
+
+static noinline notrace void test_unwind_kretprobed_func(void)
+{
+ asm volatile(" nop\n");
+}
+
+static noinline void test_unwind_kretprobed_func_caller(void)
+{
+ test_unwind_kretprobed_func();
+}
+
+static int test_unwind_kretprobe(struct unwindme *u)
+{
+ int ret;
+ struct kretprobe my_kretprobe;
+
+ if (!IS_ENABLED(CONFIG_KPROBES))
+ kunit_skip(current_test, "requires CONFIG_KPROBES");
+
+ u->ret = -1; /* make sure kprobe is called */
+ unwindme = u;
+
+ memset(&my_kretprobe, 0, sizeof(my_kretprobe));
+ my_kretprobe.handler = kretprobe_ret_handler;
+ my_kretprobe.maxactive = 1;
+ my_kretprobe.kp.addr = (kprobe_opcode_t *)test_unwind_kretprobed_func;
+
+ ret = register_kretprobe(&my_kretprobe);
+
+ if (ret < 0) {
+ kunit_err(current_test, "register_kretprobe failed %d\n", ret);
+ return -EINVAL;
+ }
+
+ test_unwind_kretprobed_func_caller();
+ unregister_kretprobe(&my_kretprobe);
+ unwindme = NULL;
+ return u->ret;
+}
+
+static int kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs)
{
struct unwindme *u = unwindme;
@@ -145,8 +201,97 @@ static int pgm_pre_handler(struct kprobe *p, struct pt_regs *regs)
(u->flags & UWM_SP) ? u->sp : 0);
return 0;
}
+
+extern const char test_unwind_kprobed_insn[];
+
+static noinline void test_unwind_kprobed_func(void)
+{
+ asm volatile(
+ " nopr %%r7\n"
+ "test_unwind_kprobed_insn:\n"
+ " nopr %%r7\n"
+ :);
+}
+
+static int test_unwind_kprobe(struct unwindme *u)
+{
+ struct kprobe kp;
+ int ret;
+
+ if (!IS_ENABLED(CONFIG_KPROBES))
+ kunit_skip(current_test, "requires CONFIG_KPROBES");
+ if (!IS_ENABLED(CONFIG_KPROBES_ON_FTRACE) && u->flags & UWM_KPROBE_ON_FTRACE)
+ kunit_skip(current_test, "requires CONFIG_KPROBES_ON_FTRACE");
+
+ u->ret = -1; /* make sure kprobe is called */
+ unwindme = u;
+ memset(&kp, 0, sizeof(kp));
+ kp.pre_handler = kprobe_pre_handler;
+ kp.addr = u->flags & UWM_KPROBE_ON_FTRACE ?
+ (kprobe_opcode_t *)test_unwind_kprobed_func :
+ (kprobe_opcode_t *)test_unwind_kprobed_insn;
+ ret = register_kprobe(&kp);
+ if (ret < 0) {
+ kunit_err(current_test, "register_kprobe failed %d\n", ret);
+ return -EINVAL;
+ }
+
+ test_unwind_kprobed_func();
+ unregister_kprobe(&kp);
+ unwindme = NULL;
+ return u->ret;
+}
+
+static void notrace __used test_unwind_ftrace_handler(unsigned long ip,
+ unsigned long parent_ip,
+ struct ftrace_ops *fops,
+ struct ftrace_regs *fregs)
+{
+ struct unwindme *u = (struct unwindme *)fregs->regs.gprs[2];
+
+ u->ret = test_unwind(NULL, (u->flags & UWM_REGS) ? &fregs->regs : NULL,
+ (u->flags & UWM_SP) ? u->sp : 0);
+}
+
+static noinline int test_unwind_ftraced_func(struct unwindme *u)
+{
+ return READ_ONCE(u)->ret;
+}
+
+static int test_unwind_ftrace(struct unwindme *u)
+{
+ int ret;
+#ifdef CONFIG_DYNAMIC_FTRACE
+ struct ftrace_ops *fops;
+
+ fops = kunit_kzalloc(current_test, sizeof(*fops), GFP_KERNEL);
+ fops->func = test_unwind_ftrace_handler;
+ fops->flags = FTRACE_OPS_FL_DYNAMIC |
+ FTRACE_OPS_FL_RECURSION |
+ FTRACE_OPS_FL_SAVE_REGS |
+ FTRACE_OPS_FL_PERMANENT;
+#else
+ kunit_skip(current_test, "requires CONFIG_DYNAMIC_FTRACE");
#endif
+ ret = ftrace_set_filter_ip(fops, (unsigned long)test_unwind_ftraced_func, 0, 0);
+ if (ret) {
+ kunit_err(current_test, "failed to set ftrace filter (%d)\n", ret);
+ return -1;
+ }
+
+ ret = register_ftrace_function(fops);
+ if (!ret) {
+ ret = test_unwind_ftraced_func(u);
+ unregister_ftrace_function(fops);
+ } else {
+ kunit_err(current_test, "failed to register ftrace handler (%d)\n", ret);
+ }
+
+ ftrace_set_filter_ip(fops, (unsigned long)test_unwind_ftraced_func, 1, 0);
+ return ret;
+}
+
/* This function may or may not appear in the backtrace. */
static noinline int unwindme_func4(struct unwindme *u)
{
@@ -157,35 +302,12 @@ static noinline int unwindme_func4(struct unwindme *u)
wait_event(u->task_wq, kthread_should_park());
kthread_parkme();
return 0;
-#ifdef CONFIG_KPROBES
- } else if (u->flags & UWM_PGM) {
- struct kprobe kp;
- int ret;
-
- unwindme = u;
- memset(&kp, 0, sizeof(kp));
- kp.symbol_name = "do_report_trap";
- kp.pre_handler = pgm_pre_handler;
- ret = register_kprobe(&kp);
- if (ret < 0) {
- kunit_err(current_test, "register_kprobe failed %d\n", ret);
- return -EINVAL;
- }
-
- /*
- * Trigger operation exception; use insn notation to bypass
- * llvm's integrated assembler sanity checks.
- */
- asm volatile(
- " .insn e,0x0000\n" /* illegal opcode */
- "0: nopr %%r7\n"
- EX_TABLE(0b, 0b)
- :);
-
- unregister_kprobe(&kp);
- unwindme = NULL;
- return u->ret;
-#endif
+ } else if (u->flags & (UWM_PGM | UWM_KPROBE_ON_FTRACE)) {
+ return test_unwind_kprobe(u);
+ } else if (u->flags & (UWM_KRETPROBE)) {
+ return test_unwind_kretprobe(u);
+ } else if (u->flags & UWM_FTRACE) {
+ return test_unwind_ftrace(u);
} else {
struct pt_regs regs;
@@ -255,7 +377,7 @@ static int test_unwind_irq(struct unwindme *u)
}
/* Spawns a task and passes it to test_unwind(). */
-static int test_unwind_task(struct kunit *test, struct unwindme *u)
+static int test_unwind_task(struct unwindme *u)
{
struct task_struct *task;
int ret;
@@ -270,7 +392,7 @@ static int test_unwind_task(struct kunit *test, struct unwindme *u)
*/
task = kthread_run(unwindme_func1, u, "%s", __func__);
if (IS_ERR(task)) {
- kunit_err(test, "kthread_run() failed\n");
+ kunit_err(current_test, "kthread_run() failed\n");
return PTR_ERR(task);
}
/*
@@ -293,49 +415,43 @@ struct test_params {
/*
* Create required parameter list for tests
*/
+#define TEST_WITH_FLAGS(f) { .flags = f, .name = #f }
static const struct test_params param_list[] = {
- {.flags = UWM_DEFAULT, .name = "UWM_DEFAULT"},
- {.flags = UWM_SP, .name = "UWM_SP"},
- {.flags = UWM_REGS, .name = "UWM_REGS"},
- {.flags = UWM_SWITCH_STACK,
- .name = "UWM_SWITCH_STACK"},
- {.flags = UWM_SP | UWM_REGS,
- .name = "UWM_SP | UWM_REGS"},
- {.flags = UWM_CALLER | UWM_SP,
- .name = "WM_CALLER | UWM_SP"},
- {.flags = UWM_CALLER | UWM_SP | UWM_REGS,
- .name = "UWM_CALLER | UWM_SP | UWM_REGS"},
- {.flags = UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK,
- .name = "UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK"},
- {.flags = UWM_THREAD, .name = "UWM_THREAD"},
- {.flags = UWM_THREAD | UWM_SP,
- .name = "UWM_THREAD | UWM_SP"},
- {.flags = UWM_THREAD | UWM_CALLER | UWM_SP,
- .name = "UWM_THREAD | UWM_CALLER | UWM_SP"},
- {.flags = UWM_IRQ, .name = "UWM_IRQ"},
- {.flags = UWM_IRQ | UWM_SWITCH_STACK,
- .name = "UWM_IRQ | UWM_SWITCH_STACK"},
- {.flags = UWM_IRQ | UWM_SP,
- .name = "UWM_IRQ | UWM_SP"},
- {.flags = UWM_IRQ | UWM_REGS,
- .name = "UWM_IRQ | UWM_REGS"},
- {.flags = UWM_IRQ | UWM_SP | UWM_REGS,
- .name = "UWM_IRQ | UWM_SP | UWM_REGS"},
- {.flags = UWM_IRQ | UWM_CALLER | UWM_SP,
- .name = "UWM_IRQ | UWM_CALLER | UWM_SP"},
- {.flags = UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS,
- .name = "UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS"},
- {.flags = UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK,
- .name = "UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK"},
- #ifdef CONFIG_KPROBES
- {.flags = UWM_PGM, .name = "UWM_PGM"},
- {.flags = UWM_PGM | UWM_SP,
- .name = "UWM_PGM | UWM_SP"},
- {.flags = UWM_PGM | UWM_REGS,
- .name = "UWM_PGM | UWM_REGS"},
- {.flags = UWM_PGM | UWM_SP | UWM_REGS,
- .name = "UWM_PGM | UWM_SP | UWM_REGS"},
- #endif
+ TEST_WITH_FLAGS(UWM_DEFAULT),
+ TEST_WITH_FLAGS(UWM_SP),
+ TEST_WITH_FLAGS(UWM_REGS),
+ TEST_WITH_FLAGS(UWM_SWITCH_STACK),
+ TEST_WITH_FLAGS(UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_CALLER | UWM_SP),
+ TEST_WITH_FLAGS(UWM_CALLER | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK),
+ TEST_WITH_FLAGS(UWM_THREAD),
+ TEST_WITH_FLAGS(UWM_THREAD | UWM_SP),
+ TEST_WITH_FLAGS(UWM_THREAD | UWM_CALLER | UWM_SP),
+ TEST_WITH_FLAGS(UWM_IRQ),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_SWITCH_STACK),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_SP),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_CALLER | UWM_SP),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_IRQ | UWM_CALLER | UWM_SP | UWM_REGS | UWM_SWITCH_STACK),
+ TEST_WITH_FLAGS(UWM_PGM),
+ TEST_WITH_FLAGS(UWM_PGM | UWM_SP),
+ TEST_WITH_FLAGS(UWM_PGM | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_PGM | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE),
+ TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE | UWM_SP),
+ TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_KPROBE_ON_FTRACE | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_FTRACE),
+ TEST_WITH_FLAGS(UWM_FTRACE | UWM_SP),
+ TEST_WITH_FLAGS(UWM_FTRACE | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_FTRACE | UWM_SP | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_KRETPROBE),
+ TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_SP),
+ TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_REGS),
+ TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_SP | UWM_REGS),
};
/*
@@ -360,7 +476,7 @@ static void test_unwind_flags(struct kunit *test)
params = (const struct test_params *)test->param_value;
u.flags = params->flags;
if (u.flags & UWM_THREAD)
- KUNIT_EXPECT_EQ(test, 0, test_unwind_task(test, &u));
+ KUNIT_EXPECT_EQ(test, 0, test_unwind_task(&u));
else if (u.flags & UWM_IRQ)
KUNIT_EXPECT_EQ(test, 0, test_unwind_irq(&u));
else
diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
index 8a5d21461889..d7b3b193d108 100644
--- a/arch/s390/lib/uaccess.c
+++ b/arch/s390/lib/uaccess.c
@@ -8,13 +8,10 @@
* Gerald Schaefer (gerald.schaefer@de.ibm.com)
*/
-#include <linux/jump_label.h>
#include <linux/uaccess.h>
#include <linux/export.h>
-#include <linux/errno.h>
#include <linux/mm.h>
-#include <asm/mmu_context.h>
-#include <asm/facility.h>
+#include <asm/asm-extable.h>
#ifdef CONFIG_DEBUG_ENTRY
void debug_user_asce(int exit)
@@ -34,43 +31,21 @@ void debug_user_asce(int exit)
}
#endif /*CONFIG_DEBUG_ENTRY */
-#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES
-static DEFINE_STATIC_KEY_FALSE(have_mvcos);
-
-static int __init uaccess_init(void)
-{
- if (test_facility(27))
- static_branch_enable(&have_mvcos);
- return 0;
-}
-early_initcall(uaccess_init);
-
-static inline int copy_with_mvcos(void)
-{
- if (static_branch_likely(&have_mvcos))
- return 1;
- return 0;
-}
-#else
-static inline int copy_with_mvcos(void)
-{
- return 1;
-}
-#endif
-
-static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr,
- unsigned long size)
+static unsigned long raw_copy_from_user_key(void *to, const void __user *from,
+ unsigned long size, unsigned long key)
{
unsigned long tmp1, tmp2;
union oac spec = {
+ .oac2.key = key,
.oac2.as = PSW_BITS_AS_SECONDARY,
+ .oac2.k = 1,
.oac2.a = 1,
};
tmp1 = -4096UL;
asm volatile(
" lr 0,%[spec]\n"
- "0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
+ "0: mvcos 0(%2),0(%1),%0\n"
"6: jz 4f\n"
"1: algr %0,%3\n"
" slgr %1,%3\n"
@@ -81,73 +56,55 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr
" slgr %4,%1\n"
" clgr %0,%4\n" /* copy crosses next page boundary? */
" jnh 5f\n"
- "3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
+ "3: mvcos 0(%2),0(%1),%4\n"
"7: slgr %0,%4\n"
" j 5f\n"
"4: slgr %0,%0\n"
"5:\n"
EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
- : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
+ : "+a" (size), "+a" (from), "+a" (to), "+a" (tmp1), "=a" (tmp2)
: [spec] "d" (spec.val)
: "cc", "memory", "0");
return size;
}
-static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr,
- unsigned long size)
+unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
- unsigned long tmp1, tmp2;
-
- tmp1 = -256UL;
- asm volatile(
- " sacf 0\n"
- "0: mvcp 0(%0,%2),0(%1),%3\n"
- "7: jz 5f\n"
- "1: algr %0,%3\n"
- " la %1,256(%1)\n"
- " la %2,256(%2)\n"
- "2: mvcp 0(%0,%2),0(%1),%3\n"
- "8: jnz 1b\n"
- " j 5f\n"
- "3: la %4,255(%1)\n" /* %4 = ptr + 255 */
- " lghi %3,-4096\n"
- " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
- " slgr %4,%1\n"
- " clgr %0,%4\n" /* copy crosses next page boundary? */
- " jnh 6f\n"
- "4: mvcp 0(%4,%2),0(%1),%3\n"
- "9: slgr %0,%4\n"
- " j 6f\n"
- "5: slgr %0,%0\n"
- "6: sacf 768\n"
- EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
- EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
- : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
- : : "cc", "memory");
- return size;
+ return raw_copy_from_user_key(to, from, n, 0);
}
+EXPORT_SYMBOL(raw_copy_from_user);
-unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+unsigned long _copy_from_user_key(void *to, const void __user *from,
+ unsigned long n, unsigned long key)
{
- if (copy_with_mvcos())
- return copy_from_user_mvcos(to, from, n);
- return copy_from_user_mvcp(to, from, n);
+ unsigned long res = n;
+
+ might_fault();
+ if (!should_fail_usercopy()) {
+ instrument_copy_from_user(to, from, n);
+ res = raw_copy_from_user_key(to, from, n, key);
+ }
+ if (unlikely(res))
+ memset(to + (n - res), 0, res);
+ return res;
}
-EXPORT_SYMBOL(raw_copy_from_user);
+EXPORT_SYMBOL(_copy_from_user_key);
-static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
- unsigned long size)
+static unsigned long raw_copy_to_user_key(void __user *to, const void *from,
+ unsigned long size, unsigned long key)
{
unsigned long tmp1, tmp2;
union oac spec = {
+ .oac1.key = key,
.oac1.as = PSW_BITS_AS_SECONDARY,
+ .oac1.k = 1,
.oac1.a = 1,
};
tmp1 = -4096UL;
asm volatile(
" lr 0,%[spec]\n"
- "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
+ "0: mvcos 0(%1),0(%2),%0\n"
"6: jz 4f\n"
"1: algr %0,%3\n"
" slgr %1,%3\n"
@@ -158,61 +115,36 @@ static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
" slgr %4,%1\n"
" clgr %0,%4\n" /* copy crosses next page boundary? */
" jnh 5f\n"
- "3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n"
+ "3: mvcos 0(%1),0(%2),%4\n"
"7: slgr %0,%4\n"
" j 5f\n"
"4: slgr %0,%0\n"
"5:\n"
EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
- : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
+ : "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2)
: [spec] "d" (spec.val)
: "cc", "memory", "0");
return size;
}
-static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x,
- unsigned long size)
+unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
- unsigned long tmp1, tmp2;
-
- tmp1 = -256UL;
- asm volatile(
- " sacf 0\n"
- "0: mvcs 0(%0,%1),0(%2),%3\n"
- "7: jz 5f\n"
- "1: algr %0,%3\n"
- " la %1,256(%1)\n"
- " la %2,256(%2)\n"
- "2: mvcs 0(%0,%1),0(%2),%3\n"
- "8: jnz 1b\n"
- " j 5f\n"
- "3: la %4,255(%1)\n" /* %4 = ptr + 255 */
- " lghi %3,-4096\n"
- " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
- " slgr %4,%1\n"
- " clgr %0,%4\n" /* copy crosses next page boundary? */
- " jnh 6f\n"
- "4: mvcs 0(%4,%1),0(%2),%3\n"
- "9: slgr %0,%4\n"
- " j 6f\n"
- "5: slgr %0,%0\n"
- "6: sacf 768\n"
- EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
- EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
- : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
- : : "cc", "memory");
- return size;
+ return raw_copy_to_user_key(to, from, n, 0);
}
+EXPORT_SYMBOL(raw_copy_to_user);
-unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+unsigned long _copy_to_user_key(void __user *to, const void *from,
+ unsigned long n, unsigned long key)
{
- if (copy_with_mvcos())
- return copy_to_user_mvcos(to, from, n);
- return copy_to_user_mvcs(to, from, n);
+ might_fault();
+ if (should_fail_usercopy())
+ return n;
+ instrument_copy_to_user(to, from, n);
+ return raw_copy_to_user_key(to, from, n, key);
}
-EXPORT_SYMBOL(raw_copy_to_user);
+EXPORT_SYMBOL(_copy_to_user_key);
-static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size)
+unsigned long __clear_user(void __user *to, unsigned long size)
{
unsigned long tmp1, tmp2;
union oac spec = {
@@ -223,7 +155,7 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size
tmp1 = -4096UL;
asm volatile(
" lr 0,%[spec]\n"
- "0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n"
+ "0: mvcos 0(%1),0(%4),%0\n"
" jz 4f\n"
"1: algr %0,%2\n"
" slgr %1,%2\n"
@@ -233,7 +165,7 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size
" slgr %3,%1\n"
" clgr %0,%3\n" /* copy crosses next page boundary? */
" jnh 5f\n"
- "3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n"
+ "3: mvcos 0(%1),0(%4),%3\n"
" slgr %0,%3\n"
" j 5f\n"
"4: slgr %0,%0\n"
@@ -244,46 +176,4 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size
: "cc", "memory", "0");
return size;
}
-
-static inline unsigned long clear_user_xc(void __user *to, unsigned long size)
-{
- unsigned long tmp1, tmp2;
-
- asm volatile(
- " sacf 256\n"
- " aghi %0,-1\n"
- " jo 5f\n"
- " bras %3,3f\n"
- " xc 0(1,%1),0(%1)\n"
- "0: aghi %0,257\n"
- " la %2,255(%1)\n" /* %2 = ptr + 255 */
- " srl %2,12\n"
- " sll %2,12\n" /* %2 = (ptr + 255) & -4096 */
- " slgr %2,%1\n"
- " clgr %0,%2\n" /* clear crosses next page boundary? */
- " jnh 5f\n"
- " aghi %2,-1\n"
- "1: ex %2,0(%3)\n"
- " aghi %2,1\n"
- " slgr %0,%2\n"
- " j 5f\n"
- "2: xc 0(256,%1),0(%1)\n"
- " la %1,256(%1)\n"
- "3: aghi %0,-256\n"
- " jnm 2b\n"
- "4: ex %0,0(%3)\n"
- "5: slgr %0,%0\n"
- "6: sacf 768\n"
- EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
- : "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2)
- : : "cc", "memory");
- return size;
-}
-
-unsigned long __clear_user(void __user *to, unsigned long size)
-{
- if (copy_with_mvcos())
- return clear_user_mvcos(to, size);
- return clear_user_xc(to, size);
-}
EXPORT_SYMBOL(__clear_user);
diff --git a/arch/s390/lib/xor.c b/arch/s390/lib/xor.c
index a963c3d8ad0d..fb924a8041dc 100644
--- a/arch/s390/lib/xor.c
+++ b/arch/s390/lib/xor.c
@@ -11,7 +11,8 @@
#include <linux/raid/xor.h>
#include <asm/xor.h>
-static void xor_xc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
+static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
{
asm volatile(
" larl 1,2f\n"
@@ -32,8 +33,9 @@ static void xor_xc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
: "0", "1", "cc", "memory");
}
-static void xor_xc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
- unsigned long *p3)
+static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
{
asm volatile(
" larl 1,2f\n"
@@ -58,8 +60,10 @@ static void xor_xc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
: : "0", "1", "cc", "memory");
}
-static void xor_xc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
- unsigned long *p3, unsigned long *p4)
+static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
{
asm volatile(
" larl 1,2f\n"
@@ -88,8 +92,11 @@ static void xor_xc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
: : "0", "1", "cc", "memory");
}
-static void xor_xc_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
- unsigned long *p3, unsigned long *p4, unsigned long *p5)
+static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
{
asm volatile(
" larl 1,2f\n"
generated by cgit v1.2.3 (git 2.39.0) at 2024-11-06 14:26:44 +0300