From f3cfb875d0fd5f4af40cbb992f436ad396f69a71 Mon Sep 17 00:00:00 2001 From: Ingo Franzki Date: Thu, 31 Aug 2023 17:36:43 +0200 Subject: s390/zcrypt: update list of EP11 operation modes Add additional operation mode strings into the EP11 operation mode table. These strings are returned by sysfs entries /sys/devices/ap/cardxx/op_modes and /sys/devices/ap/cardxx/xx.yyyy/op_modes. Signed-off-by: Ingo Franzki Reviewed-by: Harald Freudenberger Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/zcrypt_cex4.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/s390/crypto/zcrypt_cex4.c b/drivers/s390/crypto/zcrypt_cex4.c index 9cfce9ff2e65..5c4532ab0040 100644 --- a/drivers/s390/crypto/zcrypt_cex4.c +++ b/drivers/s390/crypto/zcrypt_cex4.c @@ -279,7 +279,11 @@ static const struct { { 1, "BSI2009" }, { 2, "FIPS2011" }, { 3, "BSI2011" }, + { 4, "SIGG-IMPORT" }, + { 5, "SIGG" }, { 6, "BSICC2017" }, + { 7, "FIPS2021" }, + { 8, "FIPS2024" }, { 0, NULL } }; -- cgit v1.2.3 From 4a1725281fc5b0009944b1c0e1d2c1dc311a09ec Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 5 Sep 2023 15:49:37 +0200 Subject: s390/smp,mcck: fix early IPI handling Both the external call as well as the emergency signal submask bits in control register 0 are set before any interrupt handler is registered. Change the order and first register the interrupt handler and only then enable the interrupts by setting the corresponding bits in control register 0. This prevents that the second part of the machine check handler for early machine check handling is not executed: the machine check handler sends an IPI to the CPU it runs on. If the corresponding interrupts are enabled, but no interrupt handler is present, the interrupt is ignored. Reviewed-by: Sven Schnelle Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/early.c | 12 +++--------- arch/s390/kernel/smp.c | 4 ++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 442ce0489e1a..3a54733e4fc6 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -258,15 +258,9 @@ static inline void save_vector_registers(void) #endif } -static inline void setup_control_registers(void) +static inline void setup_low_address_protection(void) { - unsigned long reg; - - __ctl_store(reg, 0, 0); - reg |= CR0_LOW_ADDRESS_PROTECTION; - reg |= CR0_EMERGENCY_SIGNAL_SUBMASK; - reg |= CR0_EXTERNAL_CALL_SUBMASK; - __ctl_load(reg, 0, 0); + __ctl_set_bit(0, 28); } static inline void setup_access_registers(void) @@ -314,7 +308,7 @@ void __init startup_init(void) save_vector_registers(); setup_topology(); sclp_early_detect(); - setup_control_registers(); + setup_low_address_protection(); setup_access_registers(); lockdep_on(); } diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index a4edb7ea66ea..c63be2efd689 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -1013,12 +1013,12 @@ void __init smp_fill_possible_mask(void) void __init smp_prepare_cpus(unsigned int max_cpus) { - /* request the 0x1201 emergency signal external interrupt */ if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1201"); - /* request the 0x1202 external call external interrupt */ + ctl_set_bit(0, 14); if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1202"); + ctl_set_bit(0, 13); } void __init smp_prepare_boot_cpu(void) -- cgit v1.2.3 From 496bb034f4ff4866452ced7fdfaa4a0736230bdf Mon Sep 17 00:00:00 2001 From: Tobias Huschle Date: Wed, 16 Aug 2023 09:28:20 +0200 Subject: s390/smp: disallow CPU hotplug of CPU 0 On s390, CPU 0 has special properties in comparison to other CPUs, as it cannot be deconfigured for example. Therefore, allowing to hotplug CPU 0 introduces additional complexity when handling these properties. Disallowing to hotplug CPU 0 allows to remove such complexities. This follows x86 which also prevents offlining of CPU0 since commit e59e74dc48a3 ("x86/topology: Remove CPU0 hotplug option"). [hca@linux.ibm.com: changed commit message] Suggested-by: Heiko Carstens Signed-off-by: Tobias Huschle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/smp.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index c63be2efd689..e8284cec36e7 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -1076,11 +1076,9 @@ static ssize_t cpu_configure_store(struct device *dev, cpus_read_lock(); mutex_lock(&smp_cpu_state_mutex); rc = -EBUSY; - /* disallow configuration changes of online cpus and cpu 0 */ + /* disallow configuration changes of online cpus */ cpu = dev->id; cpu = smp_get_base_cpu(cpu); - if (cpu == 0) - goto out; for (i = 0; i <= smp_cpu_mtid; i++) if (cpu_online(cpu + i)) goto out; @@ -1180,7 +1178,7 @@ static int smp_add_present_cpu(int cpu) return -ENOMEM; per_cpu(cpu_device, cpu) = c; s = &c->dev; - c->hotpluggable = 1; + c->hotpluggable = !!cpu; rc = register_cpu(c, cpu); if (rc) goto out; -- cgit v1.2.3 From 3570ee046c46b5dc3d077e3e60616b141a221027 Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Mon, 31 Jul 2023 17:07:11 +0200 Subject: s390/smp: keep the original lowcore for CPU 0 Now that CPU 0 is not hotpluggable, it is not necessary to support freeing its stacks. Delete all the code that migrates it to new stacks and a new lowcore. Suggested-by: Heiko Carstens Signed-off-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/smp.h | 1 - arch/s390/kernel/setup.c | 6 ----- arch/s390/kernel/smp.c | 57 --------------------------------------------- 3 files changed, 64 deletions(-) diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h index 73ed2781073b..6e5b1b4b19a9 100644 --- a/arch/s390/include/asm/smp.h +++ b/arch/s390/include/asm/smp.h @@ -59,7 +59,6 @@ static inline void smp_cpus_done(unsigned int max_cpus) { } -extern int smp_reinit_ipl_cpu(void); extern int smp_rescan_cpus(void); extern void __noreturn cpu_die(void); extern void __cpu_die(unsigned int cpu); diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index de6ad0fb2328..deb97723c890 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -381,12 +381,6 @@ void stack_free(unsigned long stack) #endif } -void __init __noreturn arch_call_rest_init(void) -{ - smp_reinit_ipl_cpu(); - rest_init(); -} - static unsigned long __init stack_alloc_early(void) { unsigned long stack; diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index e8284cec36e7..9c5e107da4b4 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -1256,60 +1256,3 @@ out: return rc; } subsys_initcall(s390_smp_init); - -static __always_inline void set_new_lowcore(struct lowcore *lc) -{ - union register_pair dst, src; - u32 pfx; - - src.even = (unsigned long) &S390_lowcore; - src.odd = sizeof(S390_lowcore); - dst.even = (unsigned long) lc; - dst.odd = sizeof(*lc); - pfx = __pa(lc); - - asm volatile( - " mvcl %[dst],%[src]\n" - " spx %[pfx]\n" - : [dst] "+&d" (dst.pair), [src] "+&d" (src.pair) - : [pfx] "Q" (pfx) - : "memory", "cc"); -} - -int __init smp_reinit_ipl_cpu(void) -{ - unsigned long async_stack, nodat_stack, mcck_stack; - struct lowcore *lc, *lc_ipl; - unsigned long flags, cr0; - u64 mcesad; - - lc_ipl = lowcore_ptr[0]; - lc = (struct lowcore *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER); - nodat_stack = __get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER); - async_stack = stack_alloc(); - mcck_stack = stack_alloc(); - if (!lc || !nodat_stack || !async_stack || !mcck_stack || nmi_alloc_mcesa(&mcesad)) - panic("Couldn't allocate memory"); - - local_irq_save(flags); - local_mcck_disable(); - set_new_lowcore(lc); - S390_lowcore.nodat_stack = nodat_stack + STACK_INIT_OFFSET; - S390_lowcore.async_stack = async_stack + STACK_INIT_OFFSET; - S390_lowcore.mcck_stack = mcck_stack + STACK_INIT_OFFSET; - __ctl_store(cr0, 0, 0); - __ctl_clear_bit(0, 28); /* disable lowcore protection */ - S390_lowcore.mcesad = mcesad; - __ctl_load(cr0, 0, 0); - if (abs_lowcore_map(0, lc, false)) - panic("Couldn't remap absolute lowcore"); - lowcore_ptr[0] = lc; - local_mcck_enable(); - local_irq_restore(flags); - - memblock_free_late(__pa(lc_ipl->mcck_stack - STACK_INIT_OFFSET), THREAD_SIZE); - memblock_free_late(__pa(lc_ipl->async_stack - STACK_INIT_OFFSET), THREAD_SIZE); - memblock_free_late(__pa(lc_ipl->nodat_stack - STACK_INIT_OFFSET), THREAD_SIZE); - memblock_free_late(__pa(lc_ipl), sizeof(*lc_ipl)); - return 0; -} -- cgit v1.2.3 From aa36d433b79657c1fed31ff5ae2bbd6f7bc123aa Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 13:03:00 +0200 Subject: s390/setup: use strlcat() instead of strcat() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use strlcat() instead of strcat() in order to get rid of this W=1 warning: In function ‘strlcat’, inlined from ‘strcat’ at ./include/linux/fortify-string.h:432:6, inlined from ‘setup_zfcpdump’ at arch/s390/kernel/setup.c:308:2, inlined from ‘setup_arch’ at arch/s390/kernel/setup.c:1002:2: ./include/linux/fortify-string.h:406:19: warning: writing 1 byte into a region of size 0 [-Wstringop-overflow=] 406 | p[actual] = '\0'; | ~~~~~~~~~~^~~~~~ As stated in fortify-string.h strcat() should not be used, since FORTIFY_SOURCE cannot figure out the size of the destination buffer. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index deb97723c890..ef158bd0ea7b 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -305,7 +305,7 @@ static void __init setup_zfcpdump(void) return; if (oldmem_data.start) return; - strcat(boot_command_line, " cio_ignore=all,!ipldev,!condev"); + strlcat(boot_command_line, " cio_ignore=all,!ipldev,!condev", COMMAND_LINE_SIZE); console_loglevel = 2; } #else -- cgit v1.2.3 From 0c4d01f3952911b766e6394e0053146c24c98357 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:39:55 +0200 Subject: s390/ctlreg: move control register code to separate file Control register handling has nothing to do with low level SMP code. Move it to a separate file. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/vmem.c | 1 + arch/s390/include/asm/ctl_reg.h | 8 +++-- arch/s390/include/asm/fpu/internal.h | 1 - arch/s390/include/asm/uaccess.h | 1 - arch/s390/kernel/Makefile | 2 +- arch/s390/kernel/ctlreg.c | 62 ++++++++++++++++++++++++++++++++++++ arch/s390/kernel/smp.c | 53 ++---------------------------- arch/s390/lib/uaccess.c | 1 + arch/s390/mm/vmem.c | 1 + drivers/s390/char/sclp_early_core.c | 1 + 10 files changed, 75 insertions(+), 56 deletions(-) create mode 100644 arch/s390/kernel/ctlreg.c diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 01257ce3b89c..36b90864d69f 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctl_reg.h index adf7d8cdac7e..f5536fdb398d 100644 --- a/arch/s390/include/asm/ctl_reg.h +++ b/arch/s390/include/asm/ctl_reg.h @@ -74,16 +74,18 @@ static __always_inline void __ctl_clear_bit(unsigned int cr, unsigned int bit) __ctl_load(reg, cr, cr); } -void smp_ctl_set_clear_bit(int cr, int bit, bool set); +void ctlreg_lock(void); +void ctlreg_unlock(void); +void ctl_set_clear_bit(int cr, int bit, bool set); static inline void ctl_set_bit(int cr, int bit) { - smp_ctl_set_clear_bit(cr, bit, true); + ctl_set_clear_bit(cr, bit, true); } static inline void ctl_clear_bit(int cr, int bit) { - smp_ctl_set_clear_bit(cr, bit, false); + ctl_set_clear_bit(cr, bit, false); } union ctlreg0 { diff --git a/arch/s390/include/asm/fpu/internal.h b/arch/s390/include/asm/fpu/internal.h index bbdadb1c9efc..8634581b9011 100644 --- a/arch/s390/include/asm/fpu/internal.h +++ b/arch/s390/include/asm/fpu/internal.h @@ -10,7 +10,6 @@ #define _ASM_S390_FPU_INTERNAL_H #include -#include #include static inline void save_vx_regs(__vector128 *vxrs) diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 8a8c64a678c4..81ae8a98e7ec 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -15,7 +15,6 @@ */ #include #include -#include #include #include #include diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 0df2b88cc0da..353def93973b 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -37,7 +37,7 @@ CFLAGS_unwind_bc.o += -fno-optimize-sibling-calls obj-y := head64.o traps.o time.o process.o earlypgm.o early.o setup.o idle.o vtime.o obj-y += processor.o syscall.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o cpufeature.o -obj-y += sysinfo.o lgr.o os_info.o +obj-y += sysinfo.o lgr.o os_info.o ctlreg.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o kdebugfs.o alternative.o obj-y += nospec-branch.o ipl_vmparm.o machine_kexec_reloc.o unwind_bc.o diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c new file mode 100644 index 000000000000..176ada8b45c2 --- /dev/null +++ b/arch/s390/kernel/ctlreg.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright IBM Corp. 1999, 2023 + */ + +#include +#include +#include +#include + +/* + * ctl_lock guards access to global control register contents which + * are kept in the control register save area within absolute lowcore + * at physical address zero. + */ +static DEFINE_SPINLOCK(ctl_lock); + +void ctlreg_lock(void) + __acquires(&ctl_lock) +{ + spin_lock(&ctl_lock); +} + +void ctlreg_unlock(void) + __releases(&ctl_lock) +{ + spin_unlock(&ctl_lock); +} + +struct ctl_bit_parms { + unsigned long orval; + unsigned long andval; + int cr; +}; + +static void ctl_bit_callback(void *info) +{ + struct ctl_bit_parms *pp = info; + unsigned long regs[16]; + + __ctl_store(regs, 0, 15); + regs[pp->cr] &= pp->andval; + regs[pp->cr] |= pp->orval; + __ctl_load(regs, 0, 15); +} + +void ctl_set_clear_bit(int cr, int bit, bool set) +{ + struct ctl_bit_parms pp = { .cr = cr, }; + struct lowcore *abs_lc; + + pp.orval = set ? 1UL << bit : 0; + pp.andval = set ? -1UL : ~(1UL << bit); + ctlreg_lock(); + abs_lc = get_abs_lowcore(); + abs_lc->cregs_save_area[cr] &= pp.andval; + abs_lc->cregs_save_area[cr] |= pp.orval; + put_abs_lowcore(abs_lc); + on_each_cpu(ctl_bit_callback, &pp, 1); + ctlreg_unlock(); +} +EXPORT_SYMBOL(ctl_set_clear_bit); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 9c5e107da4b4..d12dbba9d03c 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -567,54 +568,6 @@ void arch_irq_work_raise(void) } #endif -/* - * parameter area for the set/clear control bit callbacks - */ -struct ec_creg_mask_parms { - unsigned long orval; - unsigned long andval; - int cr; -}; - -/* - * callback for setting/clearing control bits - */ -static void smp_ctl_bit_callback(void *info) -{ - struct ec_creg_mask_parms *pp = info; - unsigned long cregs[16]; - - __ctl_store(cregs, 0, 15); - cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval; - __ctl_load(cregs, 0, 15); -} - -static DEFINE_SPINLOCK(ctl_lock); - -void smp_ctl_set_clear_bit(int cr, int bit, bool set) -{ - struct ec_creg_mask_parms parms = { .cr = cr, }; - struct lowcore *abs_lc; - u64 ctlreg; - - if (set) { - parms.orval = 1UL << bit; - parms.andval = -1UL; - } else { - parms.orval = 0; - parms.andval = ~(1UL << bit); - } - spin_lock(&ctl_lock); - abs_lc = get_abs_lowcore(); - ctlreg = abs_lc->cregs_save_area[cr]; - ctlreg = (ctlreg & parms.andval) | parms.orval; - abs_lc->cregs_save_area[cr] = ctlreg; - put_abs_lowcore(abs_lc); - on_each_cpu(smp_ctl_bit_callback, &parms, 1); - spin_unlock(&ctl_lock); -} -EXPORT_SYMBOL(smp_ctl_set_clear_bit); - #ifdef CONFIG_CRASH_DUMP int smp_store_status(int cpu) @@ -935,14 +888,14 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) * Make sure global control register contents do not change * until new CPU has initialized control registers. */ - spin_lock(&ctl_lock); + ctlreg_lock(); pcpu_prepare_secondary(pcpu, cpu); pcpu_attach_task(pcpu, tidle); pcpu_start_fn(pcpu, smp_start_secondary, NULL); /* Wait until cpu puts itself in the online & active maps */ while (!cpu_online(cpu)) cpu_relax(); - spin_unlock(&ctl_lock); + ctlreg_unlock(); return 0; } diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index e4a13d7cab6e..80b9c2d039f6 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -12,6 +12,7 @@ #include #include #include +#include #ifdef CONFIG_DEBUG_ENTRY void debug_user_asce(int exit) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 6957d2ed97bf..9d3f9fa6f498 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index dbd5c53d8edf..a191d69573fb 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include -- cgit v1.2.3 From ebe1cd530fb2f6450656758bd904cfb5767f8d33 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:39:56 +0200 Subject: s390/ctlreg: rename ctl_reg.h to ctlreg.h Rename ctl_reg.h to ctlreg.h so it matches not only ctlreg.c but also other control register related function, union, and structure names, which all come with a ctlreg prefix. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/vmem.c | 2 +- arch/s390/include/asm/ctl_reg.h | 148 ------------------------------------ arch/s390/include/asm/ctlreg.h | 148 ++++++++++++++++++++++++++++++++++++ arch/s390/include/asm/mmu_context.h | 2 +- arch/s390/kernel/ctlreg.c | 2 +- arch/s390/kernel/nmi.c | 2 +- arch/s390/kernel/perf_pai_crypto.c | 3 +- arch/s390/kernel/perf_pai_ext.c | 3 +- arch/s390/kernel/smp.c | 2 +- arch/s390/lib/uaccess.c | 2 +- arch/s390/mm/init.c | 2 +- arch/s390/mm/maccess.c | 2 +- arch/s390/mm/vmem.c | 2 +- drivers/s390/char/diag_ftp.c | 2 +- drivers/s390/char/sclp_cmd.c | 2 +- drivers/s390/char/sclp_early.c | 2 +- drivers/s390/char/sclp_early_core.c | 2 +- drivers/s390/cio/crw.c | 2 +- 18 files changed, 164 insertions(+), 166 deletions(-) delete mode 100644 arch/s390/include/asm/ctl_reg.h create mode 100644 arch/s390/include/asm/ctlreg.h diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 36b90864d69f..9ac6d00537c1 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -2,10 +2,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctl_reg.h deleted file mode 100644 index f5536fdb398d..000000000000 --- a/arch/s390/include/asm/ctl_reg.h +++ /dev/null @@ -1,148 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright IBM Corp. 1999, 2009 - * - * Author(s): Martin Schwidefsky - */ - -#ifndef __ASM_CTL_REG_H -#define __ASM_CTL_REG_H - -#include - -#define CR0_CLOCK_COMPARATOR_SIGN BIT(63 - 10) -#define CR0_LOW_ADDRESS_PROTECTION BIT(63 - 35) -#define CR0_FETCH_PROTECTION_OVERRIDE BIT(63 - 38) -#define CR0_STORAGE_PROTECTION_OVERRIDE BIT(63 - 39) -#define CR0_EMERGENCY_SIGNAL_SUBMASK BIT(63 - 49) -#define CR0_EXTERNAL_CALL_SUBMASK BIT(63 - 50) -#define CR0_CLOCK_COMPARATOR_SUBMASK BIT(63 - 52) -#define CR0_CPU_TIMER_SUBMASK BIT(63 - 53) -#define CR0_SERVICE_SIGNAL_SUBMASK BIT(63 - 54) -#define CR0_UNUSED_56 BIT(63 - 56) -#define CR0_INTERRUPT_KEY_SUBMASK BIT(63 - 57) -#define CR0_MEASUREMENT_ALERT_SUBMASK BIT(63 - 58) - -#define CR14_UNUSED_32 BIT(63 - 32) -#define CR14_UNUSED_33 BIT(63 - 33) -#define CR14_CHANNEL_REPORT_SUBMASK BIT(63 - 35) -#define CR14_RECOVERY_SUBMASK BIT(63 - 36) -#define CR14_DEGRADATION_SUBMASK BIT(63 - 37) -#define CR14_EXTERNAL_DAMAGE_SUBMASK BIT(63 - 38) -#define CR14_WARNING_SUBMASK BIT(63 - 39) - -#ifndef __ASSEMBLY__ - -#include - -#define __ctl_load(array, low, high) do { \ - typedef struct { char _[sizeof(array)]; } addrtype; \ - \ - BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ - asm volatile( \ - " lctlg %1,%2,%0\n" \ - : \ - : "Q" (*(addrtype *)(&array)), "i" (low), "i" (high) \ - : "memory"); \ -} while (0) - -#define __ctl_store(array, low, high) do { \ - typedef struct { char _[sizeof(array)]; } addrtype; \ - \ - BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ - asm volatile( \ - " stctg %1,%2,%0\n" \ - : "=Q" (*(addrtype *)(&array)) \ - : "i" (low), "i" (high)); \ -} while (0) - -static __always_inline void __ctl_set_bit(unsigned int cr, unsigned int bit) -{ - unsigned long reg; - - __ctl_store(reg, cr, cr); - reg |= 1UL << bit; - __ctl_load(reg, cr, cr); -} - -static __always_inline void __ctl_clear_bit(unsigned int cr, unsigned int bit) -{ - unsigned long reg; - - __ctl_store(reg, cr, cr); - reg &= ~(1UL << bit); - __ctl_load(reg, cr, cr); -} - -void ctlreg_lock(void); -void ctlreg_unlock(void); -void ctl_set_clear_bit(int cr, int bit, bool set); - -static inline void ctl_set_bit(int cr, int bit) -{ - ctl_set_clear_bit(cr, bit, true); -} - -static inline void ctl_clear_bit(int cr, int bit) -{ - ctl_set_clear_bit(cr, bit, false); -} - -union ctlreg0 { - unsigned long val; - struct { - unsigned long : 8; - unsigned long tcx : 1; /* Transactional-Execution control */ - unsigned long pifo : 1; /* Transactional-Execution Program- - Interruption-Filtering Override */ - unsigned long : 3; - unsigned long ccc : 1; /* Cryptography counter control */ - unsigned long pec : 1; /* PAI extension control */ - unsigned long : 17; - unsigned long : 3; - unsigned long lap : 1; /* Low-address-protection control */ - unsigned long : 4; - unsigned long edat : 1; /* Enhanced-DAT-enablement control */ - unsigned long : 2; - unsigned long iep : 1; /* Instruction-Execution-Protection */ - unsigned long : 1; - unsigned long afp : 1; /* AFP-register control */ - unsigned long vx : 1; /* Vector enablement control */ - unsigned long : 7; - unsigned long sssm : 1; /* Service signal subclass mask */ - unsigned long : 9; - }; -}; - -union ctlreg2 { - unsigned long val; - struct { - unsigned long : 33; - unsigned long ducto : 25; - unsigned long : 1; - unsigned long gse : 1; - unsigned long : 1; - unsigned long tds : 1; - unsigned long tdc : 2; - }; -}; - -union ctlreg5 { - unsigned long val; - struct { - unsigned long : 33; - unsigned long pasteo: 25; - unsigned long : 6; - }; -}; - -union ctlreg15 { - unsigned long val; - struct { - unsigned long lsea : 61; - unsigned long : 3; - }; -}; - -#endif /* __ASSEMBLY__ */ -#endif /* __ASM_CTL_REG_H */ diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h new file mode 100644 index 000000000000..25b955db8998 --- /dev/null +++ b/arch/s390/include/asm/ctlreg.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright IBM Corp. 1999, 2009 + * + * Author(s): Martin Schwidefsky + */ + +#ifndef __ASM_S390_CTLREG_H +#define __ASM_S390_CTLREG_H + +#include + +#define CR0_CLOCK_COMPARATOR_SIGN BIT(63 - 10) +#define CR0_LOW_ADDRESS_PROTECTION BIT(63 - 35) +#define CR0_FETCH_PROTECTION_OVERRIDE BIT(63 - 38) +#define CR0_STORAGE_PROTECTION_OVERRIDE BIT(63 - 39) +#define CR0_EMERGENCY_SIGNAL_SUBMASK BIT(63 - 49) +#define CR0_EXTERNAL_CALL_SUBMASK BIT(63 - 50) +#define CR0_CLOCK_COMPARATOR_SUBMASK BIT(63 - 52) +#define CR0_CPU_TIMER_SUBMASK BIT(63 - 53) +#define CR0_SERVICE_SIGNAL_SUBMASK BIT(63 - 54) +#define CR0_UNUSED_56 BIT(63 - 56) +#define CR0_INTERRUPT_KEY_SUBMASK BIT(63 - 57) +#define CR0_MEASUREMENT_ALERT_SUBMASK BIT(63 - 58) + +#define CR14_UNUSED_32 BIT(63 - 32) +#define CR14_UNUSED_33 BIT(63 - 33) +#define CR14_CHANNEL_REPORT_SUBMASK BIT(63 - 35) +#define CR14_RECOVERY_SUBMASK BIT(63 - 36) +#define CR14_DEGRADATION_SUBMASK BIT(63 - 37) +#define CR14_EXTERNAL_DAMAGE_SUBMASK BIT(63 - 38) +#define CR14_WARNING_SUBMASK BIT(63 - 39) + +#ifndef __ASSEMBLY__ + +#include + +#define __ctl_load(array, low, high) do { \ + typedef struct { char _[sizeof(array)]; } addrtype; \ + \ + BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ + asm volatile( \ + " lctlg %1,%2,%0\n" \ + : \ + : "Q" (*(addrtype *)(&array)), "i" (low), "i" (high) \ + : "memory"); \ +} while (0) + +#define __ctl_store(array, low, high) do { \ + typedef struct { char _[sizeof(array)]; } addrtype; \ + \ + BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ + asm volatile( \ + " stctg %1,%2,%0\n" \ + : "=Q" (*(addrtype *)(&array)) \ + : "i" (low), "i" (high)); \ +} while (0) + +static __always_inline void __ctl_set_bit(unsigned int cr, unsigned int bit) +{ + unsigned long reg; + + __ctl_store(reg, cr, cr); + reg |= 1UL << bit; + __ctl_load(reg, cr, cr); +} + +static __always_inline void __ctl_clear_bit(unsigned int cr, unsigned int bit) +{ + unsigned long reg; + + __ctl_store(reg, cr, cr); + reg &= ~(1UL << bit); + __ctl_load(reg, cr, cr); +} + +void ctlreg_lock(void); +void ctlreg_unlock(void); +void ctl_set_clear_bit(int cr, int bit, bool set); + +static inline void ctl_set_bit(int cr, int bit) +{ + ctl_set_clear_bit(cr, bit, true); +} + +static inline void ctl_clear_bit(int cr, int bit) +{ + ctl_set_clear_bit(cr, bit, false); +} + +union ctlreg0 { + unsigned long val; + struct { + unsigned long : 8; + unsigned long tcx : 1; /* Transactional-Execution control */ + unsigned long pifo : 1; /* Transactional-Execution Program- + Interruption-Filtering Override */ + unsigned long : 3; + unsigned long ccc : 1; /* Cryptography counter control */ + unsigned long pec : 1; /* PAI extension control */ + unsigned long : 17; + unsigned long : 3; + unsigned long lap : 1; /* Low-address-protection control */ + unsigned long : 4; + unsigned long edat : 1; /* Enhanced-DAT-enablement control */ + unsigned long : 2; + unsigned long iep : 1; /* Instruction-Execution-Protection */ + unsigned long : 1; + unsigned long afp : 1; /* AFP-register control */ + unsigned long vx : 1; /* Vector enablement control */ + unsigned long : 7; + unsigned long sssm : 1; /* Service signal subclass mask */ + unsigned long : 9; + }; +}; + +union ctlreg2 { + unsigned long val; + struct { + unsigned long : 33; + unsigned long ducto : 25; + unsigned long : 1; + unsigned long gse : 1; + unsigned long : 1; + unsigned long tds : 1; + unsigned long tdc : 2; + }; +}; + +union ctlreg5 { + unsigned long val; + struct { + unsigned long : 33; + unsigned long pasteo: 25; + unsigned long : 6; + }; +}; + +union ctlreg15 { + unsigned long val; + struct { + unsigned long lsea : 61; + unsigned long : 3; + }; +}; + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_S390_CTLREG_H */ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 2a38af5a00c2..50f247eede60 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #define init_new_context init_new_context diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 176ada8b45c2..a01b901d4866 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -6,7 +6,7 @@ #include #include #include -#include +#include /* * ctl_lock guards access to global control register contents which diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 38ec0487521c..b813d9dfa53c 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -22,13 +22,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index fe7d1774ded1..6280a0e588c5 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -16,8 +16,7 @@ #include #include #include - -#include +#include #include #include diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index c57c1a203256..b8877493dbb6 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -17,8 +17,7 @@ #include #include #include - -#include +#include #include #include diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index d12dbba9d03c..204c72428696 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -37,7 +37,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 80b9c2d039f6..478ab4c7d197 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #ifdef CONFIG_DEBUG_ENTRY void debug_user_asce(int exit) diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 8b94d2212d33..c6ab6f2707d7 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -42,7 +43,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index c805b3e2592b..632c3a55feed 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -15,10 +15,10 @@ #include #include #include -#include #include #include #include +#include unsigned long __bootdata_preserved(__memcpy_real_area); pte_t *__bootdata_preserved(memcpy_real_ptep); diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 9d3f9fa6f498..91c1f42f6a76 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/s390/char/diag_ftp.c b/drivers/s390/char/diag_ftp.c index 65c7f2d565d8..9418a9270d03 100644 --- a/drivers/s390/char/diag_ftp.c +++ b/drivers/s390/char/diag_ftp.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include "hmcdrv_ftp.h" diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index 8b4575a0db9f..b73edf0cd725 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index fdc8668f3fba..60a247fdb2a7 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index a191d69573fb..f7067a70c461 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -6,8 +6,8 @@ #include #include -#include #include +#include #include #include #include diff --git a/drivers/s390/cio/crw.c b/drivers/s390/cio/crw.c index 7b02a6349c4d..58f22270d9c4 100644 --- a/drivers/s390/cio/crw.c +++ b/drivers/s390/cio/crw.c @@ -12,8 +12,8 @@ #include #include #include +#include #include -#include #include "ioasm.h" static DEFINE_MUTEX(crw_handler_mutex); -- cgit v1.2.3 From a74e4fc168d284e1e4e7296f8cf4b884035ebd45 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:39:57 +0200 Subject: s390/ctlreg: cleanup inline assemblies Use symbolic names for operands, remove typedefs, and slightly refactor the code. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 25b955db8998..0b55a1bb3113 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -36,24 +36,37 @@ #include #define __ctl_load(array, low, high) do { \ - typedef struct { char _[sizeof(array)]; } addrtype; \ + struct addrtype { \ + char _[sizeof(array)]; \ + }; \ + int _high = high; \ + int _low = low; \ + int _esize; \ \ - BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ + _esize = (_high - _low + 1) * sizeof(unsigned long); \ + BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ asm volatile( \ - " lctlg %1,%2,%0\n" \ + " lctlg %[_low],%[_high],%[_arr]\n" \ : \ - : "Q" (*(addrtype *)(&array)), "i" (low), "i" (high) \ + : [_arr] "Q" (*(struct addrtype *)(&array)), \ + [_low] "i" (low), [_high] "i" (high) \ : "memory"); \ } while (0) #define __ctl_store(array, low, high) do { \ - typedef struct { char _[sizeof(array)]; } addrtype; \ + struct addrtype { \ + char _[sizeof(array)]; \ + }; \ + int _high = high; \ + int _low = low; \ + int _esize; \ \ - BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ + _esize = (_high - _low + 1) * sizeof(unsigned long); \ + BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ asm volatile( \ - " stctg %1,%2,%0\n" \ - : "=Q" (*(addrtype *)(&array)) \ - : "i" (low), "i" (high)); \ + " stctg %[_low],%[_high],%[_arr]\n" \ + : [_arr] "=Q" (*(struct addrtype *)(&array)) \ + : [_low] "i" (low), [_high] "i" (high)); \ } while (0) static __always_inline void __ctl_set_bit(unsigned int cr, unsigned int bit) -- cgit v1.2.3 From 8d5e98f8d6b11dd0e61323ece3b7ccceea55c281 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:39:58 +0200 Subject: s390/ctlreg: add local and system prefix to some functions Add local and system prefix to some functions to clarify they change control register contents on either the local CPU or the on all CPUs. This results in the following API: Two defines which load and save multiple control registers. The defines correlate with the following C prototypes: void __local_ctl_load(unsigned long *, unsigned int cr_low, unsigned int cr_high); void __local_ctl_store(unsigned long *, unsigned int cr_low, unsigned int cr_high); Two functions which locally set or clear one bit for a specified control register: void local_ctl_set_bit(unsigned int cr, unsigned int bit); void local_ctl_clear_bit(unsigned int cr, unsigned int bit); Two functions which set or clear one bit for a specified control register on all CPUs: void system_ctl_set_bit(unsigned int cr, unsigned int bit); void system_ctl_clear_bit(unsigend int cr, unsigned int bit); Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/startup.c | 2 +- arch/s390/boot/vmem.c | 6 +++--- arch/s390/include/asm/ctlreg.h | 30 +++++++++++++++--------------- arch/s390/include/asm/mmu_context.h | 6 +++--- arch/s390/kernel/ctlreg.c | 26 +++++++++++++------------- arch/s390/kernel/early.c | 12 ++++++------ arch/s390/kernel/guarded_storage.c | 6 +++--- arch/s390/kernel/ipl.c | 2 +- arch/s390/kernel/irq.c | 4 ++-- arch/s390/kernel/kprobes.c | 6 +++--- arch/s390/kernel/machine_kexec.c | 6 +++--- arch/s390/kernel/nmi.c | 18 +++++++++--------- arch/s390/kernel/perf_cpum_cf.c | 2 +- arch/s390/kernel/perf_pai_crypto.c | 4 ++-- arch/s390/kernel/perf_pai_ext.c | 4 ++-- arch/s390/kernel/ptrace.c | 14 +++++++------- arch/s390/kernel/setup.c | 14 +++++++------- arch/s390/kernel/smp.c | 12 ++++++------ arch/s390/kernel/time.c | 4 ++-- arch/s390/kvm/kvm-s390.c | 6 +++--- arch/s390/kvm/priv.c | 2 +- arch/s390/lib/uaccess.c | 4 ++-- arch/s390/mm/pgalloc.c | 2 +- arch/s390/mm/vmem.c | 2 +- arch/s390/pci/pci.c | 2 +- drivers/s390/char/sclp.c | 6 +++--- drivers/s390/char/sclp_early_core.c | 6 +++--- drivers/s390/cio/crw.c | 2 +- drivers/s390/cio/isc.c | 4 ++-- net/iucv/iucv.c | 4 ++-- 30 files changed, 109 insertions(+), 109 deletions(-) diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index d3e48bd9c394..cf622479647f 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -49,7 +49,7 @@ static void detect_facilities(void) { if (test_facility(8)) { machine.has_edat1 = 1; - __ctl_set_bit(0, 23); + local_ctl_set_bit(0, 23); } if (test_facility(78)) machine.has_edat2 = 1; diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 9ac6d00537c1..666fc4e50c1e 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -446,9 +446,9 @@ void setup_vmem(unsigned long asce_limit) S390_lowcore.kernel_asce = swapper_pg_dir | asce_bits; S390_lowcore.user_asce = s390_invalid_asce; - __ctl_load(S390_lowcore.kernel_asce, 1, 1); - __ctl_load(S390_lowcore.user_asce, 7, 7); - __ctl_load(S390_lowcore.kernel_asce, 13, 13); + __local_ctl_load(S390_lowcore.kernel_asce, 1, 1); + __local_ctl_load(S390_lowcore.user_asce, 7, 7); + __local_ctl_load(S390_lowcore.kernel_asce, 13, 13); init_mm.context.asce = S390_lowcore.kernel_asce; } diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 0b55a1bb3113..09d35ab3d1ce 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -35,7 +35,7 @@ #include -#define __ctl_load(array, low, high) do { \ +#define __local_ctl_load(array, low, high) do { \ struct addrtype { \ char _[sizeof(array)]; \ }; \ @@ -53,7 +53,7 @@ : "memory"); \ } while (0) -#define __ctl_store(array, low, high) do { \ +#define __local_ctl_store(array, low, high) do { \ struct addrtype { \ char _[sizeof(array)]; \ }; \ @@ -69,36 +69,36 @@ : [_low] "i" (low), [_high] "i" (high)); \ } while (0) -static __always_inline void __ctl_set_bit(unsigned int cr, unsigned int bit) +static __always_inline void local_ctl_set_bit(unsigned int cr, unsigned int bit) { unsigned long reg; - __ctl_store(reg, cr, cr); + __local_ctl_store(reg, cr, cr); reg |= 1UL << bit; - __ctl_load(reg, cr, cr); + __local_ctl_load(reg, cr, cr); } -static __always_inline void __ctl_clear_bit(unsigned int cr, unsigned int bit) +static __always_inline void local_ctl_clear_bit(unsigned int cr, unsigned int bit) { unsigned long reg; - __ctl_store(reg, cr, cr); + __local_ctl_store(reg, cr, cr); reg &= ~(1UL << bit); - __ctl_load(reg, cr, cr); + __local_ctl_load(reg, cr, cr); } -void ctlreg_lock(void); -void ctlreg_unlock(void); -void ctl_set_clear_bit(int cr, int bit, bool set); +void system_ctlreg_lock(void); +void system_ctlreg_unlock(void); +void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set); -static inline void ctl_set_bit(int cr, int bit) +static inline void system_ctl_set_bit(unsigned int cr, unsigned int bit) { - ctl_set_clear_bit(cr, bit, true); + system_ctl_set_clear_bit(cr, bit, true); } -static inline void ctl_clear_bit(int cr, int bit) +static inline void system_ctl_clear_bit(unsigned int cr, unsigned int bit) { - ctl_set_clear_bit(cr, bit, false); + system_ctl_set_clear_bit(cr, bit, false); } union ctlreg0 { diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 50f247eede60..a72865fce0ad 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -81,7 +81,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct * S390_lowcore.user_asce = next->context.asce; cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); /* Clear previous user-ASCE from CR7 */ - __ctl_load(s390_invalid_asce, 7, 7); + __local_ctl_load(s390_invalid_asce, 7, 7); if (prev != next) cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask); } @@ -111,7 +111,7 @@ static inline void finish_arch_post_lock_switch(void) __tlb_flush_mm_lazy(mm); preempt_enable(); } - __ctl_load(S390_lowcore.user_asce, 7, 7); + __local_ctl_load(S390_lowcore.user_asce, 7, 7); } #define activate_mm activate_mm @@ -120,7 +120,7 @@ static inline void activate_mm(struct mm_struct *prev, { switch_mm(prev, next, current); cpumask_set_cpu(smp_processor_id(), mm_cpumask(next)); - __ctl_load(S390_lowcore.user_asce, 7, 7); + __local_ctl_load(S390_lowcore.user_asce, 7, 7); } #include diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index a01b901d4866..795e7f4861fb 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -13,18 +13,18 @@ * are kept in the control register save area within absolute lowcore * at physical address zero. */ -static DEFINE_SPINLOCK(ctl_lock); +static DEFINE_SPINLOCK(system_ctl_lock); -void ctlreg_lock(void) - __acquires(&ctl_lock) +void system_ctlreg_lock(void) + __acquires(&system_ctl_lock) { - spin_lock(&ctl_lock); + spin_lock(&system_ctl_lock); } -void ctlreg_unlock(void) - __releases(&ctl_lock) +void system_ctlreg_unlock(void) + __releases(&system_ctl_lock) { - spin_unlock(&ctl_lock); + spin_unlock(&system_ctl_lock); } struct ctl_bit_parms { @@ -38,25 +38,25 @@ static void ctl_bit_callback(void *info) struct ctl_bit_parms *pp = info; unsigned long regs[16]; - __ctl_store(regs, 0, 15); + __local_ctl_store(regs, 0, 15); regs[pp->cr] &= pp->andval; regs[pp->cr] |= pp->orval; - __ctl_load(regs, 0, 15); + __local_ctl_load(regs, 0, 15); } -void ctl_set_clear_bit(int cr, int bit, bool set) +void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) { struct ctl_bit_parms pp = { .cr = cr, }; struct lowcore *abs_lc; pp.orval = set ? 1UL << bit : 0; pp.andval = set ? -1UL : ~(1UL << bit); - ctlreg_lock(); + system_ctlreg_lock(); abs_lc = get_abs_lowcore(); abs_lc->cregs_save_area[cr] &= pp.andval; abs_lc->cregs_save_area[cr] |= pp.orval; put_abs_lowcore(abs_lc); on_each_cpu(ctl_bit_callback, &pp, 1); - ctlreg_unlock(); + system_ctlreg_unlock(); } -EXPORT_SYMBOL(ctl_set_clear_bit); +EXPORT_SYMBOL(system_ctl_set_clear_bit); diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 3a54733e4fc6..1f359309cce6 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -216,7 +216,7 @@ static __init void detect_machine_facilities(void) { if (test_facility(8)) { S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT1; - __ctl_set_bit(0, 23); + local_ctl_set_bit(0, 23); } if (test_facility(78)) S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT2; @@ -224,13 +224,13 @@ static __init void detect_machine_facilities(void) S390_lowcore.machine_flags |= MACHINE_FLAG_IDTE; if (test_facility(50) && test_facility(73)) { S390_lowcore.machine_flags |= MACHINE_FLAG_TE; - __ctl_set_bit(0, 55); + local_ctl_set_bit(0, 55); } if (test_facility(51)) S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_LC; if (test_facility(129)) { S390_lowcore.machine_flags |= MACHINE_FLAG_VX; - __ctl_set_bit(0, 17); + local_ctl_set_bit(0, 17); } if (test_facility(130)) S390_lowcore.machine_flags |= MACHINE_FLAG_NX; @@ -240,7 +240,7 @@ static __init void detect_machine_facilities(void) /* Enabled signed clock comparator comparisons */ S390_lowcore.machine_flags |= MACHINE_FLAG_SCC; clock_comparator_max = -1ULL >> 1; - __ctl_set_bit(0, 53); + local_ctl_set_bit(0, 53); } if (IS_ENABLED(CONFIG_PCI) && test_facility(153)) { S390_lowcore.machine_flags |= MACHINE_FLAG_PCI_MIO; @@ -260,7 +260,7 @@ static inline void save_vector_registers(void) static inline void setup_low_address_protection(void) { - __ctl_set_bit(0, 28); + local_ctl_set_bit(0, 28); } static inline void setup_access_registers(void) @@ -273,7 +273,7 @@ static inline void setup_access_registers(void) static int __init disable_vector_extension(char *str) { S390_lowcore.machine_flags &= ~MACHINE_FLAG_VX; - __ctl_clear_bit(0, 17); + local_ctl_clear_bit(0, 17); return 0; } early_param("novx", disable_vector_extension); diff --git a/arch/s390/kernel/guarded_storage.c b/arch/s390/kernel/guarded_storage.c index d14dd1c2e524..945acfc77ee4 100644 --- a/arch/s390/kernel/guarded_storage.c +++ b/arch/s390/kernel/guarded_storage.c @@ -28,7 +28,7 @@ static int gs_enable(void) return -ENOMEM; gs_cb->gsd = 25; preempt_disable(); - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); load_gs_cb(gs_cb); current->thread.gs_cb = gs_cb; preempt_enable(); @@ -42,7 +42,7 @@ static int gs_disable(void) preempt_disable(); kfree(current->thread.gs_cb); current->thread.gs_cb = NULL; - __ctl_clear_bit(2, 4); + local_ctl_clear_bit(2, 4); preempt_enable(); } return 0; @@ -84,7 +84,7 @@ void gs_load_bc_cb(struct pt_regs *regs) if (gs_cb) { kfree(current->thread.gs_cb); current->thread.gs_bc_cb = NULL; - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); load_gs_cb(gs_cb); current->thread.gs_cb = gs_cb; } diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 05e51666db03..492dd8f371da 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -2381,7 +2381,7 @@ void s390_reset_system(void) set_prefix(0); /* Disable lowcore protection */ - __ctl_clear_bit(0, 28); + local_ctl_clear_bit(0, 28); diag_amode31_ops.diag308_reset(); } diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index b020ff17d206..6f71b0ce1068 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -385,7 +385,7 @@ void irq_subclass_register(enum irq_subclass subclass) { spin_lock(&irq_subclass_lock); if (!irq_subclass_refcount[subclass]) - ctl_set_bit(0, subclass); + system_ctl_set_bit(0, subclass); irq_subclass_refcount[subclass]++; spin_unlock(&irq_subclass_lock); } @@ -396,7 +396,7 @@ void irq_subclass_unregister(enum irq_subclass subclass) spin_lock(&irq_subclass_lock); irq_subclass_refcount[subclass]--; if (!irq_subclass_refcount[subclass]) - ctl_clear_bit(0, subclass); + system_ctl_clear_bit(0, subclass); spin_unlock(&irq_subclass_lock); } EXPORT_SYMBOL(irq_subclass_unregister); diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index d4b863ed0aa7..1b2e1903a54c 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -232,12 +232,12 @@ static void enable_singlestep(struct kprobe_ctlblk *kcb, per_kprobe.end = ip; /* Save control regs and psw mask */ - __ctl_store(kcb->kprobe_saved_ctl, 9, 11); + __local_ctl_store(kcb->kprobe_saved_ctl, 9, 11); kcb->kprobe_saved_imask = regs->psw.mask & (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT); /* Set PER control regs, turns on single step for the given address */ - __ctl_load(per_kprobe, 9, 11); + __local_ctl_load(per_kprobe, 9, 11); regs->psw.mask |= PSW_MASK_PER; regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); regs->psw.addr = ip; @@ -249,7 +249,7 @@ static void disable_singlestep(struct kprobe_ctlblk *kcb, unsigned long ip) { /* Restore control regs and psw mask, set new psw address */ - __ctl_load(kcb->kprobe_saved_ctl, 9, 11); + __local_ctl_load(kcb->kprobe_saved_ctl, 9, 11); regs->psw.mask &= ~PSW_MASK_PER; regs->psw.mask |= kcb->kprobe_saved_imask; regs->psw.addr = ip; diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index ce65fc01671f..07152e74afb1 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -94,12 +94,12 @@ static noinline void __machine_kdump(void *image) if (MACHINE_HAS_VX) save_vx_regs((__vector128 *) mcesa->vector_save_area); if (MACHINE_HAS_GS) { - __ctl_store(cr2_old.val, 2, 2); + __local_ctl_store(cr2_old.val, 2, 2); cr2_new = cr2_old; cr2_new.gse = 1; - __ctl_load(cr2_new.val, 2, 2); + __local_ctl_load(cr2_new.val, 2, 2); save_gs_cb((struct gs_cb *) mcesa->guarded_storage_save_area); - __ctl_load(cr2_old.val, 2, 2); + __local_ctl_load(cr2_old.val, 2, 2); } /* * To create a good backchain for this CPU in the dump store_status diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index b813d9dfa53c..927bd83ac9c0 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -131,10 +131,10 @@ static notrace void s390_handle_damage(void) * Disable low address protection and make machine check new PSW a * disabled wait PSW. Any additional machine check cannot be handled. */ - __ctl_store(cr0.val, 0, 0); + __local_ctl_store(cr0.val, 0, 0); cr0_new = cr0; cr0_new.lap = 0; - __ctl_load(cr0_new.val, 0, 0); + __local_ctl_load(cr0_new.val, 0, 0); psw_save = S390_lowcore.mcck_new_psw; psw_bits(S390_lowcore.mcck_new_psw).io = 0; psw_bits(S390_lowcore.mcck_new_psw).ext = 0; @@ -146,7 +146,7 @@ static notrace void s390_handle_damage(void) * values. This makes possible system dump analysis easier. */ S390_lowcore.mcck_new_psw = psw_save; - __ctl_load(cr0.val, 0, 0); + __local_ctl_load(cr0.val, 0, 0); disabled_wait(); while (1); } @@ -185,7 +185,7 @@ void s390_handle_mcck(void) static int mchchk_wng_posted = 0; /* Use single cpu clear, as we cannot handle smp here. */ - __ctl_clear_bit(14, 24); /* Disable WARNING MCH */ + local_ctl_clear_bit(14, 24); /* Disable WARNING MCH */ if (xchg(&mchchk_wng_posted, 1) == 0) kill_cad_pid(SIGPWR, 1); } @@ -271,7 +271,7 @@ static int notrace s390_validate_registers(union mci mci) kill_task = 1; cr0.val = S390_lowcore.cregs_save_area[0]; cr0.afp = cr0.vx = 1; - __ctl_load(cr0.val, 0, 0); + __local_ctl_load(cr0.val, 0, 0); asm volatile( " la 1,%0\n" " VLM 0,15,0,1\n" @@ -279,7 +279,7 @@ static int notrace s390_validate_registers(union mci mci) : : "Q" (*(struct vx_array *)mcesa->vector_save_area) : "1"); - __ctl_load(S390_lowcore.cregs_save_area[0], 0, 0); + __local_ctl_load(S390_lowcore.cregs_save_area[0], 0, 0); } /* Validate access registers */ asm volatile( @@ -505,9 +505,9 @@ NOKPROBE_SYMBOL(s390_do_machine_check); static int __init machine_check_init(void) { - ctl_set_bit(14, 25); /* enable external damage MCH */ - ctl_set_bit(14, 27); /* enable system recovery MCH */ - ctl_set_bit(14, 24); /* enable warning MCH */ + system_ctl_set_bit(14, 25); /* enable external damage MCH */ + system_ctl_set_bit(14, 27); /* enable system recovery MCH */ + system_ctl_set_bit(14, 24); /* enable warning MCH */ return 0; } early_initcall(machine_check_init); diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 850c11ea631a..4e05c0cb1648 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -1193,7 +1193,7 @@ static int __init cpumf_pmu_init(void) * Clear bit 15 of cr0 to unauthorize problem-state to * extract measurement counters */ - ctl_clear_bit(0, 48); + system_ctl_clear_bit(0, 48); /* register handler for measurement-alert interruptions */ rc = register_external_irq(EXT_IRQ_MEASURE_ALERT, diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 6280a0e588c5..6612fd13e95b 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -265,7 +265,7 @@ static int paicrypt_add(struct perf_event *event, int flags) if (++cpump->active_events == 1) { ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET; WRITE_ONCE(S390_lowcore.ccd, ccd); - __ctl_set_bit(0, 50); + local_ctl_set_bit(0, 50); } cpump->event = event; if (flags & PERF_EF_START && !event->attr.sample_period) { @@ -294,7 +294,7 @@ static void paicrypt_del(struct perf_event *event, int flags) /* Only counting needs to read counter */ paicrypt_stop(event, PERF_EF_UPDATE); if (--cpump->active_events == 0) { - __ctl_clear_bit(0, 50); + local_ctl_clear_bit(0, 50); WRITE_ONCE(S390_lowcore.ccd, 0); } } diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index b8877493dbb6..9f724ae44fc7 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -339,7 +339,7 @@ static int paiext_add(struct perf_event *event, int flags) S390_lowcore.aicd = virt_to_phys(cpump->paiext_cb); pcb->acc = virt_to_phys(cpump->area) | 0x1; /* Enable CPU instruction lookup for PAIE1 control block */ - __ctl_set_bit(0, 49); + local_ctl_set_bit(0, 49); debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", __func__, S390_lowcore.aicd, pcb->acc); } @@ -375,7 +375,7 @@ static void paiext_del(struct perf_event *event, int flags) } if (--cpump->active_events == 0) { /* Disable CPU instruction lookup for PAIE1 control block */ - __ctl_clear_bit(0, 49); + local_ctl_clear_bit(0, 49); pcb->acc = 0; S390_lowcore.aicd = 0; debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index ea244a73efad..1824c1208852 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -46,8 +46,8 @@ void update_cr_regs(struct task_struct *task) union ctlreg2 cr2_old, cr2_new; int cr0_changed, cr2_changed; - __ctl_store(cr0_old.val, 0, 0); - __ctl_store(cr2_old.val, 2, 2); + __local_ctl_store(cr0_old.val, 0, 0); + __local_ctl_store(cr2_old.val, 2, 2); cr0_new = cr0_old; cr2_new = cr2_old; /* Take care of the enable/disable of transactional execution. */ @@ -75,9 +75,9 @@ void update_cr_regs(struct task_struct *task) cr0_changed = cr0_new.val != cr0_old.val; cr2_changed = cr2_new.val != cr2_old.val; if (cr0_changed) - __ctl_load(cr0_new.val, 0, 0); + __local_ctl_load(cr0_new.val, 0, 0); if (cr2_changed) - __ctl_load(cr2_new.val, 2, 2); + __local_ctl_load(cr2_new.val, 2, 2); /* Copy user specified PER registers */ new.control = thread->per_user.control; new.start = thread->per_user.start; @@ -104,9 +104,9 @@ void update_cr_regs(struct task_struct *task) return; } regs->psw.mask |= PSW_MASK_PER; - __ctl_store(old, 9, 11); + __local_ctl_store(old, 9, 11); if (memcmp(&new, &old, sizeof(struct per_regs)) != 0) - __ctl_load(new, 9, 11); + __local_ctl_load(new, 9, 11); } void user_enable_single_step(struct task_struct *task) @@ -1107,7 +1107,7 @@ static int s390_gs_cb_set(struct task_struct *target, target->thread.gs_cb = data; *target->thread.gs_cb = gs_cb; if (target == current) { - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); restore_gs_cb(target->thread.gs_cb); } preempt_enable(); diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index ef158bd0ea7b..c516e2e909e6 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -449,7 +449,7 @@ static void __init setup_lowcore(void) lc->restart_fn = (unsigned long) do_restart; lc->restart_data = 0; lc->restart_source = -1U; - __ctl_store(lc->cregs_save_area, 0, 15); + __local_ctl_store(lc->cregs_save_area, 0, 15); lc->spinlock_lockval = arch_spin_lockval(0); lc->spinlock_index = 0; arch_spin_lock_setup(0); @@ -791,15 +791,15 @@ static void __init setup_cr(void) __ctl_duct[4] = (unsigned long)__ctl_duald; /* Update control registers CR2, CR5 and CR15 */ - __ctl_store(cr2.val, 2, 2); - __ctl_store(cr5.val, 5, 5); - __ctl_store(cr15.val, 15, 15); + __local_ctl_store(cr2.val, 2, 2); + __local_ctl_store(cr5.val, 5, 5); + __local_ctl_store(cr15.val, 15, 15); cr2.ducto = (unsigned long)__ctl_duct >> 6; cr5.pasteo = (unsigned long)__ctl_duct >> 6; cr15.lsea = (unsigned long)__ctl_linkage_stack >> 3; - __ctl_load(cr2.val, 2, 2); - __ctl_load(cr5.val, 5, 5); - __ctl_load(cr15.val, 15, 15); + __local_ctl_load(cr2.val, 2, 2); + __local_ctl_load(cr5.val, 5, 5); + __local_ctl_load(cr15.val, 15, 15); } /* diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 204c72428696..1c65bbd411c7 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -888,14 +888,14 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) * Make sure global control register contents do not change * until new CPU has initialized control registers. */ - ctlreg_lock(); + system_ctlreg_lock(); pcpu_prepare_secondary(pcpu, cpu); pcpu_attach_task(pcpu, tidle); pcpu_start_fn(pcpu, smp_start_secondary, NULL); /* Wait until cpu puts itself in the online & active maps */ while (!cpu_online(cpu)) cpu_relax(); - ctlreg_unlock(); + system_ctlreg_unlock(); return 0; } @@ -922,11 +922,11 @@ int __cpu_disable(void) /* Disable pseudo page faults on this cpu. */ pfault_fini(); /* Disable interrupt sources via control register. */ - __ctl_store(cregs, 0, 15); + __local_ctl_store(cregs, 0, 15); cregs[0] &= ~0x0000ee70UL; /* disable all external interrupts */ cregs[6] &= ~0xff000000UL; /* disable all I/O interrupts */ cregs[14] &= ~0x1f000000UL; /* disable most machine checks */ - __ctl_load(cregs, 0, 15); + __local_ctl_load(cregs, 0, 15); clear_cpu_flag(CIF_NOHZ_DELAY); return 0; } @@ -968,10 +968,10 @@ void __init smp_prepare_cpus(unsigned int max_cpus) { if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1201"); - ctl_set_bit(0, 14); + system_ctl_set_bit(0, 14); if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1202"); - ctl_set_bit(0, 13); + system_ctl_set_bit(0, 13); } void __init smp_prepare_boot_cpu(void) diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index d34d3548c046..34c4fbf4e555 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -173,10 +173,10 @@ void init_cpu_timer(void) clockevents_register_device(cd); /* Enable clock comparator timer interrupt. */ - __ctl_set_bit(0,11); + local_ctl_set_bit(0, 11); /* Always allow the timing alert external interrupt. */ - __ctl_set_bit(0, 4); + local_ctl_set_bit(0, 4); } static void clock_comparator_interrupt(struct ext_code ext_code, diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index b3f17e014cab..92bd15fe769a 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4918,7 +4918,7 @@ static void sync_regs_fmt2(struct kvm_vcpu *vcpu) } if (MACHINE_HAS_GS) { preempt_disable(); - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); if (current->thread.gs_cb) { vcpu->arch.host_gscb = current->thread.gs_cb; save_gs_cb(vcpu->arch.host_gscb); @@ -4995,13 +4995,13 @@ static void store_regs_fmt2(struct kvm_vcpu *vcpu) kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val; if (MACHINE_HAS_GS) { preempt_disable(); - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); if (vcpu->arch.gs_enabled) save_gs_cb(current->thread.gs_cb); current->thread.gs_cb = vcpu->arch.host_gscb; restore_gs_cb(vcpu->arch.host_gscb); if (!vcpu->arch.host_gscb) - __ctl_clear_bit(2, 4); + local_ctl_clear_bit(2, 4); vcpu->arch.host_gscb = NULL; preempt_enable(); } diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index dc4cfa8795c0..c199e4b5e5ce 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -57,7 +57,7 @@ static int handle_gs(struct kvm_vcpu *vcpu) if (test_kvm_facility(vcpu->kvm, 133)) { VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)"); preempt_disable(); - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); current->thread.gs_cb = (struct gs_cb *)&vcpu->run->s.regs.gscb; restore_gs_cb(current->thread.gs_cb); preempt_enable(); diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 478ab4c7d197..551de5394be6 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -19,8 +19,8 @@ void debug_user_asce(int exit) { unsigned long cr1, cr7; - __ctl_store(cr1, 1, 1); - __ctl_store(cr7, 7, 7); + __local_ctl_store(cr1, 1, 1); + __local_ctl_store(cr7, 7, 7); if (cr1 == S390_lowcore.kernel_asce && cr7 == S390_lowcore.user_asce) return; panic("incorrect ASCE on kernel %s\n" diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 07fc660a24aa..43bef9cce9b2 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -63,7 +63,7 @@ static void __crst_table_upgrade(void *arg) /* change all active ASCEs to avoid the creation of new TLBs */ if (current->active_mm == mm) { S390_lowcore.user_asce = mm->context.asce; - __ctl_load(S390_lowcore.user_asce, 7, 7); + __local_ctl_load(S390_lowcore.user_asce, 7, 7); } __tlb_flush_local(); } diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 91c1f42f6a76..95f04a502e62 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -660,7 +660,7 @@ void __init vmem_map_init(void) __set_memory_4k(__va(0), RELOC_HIDE(__va(0), ident_map_size)); } if (MACHINE_HAS_NX) - ctl_set_bit(0, 20); + system_ctl_set_bit(0, 20); pr_info("Write protected kernel read-only data: %luk\n", (unsigned long)(__end_rodata - _stext) >> 10); } diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index d34d5813d006..a0aecc94e284 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -1094,7 +1094,7 @@ static int __init pci_base_init(void) if (MACHINE_HAS_PCI_MIO) { static_branch_enable(&have_mio); - ctl_set_bit(2, 5); + system_ctl_set_bit(2, 5); } rc = zpci_debug_init(); diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index 8f74db689a0c..1ebc60d9cedc 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -732,10 +732,10 @@ sclp_sync_wait(void) /* Enable service-signal interruption, disable timer interrupts */ old_tick = local_tick_disable(); trace_hardirqs_on(); - __ctl_store(cr0, 0, 0); + __local_ctl_store(cr0, 0, 0); cr0_sync = cr0 & ~CR0_IRQ_SUBCLASS_MASK; cr0_sync |= 1UL << (63 - 54); - __ctl_load(cr0_sync, 0, 0); + __local_ctl_load(cr0_sync, 0, 0); __arch_local_irq_stosm(0x01); /* Loop until driver state indicates finished request */ while (sclp_running_state != sclp_running_state_idle) { @@ -745,7 +745,7 @@ sclp_sync_wait(void) cpu_relax(); } local_irq_disable(); - __ctl_load(cr0, 0, 0); + __local_ctl_load(cr0, 0, 0); if (!irq_context) _local_bh_enable(); local_tick_enable(old_tick); diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index f7067a70c461..fa5b7d63a254 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -32,11 +32,11 @@ void sclp_early_wait_irq(void) psw_t psw_ext_save, psw_wait; union ctlreg0 cr0, cr0_new; - __ctl_store(cr0.val, 0, 0); + __local_ctl_store(cr0.val, 0, 0); cr0_new.val = cr0.val & ~CR0_IRQ_SUBCLASS_MASK; cr0_new.lap = 0; cr0_new.sssm = 1; - __ctl_load(cr0_new.val, 0, 0); + __local_ctl_load(cr0_new.val, 0, 0); psw_ext_save = S390_lowcore.external_new_psw; psw_mask = __extract_psw(); @@ -59,7 +59,7 @@ void sclp_early_wait_irq(void) } while (S390_lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG); S390_lowcore.external_new_psw = psw_ext_save; - __ctl_load(cr0.val, 0, 0); + __local_ctl_load(cr0.val, 0, 0); } int sclp_early_cmd(sclp_cmdw_t cmd, void *sccb) diff --git a/drivers/s390/cio/crw.c b/drivers/s390/cio/crw.c index 58f22270d9c4..872b1f2855cb 100644 --- a/drivers/s390/cio/crw.c +++ b/drivers/s390/cio/crw.c @@ -156,7 +156,7 @@ static int __init crw_machine_check_init(void) task = kthread_run(crw_collect_info, NULL, "kmcheck"); if (IS_ERR(task)) return PTR_ERR(task); - ctl_set_bit(14, 28); /* enable channel report MCH */ + system_ctl_set_bit(14, 28); /* enable channel report MCH */ return 0; } device_initcall(crw_machine_check_init); diff --git a/drivers/s390/cio/isc.c b/drivers/s390/cio/isc.c index 77fde9f5ea8b..dbc2ac7711e5 100644 --- a/drivers/s390/cio/isc.c +++ b/drivers/s390/cio/isc.c @@ -33,7 +33,7 @@ void isc_register(unsigned int isc) spin_lock(&isc_ref_lock); if (isc_refs[isc] == 0) - ctl_set_bit(6, 31 - isc); + system_ctl_set_bit(6, 31 - isc); isc_refs[isc]++; spin_unlock(&isc_ref_lock); } @@ -61,7 +61,7 @@ void isc_unregister(unsigned int isc) goto out_unlock; } if (isc_refs[isc] == 1) - ctl_clear_bit(6, 31 - isc); + system_ctl_clear_bit(6, 31 - isc); isc_refs[isc]--; out_unlock: spin_unlock(&isc_ref_lock); diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c index fc3fddeb6f36..b5cee8dad926 100644 --- a/net/iucv/iucv.c +++ b/net/iucv/iucv.c @@ -1823,7 +1823,7 @@ static int __init iucv_init(void) rc = -EPROTONOSUPPORT; goto out; } - ctl_set_bit(0, 1); + system_ctl_set_bit(0, 1); rc = iucv_query_maxconn(); if (rc) goto out_ctl; @@ -1871,7 +1871,7 @@ out_dev: out_int: unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt); out_ctl: - ctl_clear_bit(0, 1); + system_ctl_clear_bit(0, 1); out: return rc; } -- cgit v1.2.3 From dfa33ce1245a4b88402947fa0a847e179044d2fc Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:39:59 +0200 Subject: s390/ctlreg: add local_ctl_load() and local_ctl_store() Add local_ctl_load() and local_ctl_store() which load and store contents for only a single control register. This allows for easier to read code, but also better type checking, since __local_ctl_load() and __local_ctl_store() do not come with any type checking at all (which will be changed with subsequent patches). Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 09d35ab3d1ce..e8e5aef08bfd 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -69,6 +69,23 @@ : [_low] "i" (low), [_high] "i" (high)); \ } while (0) +static __always_inline void local_ctl_load(unsigned int cr, unsigned long *reg) +{ + asm volatile( + " lctlg %[cr],%[cr],%[reg]\n" + : + : [reg] "Q" (*reg), [cr] "i" (cr) + : "memory"); +} + +static __always_inline void local_ctl_store(unsigned int cr, unsigned long *reg) +{ + asm volatile( + " stctg %[cr],%[cr],%[reg]\n" + : [reg] "=Q" (*reg) + : [cr] "i" (cr)); +} + static __always_inline void local_ctl_set_bit(unsigned int cr, unsigned int bit) { unsigned long reg; -- cgit v1.2.3 From 2372d391421350e318c98844d21ab9ad16e3eac0 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:00 +0200 Subject: s390/ctlreg: use local_ctl_load() and local_ctl_store() where possible Convert all single control register usages of __local_ctl_load() and __local_ctl_store() to local_ctl_load() and local_ctl_store(). This also requires to change the type of some struct lowcore members from __u64 to unsigned long. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/vmem.c | 6 +++--- arch/s390/include/asm/ctlreg.h | 8 ++++---- arch/s390/include/asm/lowcore.h | 6 +++--- arch/s390/include/asm/mmu_context.h | 6 +++--- arch/s390/kernel/machine_kexec.c | 6 +++--- arch/s390/kernel/nmi.c | 10 +++++----- arch/s390/kernel/ptrace.c | 8 ++++---- arch/s390/kernel/setup.c | 12 ++++++------ arch/s390/lib/uaccess.c | 6 +++--- arch/s390/mm/pgalloc.c | 2 +- drivers/s390/char/sclp.c | 6 +++--- drivers/s390/char/sclp_early_core.c | 6 +++--- 12 files changed, 41 insertions(+), 41 deletions(-) diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 666fc4e50c1e..44278ae2512f 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -446,9 +446,9 @@ void setup_vmem(unsigned long asce_limit) S390_lowcore.kernel_asce = swapper_pg_dir | asce_bits; S390_lowcore.user_asce = s390_invalid_asce; - __local_ctl_load(S390_lowcore.kernel_asce, 1, 1); - __local_ctl_load(S390_lowcore.user_asce, 7, 7); - __local_ctl_load(S390_lowcore.kernel_asce, 13, 13); + local_ctl_load(1, &S390_lowcore.kernel_asce); + local_ctl_load(7, &S390_lowcore.user_asce); + local_ctl_load(13, &S390_lowcore.kernel_asce); init_mm.context.asce = S390_lowcore.kernel_asce; } diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index e8e5aef08bfd..86887b490bb5 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -90,18 +90,18 @@ static __always_inline void local_ctl_set_bit(unsigned int cr, unsigned int bit) { unsigned long reg; - __local_ctl_store(reg, cr, cr); + local_ctl_store(cr, ®); reg |= 1UL << bit; - __local_ctl_load(reg, cr, cr); + local_ctl_load(cr, ®); } static __always_inline void local_ctl_clear_bit(unsigned int cr, unsigned int bit) { unsigned long reg; - __local_ctl_store(reg, cr, cr); + local_ctl_store(cr, ®); reg &= ~(1UL << bit); - __local_ctl_load(reg, cr, cr); + local_ctl_load(cr, ®); } void system_ctlreg_lock(void); diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 69ccc464a430..2174f00e188b 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -139,8 +139,8 @@ struct lowcore { __u32 restart_flags; /* 0x0384 */ /* Address space pointer. */ - __u64 kernel_asce; /* 0x0388 */ - __u64 user_asce; /* 0x0390 */ + unsigned long kernel_asce; /* 0x0388 */ + unsigned long user_asce; /* 0x0390 */ /* * The lpp and current_pid fields form a @@ -199,7 +199,7 @@ struct lowcore { __u32 clock_comp_save_area[2]; /* 0x1330 */ __u64 last_break_save_area; /* 0x1338 */ __u32 access_regs_save_area[16]; /* 0x1340 */ - __u64 cregs_save_area[16]; /* 0x1380 */ + unsigned long cregs_save_area[16]; /* 0x1380 */ __u8 pad_0x1400[0x1500-0x1400]; /* 0x1400 */ /* Cryptography-counter designation */ __u64 ccd; /* 0x1500 */ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index a72865fce0ad..0e93275f80f0 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -81,7 +81,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct * S390_lowcore.user_asce = next->context.asce; cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); /* Clear previous user-ASCE from CR7 */ - __local_ctl_load(s390_invalid_asce, 7, 7); + local_ctl_load(7, &s390_invalid_asce); if (prev != next) cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask); } @@ -111,7 +111,7 @@ static inline void finish_arch_post_lock_switch(void) __tlb_flush_mm_lazy(mm); preempt_enable(); } - __local_ctl_load(S390_lowcore.user_asce, 7, 7); + local_ctl_load(7, &S390_lowcore.user_asce); } #define activate_mm activate_mm @@ -120,7 +120,7 @@ static inline void activate_mm(struct mm_struct *prev, { switch_mm(prev, next, current); cpumask_set_cpu(smp_processor_id(), mm_cpumask(next)); - __local_ctl_load(S390_lowcore.user_asce, 7, 7); + local_ctl_load(7, &S390_lowcore.user_asce); } #include diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index 07152e74afb1..fe9d65060fa4 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -94,12 +94,12 @@ static noinline void __machine_kdump(void *image) if (MACHINE_HAS_VX) save_vx_regs((__vector128 *) mcesa->vector_save_area); if (MACHINE_HAS_GS) { - __local_ctl_store(cr2_old.val, 2, 2); + local_ctl_store(2, &cr2_old.val); cr2_new = cr2_old; cr2_new.gse = 1; - __local_ctl_load(cr2_new.val, 2, 2); + local_ctl_load(2, &cr2_new.val); save_gs_cb((struct gs_cb *) mcesa->guarded_storage_save_area); - __local_ctl_load(cr2_old.val, 2, 2); + local_ctl_load(2, &cr2_old.val); } /* * To create a good backchain for this CPU in the dump store_status diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 927bd83ac9c0..7880a42896a3 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -131,10 +131,10 @@ static notrace void s390_handle_damage(void) * Disable low address protection and make machine check new PSW a * disabled wait PSW. Any additional machine check cannot be handled. */ - __local_ctl_store(cr0.val, 0, 0); + local_ctl_store(0, &cr0.val); cr0_new = cr0; cr0_new.lap = 0; - __local_ctl_load(cr0_new.val, 0, 0); + local_ctl_load(0, &cr0_new.val); psw_save = S390_lowcore.mcck_new_psw; psw_bits(S390_lowcore.mcck_new_psw).io = 0; psw_bits(S390_lowcore.mcck_new_psw).ext = 0; @@ -146,7 +146,7 @@ static notrace void s390_handle_damage(void) * values. This makes possible system dump analysis easier. */ S390_lowcore.mcck_new_psw = psw_save; - __local_ctl_load(cr0.val, 0, 0); + local_ctl_load(0, &cr0.val); disabled_wait(); while (1); } @@ -271,7 +271,7 @@ static int notrace s390_validate_registers(union mci mci) kill_task = 1; cr0.val = S390_lowcore.cregs_save_area[0]; cr0.afp = cr0.vx = 1; - __local_ctl_load(cr0.val, 0, 0); + local_ctl_load(0, &cr0.val); asm volatile( " la 1,%0\n" " VLM 0,15,0,1\n" @@ -279,7 +279,7 @@ static int notrace s390_validate_registers(union mci mci) : : "Q" (*(struct vx_array *)mcesa->vector_save_area) : "1"); - __local_ctl_load(S390_lowcore.cregs_save_area[0], 0, 0); + local_ctl_load(0, &S390_lowcore.cregs_save_area[0]); } /* Validate access registers */ asm volatile( diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 1824c1208852..a1fb4536784f 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -46,8 +46,8 @@ void update_cr_regs(struct task_struct *task) union ctlreg2 cr2_old, cr2_new; int cr0_changed, cr2_changed; - __local_ctl_store(cr0_old.val, 0, 0); - __local_ctl_store(cr2_old.val, 2, 2); + local_ctl_store(0, &cr0_old.val); + local_ctl_store(2, &cr2_old.val); cr0_new = cr0_old; cr2_new = cr2_old; /* Take care of the enable/disable of transactional execution. */ @@ -75,9 +75,9 @@ void update_cr_regs(struct task_struct *task) cr0_changed = cr0_new.val != cr0_old.val; cr2_changed = cr2_new.val != cr2_old.val; if (cr0_changed) - __local_ctl_load(cr0_new.val, 0, 0); + local_ctl_load(0, &cr0_new.val); if (cr2_changed) - __local_ctl_load(cr2_new.val, 2, 2); + local_ctl_load(2, &cr2_new.val); /* Copy user specified PER registers */ new.control = thread->per_user.control; new.start = thread->per_user.start; diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index c516e2e909e6..48a456a10206 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -791,15 +791,15 @@ static void __init setup_cr(void) __ctl_duct[4] = (unsigned long)__ctl_duald; /* Update control registers CR2, CR5 and CR15 */ - __local_ctl_store(cr2.val, 2, 2); - __local_ctl_store(cr5.val, 5, 5); - __local_ctl_store(cr15.val, 15, 15); + local_ctl_store(2, &cr2.val); + local_ctl_store(5, &cr5.val); + local_ctl_store(15, &cr15.val); cr2.ducto = (unsigned long)__ctl_duct >> 6; cr5.pasteo = (unsigned long)__ctl_duct >> 6; cr15.lsea = (unsigned long)__ctl_linkage_stack >> 3; - __local_ctl_load(cr2.val, 2, 2); - __local_ctl_load(cr5.val, 5, 5); - __local_ctl_load(cr15.val, 15, 15); + local_ctl_load(2, &cr2.val); + local_ctl_load(5, &cr5.val); + local_ctl_load(15, &cr15.val); } /* diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 551de5394be6..56f7eb90b44d 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -19,13 +19,13 @@ void debug_user_asce(int exit) { unsigned long cr1, cr7; - __local_ctl_store(cr1, 1, 1); - __local_ctl_store(cr7, 7, 7); + local_ctl_store(1, &cr1); + local_ctl_store(7, &cr7); if (cr1 == S390_lowcore.kernel_asce && cr7 == S390_lowcore.user_asce) return; panic("incorrect ASCE on kernel %s\n" "cr1: %016lx cr7: %016lx\n" - "kernel: %016llx user: %016llx\n", + "kernel: %016lx user: %016lx\n", exit ? "exit" : "entry", cr1, cr7, S390_lowcore.kernel_asce, S390_lowcore.user_asce); } diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 43bef9cce9b2..27720a687022 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -63,7 +63,7 @@ static void __crst_table_upgrade(void *arg) /* change all active ASCEs to avoid the creation of new TLBs */ if (current->active_mm == mm) { S390_lowcore.user_asce = mm->context.asce; - __local_ctl_load(S390_lowcore.user_asce, 7, 7); + local_ctl_load(7, &S390_lowcore.user_asce); } __tlb_flush_local(); } diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index 1ebc60d9cedc..30a7cd9748fe 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -732,10 +732,10 @@ sclp_sync_wait(void) /* Enable service-signal interruption, disable timer interrupts */ old_tick = local_tick_disable(); trace_hardirqs_on(); - __local_ctl_store(cr0, 0, 0); + local_ctl_store(0, &cr0); cr0_sync = cr0 & ~CR0_IRQ_SUBCLASS_MASK; cr0_sync |= 1UL << (63 - 54); - __local_ctl_load(cr0_sync, 0, 0); + local_ctl_load(0, &cr0_sync); __arch_local_irq_stosm(0x01); /* Loop until driver state indicates finished request */ while (sclp_running_state != sclp_running_state_idle) { @@ -745,7 +745,7 @@ sclp_sync_wait(void) cpu_relax(); } local_irq_disable(); - __local_ctl_load(cr0, 0, 0); + local_ctl_load(0, &cr0); if (!irq_context) _local_bh_enable(); local_tick_enable(old_tick); diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index fa5b7d63a254..8cb9fb7098e2 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -32,11 +32,11 @@ void sclp_early_wait_irq(void) psw_t psw_ext_save, psw_wait; union ctlreg0 cr0, cr0_new; - __local_ctl_store(cr0.val, 0, 0); + local_ctl_store(0, &cr0.val); cr0_new.val = cr0.val & ~CR0_IRQ_SUBCLASS_MASK; cr0_new.lap = 0; cr0_new.sssm = 1; - __local_ctl_load(cr0_new.val, 0, 0); + local_ctl_load(0, &cr0_new.val); psw_ext_save = S390_lowcore.external_new_psw; psw_mask = __extract_psw(); @@ -59,7 +59,7 @@ void sclp_early_wait_irq(void) } while (S390_lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG); S390_lowcore.external_new_psw = psw_ext_save; - __local_ctl_load(cr0.val, 0, 0); + local_ctl_load(0, &cr0.val); } int sclp_early_cmd(sclp_cmdw_t cmd, void *sccb) -- cgit v1.2.3 From 80725978260ff823e1c442e75d66cf1b61684bba Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:01 +0200 Subject: s390/ctlreg: change parameters of __local_ctl_load() and __local_ctl_store() Change __local_ctl_load() and __local_ctl_store(), so that control register parameters come first. This way all control handling functions consistently have control register(s) parameter first. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 4 ++-- arch/s390/kernel/ctlreg.c | 4 ++-- arch/s390/kernel/kprobes.c | 6 +++--- arch/s390/kernel/ptrace.c | 4 ++-- arch/s390/kernel/setup.c | 2 +- arch/s390/kernel/smp.c | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 86887b490bb5..e60c672ffb9c 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -35,7 +35,7 @@ #include -#define __local_ctl_load(array, low, high) do { \ +#define __local_ctl_load(low, high, array) do { \ struct addrtype { \ char _[sizeof(array)]; \ }; \ @@ -53,7 +53,7 @@ : "memory"); \ } while (0) -#define __local_ctl_store(array, low, high) do { \ +#define __local_ctl_store(low, high, array) do { \ struct addrtype { \ char _[sizeof(array)]; \ }; \ diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 795e7f4861fb..1c9fee1ca4a3 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -38,10 +38,10 @@ static void ctl_bit_callback(void *info) struct ctl_bit_parms *pp = info; unsigned long regs[16]; - __local_ctl_store(regs, 0, 15); + __local_ctl_store(0, 15, regs); regs[pp->cr] &= pp->andval; regs[pp->cr] |= pp->orval; - __local_ctl_load(regs, 0, 15); + __local_ctl_load(0, 15, regs); } void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index 1b2e1903a54c..ef556b93596d 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -232,12 +232,12 @@ static void enable_singlestep(struct kprobe_ctlblk *kcb, per_kprobe.end = ip; /* Save control regs and psw mask */ - __local_ctl_store(kcb->kprobe_saved_ctl, 9, 11); + __local_ctl_store(9, 11, kcb->kprobe_saved_ctl); kcb->kprobe_saved_imask = regs->psw.mask & (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT); /* Set PER control regs, turns on single step for the given address */ - __local_ctl_load(per_kprobe, 9, 11); + __local_ctl_load(9, 11, per_kprobe); regs->psw.mask |= PSW_MASK_PER; regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); regs->psw.addr = ip; @@ -249,7 +249,7 @@ static void disable_singlestep(struct kprobe_ctlblk *kcb, unsigned long ip) { /* Restore control regs and psw mask, set new psw address */ - __local_ctl_load(kcb->kprobe_saved_ctl, 9, 11); + __local_ctl_load(9, 11, kcb->kprobe_saved_ctl); regs->psw.mask &= ~PSW_MASK_PER; regs->psw.mask |= kcb->kprobe_saved_imask; regs->psw.addr = ip; diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index a1fb4536784f..c03fda09f689 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -104,9 +104,9 @@ void update_cr_regs(struct task_struct *task) return; } regs->psw.mask |= PSW_MASK_PER; - __local_ctl_store(old, 9, 11); + __local_ctl_store(9, 11, old); if (memcmp(&new, &old, sizeof(struct per_regs)) != 0) - __local_ctl_load(new, 9, 11); + __local_ctl_load(9, 11, new); } void user_enable_single_step(struct task_struct *task) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 48a456a10206..2c4bfe41d284 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -449,7 +449,7 @@ static void __init setup_lowcore(void) lc->restart_fn = (unsigned long) do_restart; lc->restart_data = 0; lc->restart_source = -1U; - __local_ctl_store(lc->cregs_save_area, 0, 15); + __local_ctl_store(0, 15, lc->cregs_save_area); lc->spinlock_lockval = arch_spin_lockval(0); lc->spinlock_index = 0; arch_spin_lock_setup(0); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 1c65bbd411c7..90d3502379aa 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -922,11 +922,11 @@ int __cpu_disable(void) /* Disable pseudo page faults on this cpu. */ pfault_fini(); /* Disable interrupt sources via control register. */ - __local_ctl_store(cregs, 0, 15); + __local_ctl_store(0, 15, cregs); cregs[0] &= ~0x0000ee70UL; /* disable all external interrupts */ cregs[6] &= ~0xff000000UL; /* disable all I/O interrupts */ cregs[14] &= ~0x1f000000UL; /* disable most machine checks */ - __local_ctl_load(cregs, 0, 15); + __local_ctl_load(0, 15, cregs); clear_cpu_flag(CIF_NOHZ_DELAY); return 0; } -- cgit v1.2.3 From 4b440e01da51da0cc651c06a036b8914b7b61ccf Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:02 +0200 Subject: s390/kprobes,ptrace: open code struct per_reg Open code struct per_regs within kprobes and ptrace code, since at both locations a struct per_regs is passed to __local_ctl_load() and __local_ctl_store() which prevents to implement type checking for both functions. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/kprobes.c | 11 +++++++++-- arch/s390/kernel/ptrace.c | 13 ++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index ef556b93596d..d4c2ece4f839 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -224,7 +224,14 @@ static void enable_singlestep(struct kprobe_ctlblk *kcb, struct pt_regs *regs, unsigned long ip) { - struct per_regs per_kprobe; + union { + unsigned long regs[3]; + struct { + unsigned long control; + unsigned long start; + unsigned long end; + }; + } per_kprobe; /* Set up the PER control registers %cr9-%cr11 */ per_kprobe.control = PER_EVENT_IFETCH; @@ -237,7 +244,7 @@ static void enable_singlestep(struct kprobe_ctlblk *kcb, (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT); /* Set PER control regs, turns on single step for the given address */ - __local_ctl_load(9, 11, per_kprobe); + __local_ctl_load(9, 11, per_kprobe.regs); regs->psw.mask |= PSW_MASK_PER; regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); regs->psw.addr = ip; diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index c03fda09f689..1e1de907f24d 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -41,10 +41,17 @@ void update_cr_regs(struct task_struct *task) { struct pt_regs *regs = task_pt_regs(task); struct thread_struct *thread = &task->thread; - struct per_regs old, new; union ctlreg0 cr0_old, cr0_new; union ctlreg2 cr2_old, cr2_new; int cr0_changed, cr2_changed; + union { + unsigned long regs[3]; + struct { + unsigned long control; + unsigned long start; + unsigned long end; + }; + } old, new; local_ctl_store(0, &cr0_old.val); local_ctl_store(2, &cr2_old.val); @@ -104,9 +111,9 @@ void update_cr_regs(struct task_struct *task) return; } regs->psw.mask |= PSW_MASK_PER; - __local_ctl_store(9, 11, old); + __local_ctl_store(9, 11, old.regs); if (memcmp(&new, &old, sizeof(struct per_regs)) != 0) - __local_ctl_load(9, 11, new); + __local_ctl_load(9, 11, new.regs); } void user_enable_single_step(struct task_struct *task) -- cgit v1.2.3 From ecc53818f60447177e24ea11b7f136c405150976 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:03 +0200 Subject: s390/ctlreg: add type checking to __local_ctl_load() and __local_ctl_store() Add type checking to __local_ctl_load() and __local_ctl_store(). For both functions enforce to pass an array consisting of unsigned longs. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index e60c672ffb9c..a49459adba9d 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -45,6 +45,7 @@ \ _esize = (_high - _low + 1) * sizeof(unsigned long); \ BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ + typecheck(unsigned long, array[0]); \ asm volatile( \ " lctlg %[_low],%[_high],%[_arr]\n" \ : \ @@ -63,6 +64,7 @@ \ _esize = (_high - _low + 1) * sizeof(unsigned long); \ BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ + typecheck(unsigned long, array[0]); \ asm volatile( \ " stctg %[_low],%[_high],%[_arr]\n" \ : [_arr] "=Q" (*(struct addrtype *)(&array)) \ -- cgit v1.2.3 From 527618abb92793b9d4dba548d55822dcebd95317 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:04 +0200 Subject: s390/ctlreg: add struct ctlreg Add struct ctlreg to enforce strict type checking / usage for control register functions. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/vmem.c | 8 ++++---- arch/s390/include/asm/ctlreg.h | 28 +++++++++++++++++---------- arch/s390/include/asm/kprobes.h | 3 ++- arch/s390/include/asm/lowcore.h | 7 ++++--- arch/s390/include/asm/mmu_context.h | 2 +- arch/s390/include/asm/pgtable.h | 3 ++- arch/s390/kernel/ctlreg.c | 10 +++++----- arch/s390/kernel/kprobes.c | 14 +++++++------- arch/s390/kernel/machine_kexec.c | 6 +++--- arch/s390/kernel/nmi.c | 12 ++++++------ arch/s390/kernel/ptrace.c | 38 ++++++++++++++++++------------------- arch/s390/kernel/setup.c | 12 ++++++------ arch/s390/kernel/smp.c | 8 ++++---- arch/s390/lib/uaccess.c | 8 ++++---- arch/s390/mm/dump_pagetables.c | 2 +- arch/s390/mm/fault.c | 4 ++-- arch/s390/mm/init.c | 2 +- arch/s390/mm/pageattr.c | 2 +- arch/s390/mm/pgalloc.c | 2 +- drivers/s390/char/sclp.c | 6 +++--- drivers/s390/char/sclp_early_core.c | 6 +++--- 21 files changed, 97 insertions(+), 86 deletions(-) diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 44278ae2512f..bdbfee86d1ac 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -12,7 +12,7 @@ #include "decompressor.h" #include "boot.h" -unsigned long __bootdata_preserved(s390_invalid_asce); +struct ctlreg __bootdata_preserved(s390_invalid_asce); #ifdef CONFIG_PROC_FS atomic_long_t __bootdata_preserved(direct_pages_count[PG_DIRECT_MAP_MAX]); @@ -422,7 +422,7 @@ void setup_vmem(unsigned long asce_limit) asce_type = _REGION3_ENTRY_EMPTY; asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; } - s390_invalid_asce = invalid_pg_dir | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; + s390_invalid_asce.val = invalid_pg_dir | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; crst_table_init((unsigned long *)swapper_pg_dir, asce_type); crst_table_init((unsigned long *)invalid_pg_dir, _REGION3_ENTRY_EMPTY); @@ -443,12 +443,12 @@ void setup_vmem(unsigned long asce_limit) kasan_populate_shadow(); - S390_lowcore.kernel_asce = swapper_pg_dir | asce_bits; + S390_lowcore.kernel_asce.val = swapper_pg_dir | asce_bits; S390_lowcore.user_asce = s390_invalid_asce; local_ctl_load(1, &S390_lowcore.kernel_asce); local_ctl_load(7, &S390_lowcore.user_asce); local_ctl_load(13, &S390_lowcore.kernel_asce); - init_mm.context.asce = S390_lowcore.kernel_asce; + init_mm.context.asce = S390_lowcore.kernel_asce.val; } diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index a49459adba9d..57cc610dd997 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -35,6 +35,10 @@ #include +struct ctlreg { + unsigned long val; +}; + #define __local_ctl_load(low, high, array) do { \ struct addrtype { \ char _[sizeof(array)]; \ @@ -43,9 +47,9 @@ int _low = low; \ int _esize; \ \ - _esize = (_high - _low + 1) * sizeof(unsigned long); \ + _esize = (_high - _low + 1) * sizeof(struct ctlreg); \ BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ - typecheck(unsigned long, array[0]); \ + typecheck(struct ctlreg, array[0]); \ asm volatile( \ " lctlg %[_low],%[_high],%[_arr]\n" \ : \ @@ -62,16 +66,16 @@ int _low = low; \ int _esize; \ \ - _esize = (_high - _low + 1) * sizeof(unsigned long); \ + _esize = (_high - _low + 1) * sizeof(struct ctlreg); \ BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ - typecheck(unsigned long, array[0]); \ + typecheck(struct ctlreg, array[0]); \ asm volatile( \ " stctg %[_low],%[_high],%[_arr]\n" \ : [_arr] "=Q" (*(struct addrtype *)(&array)) \ : [_low] "i" (low), [_high] "i" (high)); \ } while (0) -static __always_inline void local_ctl_load(unsigned int cr, unsigned long *reg) +static __always_inline void local_ctl_load(unsigned int cr, struct ctlreg *reg) { asm volatile( " lctlg %[cr],%[cr],%[reg]\n" @@ -80,7 +84,7 @@ static __always_inline void local_ctl_load(unsigned int cr, unsigned long *reg) : "memory"); } -static __always_inline void local_ctl_store(unsigned int cr, unsigned long *reg) +static __always_inline void local_ctl_store(unsigned int cr, struct ctlreg *reg) { asm volatile( " stctg %[cr],%[cr],%[reg]\n" @@ -90,19 +94,19 @@ static __always_inline void local_ctl_store(unsigned int cr, unsigned long *reg) static __always_inline void local_ctl_set_bit(unsigned int cr, unsigned int bit) { - unsigned long reg; + struct ctlreg reg; local_ctl_store(cr, ®); - reg |= 1UL << bit; + reg.val |= 1UL << bit; local_ctl_load(cr, ®); } static __always_inline void local_ctl_clear_bit(unsigned int cr, unsigned int bit) { - unsigned long reg; + struct ctlreg reg; local_ctl_store(cr, ®); - reg &= ~(1UL << bit); + reg.val &= ~(1UL << bit); local_ctl_load(cr, ®); } @@ -122,6 +126,7 @@ static inline void system_ctl_clear_bit(unsigned int cr, unsigned int bit) union ctlreg0 { unsigned long val; + struct ctlreg reg; struct { unsigned long : 8; unsigned long tcx : 1; /* Transactional-Execution control */ @@ -148,6 +153,7 @@ union ctlreg0 { union ctlreg2 { unsigned long val; + struct ctlreg reg; struct { unsigned long : 33; unsigned long ducto : 25; @@ -161,6 +167,7 @@ union ctlreg2 { union ctlreg5 { unsigned long val; + struct ctlreg reg; struct { unsigned long : 33; unsigned long pasteo: 25; @@ -170,6 +177,7 @@ union ctlreg5 { union ctlreg15 { unsigned long val; + struct ctlreg reg; struct { unsigned long lsea : 61; unsigned long : 3; diff --git a/arch/s390/include/asm/kprobes.h b/arch/s390/include/asm/kprobes.h index 83f732ca3af4..21b9e5290c04 100644 --- a/arch/s390/include/asm/kprobes.h +++ b/arch/s390/include/asm/kprobes.h @@ -15,6 +15,7 @@ * */ #include +#include #include #define BREAKPOINT_INSTRUCTION 0x0002 @@ -65,7 +66,7 @@ struct prev_kprobe { struct kprobe_ctlblk { unsigned long kprobe_status; unsigned long kprobe_saved_imask; - unsigned long kprobe_saved_ctl[3]; + struct ctlreg kprobe_saved_ctl[3]; struct prev_kprobe prev_kprobe; }; diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 2174f00e188b..3366431dcad5 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -139,8 +140,8 @@ struct lowcore { __u32 restart_flags; /* 0x0384 */ /* Address space pointer. */ - unsigned long kernel_asce; /* 0x0388 */ - unsigned long user_asce; /* 0x0390 */ + struct ctlreg kernel_asce; /* 0x0388 */ + struct ctlreg user_asce; /* 0x0390 */ /* * The lpp and current_pid fields form a @@ -199,7 +200,7 @@ struct lowcore { __u32 clock_comp_save_area[2]; /* 0x1330 */ __u64 last_break_save_area; /* 0x1338 */ __u32 access_regs_save_area[16]; /* 0x1340 */ - unsigned long cregs_save_area[16]; /* 0x1380 */ + struct ctlreg cregs_save_area[16]; /* 0x1380 */ __u8 pad_0x1400[0x1500-0x1400]; /* 0x1400 */ /* Cryptography-counter designation */ __u64 ccd; /* 0x1500 */ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 0e93275f80f0..757fe6f0d802 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -78,7 +78,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct * if (next == &init_mm) S390_lowcore.user_asce = s390_invalid_asce; else - S390_lowcore.user_asce = next->context.asce; + S390_lowcore.user_asce.val = next->context.asce; cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); /* Clear previous user-ASCE from CR7 */ local_ctl_load(7, &s390_invalid_asce); diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index fb3ee7758b76..601e87fa8a9a 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -25,7 +26,7 @@ extern pgd_t swapper_pg_dir[]; extern pgd_t invalid_pg_dir[]; extern void paging_init(void); -extern unsigned long s390_invalid_asce; +extern struct ctlreg s390_invalid_asce; enum { PG_DIRECT_MAP_4K = 0, diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 1c9fee1ca4a3..14f715bc5f73 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -36,11 +36,11 @@ struct ctl_bit_parms { static void ctl_bit_callback(void *info) { struct ctl_bit_parms *pp = info; - unsigned long regs[16]; + struct ctlreg regs[16]; __local_ctl_store(0, 15, regs); - regs[pp->cr] &= pp->andval; - regs[pp->cr] |= pp->orval; + regs[pp->cr].val &= pp->andval; + regs[pp->cr].val |= pp->orval; __local_ctl_load(0, 15, regs); } @@ -53,8 +53,8 @@ void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) pp.andval = set ? -1UL : ~(1UL << bit); system_ctlreg_lock(); abs_lc = get_abs_lowcore(); - abs_lc->cregs_save_area[cr] &= pp.andval; - abs_lc->cregs_save_area[cr] |= pp.orval; + abs_lc->cregs_save_area[cr].val &= pp.andval; + abs_lc->cregs_save_area[cr].val |= pp.orval; put_abs_lowcore(abs_lc); on_each_cpu(ctl_bit_callback, &pp, 1); system_ctlreg_unlock(); diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index d4c2ece4f839..f0cf20d4b3c5 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -225,18 +225,18 @@ static void enable_singlestep(struct kprobe_ctlblk *kcb, unsigned long ip) { union { - unsigned long regs[3]; + struct ctlreg regs[3]; struct { - unsigned long control; - unsigned long start; - unsigned long end; + struct ctlreg control; + struct ctlreg start; + struct ctlreg end; }; } per_kprobe; /* Set up the PER control registers %cr9-%cr11 */ - per_kprobe.control = PER_EVENT_IFETCH; - per_kprobe.start = ip; - per_kprobe.end = ip; + per_kprobe.control.val = PER_EVENT_IFETCH; + per_kprobe.start.val = ip; + per_kprobe.end.val = ip; /* Save control regs and psw mask */ __local_ctl_store(9, 11, kcb->kprobe_saved_ctl); diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index fe9d65060fa4..bb0d4d68fcbe 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -94,12 +94,12 @@ static noinline void __machine_kdump(void *image) if (MACHINE_HAS_VX) save_vx_regs((__vector128 *) mcesa->vector_save_area); if (MACHINE_HAS_GS) { - local_ctl_store(2, &cr2_old.val); + local_ctl_store(2, &cr2_old.reg); cr2_new = cr2_old; cr2_new.gse = 1; - local_ctl_load(2, &cr2_new.val); + local_ctl_load(2, &cr2_new.reg); save_gs_cb((struct gs_cb *) mcesa->guarded_storage_save_area); - local_ctl_load(2, &cr2_old.val); + local_ctl_load(2, &cr2_old.reg); } /* * To create a good backchain for this CPU in the dump store_status diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 7880a42896a3..579cebc58d8c 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -131,10 +131,10 @@ static notrace void s390_handle_damage(void) * Disable low address protection and make machine check new PSW a * disabled wait PSW. Any additional machine check cannot be handled. */ - local_ctl_store(0, &cr0.val); + local_ctl_store(0, &cr0.reg); cr0_new = cr0; cr0_new.lap = 0; - local_ctl_load(0, &cr0_new.val); + local_ctl_load(0, &cr0_new.reg); psw_save = S390_lowcore.mcck_new_psw; psw_bits(S390_lowcore.mcck_new_psw).io = 0; psw_bits(S390_lowcore.mcck_new_psw).ext = 0; @@ -146,7 +146,7 @@ static notrace void s390_handle_damage(void) * values. This makes possible system dump analysis easier. */ S390_lowcore.mcck_new_psw = psw_save; - local_ctl_load(0, &cr0.val); + local_ctl_load(0, &cr0.reg); disabled_wait(); while (1); } @@ -269,9 +269,9 @@ static int notrace s390_validate_registers(union mci mci) */ if (!mci.vr && !test_cpu_flag(CIF_MCCK_GUEST)) kill_task = 1; - cr0.val = S390_lowcore.cregs_save_area[0]; + cr0.reg = S390_lowcore.cregs_save_area[0]; cr0.afp = cr0.vx = 1; - local_ctl_load(0, &cr0.val); + local_ctl_load(0, &cr0.reg); asm volatile( " la 1,%0\n" " VLM 0,15,0,1\n" @@ -290,7 +290,7 @@ static int notrace s390_validate_registers(union mci mci) if (!mci.ar) kill_task = 1; /* Validate guarded storage registers */ - cr2.val = S390_lowcore.cregs_save_area[2]; + cr2.reg = S390_lowcore.cregs_save_area[2]; if (cr2.gse) { if (!mci.gs) { /* diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 1e1de907f24d..6a825351ff41 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -45,16 +45,16 @@ void update_cr_regs(struct task_struct *task) union ctlreg2 cr2_old, cr2_new; int cr0_changed, cr2_changed; union { - unsigned long regs[3]; + struct ctlreg regs[3]; struct { - unsigned long control; - unsigned long start; - unsigned long end; + struct ctlreg control; + struct ctlreg start; + struct ctlreg end; }; } old, new; - local_ctl_store(0, &cr0_old.val); - local_ctl_store(2, &cr2_old.val); + local_ctl_store(0, &cr0_old.reg); + local_ctl_store(2, &cr2_old.reg); cr0_new = cr0_old; cr2_new = cr2_old; /* Take care of the enable/disable of transactional execution. */ @@ -82,31 +82,31 @@ void update_cr_regs(struct task_struct *task) cr0_changed = cr0_new.val != cr0_old.val; cr2_changed = cr2_new.val != cr2_old.val; if (cr0_changed) - local_ctl_load(0, &cr0_new.val); + local_ctl_load(0, &cr0_new.reg); if (cr2_changed) - local_ctl_load(2, &cr2_new.val); + local_ctl_load(2, &cr2_new.reg); /* Copy user specified PER registers */ - new.control = thread->per_user.control; - new.start = thread->per_user.start; - new.end = thread->per_user.end; + new.control.val = thread->per_user.control; + new.start.val = thread->per_user.start; + new.end.val = thread->per_user.end; /* merge TIF_SINGLE_STEP into user specified PER registers. */ if (test_tsk_thread_flag(task, TIF_SINGLE_STEP) || test_tsk_thread_flag(task, TIF_UPROBE_SINGLESTEP)) { if (test_tsk_thread_flag(task, TIF_BLOCK_STEP)) - new.control |= PER_EVENT_BRANCH; + new.control.val |= PER_EVENT_BRANCH; else - new.control |= PER_EVENT_IFETCH; - new.control |= PER_CONTROL_SUSPENSION; - new.control |= PER_EVENT_TRANSACTION_END; + new.control.val |= PER_EVENT_IFETCH; + new.control.val |= PER_CONTROL_SUSPENSION; + new.control.val |= PER_EVENT_TRANSACTION_END; if (test_tsk_thread_flag(task, TIF_UPROBE_SINGLESTEP)) - new.control |= PER_EVENT_IFETCH; - new.start = 0; - new.end = -1UL; + new.control.val |= PER_EVENT_IFETCH; + new.start.val = 0; + new.end.val = -1UL; } /* Take care of the PER enablement bit in the PSW. */ - if (!(new.control & PER_EVENT_MASK)) { + if (!(new.control.val & PER_EVENT_MASK)) { regs->psw.mask &= ~PSW_MASK_PER; return; } diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 2c4bfe41d284..1c049a65c769 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -791,15 +791,15 @@ static void __init setup_cr(void) __ctl_duct[4] = (unsigned long)__ctl_duald; /* Update control registers CR2, CR5 and CR15 */ - local_ctl_store(2, &cr2.val); - local_ctl_store(5, &cr5.val); - local_ctl_store(15, &cr15.val); + local_ctl_store(2, &cr2.reg); + local_ctl_store(5, &cr5.reg); + local_ctl_store(15, &cr15.reg); cr2.ducto = (unsigned long)__ctl_duct >> 6; cr5.pasteo = (unsigned long)__ctl_duct >> 6; cr15.lsea = (unsigned long)__ctl_linkage_stack >> 3; - local_ctl_load(2, &cr2.val); - local_ctl_load(5, &cr5.val); - local_ctl_load(15, &cr15.val); + local_ctl_load(2, &cr2.reg); + local_ctl_load(5, &cr5.reg); + local_ctl_load(15, &cr15.reg); } /* diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 90d3502379aa..fbd2801c7061 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -910,7 +910,7 @@ early_param("possible_cpus", _setup_possible_cpus); int __cpu_disable(void) { - unsigned long cregs[16]; + struct ctlreg cregs[16]; int cpu; /* Handle possible pending IPIs */ @@ -923,9 +923,9 @@ int __cpu_disable(void) pfault_fini(); /* Disable interrupt sources via control register. */ __local_ctl_store(0, 15, cregs); - cregs[0] &= ~0x0000ee70UL; /* disable all external interrupts */ - cregs[6] &= ~0xff000000UL; /* disable all I/O interrupts */ - cregs[14] &= ~0x1f000000UL; /* disable most machine checks */ + cregs[0].val &= ~0x0000ee70UL; /* disable all external interrupts */ + cregs[6].val &= ~0xff000000UL; /* disable all I/O interrupts */ + cregs[14].val &= ~0x1f000000UL; /* disable most machine checks */ __local_ctl_load(0, 15, cregs); clear_cpu_flag(CIF_NOHZ_DELAY); return 0; diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 56f7eb90b44d..61d8dcd95bbc 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -17,17 +17,17 @@ #ifdef CONFIG_DEBUG_ENTRY void debug_user_asce(int exit) { - unsigned long cr1, cr7; + struct ctlreg cr1, cr7; local_ctl_store(1, &cr1); local_ctl_store(7, &cr7); - if (cr1 == S390_lowcore.kernel_asce && cr7 == S390_lowcore.user_asce) + if (cr1.val == S390_lowcore.kernel_asce.val && cr7.val == S390_lowcore.user_asce.val) return; panic("incorrect ASCE on kernel %s\n" "cr1: %016lx cr7: %016lx\n" "kernel: %016lx user: %016lx\n", - exit ? "exit" : "entry", cr1, cr7, - S390_lowcore.kernel_asce, S390_lowcore.user_asce); + exit ? "exit" : "entry", cr1.val, cr7.val, + S390_lowcore.kernel_asce.val, S390_lowcore.user_asce.val); } #endif /*CONFIG_DEBUG_ENTRY */ diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index b51666967aa1..d37a8f607b71 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -287,7 +287,7 @@ static int pt_dump_init(void) * kernel ASCE. We need this to keep the page table walker functions * from accessing non-existent entries. */ - max_addr = (S390_lowcore.kernel_asce & _REGION_ENTRY_TYPE_MASK) >> 2; + max_addr = (S390_lowcore.kernel_asce.val & _REGION_ENTRY_TYPE_MASK) >> 2; max_addr = 1UL << (max_addr * 11 + 31); address_markers[IDENTITY_AFTER_END_NR].start_address = ident_map_size; address_markers[AMODE31_START_NR].start_address = (unsigned long)__samode31; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index b678295931c3..587b6b64185f 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -196,7 +196,7 @@ static void dump_fault_info(struct pt_regs *regs) pr_cont("mode while using "); switch (get_fault_type(regs)) { case USER_FAULT: - asce = S390_lowcore.user_asce; + asce = S390_lowcore.user_asce.val; pr_cont("user "); break; case GMAP_FAULT: @@ -204,7 +204,7 @@ static void dump_fault_info(struct pt_regs *regs) pr_cont("gmap "); break; case KERNEL_FAULT: - asce = S390_lowcore.kernel_asce; + asce = S390_lowcore.kernel_asce.val; pr_cont("kernel "); break; default: diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index c6ab6f2707d7..7eca10c32caa 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -54,7 +54,7 @@ pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir"); -unsigned long __bootdata_preserved(s390_invalid_asce); +struct ctlreg __bootdata_preserved(s390_invalid_asce); unsigned long empty_zero_page, zero_page_mask; EXPORT_SYMBOL(empty_zero_page); diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index b87e96c64b61..631e3a4ee2de 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -75,7 +75,7 @@ static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr, break; } table = (unsigned long *)((unsigned long)old & mask); - crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce); + crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce.val); } else if (MACHINE_HAS_IDTE) { cspg(old, *old, new); } else { diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 27720a687022..61fb157029c8 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -62,7 +62,7 @@ static void __crst_table_upgrade(void *arg) /* change all active ASCEs to avoid the creation of new TLBs */ if (current->active_mm == mm) { - S390_lowcore.user_asce = mm->context.asce; + S390_lowcore.user_asce.val = mm->context.asce; local_ctl_load(7, &S390_lowcore.user_asce); } __tlb_flush_local(); diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index 30a7cd9748fe..ba9b202c5dee 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -706,8 +706,8 @@ void sclp_sync_wait(void) { unsigned long long old_tick; + struct ctlreg cr0, cr0_sync; unsigned long flags; - unsigned long cr0, cr0_sync; static u64 sync_count; u64 timeout; int irq_context; @@ -733,8 +733,8 @@ sclp_sync_wait(void) old_tick = local_tick_disable(); trace_hardirqs_on(); local_ctl_store(0, &cr0); - cr0_sync = cr0 & ~CR0_IRQ_SUBCLASS_MASK; - cr0_sync |= 1UL << (63 - 54); + cr0_sync.val = cr0.val & ~CR0_IRQ_SUBCLASS_MASK; + cr0_sync.val |= 1UL << (63 - 54); local_ctl_load(0, &cr0_sync); __arch_local_irq_stosm(0x01); /* Loop until driver state indicates finished request */ diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index 8cb9fb7098e2..9f6165cafdc3 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -32,11 +32,11 @@ void sclp_early_wait_irq(void) psw_t psw_ext_save, psw_wait; union ctlreg0 cr0, cr0_new; - local_ctl_store(0, &cr0.val); + local_ctl_store(0, &cr0.reg); cr0_new.val = cr0.val & ~CR0_IRQ_SUBCLASS_MASK; cr0_new.lap = 0; cr0_new.sssm = 1; - local_ctl_load(0, &cr0_new.val); + local_ctl_load(0, &cr0_new.reg); psw_ext_save = S390_lowcore.external_new_psw; psw_mask = __extract_psw(); @@ -59,7 +59,7 @@ void sclp_early_wait_irq(void) } while (S390_lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG); S390_lowcore.external_new_psw = psw_ext_save; - local_ctl_load(0, &cr0.val); + local_ctl_load(0, &cr0.reg); } int sclp_early_cmd(sclp_cmdw_t cmd, void *sccb) -- cgit v1.2.3 From d11d5c8c8426a4abdcb2252999f1b57e3aa79e5b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:05 +0200 Subject: s390/ctltreg: make initialization of control register save area explicit Commit e1b9c2749af0 ("s390/smp: ensure global control register contents are in sync") made the control register save area contained within the lowcore at absolute address zero a resource which is used when initializing CPUs. However this is anything but obvious from the code. Add an ctlreg_init_save_area() function in order to make this explicit. Signed-off-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 3 +++ arch/s390/kernel/ctlreg.c | 11 +++++++++++ arch/s390/kernel/setup.c | 3 +-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 57cc610dd997..1f25601d75a0 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -110,9 +110,12 @@ static __always_inline void local_ctl_clear_bit(unsigned int cr, unsigned int bi local_ctl_load(cr, ®); } +struct lowcore; + void system_ctlreg_lock(void); void system_ctlreg_unlock(void); void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set); +void system_ctlreg_init_save_area(struct lowcore *lc); static inline void system_ctl_set_bit(unsigned int cr, unsigned int bit) { diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 14f715bc5f73..7acc40804874 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -4,6 +4,7 @@ */ #include +#include #include #include #include @@ -27,6 +28,16 @@ void system_ctlreg_unlock(void) spin_unlock(&system_ctl_lock); } +void __init system_ctlreg_init_save_area(struct lowcore *lc) +{ + struct lowcore *abs_lc; + + abs_lc = get_abs_lowcore(); + __local_ctl_store(0, 15, lc->cregs_save_area); + __local_ctl_store(0, 15, abs_lc->cregs_save_area); + put_abs_lowcore(abs_lc); +} + struct ctl_bit_parms { unsigned long orval; unsigned long andval; diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 1c049a65c769..d399f18d0379 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -449,7 +449,6 @@ static void __init setup_lowcore(void) lc->restart_fn = (unsigned long) do_restart; lc->restart_data = 0; lc->restart_source = -1U; - __local_ctl_store(0, 15, lc->cregs_save_area); lc->spinlock_lockval = arch_spin_lockval(0); lc->spinlock_index = 0; arch_spin_lock_setup(0); @@ -459,6 +458,7 @@ static void __init setup_lowcore(void) lc->kernel_asce = S390_lowcore.kernel_asce; lc->user_asce = S390_lowcore.user_asce; + system_ctlreg_init_save_area(lc); abs_lc = get_abs_lowcore(); abs_lc->restart_stack = lc->restart_stack; abs_lc->restart_fn = lc->restart_fn; @@ -466,7 +466,6 @@ static void __init setup_lowcore(void) abs_lc->restart_source = lc->restart_source; abs_lc->restart_psw = lc->restart_psw; abs_lc->restart_flags = RESTART_FLAG_CTLREGS; - memcpy(abs_lc->cregs_save_area, lc->cregs_save_area, sizeof(abs_lc->cregs_save_area)); abs_lc->program_new_psw = lc->program_new_psw; abs_lc->mcesad = lc->mcesad; put_abs_lowcore(abs_lc); -- cgit v1.2.3 From cce2c8606f8ed3b0285842f4ad21e906d511dba1 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:06 +0200 Subject: s390/ctlreg: allow to call system_ctl_set/clear_bit() early Allow to call system_ctl_set_bit() and system_clt_clear_bit() early, so that users do not have to take care when the control register save area has been initialized. Users are supposed to use system_ctl_set_bit() and system:clt_clear_bit() for all control register changes which are supposed to be seen globally. Depending on the system state such calls will change: - local control register contents - save area and local control register contents - save area and global control register contents Signed-off-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/ctlreg.c | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 7acc40804874..27ba8db1d038 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -3,9 +3,12 @@ * Copyright IBM Corp. 1999, 2023 */ +#include #include +#include #include #include +#include #include #include @@ -28,6 +31,8 @@ void system_ctlreg_unlock(void) spin_unlock(&system_ctl_lock); } +static bool system_ctlreg_area_init __ro_after_init; + void __init system_ctlreg_init_save_area(struct lowcore *lc) { struct lowcore *abs_lc; @@ -36,6 +41,7 @@ void __init system_ctlreg_init_save_area(struct lowcore *lc) __local_ctl_store(0, 15, lc->cregs_save_area); __local_ctl_store(0, 15, abs_lc->cregs_save_area); put_abs_lowcore(abs_lc); + system_ctlreg_area_init = true; } struct ctl_bit_parms { @@ -55,6 +61,23 @@ static void ctl_bit_callback(void *info) __local_ctl_load(0, 15, regs); } +static void system_ctl_bit_update(void *info) +{ + unsigned long flags; + + if (system_state == SYSTEM_BOOTING) { + /* + * For very early calls do not call on_each_cpu() + * since not everything might be setup. + */ + local_irq_save(flags); + ctl_bit_callback(info); + local_irq_restore(flags); + } else { + on_each_cpu(ctl_bit_callback, info, 1); + } +} + void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) { struct ctl_bit_parms pp = { .cr = cr, }; @@ -62,12 +85,16 @@ void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) pp.orval = set ? 1UL << bit : 0; pp.andval = set ? -1UL : ~(1UL << bit); - system_ctlreg_lock(); - abs_lc = get_abs_lowcore(); - abs_lc->cregs_save_area[cr].val &= pp.andval; - abs_lc->cregs_save_area[cr].val |= pp.orval; - put_abs_lowcore(abs_lc); - on_each_cpu(ctl_bit_callback, &pp, 1); - system_ctlreg_unlock(); + if (system_ctlreg_area_init) { + system_ctlreg_lock(); + abs_lc = get_abs_lowcore(); + abs_lc->cregs_save_area[cr].val &= pp.andval; + abs_lc->cregs_save_area[cr].val |= pp.orval; + put_abs_lowcore(abs_lc); + system_ctl_bit_update(&pp); + system_ctlreg_unlock(); + } else { + system_ctl_bit_update(&pp); + } } EXPORT_SYMBOL(system_ctl_set_clear_bit); -- cgit v1.2.3 From 103dde702d698c2e17529550650407a386711cdf Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:07 +0200 Subject: s390/early: use system_ctl_set_bit() instead of local_ctl_set_bit() Use system_ctl_set_bit() instead of local_ctl_set_bit() to reflect that the control register changes are supposed to be global. This change is just for documentation purposes, since it still results only in local control register contents being changed. Signed-off-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/early.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 1f359309cce6..0beaa0ba58a8 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -216,7 +216,7 @@ static __init void detect_machine_facilities(void) { if (test_facility(8)) { S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT1; - local_ctl_set_bit(0, 23); + system_ctl_set_bit(0, 23); } if (test_facility(78)) S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT2; @@ -224,13 +224,13 @@ static __init void detect_machine_facilities(void) S390_lowcore.machine_flags |= MACHINE_FLAG_IDTE; if (test_facility(50) && test_facility(73)) { S390_lowcore.machine_flags |= MACHINE_FLAG_TE; - local_ctl_set_bit(0, 55); + system_ctl_set_bit(0, 55); } if (test_facility(51)) S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_LC; if (test_facility(129)) { S390_lowcore.machine_flags |= MACHINE_FLAG_VX; - local_ctl_set_bit(0, 17); + system_ctl_set_bit(0, 17); } if (test_facility(130)) S390_lowcore.machine_flags |= MACHINE_FLAG_NX; @@ -240,7 +240,7 @@ static __init void detect_machine_facilities(void) /* Enabled signed clock comparator comparisons */ S390_lowcore.machine_flags |= MACHINE_FLAG_SCC; clock_comparator_max = -1ULL >> 1; - local_ctl_set_bit(0, 53); + system_ctl_set_bit(0, 53); } if (IS_ENABLED(CONFIG_PCI) && test_facility(153)) { S390_lowcore.machine_flags |= MACHINE_FLAG_PCI_MIO; @@ -260,7 +260,7 @@ static inline void save_vector_registers(void) static inline void setup_low_address_protection(void) { - local_ctl_set_bit(0, 28); + system_ctl_set_bit(0, 28); } static inline void setup_access_registers(void) @@ -273,7 +273,7 @@ static inline void setup_access_registers(void) static int __init disable_vector_extension(char *str) { S390_lowcore.machine_flags &= ~MACHINE_FLAG_VX; - local_ctl_clear_bit(0, 17); + system_ctl_clear_bit(0, 17); return 0; } early_param("novx", disable_vector_extension); -- cgit v1.2.3 From 305a5551af34e1848d80e095c23e94c985cc4ebc Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:08 +0200 Subject: s390/ctlreg: add system_ctl_load() Add system_ctl_load() which can be used to load a value to a control register system wide. Refactor system_ctl_set_clear_bit() so it can handle all different request types, and also rename it to system_ctlreg_modify() Signed-off-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 17 ++++++++++--- arch/s390/kernel/ctlreg.c | 57 +++++++++++++++++++++++++++++------------- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 1f25601d75a0..6582b1b9b97b 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -114,17 +114,28 @@ struct lowcore; void system_ctlreg_lock(void); void system_ctlreg_unlock(void); -void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set); void system_ctlreg_init_save_area(struct lowcore *lc); +void system_ctlreg_modify(unsigned int cr, unsigned long data, int request); + +enum { + CTLREG_SET_BIT, + CTLREG_CLEAR_BIT, + CTLREG_LOAD, +}; static inline void system_ctl_set_bit(unsigned int cr, unsigned int bit) { - system_ctl_set_clear_bit(cr, bit, true); + system_ctlreg_modify(cr, bit, CTLREG_SET_BIT); } static inline void system_ctl_clear_bit(unsigned int cr, unsigned int bit) { - system_ctl_set_clear_bit(cr, bit, false); + system_ctlreg_modify(cr, bit, CTLREG_CLEAR_BIT); +} + +static inline void system_ctl_load(unsigned int cr, struct ctlreg *reg) +{ + system_ctlreg_modify(cr, reg->val, CTLREG_LOAD); } union ctlreg0 { diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 27ba8db1d038..8cc26cf2c64a 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -44,24 +44,30 @@ void __init system_ctlreg_init_save_area(struct lowcore *lc) system_ctlreg_area_init = true; } -struct ctl_bit_parms { - unsigned long orval; +struct ctlreg_parms { unsigned long andval; + unsigned long orval; + unsigned long val; + int request; int cr; }; -static void ctl_bit_callback(void *info) +static void ctlreg_callback(void *info) { - struct ctl_bit_parms *pp = info; + struct ctlreg_parms *pp = info; struct ctlreg regs[16]; __local_ctl_store(0, 15, regs); - regs[pp->cr].val &= pp->andval; - regs[pp->cr].val |= pp->orval; + if (pp->request == CTLREG_LOAD) { + regs[pp->cr].val = pp->val; + } else { + regs[pp->cr].val &= pp->andval; + regs[pp->cr].val |= pp->orval; + } __local_ctl_load(0, 15, regs); } -static void system_ctl_bit_update(void *info) +static void system_ctlreg_update(void *info) { unsigned long flags; @@ -71,30 +77,45 @@ static void system_ctl_bit_update(void *info) * since not everything might be setup. */ local_irq_save(flags); - ctl_bit_callback(info); + ctlreg_callback(info); local_irq_restore(flags); } else { - on_each_cpu(ctl_bit_callback, info, 1); + on_each_cpu(ctlreg_callback, info, 1); } } -void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) +void system_ctlreg_modify(unsigned int cr, unsigned long data, int request) { - struct ctl_bit_parms pp = { .cr = cr, }; + struct ctlreg_parms pp = { .cr = cr, .request = request, }; struct lowcore *abs_lc; - pp.orval = set ? 1UL << bit : 0; - pp.andval = set ? -1UL : ~(1UL << bit); + switch (request) { + case CTLREG_SET_BIT: + pp.orval = 1UL << data; + pp.andval = -1UL; + break; + case CTLREG_CLEAR_BIT: + pp.orval = 0; + pp.andval = ~(1UL << data); + break; + case CTLREG_LOAD: + pp.val = data; + break; + } if (system_ctlreg_area_init) { system_ctlreg_lock(); abs_lc = get_abs_lowcore(); - abs_lc->cregs_save_area[cr].val &= pp.andval; - abs_lc->cregs_save_area[cr].val |= pp.orval; + if (request == CTLREG_LOAD) { + abs_lc->cregs_save_area[cr].val = pp.val; + } else { + abs_lc->cregs_save_area[cr].val &= pp.andval; + abs_lc->cregs_save_area[cr].val |= pp.orval; + } put_abs_lowcore(abs_lc); - system_ctl_bit_update(&pp); + system_ctlreg_update(&pp); system_ctlreg_unlock(); } else { - system_ctl_bit_update(&pp); + system_ctlreg_update(&pp); } } -EXPORT_SYMBOL(system_ctl_set_clear_bit); +EXPORT_SYMBOL(system_ctlreg_modify); -- cgit v1.2.3 From 0b6529e3dc1c60565f10ccf40533b127ab3262a1 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:09 +0200 Subject: s390/setup: make use of system_ctl_load() Use system_ctl_load() instead of local_ctl_load() to reflect that control register changes are supposed to be global. Even though setup_cr() was ok, note that the usage of local_ctl_load() would have been wrong, if it would have happened after the control register save area was initialized: only local control register contents would have been changed, but wouldn't be used for new CPUs. With using system_ctl_load() the caller doesn't need to take care. Signed-off-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/setup.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index d399f18d0379..558bcf18df52 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -796,9 +796,9 @@ static void __init setup_cr(void) cr2.ducto = (unsigned long)__ctl_duct >> 6; cr5.pasteo = (unsigned long)__ctl_duct >> 6; cr15.lsea = (unsigned long)__ctl_linkage_stack >> 3; - local_ctl_load(2, &cr2.reg); - local_ctl_load(5, &cr5.reg); - local_ctl_load(15, &cr15.reg); + system_ctl_load(2, &cr2.reg); + system_ctl_load(5, &cr5.reg); + system_ctl_load(15, &cr15.reg); } /* -- cgit v1.2.3 From 4f4cee9619edc877e8da913193968497e4e42cee Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:10 +0200 Subject: s390/ctlreg: add missing defines Add a couple of missing control register defines which otherwise would prevent to convert other open-coded usages. Acked-by: Thomas Richter Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 6582b1b9b97b..76ed70cca43c 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -10,10 +10,19 @@ #include +#define CR0_TRANSACTIONAL_EXECUTION BIT(63 - 8) #define CR0_CLOCK_COMPARATOR_SIGN BIT(63 - 10) +#define CR0_CRYPTOGRAPHY_COUNTER BIT(63 - 13) +#define CR0_PAI_EXTENSION BIT(63 - 14) +#define CR0_CPUMF_EXTRACTION_AUTH BIT(63 - 15) +#define CR0_WARNING_TRACK BIT(63 - 30) #define CR0_LOW_ADDRESS_PROTECTION BIT(63 - 35) #define CR0_FETCH_PROTECTION_OVERRIDE BIT(63 - 38) #define CR0_STORAGE_PROTECTION_OVERRIDE BIT(63 - 39) +#define CR0_EDAT BIT(63 - 40) +#define CR0_INSTRUCTION_EXEC_PROTECTION BIT(63 - 43) +#define CR0_VECTOR BIT(63 - 46) +#define CR0_MALFUNCTION_ALERT_SUBMASK BIT(63 - 48) #define CR0_EMERGENCY_SIGNAL_SUBMASK BIT(63 - 49) #define CR0_EXTERNAL_CALL_SUBMASK BIT(63 - 50) #define CR0_CLOCK_COMPARATOR_SUBMASK BIT(63 - 52) @@ -22,6 +31,11 @@ #define CR0_UNUSED_56 BIT(63 - 56) #define CR0_INTERRUPT_KEY_SUBMASK BIT(63 - 57) #define CR0_MEASUREMENT_ALERT_SUBMASK BIT(63 - 58) +#define CR0_ETR_SUBMASK BIT(63 - 59) +#define CR0_IUCV BIT(63 - 62) + +#define CR2_MIO_ADDRESSING BIT(63 - 58) +#define CR2_GUARDED_STORAGE BIT(63 - 59) #define CR14_UNUSED_32 BIT(63 - 32) #define CR14_UNUSED_33 BIT(63 - 33) -- cgit v1.2.3 From da290f4382eb98e152af59e416a08d43a5c31157 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:11 +0200 Subject: s390/irq: use CR0 defines to define CR0_IRQ_SUBCLASS_MASK Use existing CR0 defines to define CR0_IRQ_SUBCLASS_MASK instead of open-coding the defines again. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/irq.h | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h index 89902f754740..54b42817f70a 100644 --- a/arch/s390/include/asm/irq.h +++ b/arch/s390/include/asm/irq.h @@ -31,6 +31,7 @@ #include #include #include +#include enum interruption_class { IRQEXT_CLK, @@ -101,17 +102,17 @@ enum irq_subclass { }; #define CR0_IRQ_SUBCLASS_MASK \ - ((1UL << (63 - 30)) /* Warning Track */ | \ - (1UL << (63 - 48)) /* Malfunction Alert */ | \ - (1UL << (63 - 49)) /* Emergency Signal */ | \ - (1UL << (63 - 50)) /* External Call */ | \ - (1UL << (63 - 52)) /* Clock Comparator */ | \ - (1UL << (63 - 53)) /* CPU Timer */ | \ - (1UL << (63 - 54)) /* Service Signal */ | \ - (1UL << (63 - 57)) /* Interrupt Key */ | \ - (1UL << (63 - 58)) /* Measurement Alert */ | \ - (1UL << (63 - 59)) /* Timing Alert */ | \ - (1UL << (63 - 62))) /* IUCV */ + (CR0_WARNING_TRACK | \ + CR0_MALFUNCTION_ALERT_SUBMASK | \ + CR0_EMERGENCY_SIGNAL_SUBMASK | \ + CR0_EXTERNAL_CALL_SUBMASK | \ + CR0_CLOCK_COMPARATOR_SUBMASK | \ + CR0_CPU_TIMER_SUBMASK | \ + CR0_SERVICE_SIGNAL_SUBMASK | \ + CR0_INTERRUPT_KEY_SUBMASK | \ + CR0_MEASUREMENT_ALERT_SUBMASK | \ + CR0_ETR_SUBMASK | \ + CR0_IUCV) void irq_subclass_register(enum irq_subclass subclass); void irq_subclass_unregister(enum irq_subclass subclass); -- cgit v1.2.3 From 9c66cc5609b13899ef7d143f51c59bdcfd63484d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:12 +0200 Subject: s390/ctlreg: add control register bits Instead of having only masks for specific bit locations within control registers, also define the bit numbers and use them to define the masks. The bit defines can be used to convert plain numbers to defines. Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 103 +++++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 34 deletions(-) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 76ed70cca43c..6d4b85f2b541 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -10,40 +10,75 @@ #include -#define CR0_TRANSACTIONAL_EXECUTION BIT(63 - 8) -#define CR0_CLOCK_COMPARATOR_SIGN BIT(63 - 10) -#define CR0_CRYPTOGRAPHY_COUNTER BIT(63 - 13) -#define CR0_PAI_EXTENSION BIT(63 - 14) -#define CR0_CPUMF_EXTRACTION_AUTH BIT(63 - 15) -#define CR0_WARNING_TRACK BIT(63 - 30) -#define CR0_LOW_ADDRESS_PROTECTION BIT(63 - 35) -#define CR0_FETCH_PROTECTION_OVERRIDE BIT(63 - 38) -#define CR0_STORAGE_PROTECTION_OVERRIDE BIT(63 - 39) -#define CR0_EDAT BIT(63 - 40) -#define CR0_INSTRUCTION_EXEC_PROTECTION BIT(63 - 43) -#define CR0_VECTOR BIT(63 - 46) -#define CR0_MALFUNCTION_ALERT_SUBMASK BIT(63 - 48) -#define CR0_EMERGENCY_SIGNAL_SUBMASK BIT(63 - 49) -#define CR0_EXTERNAL_CALL_SUBMASK BIT(63 - 50) -#define CR0_CLOCK_COMPARATOR_SUBMASK BIT(63 - 52) -#define CR0_CPU_TIMER_SUBMASK BIT(63 - 53) -#define CR0_SERVICE_SIGNAL_SUBMASK BIT(63 - 54) -#define CR0_UNUSED_56 BIT(63 - 56) -#define CR0_INTERRUPT_KEY_SUBMASK BIT(63 - 57) -#define CR0_MEASUREMENT_ALERT_SUBMASK BIT(63 - 58) -#define CR0_ETR_SUBMASK BIT(63 - 59) -#define CR0_IUCV BIT(63 - 62) - -#define CR2_MIO_ADDRESSING BIT(63 - 58) -#define CR2_GUARDED_STORAGE BIT(63 - 59) - -#define CR14_UNUSED_32 BIT(63 - 32) -#define CR14_UNUSED_33 BIT(63 - 33) -#define CR14_CHANNEL_REPORT_SUBMASK BIT(63 - 35) -#define CR14_RECOVERY_SUBMASK BIT(63 - 36) -#define CR14_DEGRADATION_SUBMASK BIT(63 - 37) -#define CR14_EXTERNAL_DAMAGE_SUBMASK BIT(63 - 38) -#define CR14_WARNING_SUBMASK BIT(63 - 39) +#define CR0_TRANSACTIONAL_EXECUTION_BIT (63 - 8) +#define CR0_CLOCK_COMPARATOR_SIGN_BIT (63 - 10) +#define CR0_CRYPTOGRAPHY_COUNTER_BIT (63 - 13) +#define CR0_PAI_EXTENSION_BIT (63 - 14) +#define CR0_CPUMF_EXTRACTION_AUTH_BIT (63 - 15) +#define CR0_WARNING_TRACK_BIT (63 - 30) +#define CR0_LOW_ADDRESS_PROTECTION_BIT (63 - 35) +#define CR0_FETCH_PROTECTION_OVERRIDE_BIT (63 - 38) +#define CR0_STORAGE_PROTECTION_OVERRIDE_BIT (63 - 39) +#define CR0_EDAT_BIT (63 - 40) +#define CR0_INSTRUCTION_EXEC_PROTECTION_BIT (63 - 43) +#define CR0_VECTOR_BIT (63 - 46) +#define CR0_MALFUNCTION_ALERT_SUBMASK_BIT (63 - 48) +#define CR0_EMERGENCY_SIGNAL_SUBMASK_BIT (63 - 49) +#define CR0_EXTERNAL_CALL_SUBMASK_BIT (63 - 50) +#define CR0_CLOCK_COMPARATOR_SUBMASK_BIT (63 - 52) +#define CR0_CPU_TIMER_SUBMASK_BIT (63 - 53) +#define CR0_SERVICE_SIGNAL_SUBMASK_BIT (63 - 54) +#define CR0_UNUSED_56_BIT (63 - 56) +#define CR0_INTERRUPT_KEY_SUBMASK_BIT (63 - 57) +#define CR0_MEASUREMENT_ALERT_SUBMASK_BIT (63 - 58) +#define CR0_ETR_SUBMASK_BIT (63 - 59) +#define CR0_IUCV_BIT (63 - 62) + +#define CR0_TRANSACTIONAL_EXECUTION BIT(CR0_TRANSACTIONAL_EXECUTION_BIT) +#define CR0_CLOCK_COMPARATOR_SIGN BIT(CR0_CLOCK_COMPARATOR_SIGN_BIT) +#define CR0_CRYPTOGRAPHY_COUNTER BIT(CR0_CRYPTOGRAPHY_COUNTER_BIT) +#define CR0_PAI_EXTENSION BIT(CR0_PAI_EXTENSION_BIT) +#define CR0_CPUMF_EXTRACTION_AUTH BIT(CR0_CPUMF_EXTRACTION_AUTH_BIT) +#define CR0_WARNING_TRACK BIT(CR0_WARNING_TRACK_BIT) +#define CR0_LOW_ADDRESS_PROTECTION BIT(CR0_LOW_ADDRESS_PROTECTION_BIT) +#define CR0_FETCH_PROTECTION_OVERRIDE BIT(CR0_FETCH_PROTECTION_OVERRIDE_BIT) +#define CR0_STORAGE_PROTECTION_OVERRIDE BIT(CR0_STORAGE_PROTECTION_OVERRIDE_BIT) +#define CR0_EDAT BIT(CR0_EDAT_BIT) +#define CR0_INSTRUCTION_EXEC_PROTECTION BIT(CR0_INSTRUCTION_EXEC_PROTECTION_BIT) +#define CR0_VECTOR BIT(CR0_VECTOR_BIT) +#define CR0_MALFUNCTION_ALERT_SUBMASK BIT(CR0_MALFUNCTION_ALERT_SUBMASK_BIT) +#define CR0_EMERGENCY_SIGNAL_SUBMASK BIT(CR0_EMERGENCY_SIGNAL_SUBMASK_BIT) +#define CR0_EXTERNAL_CALL_SUBMASK BIT(CR0_EXTERNAL_CALL_SUBMASK_BIT) +#define CR0_CLOCK_COMPARATOR_SUBMASK BIT(CR0_CLOCK_COMPARATOR_SUBMASK_BIT) +#define CR0_CPU_TIMER_SUBMASK BIT(CR0_CPU_TIMER_SUBMASK_BIT) +#define CR0_SERVICE_SIGNAL_SUBMASK BIT(CR0_SERVICE_SIGNAL_SUBMASK_BIT) +#define CR0_UNUSED_56 BIT(CR0_UNUSED_56_BIT) +#define CR0_INTERRUPT_KEY_SUBMASK BIT(CR0_INTERRUPT_KEY_SUBMASK_BIT) +#define CR0_MEASUREMENT_ALERT_SUBMASK BIT(CR0_MEASUREMENT_ALERT_SUBMASK_BIT) +#define CR0_ETR_SUBMASK BIT(CR0_ETR_SUBMASK_BIT) +#define CR0_IUCV BIT(CR0_IUCV_BIT) + +#define CR2_MIO_ADDRESSING_BIT (63 - 58) +#define CR2_GUARDED_STORAGE_BIT (63 - 59) + +#define CR2_MIO_ADDRESSING BIT(CR2_MIO_ADDRESSING_BIT) +#define CR2_GUARDED_STORAGE BIT(CR2_GUARDED_STORAGE_BIT) + +#define CR14_UNUSED_32_BIT (63 - 32) +#define CR14_UNUSED_33_BIT (63 - 33) +#define CR14_CHANNEL_REPORT_SUBMASK_BIT (63 - 35) +#define CR14_RECOVERY_SUBMASK_BIT (63 - 36) +#define CR14_DEGRADATION_SUBMASK_BIT (63 - 37) +#define CR14_EXTERNAL_DAMAGE_SUBMASK_BIT (63 - 38) +#define CR14_WARNING_SUBMASK_BIT (63 - 39) + +#define CR14_UNUSED_32 BIT(CR14_UNUSED_32_BIT) +#define CR14_UNUSED_33 BIT(CR14_UNUSED_33_BIT) +#define CR14_CHANNEL_REPORT_SUBMASK BIT(CR14_CHANNEL_REPORT_SUBMASK_BIT) +#define CR14_RECOVERY_SUBMASK BIT(CR14_RECOVERY_SUBMASK_BIT) +#define CR14_DEGRADATION_SUBMASK BIT(CR14_DEGRADATION_SUBMASK_BIT) +#define CR14_EXTERNAL_DAMAGE_SUBMASK BIT(CR14_EXTERNAL_DAMAGE_SUBMASK_BIT) +#define CR14_WARNING_SUBMASK BIT(CR14_WARNING_SUBMASK_BIT) #ifndef __ASSEMBLY__ -- cgit v1.2.3 From 99441a38c391b1115e405d1f47ede237fca37f1b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:13 +0200 Subject: s390: use control register bit defines Use control register bit defines instead of plain numbers where possible. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/startup.c | 2 +- arch/s390/kernel/early.c | 12 ++++++------ arch/s390/kernel/guarded_storage.c | 6 +++--- arch/s390/kernel/ipl.c | 2 +- arch/s390/kernel/nmi.c | 8 ++++---- arch/s390/kernel/perf_cpum_cf.c | 2 +- arch/s390/kernel/perf_pai_crypto.c | 4 ++-- arch/s390/kernel/perf_pai_ext.c | 4 ++-- arch/s390/kernel/ptrace.c | 2 +- arch/s390/kernel/time.c | 4 ++-- arch/s390/kvm/kvm-s390.c | 6 +++--- arch/s390/kvm/priv.c | 2 +- arch/s390/mm/vmem.c | 2 +- arch/s390/pci/pci.c | 2 +- drivers/s390/cio/crw.c | 2 +- net/iucv/iucv.c | 2 +- 16 files changed, 31 insertions(+), 31 deletions(-) diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index cf622479647f..8826c4f18645 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -49,7 +49,7 @@ static void detect_facilities(void) { if (test_facility(8)) { machine.has_edat1 = 1; - local_ctl_set_bit(0, 23); + local_ctl_set_bit(0, CR0_EDAT_BIT); } if (test_facility(78)) machine.has_edat2 = 1; diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 0beaa0ba58a8..ff1f02b54771 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -216,7 +216,7 @@ static __init void detect_machine_facilities(void) { if (test_facility(8)) { S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT1; - system_ctl_set_bit(0, 23); + system_ctl_set_bit(0, CR0_EDAT_BIT); } if (test_facility(78)) S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT2; @@ -224,13 +224,13 @@ static __init void detect_machine_facilities(void) S390_lowcore.machine_flags |= MACHINE_FLAG_IDTE; if (test_facility(50) && test_facility(73)) { S390_lowcore.machine_flags |= MACHINE_FLAG_TE; - system_ctl_set_bit(0, 55); + system_ctl_set_bit(0, CR0_TRANSACTIONAL_EXECUTION_BIT); } if (test_facility(51)) S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_LC; if (test_facility(129)) { S390_lowcore.machine_flags |= MACHINE_FLAG_VX; - system_ctl_set_bit(0, 17); + system_ctl_set_bit(0, CR0_VECTOR_BIT); } if (test_facility(130)) S390_lowcore.machine_flags |= MACHINE_FLAG_NX; @@ -240,7 +240,7 @@ static __init void detect_machine_facilities(void) /* Enabled signed clock comparator comparisons */ S390_lowcore.machine_flags |= MACHINE_FLAG_SCC; clock_comparator_max = -1ULL >> 1; - system_ctl_set_bit(0, 53); + system_ctl_set_bit(0, CR0_CLOCK_COMPARATOR_SIGN_BIT); } if (IS_ENABLED(CONFIG_PCI) && test_facility(153)) { S390_lowcore.machine_flags |= MACHINE_FLAG_PCI_MIO; @@ -260,7 +260,7 @@ static inline void save_vector_registers(void) static inline void setup_low_address_protection(void) { - system_ctl_set_bit(0, 28); + system_ctl_set_bit(0, CR0_LOW_ADDRESS_PROTECTION_BIT); } static inline void setup_access_registers(void) @@ -273,7 +273,7 @@ static inline void setup_access_registers(void) static int __init disable_vector_extension(char *str) { S390_lowcore.machine_flags &= ~MACHINE_FLAG_VX; - system_ctl_clear_bit(0, 17); + system_ctl_clear_bit(0, CR0_VECTOR_BIT); return 0; } early_param("novx", disable_vector_extension); diff --git a/arch/s390/kernel/guarded_storage.c b/arch/s390/kernel/guarded_storage.c index 945acfc77ee4..0b68168d9566 100644 --- a/arch/s390/kernel/guarded_storage.c +++ b/arch/s390/kernel/guarded_storage.c @@ -28,7 +28,7 @@ static int gs_enable(void) return -ENOMEM; gs_cb->gsd = 25; preempt_disable(); - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); load_gs_cb(gs_cb); current->thread.gs_cb = gs_cb; preempt_enable(); @@ -42,7 +42,7 @@ static int gs_disable(void) preempt_disable(); kfree(current->thread.gs_cb); current->thread.gs_cb = NULL; - local_ctl_clear_bit(2, 4); + local_ctl_clear_bit(2, CR2_GUARDED_STORAGE_BIT); preempt_enable(); } return 0; @@ -84,7 +84,7 @@ void gs_load_bc_cb(struct pt_regs *regs) if (gs_cb) { kfree(current->thread.gs_cb); current->thread.gs_bc_cb = NULL; - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); load_gs_cb(gs_cb); current->thread.gs_cb = gs_cb; } diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 492dd8f371da..cc364fce6aa9 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -2381,7 +2381,7 @@ void s390_reset_system(void) set_prefix(0); /* Disable lowcore protection */ - local_ctl_clear_bit(0, 28); + local_ctl_clear_bit(0, CR0_LOW_ADDRESS_PROTECTION_BIT); diag_amode31_ops.diag308_reset(); } diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 579cebc58d8c..0daf0f1cdfc9 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -185,7 +185,7 @@ void s390_handle_mcck(void) static int mchchk_wng_posted = 0; /* Use single cpu clear, as we cannot handle smp here. */ - local_ctl_clear_bit(14, 24); /* Disable WARNING MCH */ + local_ctl_clear_bit(14, CR14_WARNING_SUBMASK_BIT); if (xchg(&mchchk_wng_posted, 1) == 0) kill_cad_pid(SIGPWR, 1); } @@ -505,9 +505,9 @@ NOKPROBE_SYMBOL(s390_do_machine_check); static int __init machine_check_init(void) { - system_ctl_set_bit(14, 25); /* enable external damage MCH */ - system_ctl_set_bit(14, 27); /* enable system recovery MCH */ - system_ctl_set_bit(14, 24); /* enable warning MCH */ + system_ctl_set_bit(14, CR14_EXTERNAL_DAMAGE_SUBMASK_BIT); + system_ctl_set_bit(14, CR14_RECOVERY_SUBMASK_BIT); + system_ctl_set_bit(14, CR14_WARNING_SUBMASK_BIT); return 0; } early_initcall(machine_check_init); diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 4e05c0cb1648..41ed6e0f0a2a 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -1193,7 +1193,7 @@ static int __init cpumf_pmu_init(void) * Clear bit 15 of cr0 to unauthorize problem-state to * extract measurement counters */ - system_ctl_clear_bit(0, 48); + system_ctl_clear_bit(0, CR0_CPUMF_EXTRACTION_AUTH_BIT); /* register handler for measurement-alert interruptions */ rc = register_external_irq(EXT_IRQ_MEASURE_ALERT, diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 6612fd13e95b..db470243966f 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -265,7 +265,7 @@ static int paicrypt_add(struct perf_event *event, int flags) if (++cpump->active_events == 1) { ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET; WRITE_ONCE(S390_lowcore.ccd, ccd); - local_ctl_set_bit(0, 50); + local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); } cpump->event = event; if (flags & PERF_EF_START && !event->attr.sample_period) { @@ -294,7 +294,7 @@ static void paicrypt_del(struct perf_event *event, int flags) /* Only counting needs to read counter */ paicrypt_stop(event, PERF_EF_UPDATE); if (--cpump->active_events == 0) { - local_ctl_clear_bit(0, 50); + local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); WRITE_ONCE(S390_lowcore.ccd, 0); } } diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index 9f724ae44fc7..e91c77155f08 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -339,7 +339,7 @@ static int paiext_add(struct perf_event *event, int flags) S390_lowcore.aicd = virt_to_phys(cpump->paiext_cb); pcb->acc = virt_to_phys(cpump->area) | 0x1; /* Enable CPU instruction lookup for PAIE1 control block */ - local_ctl_set_bit(0, 49); + local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT); debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", __func__, S390_lowcore.aicd, pcb->acc); } @@ -375,7 +375,7 @@ static void paiext_del(struct perf_event *event, int flags) } if (--cpump->active_events == 0) { /* Disable CPU instruction lookup for PAIE1 control block */ - local_ctl_clear_bit(0, 49); + local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT); pcb->acc = 0; S390_lowcore.aicd = 0; debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 6a825351ff41..046403471c5d 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -1114,7 +1114,7 @@ static int s390_gs_cb_set(struct task_struct *target, target->thread.gs_cb = data; *target->thread.gs_cb = gs_cb; if (target == current) { - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); restore_gs_cb(target->thread.gs_cb); } preempt_enable(); diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 34c4fbf4e555..14abad953c02 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -173,10 +173,10 @@ void init_cpu_timer(void) clockevents_register_device(cd); /* Enable clock comparator timer interrupt. */ - local_ctl_set_bit(0, 11); + local_ctl_set_bit(0, CR0_CLOCK_COMPARATOR_SUBMASK_BIT); /* Always allow the timing alert external interrupt. */ - local_ctl_set_bit(0, 4); + local_ctl_set_bit(0, CR0_ETR_SUBMASK_BIT); } static void clock_comparator_interrupt(struct ext_code ext_code, diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 92bd15fe769a..e44e83f004ce 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4918,7 +4918,7 @@ static void sync_regs_fmt2(struct kvm_vcpu *vcpu) } if (MACHINE_HAS_GS) { preempt_disable(); - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); if (current->thread.gs_cb) { vcpu->arch.host_gscb = current->thread.gs_cb; save_gs_cb(vcpu->arch.host_gscb); @@ -4995,13 +4995,13 @@ static void store_regs_fmt2(struct kvm_vcpu *vcpu) kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val; if (MACHINE_HAS_GS) { preempt_disable(); - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); if (vcpu->arch.gs_enabled) save_gs_cb(current->thread.gs_cb); current->thread.gs_cb = vcpu->arch.host_gscb; restore_gs_cb(vcpu->arch.host_gscb); if (!vcpu->arch.host_gscb) - local_ctl_clear_bit(2, 4); + local_ctl_clear_bit(2, CR2_GUARDED_STORAGE_BIT); vcpu->arch.host_gscb = NULL; preempt_enable(); } diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index c199e4b5e5ce..621a17fd1a1b 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -57,7 +57,7 @@ static int handle_gs(struct kvm_vcpu *vcpu) if (test_kvm_facility(vcpu->kvm, 133)) { VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)"); preempt_disable(); - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); current->thread.gs_cb = (struct gs_cb *)&vcpu->run->s.regs.gscb; restore_gs_cb(current->thread.gs_cb); preempt_enable(); diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 95f04a502e62..5ed2e9254649 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -660,7 +660,7 @@ void __init vmem_map_init(void) __set_memory_4k(__va(0), RELOC_HIDE(__va(0), ident_map_size)); } if (MACHINE_HAS_NX) - system_ctl_set_bit(0, 20); + system_ctl_set_bit(0, CR0_INSTRUCTION_EXEC_PROTECTION_BIT); pr_info("Write protected kernel read-only data: %luk\n", (unsigned long)(__end_rodata - _stext) >> 10); } diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index a0aecc94e284..6fab5c085565 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -1094,7 +1094,7 @@ static int __init pci_base_init(void) if (MACHINE_HAS_PCI_MIO) { static_branch_enable(&have_mio); - system_ctl_set_bit(2, 5); + system_ctl_set_bit(2, CR2_MIO_ADDRESSING_BIT); } rc = zpci_debug_init(); diff --git a/drivers/s390/cio/crw.c b/drivers/s390/cio/crw.c index 872b1f2855cb..4916dd0a7eb1 100644 --- a/drivers/s390/cio/crw.c +++ b/drivers/s390/cio/crw.c @@ -156,7 +156,7 @@ static int __init crw_machine_check_init(void) task = kthread_run(crw_collect_info, NULL, "kmcheck"); if (IS_ERR(task)) return PTR_ERR(task); - system_ctl_set_bit(14, 28); /* enable channel report MCH */ + system_ctl_set_bit(14, CR14_CHANNEL_REPORT_SUBMASK_BIT); return 0; } device_initcall(crw_machine_check_init); diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c index b5cee8dad926..0ed6e34d6edd 100644 --- a/net/iucv/iucv.c +++ b/net/iucv/iucv.c @@ -1823,7 +1823,7 @@ static int __init iucv_init(void) rc = -EPROTONOSUPPORT; goto out; } - system_ctl_set_bit(0, 1); + system_ctl_set_bit(0, CR0_IUCV_BIT); rc = iucv_query_maxconn(); if (rc) goto out_ctl; -- cgit v1.2.3 From 32d1d9204f8db3360be55e65bd182a1a68f93308 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 12 Sep 2023 09:54:25 +0200 Subject: s390/ap: re-init AP queues on config on On a state toggle from config off to config on and on the state toggle from checkstop to not checkstop the queue's internal states was set but the state machine was not nudged. This did not care as on the first enqueue of a request the state machine kick ran. However, within an Secure Execution guest a queue is only chosen by the scheduler when it has been bound. But to bind a queue, it needs to run through the initial states (reset, enable interrupts, ...). So this is like a chicken-and-egg problem and the result was in fact that a queue was unusable after a config off/on toggle. With some slight rework of the handling of these states now the new function _ap_queue_init_state() is called which is the core of the ap_queue_init_state() function but without locking handling. This has the benefit that it can be called on all the places where a (re-)init of the AP queue's state machine is needed. Fixes: 2d72eaf036d2 ("s390/ap: implement SE AP bind, unbind and associate") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/ap_bus.c | 21 ++++++++++----------- drivers/s390/crypto/ap_bus.h | 1 + drivers/s390/crypto/ap_queue.c | 9 +++++++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 339812efe822..d09e08b71cfb 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -1865,15 +1865,18 @@ static inline void ap_scan_domains(struct ap_card *ac) } /* get it and thus adjust reference counter */ get_device(dev); - if (decfg) + if (decfg) { AP_DBF_INFO("%s(%d,%d) new (decfg) queue dev created\n", __func__, ac->id, dom); - else if (chkstop) + } else if (chkstop) { AP_DBF_INFO("%s(%d,%d) new (chkstop) queue dev created\n", __func__, ac->id, dom); - else + } else { + /* nudge the queue's state machine */ + ap_queue_init_state(aq); AP_DBF_INFO("%s(%d,%d) new queue dev created\n", __func__, ac->id, dom); + } goto put_dev_and_continue; } /* handle state changes on already existing queue device */ @@ -1895,10 +1898,8 @@ static inline void ap_scan_domains(struct ap_card *ac) } else if (!chkstop && aq->chkstop) { /* checkstop off */ aq->chkstop = false; - if (aq->dev_state > AP_DEV_STATE_UNINITIATED) { - aq->dev_state = AP_DEV_STATE_OPERATING; - aq->sm_state = AP_SM_STATE_RESET_START; - } + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) + _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); AP_DBF_DBG("%s(%d,%d) queue dev checkstop off\n", __func__, ac->id, dom); @@ -1922,10 +1923,8 @@ static inline void ap_scan_domains(struct ap_card *ac) } else if (!decfg && !aq->config) { /* config on this queue device */ aq->config = true; - if (aq->dev_state > AP_DEV_STATE_UNINITIATED) { - aq->dev_state = AP_DEV_STATE_OPERATING; - aq->sm_state = AP_SM_STATE_RESET_START; - } + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) + _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); AP_DBF_DBG("%s(%d,%d) queue dev config on\n", __func__, ac->id, dom); diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index be54b070c031..3e34912a6050 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -287,6 +287,7 @@ struct ap_queue *ap_queue_create(ap_qid_t qid, int device_type); void ap_queue_prepare_remove(struct ap_queue *aq); void ap_queue_remove(struct ap_queue *aq); void ap_queue_init_state(struct ap_queue *aq); +void _ap_queue_init_state(struct ap_queue *aq); struct ap_card *ap_card_create(int id, int queue_depth, int raw_type, int comp_type, unsigned int functions, int ml); diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 1336e632adc4..2943b2529d3a 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -1160,14 +1160,19 @@ void ap_queue_remove(struct ap_queue *aq) spin_unlock_bh(&aq->lock); } -void ap_queue_init_state(struct ap_queue *aq) +void _ap_queue_init_state(struct ap_queue *aq) { - spin_lock_bh(&aq->lock); aq->dev_state = AP_DEV_STATE_OPERATING; aq->sm_state = AP_SM_STATE_RESET_START; aq->last_err_rc = 0; aq->assoc_idx = ASSOC_IDX_INVALID; ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL)); +} + +void ap_queue_init_state(struct ap_queue *aq) +{ + spin_lock_bh(&aq->lock); + _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); } EXPORT_SYMBOL(ap_queue_init_state); -- cgit v1.2.3 From a19a161482b1739c2207861d086403389dc1bd47 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 12 Sep 2023 10:08:51 +0200 Subject: s390/zcrypt: introduce new internal AP queue se_bound attribute This patch introduces a new AP queue internal attribute se_bound which reflects the bound state of an APQN within a Secure Execution environment. With introduction of Secure Execution guests now an AP firmware queue needs to be bound to the guest before usage. This patch introduces a new internal attribute reflecting this bound state and some glue code to handle this new field during lifetime of an AP queue device. Together with that now the zcrypt scheduler considers the state of the AP queues when a message is about to be distributed among the existing queues. There is a new function ap_queue_usable() which returns true only when all conditions for using this AP queue device are fulfilled. In details this means: the AP queue needs to be configured, not checkstopped and within an SE environment it needs to be bound. So the new function gives and indication if the AP queue device is ready to serve requests or not. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/ap_bus.h | 2 ++ drivers/s390/crypto/ap_queue.c | 49 +++++++++++++++++++++++++++++++++++++++- drivers/s390/crypto/zcrypt_api.c | 10 ++++---- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 3e34912a6050..359a35f894d5 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -207,6 +207,7 @@ struct ap_queue { bool chkstop; /* checkstop state */ ap_qid_t qid; /* AP queue id. */ bool interrupt; /* indicate if interrupts are enabled */ + bool se_bound; /* SE bound state */ unsigned int assoc_idx; /* SE association index */ int queue_count; /* # messages currently on AP queue. */ int pendingq_count; /* # requests on pendingq list. */ @@ -271,6 +272,7 @@ enum ap_sm_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_sm_event event); int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg); void ap_cancel_message(struct ap_queue *aq, struct ap_message *ap_msg); void ap_flush_queue(struct ap_queue *aq); +bool ap_queue_usable(struct ap_queue *aq); void *ap_airq_ptr(void); int ap_sb_available(void); diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 2943b2529d3a..993240370ecf 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -33,6 +33,11 @@ static inline bool ap_q_supports_assoc(struct ap_queue *aq) return ap_test_bit(&aq->card->functions, AP_FUNC_EP11); } +static inline bool ap_q_needs_bind(struct ap_queue *aq) +{ + return ap_q_supports_bind(aq) && ap_sb_available(); +} + /** * ap_queue_enable_irq(): Enable interrupt support on this AP queue. * @aq: The AP queue @@ -304,6 +309,7 @@ static enum ap_sm_wait ap_sm_reset(struct ap_queue *aq) aq->sm_state = AP_SM_STATE_RESET_WAIT; aq->interrupt = false; aq->rapq_fbit = 0; + aq->se_bound = false; return AP_SM_WAIT_LOW_TIMEOUT; default: aq->dev_state = AP_DEV_STATE_ERROR; @@ -868,7 +874,12 @@ static ssize_t se_bind_store(struct device *dev, } status = ap_bapq(aq->qid); spin_unlock_bh(&aq->lock); - if (status.response_code) { + if (!status.response_code) { + aq->se_bound = true; + AP_DBF_INFO("%s bapq(0x%02x.%04x) success\n", __func__, + AP_QID_CARD(aq->qid), + AP_QID_QUEUE(aq->qid)); + } else { AP_DBF_WARN("%s RC 0x%02x on bapq(0x%02x.%04x)\n", __func__, status.response_code, AP_QID_CARD(aq->qid), @@ -1073,6 +1084,42 @@ int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg) } EXPORT_SYMBOL(ap_queue_message); +/** + * ap_queue_usable(): Check if queue is usable just now. + * @aq: The AP queue device to test for usability. + * This function is intended for the scheduler to query if it makes + * sense to enqueue a message into this AP queue device by calling + * ap_queue_message(). The perspective is very short-term as the + * state machine and device state(s) may change at any time. + */ +bool ap_queue_usable(struct ap_queue *aq) +{ + bool rc = true; + + spin_lock_bh(&aq->lock); + + /* check for not configured or checkstopped */ + if (!aq->config || aq->chkstop) { + rc = false; + goto unlock_and_out; + } + + /* device state needs to be ok */ + if (aq->dev_state != AP_DEV_STATE_OPERATING) { + rc = false; + goto unlock_and_out; + } + + /* SE guest's queues additionally need to be bound */ + if (ap_q_needs_bind(aq) && !aq->se_bound) + rc = false; + +unlock_and_out: + spin_unlock_bh(&aq->lock); + return rc; +} +EXPORT_SYMBOL(ap_queue_usable); + /** * ap_cancel_message(): Cancel a crypto request. * @aq: The AP device that has the message queued diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index ce04caa7913f..dcd6c7299fa9 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -693,7 +693,7 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->rsa_modexpo || - !zq->queue->config || zq->queue->chkstop) + !ap_queue_usable(zq->queue)) continue; /* check if device node has admission for this queue */ if (!zcrypt_check_queue(perms, @@ -798,7 +798,7 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->rsa_modexpo_crt || - !zq->queue->config || zq->queue->chkstop) + !ap_queue_usable(zq->queue)) continue; /* check if device node has admission for this queue */ if (!zcrypt_check_queue(perms, @@ -916,7 +916,7 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check for device usable and eligible */ if (!zq->online || !zq->ops->send_cprb || - !zq->queue->config || zq->queue->chkstop || + !ap_queue_usable(zq->queue) || (tdom != AUTOSEL_DOM && tdom != AP_QID_QUEUE(zq->queue->qid))) continue; @@ -1087,7 +1087,7 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->send_ep11_cprb || - !zq->queue->config || zq->queue->chkstop || + !ap_queue_usable(zq->queue) || (targets && !is_desired_ep11_queue(zq->queue->qid, target_num, targets))) @@ -1186,7 +1186,7 @@ static long zcrypt_rng(char *buffer) for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->rng || - !zq->queue->config || zq->queue->chkstop) + !ap_queue_usable(zq->queue)) continue; if (!zcrypt_queue_compare(zq, pref_zq, wgt, pref_wgt)) continue; -- cgit v1.2.3 From 8d6be876bbd7c09190b025ccc478260f5a40c216 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 12 Sep 2023 09:02:07 +0200 Subject: s390/ap: show APFS value on error reply 0x8B With Secure Execution the error reply RX 0x8B now carries an APFS value indicating why the request has been filtered by a lower layer of the firmware. So display this value as a warning line in the s390 debug feature trace. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/zcrypt_error.h | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/s390/crypto/zcrypt_error.h b/drivers/s390/crypto/zcrypt_error.h index d36177e65a3d..a44fcfcec938 100644 --- a/drivers/s390/crypto/zcrypt_error.h +++ b/drivers/s390/crypto/zcrypt_error.h @@ -98,8 +98,22 @@ static inline int convert_error(struct zcrypt_queue *zq, case REP88_ERROR_MESSAGE_MALFORMD: /* 0x22 */ case REP88_ERROR_KEY_TYPE: /* 0x34 */ /* RY indicates malformed request */ - ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x => rc=EINVAL\n", - __func__, card, queue, ehdr->reply_code); + if (ehdr->reply_code == REP82_ERROR_FILTERED_BY_HYPERVISOR && + ehdr->type == TYPE86_RSP_CODE) { + struct { + struct type86_hdr hdr; + struct type86_fmt2_ext fmt2; + } __packed * head = reply->msg; + unsigned int apfs = *((u32 *)head->fmt2.apfs); + + ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x apfs=0x%x => rc=EINVAL\n", + __func__, card, queue, + ehdr->reply_code, apfs); + } else { + ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x => rc=EINVAL\n", + __func__, card, queue, + ehdr->reply_code); + } return -EINVAL; case REP82_ERROR_MACHINE_FAILURE: /* 0x10 */ case REP82_ERROR_MESSAGE_TYPE: /* 0x20 */ -- cgit v1.2.3 From 4ff1261354d33caf7b2aa1f47db00deef27863ad Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Sun, 1 Oct 2023 10:09:34 +0200 Subject: s390/crash: remove unused parameter Funciton loads_init() does not use loads_offset parameter. Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/crash_dump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 7af69948b290..237c5a9199d9 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -498,7 +498,7 @@ static int get_mem_chunk_cnt(void) /* * Initialize ELF loads (new kernel) */ -static void loads_init(Elf64_Phdr *phdr, u64 loads_offset) +static void loads_init(Elf64_Phdr *phdr) { phys_addr_t start, end; u64 idx; @@ -612,7 +612,7 @@ int elfcorehdr_alloc(unsigned long long *addr, unsigned long long *size) ptr = notes_init(phdr_notes, ptr, ((unsigned long) hdr) + hdr_off); /* Init loads */ hdr_off = PTR_DIFF(ptr, hdr); - loads_init(phdr_loads, hdr_off); + loads_init(phdr_loads); *addr = (unsigned long long) hdr; *size = (unsigned long long) hdr_off; BUG_ON(elfcorehdr_size > alloc_size); -- cgit v1.2.3 From ecfe7f53286748bca0ef1646a45f428b915b0916 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Sun, 1 Oct 2023 10:01:31 +0200 Subject: s390/crash: fix virtual vs physical address confusion Fix virtual vs physical address confusion (which currently are the same). Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/crash_dump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 237c5a9199d9..514feadd4c58 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -507,7 +507,7 @@ static void loads_init(Elf64_Phdr *phdr) phdr->p_filesz = end - start; phdr->p_type = PT_LOAD; phdr->p_offset = start; - phdr->p_vaddr = start; + phdr->p_vaddr = (unsigned long)__va(start); phdr->p_paddr = start; phdr->p_memsz = end - start; phdr->p_flags = PF_R | PF_W | PF_X; -- cgit v1.2.3 From 88b2c332fcecbcf71a5d69a214a8805cf916fbf8 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 6 Oct 2023 15:42:40 +0200 Subject: s390/extable: reduce number of extable macros Get rid of __EX_TABLE() macro, rename __EX_TABLE_UA() to __EX_TABLE() and convert users of old __EX_TABLE() macro so they pass more parameters to the changed __EX_TABLE() semantics. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/asm-extable.h | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index e6532477f126..d1ecc2b6e5d3 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -23,16 +23,7 @@ #define EX_DATA_LEN_SHIFT 8 #define EX_DATA_LEN GENMASK(11, 8) -#define __EX_TABLE(_section, _fault, _target, _type) \ - stringify_in_c(.section _section,"a";) \ - stringify_in_c(.balign 4;) \ - stringify_in_c(.long (_fault) - .;) \ - stringify_in_c(.long (_target) - .;) \ - stringify_in_c(.short (_type);) \ - stringify_in_c(.short 0;) \ - stringify_in_c(.previous) - -#define __EX_TABLE_UA(_section, _fault, _target, _type, _regerr, _regaddr, _len)\ +#define __EX_TABLE(_section, _fault, _target, _type, _regerr, _regaddr, _len) \ stringify_in_c(.section _section,"a";) \ stringify_in_c(.balign 4;) \ stringify_in_c(.long (_fault) - .;) \ @@ -72,21 +63,21 @@ stringify_in_c(.previous) #define EX_TABLE(_fault, _target) \ - __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP) + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP, __stringify(%%r0), __stringify(%%r0), 0) #define EX_TABLE_AMODE31(_fault, _target) \ - __EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP) + __EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP, __stringify(%%r0), __stringify(%%r0), 0) #define EX_TABLE_UA_STORE(_fault, _target, _regerr) \ - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_STORE, _regerr, _regerr, 0) + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_STORE, _regerr, _regerr, 0) #define EX_TABLE_UA_LOAD_MEM(_fault, _target, _regerr, _regmem, _len) \ - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_MEM, _regerr, _regmem, _len) + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_MEM, _regerr, _regmem, _len) #define EX_TABLE_UA_LOAD_REG(_fault, _target, _regerr, _regzero) \ - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REG, _regerr, _regzero, 0) + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REG, _regerr, _regzero, 0) #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) #endif /* __ASM_EXTABLE_H */ -- cgit v1.2.3 From 76292d7243f809486ec26dd6cd3436dbcb9e5319 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 6 Oct 2023 15:42:41 +0200 Subject: s390: provide word-at-a-time implementation Provide an s390 specific word-at-a-time implementation. Compared to the generic variant the generated code for has_zero() is slightly better. However find_zero() is much simpler since it reuses the result of __fls() aka flogr() and now comes without any conditional branches, while the generic variant has three of them. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/word-at-a-time.h | 42 ++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 arch/s390/include/asm/word-at-a-time.h diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h new file mode 100644 index 000000000000..09b37541a175 --- /dev/null +++ b/arch/s390/include/asm/word-at-a-time.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_WORD_AT_A_TIME_H +#define _ASM_WORD_AT_A_TIME_H + +#include +#include + +struct word_at_a_time { + const unsigned long bits; +}; + +#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x7f) } + +static inline unsigned long prep_zero_mask(unsigned long val, unsigned long data, const struct word_at_a_time *c) +{ + return data; +} + +static inline unsigned long create_zero_mask(unsigned long data) +{ + return __fls(data); +} + +static inline unsigned long find_zero(unsigned long data) +{ + return (data ^ (BITS_PER_LONG - 1)) >> 3; +} + +static inline unsigned long has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c) +{ + unsigned long mask = (val & c->bits) + c->bits; + + *data = ~(mask | val | c->bits); + return *data; +} + +static inline unsigned long zero_bytemask(unsigned long data) +{ + return ~1UL << data; +} + +#endif /* _ASM_WORD_AT_A_TIME_H */ -- cgit v1.2.3 From 802ba53eefc592a6a82231f74e19bafe3256f172 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 6 Oct 2023 15:42:42 +0200 Subject: s390: add support for DCACHE_WORD_ACCESS Implement load_unaligned_zeropad() and enable DCACHE_WORD_ACCESS to speed up string operations in fs/dcache.c and fs/namei.c. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/Kconfig | 1 + arch/s390/include/asm/asm-extable.h | 4 ++++ arch/s390/include/asm/word-at-a-time.h | 22 ++++++++++++++++++++++ arch/s390/mm/extable.c | 18 ++++++++++++++++++ 4 files changed, 45 insertions(+) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index ae29e4392664..b0d67ac8695f 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -131,6 +131,7 @@ config S390 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 + select DCACHE_WORD_ACCESS if !KMSAN select DMA_OPS if PCI select DYNAMIC_FTRACE if FUNCTION_TRACER select FUNCTION_ALIGNMENT_8B if CC_IS_GCC diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index d1ecc2b6e5d3..4a6b0a8b6412 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -13,6 +13,7 @@ #define EX_TYPE_UA_LOAD_MEM 4 #define EX_TYPE_UA_LOAD_REG 5 #define EX_TYPE_UA_LOAD_REGPAIR 6 +#define EX_TYPE_ZEROPAD 7 #define EX_DATA_REG_ERR_SHIFT 0 #define EX_DATA_REG_ERR GENMASK(3, 0) @@ -80,4 +81,7 @@ #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) +#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) + #endif /* __ASM_EXTABLE_H */ diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index 09b37541a175..2579f1694b82 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -3,6 +3,7 @@ #define _ASM_WORD_AT_A_TIME_H #include +#include #include struct word_at_a_time { @@ -39,4 +40,25 @@ static inline unsigned long zero_bytemask(unsigned long data) return ~1UL << data; } +/* + * Load an unaligned word from kernel space. + * + * In the (very unlikely) case of the word being a page-crosser + * and the next page not being mapped, take the exception and + * return zeroes in the non-existing part. + */ +static inline unsigned long load_unaligned_zeropad(const void *addr) +{ + unsigned long data; + + asm volatile( + "0: lg %[data],0(%[addr])\n" + "1: nopr %%r7\n" + EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) + EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) + : [data] "=d" (data) + : [addr] "a" (addr), "m" (*(unsigned long *)addr)); + return data; +} + #endif /* _ASM_WORD_AT_A_TIME_H */ diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index fe87291df95d..0a0738a473af 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -61,6 +61,22 @@ static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex, return true; } +static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs) +{ + unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data); + unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data); + unsigned long data, addr, offset; + + addr = regs->gprs[reg_addr]; + offset = addr & (sizeof(unsigned long) - 1); + addr &= ~(sizeof(unsigned long) - 1); + data = *(unsigned long *)addr; + data <<= BITS_PER_BYTE * offset; + regs->gprs[reg_data] = data; + regs->psw.addr = extable_fixup(ex); + return true; +} + bool fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *ex; @@ -81,6 +97,8 @@ bool fixup_exception(struct pt_regs *regs) return ex_handler_ua_load_reg(ex, false, regs); case EX_TYPE_UA_LOAD_REGPAIR: return ex_handler_ua_load_reg(ex, true, regs); + case EX_TYPE_ZEROPAD: + return ex_handler_zeropad(ex, regs); } panic("invalid exception table entry"); } -- cgit v1.2.3 From 19ba9ead8a88ab02f3b00d5a6cab7853f03445e7 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Thu, 12 Oct 2023 11:39:29 +0200 Subject: s390/vmem: remove unused variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the follow warning reported by sparse: arch/s390/boot/vmem.c:170:15: warning: unused variable ‘entry’ [-Wunused-variable] 170 | pte_t entry; | ^~~~~ Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/boot/vmem.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index bdbfee86d1ac..007e87239bbf 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -162,8 +162,6 @@ static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) { - pte_t entry; - if (mode == POPULATE_KASAN_ZERO_SHADOW) { set_pte(pte, pte_z); return true; -- cgit v1.2.3 From 7bc8b8eb2b1e7b06d14bacf964fa25f74a2b87e5 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 16 Oct 2023 12:10:04 +0200 Subject: s390/mm: remove __GFP_HIGHMEM masking Remove unnecessary __GFP_HIGHMEM masking, which was introduced with commit 6326c26c1514 ("s390: convert various pgalloc functions to use ptdescs"). Also remove a whitespace change which was introduced with the same commit. Link: https://lore.kernel.org/all/CAOzc2px-SFSnmjcPriiB3cm1fNj3+YC8S0VSp4t1QvDR0f4E2A@mail.gmail.com Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/pgalloc.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 61fb157029c8..a6b71bc71896 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -488,11 +488,10 @@ static unsigned long *base_crst_alloc(unsigned long val) unsigned long *table; struct ptdesc *ptdesc; - ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, CRST_ALLOC_ORDER); + ptdesc = pagetable_alloc(GFP_KERNEL, CRST_ALLOC_ORDER); if (!ptdesc) return NULL; table = ptdesc_address(ptdesc); - crst_table_init(table, val); return table; } -- cgit v1.2.3 From 4f62c6e30155c7a85ed74e52bb8c71e7b0879cb3 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 16 Oct 2023 12:17:59 +0200 Subject: s390/mm: make vmemmap_free() only for CONFIG_MEMORY_HOTPLUG available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Get rid of this W=1 compile warning: arch/s390/mm/vmem.c:502:6: warning: no previous prototype for ‘vmemmap_free’ [-Wmissing-prototypes] 502 | void vmemmap_free(unsigned long start, unsigned long end, | ^~~~~~~~~~~~ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/vmem.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 5ed2e9254649..86384586eb20 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -499,6 +499,8 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, return ret; } +#ifdef CONFIG_MEMORY_HOTPLUG + void vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap) { @@ -507,6 +509,8 @@ void vmemmap_free(unsigned long start, unsigned long end, mutex_unlock(&vmem_mutex); } +#endif + void vmem_remove_mapping(unsigned long start, unsigned long size) { mutex_lock(&vmem_mutex); -- cgit v1.2.3 From 60f8f641f3db85edf20f9612f4801563119d7fd6 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Mon, 21 Aug 2023 16:49:05 +0200 Subject: s390/pai_crypto: dynamically allocate percpu pai crypto map data structure Struct paicrypt_map is a data structure and is statically defined for each possible CPU. Rework this and replace it by dynamically allocated data structures created when a perf_event_open() system call is invoked. It is replaced by an array of pointers to all possible CPUs and reference counting. The array of pointers is allocated when the first event is created. For each online CPU an event is installed on, a struct paicrypt_map is allocated and a pointer to struct cpu_cf_events is stored in the array: CPU 0 1 2 3 ... N +---+---+---+---+---+---+ paicrypt_root::mapptr--> | * | | | |...| | +-|-+---+---+---+---+---+ | | \|/ +--------------+ | paicrypt_map | +--------------+ With this approach the large data structure is only allocated when an event is actually installed and used. Also implement proper reference counting for allocation and removal. PAI crypto counter events can not be created when a CPU hot plug add is processed. This means a CPU hot plug add does not get the necessary PAI event to record PAI cryptography counter increments on the newly added CPU. There is no possibility to notify user space of a new CPU and the necessary event infrastructure assoiciated with the file descriptor returned by perf_event_open() system call. However system call perf_event_open() can use the newly added CPU when issued after the CPU hot plug add. Kernel CPU hot plug remove deletes the CPU and stops the PAI counters on that CPU. When the process closes the file descriptor associated with that event, the event's destroy() function removes any allocated data structures and adjusts the reference counts. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Acked-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_pai_crypto.c | 127 +++++++++++++++++++++++++++++-------- 1 file changed, 100 insertions(+), 27 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index db470243966f..19d53edf50d2 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -40,7 +40,43 @@ struct paicrypt_map { struct perf_event *event; /* Perf event for sampling */ }; -static DEFINE_PER_CPU(struct paicrypt_map, paicrypt_map); +struct paicrypt_mapptr { + struct paicrypt_map *mapptr; +}; + +static struct paicrypt_root { /* Anchor to per CPU data */ + refcount_t refcnt; /* Overall active events */ + struct paicrypt_mapptr __percpu *mapptr; +} paicrypt_root; + +/* Free per CPU data when the last event is removed. */ +static void paicrypt_root_free(void) +{ + if (refcount_dec_and_test(&paicrypt_root.refcnt)) { + free_percpu(paicrypt_root.mapptr); + paicrypt_root.mapptr = NULL; + } + debug_sprintf_event(cfm_dbg, 5, "%s root.refcount %d\n", __func__, + refcount_read(&paicrypt_root.refcnt)); +} + +/* + * On initialization of first event also allocate per CPU data dynamically. + * Start with an array of pointers, the array size is the maximum number of + * CPUs possible, which might be larger than the number of CPUs currently + * online. + */ +static int paicrypt_root_alloc(void) +{ + if (!refcount_inc_not_zero(&paicrypt_root.refcnt)) { + /* The memory is already zeroed. */ + paicrypt_root.mapptr = alloc_percpu(struct paicrypt_mapptr); + if (!paicrypt_root.mapptr) + return -ENOMEM; + refcount_set(&paicrypt_root.refcnt, 1); + } + return 0; +} /* Release the PMU if event is the last perf event */ static DEFINE_MUTEX(pai_reserve_mutex); @@ -50,7 +86,9 @@ static DEFINE_MUTEX(pai_reserve_mutex); */ static void paicrypt_event_destroy(struct perf_event *event) { - struct paicrypt_map *cpump = per_cpu_ptr(&paicrypt_map, event->cpu); + struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr, + event->cpu); + struct paicrypt_map *cpump = mp->mapptr; cpump->event = NULL; static_branch_dec(&pai_key); @@ -65,11 +103,11 @@ static void paicrypt_event_destroy(struct perf_event *event) __func__, (unsigned long)cpump->page, cpump->save); free_page((unsigned long)cpump->page); - cpump->page = NULL; kvfree(cpump->save); - cpump->save = NULL; - cpump->mode = PAI_MODE_NONE; + kfree(cpump); + mp->mapptr = NULL; } + paicrypt_root_free(); mutex_unlock(&pai_reserve_mutex); } @@ -85,7 +123,8 @@ static u64 paicrypt_getctr(struct paicrypt_map *cpump, int nr, bool kernel) */ static u64 paicrypt_getdata(struct perf_event *event, bool kernel) { - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); + struct paicrypt_map *cpump = mp->mapptr; u64 sum = 0; int i; @@ -131,11 +170,31 @@ static u64 paicrypt_getall(struct perf_event *event) * * Allocate the memory for the event. */ -static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump) +static struct paicrypt_map *paicrypt_busy(struct perf_event *event) { - int rc = 0; + struct perf_event_attr *a = &event->attr; + struct paicrypt_map *cpump = NULL; + struct paicrypt_mapptr *mp; + int rc; mutex_lock(&pai_reserve_mutex); + + /* Allocate root node */ + rc = paicrypt_root_alloc(); + if (rc) + goto unlock; + + /* Allocate node for this event */ + mp = per_cpu_ptr(paicrypt_root.mapptr, event->cpu); + cpump = mp->mapptr; + if (!cpump) { /* Paicrypt_map allocated? */ + cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); + if (!cpump) { + rc = -ENOMEM; + goto free_root; + } + } + if (a->sample_period) { /* Sampling requested */ if (cpump->mode != PAI_MODE_NONE) rc = -EBUSY; /* ... sampling/counting active */ @@ -143,8 +202,15 @@ static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump) if (cpump->mode == PAI_MODE_SAMPLING) rc = -EBUSY; /* ... and sampling active */ } + /* + * This error case triggers when there is a conflict: + * Either sampling requested and counting already active, or visa + * versa. Therefore the struct paicrypto_map for this CPU is + * needed or the error could not have occurred. Only adjust root + * node refcount. + */ if (rc) - goto unlock; + goto free_root; /* Allocate memory for counter page and counter extraction. * Only the first counting event has to allocate a page. @@ -157,30 +223,36 @@ static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump) rc = -ENOMEM; cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL); if (!cpump->page) - goto unlock; + goto free_paicrypt_map; cpump->save = kvmalloc_array(paicrypt_cnt + 1, sizeof(struct pai_userdata), GFP_KERNEL); if (!cpump->save) { free_page((unsigned long)cpump->page); cpump->page = NULL; - goto unlock; + goto free_paicrypt_map; } + + /* Set mode and reference count */ rc = 0; refcount_set(&cpump->refcnt, 1); - -unlock: - /* If rc is non-zero, do not set mode and reference count */ - if (!rc) { - cpump->mode = a->sample_period ? PAI_MODE_SAMPLING - : PAI_MODE_COUNTING; - } + cpump->mode = a->sample_period ? PAI_MODE_SAMPLING : PAI_MODE_COUNTING; + mp->mapptr = cpump; debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d" " mode %d refcnt %u page %#lx save %p rc %d\n", __func__, a->sample_period, cpump->active_events, cpump->mode, refcount_read(&cpump->refcnt), (unsigned long)cpump->page, cpump->save, rc); + goto unlock; + +free_paicrypt_map: + kfree(cpump); + mp->mapptr = NULL; +free_root: + paicrypt_root_free(); + +unlock: mutex_unlock(&pai_reserve_mutex); - return rc; + return rc ? ERR_PTR(rc) : cpump; } /* Might be called on different CPU than the one the event is intended for. */ @@ -188,7 +260,6 @@ static int paicrypt_event_init(struct perf_event *event) { struct perf_event_attr *a = &event->attr; struct paicrypt_map *cpump; - int rc; /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) @@ -204,10 +275,9 @@ static int paicrypt_event_init(struct perf_event *event) if (a->sample_period && a->config != PAI_CRYPTO_BASE) return -EINVAL; - cpump = per_cpu_ptr(&paicrypt_map, event->cpu); - rc = paicrypt_busy(a, cpump); - if (rc) - return rc; + cpump = paicrypt_busy(event); + if (IS_ERR(cpump)) + return PTR_ERR(cpump); /* Event initialization sets last_tag to 0. When later on the events * are deleted and re-added, do not reset the event count value to zero. @@ -259,7 +329,8 @@ static void paicrypt_start(struct perf_event *event, int flags) static int paicrypt_add(struct perf_event *event, int flags) { - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); + struct paicrypt_map *cpump = mp->mapptr; unsigned long ccd; if (++cpump->active_events == 1) { @@ -286,7 +357,8 @@ static void paicrypt_stop(struct perf_event *event, int flags) static void paicrypt_del(struct perf_event *event, int flags) { - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); + struct paicrypt_map *cpump = mp->mapptr; if (event->attr.sample_period) perf_sched_cb_dec(event->pmu); @@ -328,7 +400,8 @@ static size_t paicrypt_copy(struct pai_userdata *userdata, static int paicrypt_push_sample(void) { - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); + struct paicrypt_map *cpump = mp->mapptr; struct perf_event *event = cpump->event; struct perf_sample_data data; struct perf_raw_record raw; -- cgit v1.2.3 From 3aad8c044297799bee37492fc9695aafd7513d0b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:30 +0200 Subject: s390/mm,fault: remove and improve comments, adjust whitespace Remove wrong, outdated, and pointless comments. Adjust wording for some comments, and adjust whitespace at some places. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 86 ++++++++++++++-------------------------------------- 1 file changed, 23 insertions(+), 63 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 587b6b64185f..797e6e2f1d11 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -3,7 +3,7 @@ * S390 version * Copyright IBM Corp. 1999 * Author(s): Hartmut Penner (hp@de.ibm.com) - * Ulrich Weigand (uweigand@de.ibm.com) + * Ulrich Weigand (uweigand@de.ibm.com) * * Derived from "arch/i386/mm/fault.c" * Copyright (C) 1995 Linus Torvalds @@ -45,8 +45,8 @@ #define __FAIL_ADDR_MASK -4096L /* - * Allocate private vm_fault_reason from top. Please make sure it won't - * collide with vm_fault_reason. + * Allocate private vm_fault_reason from top. + * Please make sure it won't collide with vm_fault_reason. */ #define VM_FAULT_BADCONTEXT ((__force vm_fault_t)0x80000000) #define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000) @@ -78,8 +78,8 @@ static enum fault_type get_fault_type(struct pt_regs *regs) unsigned long trans_exc_code; trans_exc_code = regs->int_parm_long & 3; + /* Primary address space */ if (likely(trans_exc_code == 0)) { - /* primary space exception */ if (user_mode(regs)) return USER_FAULT; if (!IS_ENABLED(CONFIG_PGSTE)) @@ -90,11 +90,11 @@ static enum fault_type get_fault_type(struct pt_regs *regs) } if (trans_exc_code == 2) return USER_FAULT; + /* Access register mode, not used in the kernel */ if (trans_exc_code == 1) { - /* access register mode, not used in the kernel */ return USER_FAULT; } - /* home space exception -> access via kernel ASCE */ + /* Home space -> access via kernel ASCE */ return KERNEL_FAULT; } @@ -233,10 +233,6 @@ void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault) show_regs(regs); } -/* - * Send SIGSEGV to task. This is an external routine - * to keep the stack usage of do_page_fault small. - */ static noinline void do_sigsegv(struct pt_regs *regs, int si_code) { report_user_fault(regs, SIGSEGV, 1); @@ -259,10 +255,6 @@ static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault) if (kfence_handle_page_fault(address, is_write, regs)) return; } - /* - * Oops. The kernel tried to access some bad page. We'll have to - * terminate things with extreme prejudice. - */ if (fault_type == KERNEL_FAULT) printk(KERN_ALERT "Unable to handle kernel pointer dereference" " in virtual kernel address space\n"); @@ -275,10 +267,12 @@ static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault) static noinline void do_low_address(struct pt_regs *regs) { - /* Low-address protection hit in kernel mode means - NULL pointer write access in kernel mode. */ + /* + * Low-address protection hit in kernel mode means + * NULL pointer write access in kernel mode. + */ if (regs->psw.mask & PSW_MASK_PSTATE) { - /* Low-address protection hit in user mode 'cannot happen'. */ + /* Low-address protection hit in user mode: 'cannot happen' */ die (regs, "Low-address protection"); } @@ -287,10 +281,6 @@ static noinline void do_low_address(struct pt_regs *regs) static noinline void do_sigbus(struct pt_regs *regs) { - /* - * Send a sigbus, regardless of whether we were in kernel - * or user mode. - */ force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); } @@ -349,10 +339,10 @@ static noinline void do_fault_error(struct pt_regs *regs, vm_fault_t fault) * routines. * * interruption code (int_code): - * 04 Protection -> Write-Protection (suppression) - * 10 Segment translation -> Not present (nullification) - * 11 Page translation -> Not present (nullification) - * 3b Region third trans. -> Not present (nullification) + * 04 Protection -> Write-Protection (suppression) + * 10 Segment translation -> Not present (nullification) + * 11 Page translation -> Not present (nullification) + * 3b Region third trans. -> Not present (nullification) */ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) { @@ -372,19 +362,11 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) * been nullified. Don't signal single step via SIGTRAP. */ clear_thread_flag(TIF_PER_TRAP); - if (kprobe_page_fault(regs, 14)) return 0; - mm = tsk->mm; address = get_fault_address(regs); is_write = fault_is_write(regs); - - /* - * Verify that the fault happened in user space, that - * we are not in an interrupt and that there is a - * user context. - */ fault = VM_FAULT_BADCONTEXT; type = get_fault_type(regs); switch (type) { @@ -396,7 +378,6 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) goto out; break; } - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); flags = FAULT_FLAG_DEFAULT; if (user_mode(regs)) @@ -431,7 +412,6 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) } lock_mmap: mmap_read_lock(mm); - gmap = NULL; if (IS_ENABLED(CONFIG_PGSTE) && type == GMAP_FAULT) { gmap = (struct gmap *) S390_lowcore.gmap; @@ -446,13 +426,11 @@ lock_mmap: if (gmap->pfault_enabled) flags |= FAULT_FLAG_RETRY_NOWAIT; } - retry: fault = VM_FAULT_BADMAP; vma = find_vma(mm, address); if (!vma) goto out_up; - if (unlikely(vma->vm_start > address)) { if (!(vma->vm_flags & VM_GROWSDOWN)) goto out_up; @@ -460,20 +438,9 @@ retry: if (!vma) goto out; } - - /* - * Ok, we have a good vm_area for this memory access, so - * we can handle it.. - */ fault = VM_FAULT_BADACCESS; if (unlikely(!(vma->vm_flags & access))) goto out_up; - - /* - * If for any reason at all we couldn't handle the fault, - * make sure we exit gracefully rather than endlessly redo - * the fault. - */ fault = handle_mm_fault(vma, address, flags, regs); if (fault_signal_pending(fault, regs)) { fault = VM_FAULT_SIGNAL; @@ -481,7 +448,6 @@ retry: goto out_up; goto out; } - /* The fault is fully completed (including releasing mmap lock) */ if (fault & VM_FAULT_COMPLETED) { if (gmap) { @@ -491,16 +457,14 @@ retry: fault = 0; goto out; } - if (unlikely(fault & VM_FAULT_ERROR)) goto out_up; - if (fault & VM_FAULT_RETRY) { if (IS_ENABLED(CONFIG_PGSTE) && gmap && (flags & FAULT_FLAG_RETRY_NOWAIT)) { /* - * FAULT_FLAG_RETRY_NOWAIT has been set, mmap_lock has - * not been released + * FAULT_FLAG_RETRY_NOWAIT has been set, + * mmap_lock has not been released */ current->thread.gmap_pfault = 1; fault = VM_FAULT_PFAULT; @@ -592,11 +556,10 @@ void do_secure_storage_access(struct pt_regs *regs) int rc; /* - * bit 61 tells us if the address is valid, if it's not we - * have a major problem and should stop the kernel or send a - * SIGSEGV to the process. Unfortunately bit 61 is not - * reliable without the misc UV feature so we need to check - * for that as well. + * Bit 61 indicates if the address is valid, if it is not the + * kernel should be stopped or SIGSEGV should be sent to the + * process. Bit 61 is not reliable without the misc UV feature, + * therefore this needs to be checked too. */ if (uv_has_feature(BIT_UV_FEAT_MISC) && !test_bit_inv(61, ®s->int_parm_long)) { @@ -609,14 +572,12 @@ void do_secure_storage_access(struct pt_regs *regs) send_sig(SIGSEGV, current, 0); return; } - /* - * The kernel should never run into this case and we - * have no way out of this situation. + * The kernel should never run into this case and + * there is no way out of this situation. */ panic("Unexpected PGM 0x3d with TEID bit 61=0"); } - switch (get_fault_type(regs)) { case GMAP_FAULT: mm = current->mm; @@ -674,7 +635,6 @@ void do_non_secure_storage_access(struct pt_regs *regs) WARN_ON_ONCE(1); return; } - if (gmap_convert_to_secure(gmap, gaddr) == -EINVAL) send_sig(SIGSEGV, current, 0); } -- cgit v1.2.3 From 7c915a84e5e2cacf71f7328b940aa783c1b7f2e4 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:31 +0200 Subject: s390/mm,fault: reverse x-mas tree coding style Have reverse x-mas tree coding style for variables everywhere. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 797e6e2f1d11..c13999aa0526 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -346,13 +346,13 @@ static noinline void do_fault_error(struct pt_regs *regs, vm_fault_t fault) */ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) { - struct gmap *gmap; + struct vm_area_struct *vma; struct task_struct *tsk; + unsigned long address; struct mm_struct *mm; - struct vm_area_struct *vma; enum fault_type type; - unsigned long address; unsigned int flags; + struct gmap *gmap; vm_fault_t fault; bool is_write; @@ -498,8 +498,8 @@ out: void do_protection_exception(struct pt_regs *regs) { unsigned long trans_exc_code; - int access; vm_fault_t fault; + int access; trans_exc_code = regs->int_parm_long; /* @@ -534,8 +534,8 @@ NOKPROBE_SYMBOL(do_protection_exception); void do_dat_exception(struct pt_regs *regs) { - int access; vm_fault_t fault; + int access; access = VM_ACCESS_FLAGS; fault = do_exception(regs, access); -- cgit v1.2.3 From 28f3e0002aeaf9b02ace41551c92dc1c04f0de6e Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:32 +0200 Subject: s390/mm,fault: use __ratelimit() instead of printk_ratelimit() Just like other architectures make use __ratelimit() instead of printk_ratelimit(). Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index c13999aa0526..d22a8fe5e4fc 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -218,11 +218,13 @@ int show_unhandled_signals = 1; void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault) { + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); + if ((task_pid_nr(current) > 1) && !show_unhandled_signals) return; if (!unhandled_signal(current, signr)) return; - if (!printk_ratelimit()) + if (!__ratelimit(&rs)) return; printk(KERN_ALERT "User process fault: interruption code %04x ilc:%d ", regs->int_code & 0xffff, regs->int_code >> 17); -- cgit v1.2.3 From c9b611bf6e576aa1a03ac097b74322af56cb7052 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:33 +0200 Subject: s390/mm,fault: use pr_warn_ratelimited() Use pr_warn_ratelimited() instead of printk_ratelimited(). checkpatch reports: WARNING: Prefer ... pr_warn_ratelimited(... to printk_ratelimited(KERN_WARNING ... + printk_ratelimited(KERN_WARNING Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index d22a8fe5e4fc..dcea87f9ef68 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -661,9 +661,8 @@ void do_secure_storage_violation(struct pt_regs *regs) * This exception is only triggered when a guest 2 is running * and can therefore never occur in kernel context. */ - printk_ratelimited(KERN_WARNING - "Secure storage violation in task: %s, pid %d\n", - current->comm, current->pid); + pr_warn_ratelimited("Secure storage violation in task: %s, pid %d\n", + current->comm, current->pid); send_sig(SIGSEGV, current, 0); } -- cgit v1.2.3 From 760f6511096c5e37e9c9cf240d1ad525259bac86 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:34 +0200 Subject: s390/mm,fault: use pr_warn(), pr_cont(), ... instead of open-coding Use pr_warn() and friends instead of open-coding with printk(). Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index dcea87f9ef68..2d629c1693e0 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -226,10 +226,10 @@ void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault) return; if (!__ratelimit(&rs)) return; - printk(KERN_ALERT "User process fault: interruption code %04x ilc:%d ", - regs->int_code & 0xffff, regs->int_code >> 17); + pr_alert("User process fault: interruption code %04x ilc:%d ", + regs->int_code & 0xffff, regs->int_code >> 17); print_vma_addr(KERN_CONT "in ", regs->psw.addr); - printk(KERN_CONT "\n"); + pr_cont("\n"); if (is_mm_fault) dump_fault_info(regs); show_regs(regs); @@ -258,11 +258,9 @@ static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault) return; } if (fault_type == KERNEL_FAULT) - printk(KERN_ALERT "Unable to handle kernel pointer dereference" - " in virtual kernel address space\n"); + pr_alert("Unable to handle kernel pointer dereference in virtual kernel address space\n"); else - printk(KERN_ALERT "Unable to handle kernel paging request" - " in virtual user address space\n"); + pr_alert("Unable to handle kernel paging request in virtual user address space\n"); dump_fault_info(regs); die(regs, "Oops"); } -- cgit v1.2.3 From 8dbc33dc81638cfcd8d3b4103ba315d76c3c2aff Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:35 +0200 Subject: s390/mm,fault: have balanced braces, remove unnecessary blanks Remove unnecessary braces and also blanks after casts. Add braces to have balanced braces where missing. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 2d629c1693e0..e7f5330dd363 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -91,9 +91,8 @@ static enum fault_type get_fault_type(struct pt_regs *regs) if (trans_exc_code == 2) return USER_FAULT; /* Access register mode, not used in the kernel */ - if (trans_exc_code == 1) { + if (trans_exc_code == 1) return USER_FAULT; - } /* Home space -> access via kernel ASCE */ return KERNEL_FAULT; } @@ -200,7 +199,7 @@ static void dump_fault_info(struct pt_regs *regs) pr_cont("user "); break; case GMAP_FAULT: - asce = ((struct gmap *) S390_lowcore.gmap)->asce; + asce = ((struct gmap *)S390_lowcore.gmap)->asce; pr_cont("gmap "); break; case KERNEL_FAULT: @@ -273,7 +272,7 @@ static noinline void do_low_address(struct pt_regs *regs) */ if (regs->psw.mask & PSW_MASK_PSTATE) { /* Low-address protection hit in user mode: 'cannot happen' */ - die (regs, "Low-address protection"); + die(regs, "Low-address protection"); } do_no_context(regs, VM_FAULT_BADACCESS); @@ -327,8 +326,9 @@ static noinline void do_fault_error(struct pt_regs *regs, vm_fault_t fault) do_no_context(regs, fault); else do_sigbus(regs); - } else + } else { BUG(); + } break; } } @@ -414,7 +414,7 @@ lock_mmap: mmap_read_lock(mm); gmap = NULL; if (IS_ENABLED(CONFIG_PGSTE) && type == GMAP_FAULT) { - gmap = (struct gmap *) S390_lowcore.gmap; + gmap = (struct gmap *)S390_lowcore.gmap; current->thread.gmap_addr = address; current->thread.gmap_write_flag = !!(flags & FAULT_FLAG_WRITE); current->thread.gmap_int_code = regs->int_code & 0xffff; -- cgit v1.2.3 From e23c53467ba03a538f4e8f837ca80f0bb058a4ee Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:36 +0200 Subject: s390/mm,fault: include linux/mmu_context.h Include linux/mmu_context.h instead asm/mmu_context.h. checkpatch reports: CHECK: Consider using #include instead of +#include Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index e7f5330dd363..e457d87e53d2 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -37,7 +38,6 @@ #include #include #include -#include #include #include #include "../kernel/entry.h" -- cgit v1.2.3 From 4df5ec98581eb2271d940975186fbe5716158491 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:37 +0200 Subject: s390/mm,fault: remove line break chechpatch reports: CHECK: Alignment should match open parenthesis + if (IS_ENABLED(CONFIG_PGSTE) && gmap && + (flags & FAULT_FLAG_RETRY_NOWAIT)) { Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index e457d87e53d2..9545789d34ef 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -460,8 +460,7 @@ retry: if (unlikely(fault & VM_FAULT_ERROR)) goto out_up; if (fault & VM_FAULT_RETRY) { - if (IS_ENABLED(CONFIG_PGSTE) && gmap && - (flags & FAULT_FLAG_RETRY_NOWAIT)) { + if (IS_ENABLED(CONFIG_PGSTE) && gmap && (flags & FAULT_FLAG_RETRY_NOWAIT)) { /* * FAULT_FLAG_RETRY_NOWAIT has been set, * mmap_lock has not been released -- cgit v1.2.3 From 5c845de331d980d90c6d18028942812fa491fe69 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:38 +0200 Subject: s390/mm,fault: remove noinline attribute from all functions Remove all noinline attribute from all functions and leave the inlining decisions up to the compiler. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 9545789d34ef..4ab698988618 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -234,14 +234,14 @@ void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault) show_regs(regs); } -static noinline void do_sigsegv(struct pt_regs *regs, int si_code) +static void do_sigsegv(struct pt_regs *regs, int si_code) { report_user_fault(regs, SIGSEGV, 1); force_sig_fault(SIGSEGV, si_code, (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); } -static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault) +static void do_no_context(struct pt_regs *regs, vm_fault_t fault) { enum fault_type fault_type; unsigned long address; @@ -264,7 +264,7 @@ static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault) die(regs, "Oops"); } -static noinline void do_low_address(struct pt_regs *regs) +static void do_low_address(struct pt_regs *regs) { /* * Low-address protection hit in kernel mode means @@ -278,13 +278,13 @@ static noinline void do_low_address(struct pt_regs *regs) do_no_context(regs, VM_FAULT_BADACCESS); } -static noinline void do_sigbus(struct pt_regs *regs) +static void do_sigbus(struct pt_regs *regs) { force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); } -static noinline void do_fault_error(struct pt_regs *regs, vm_fault_t fault) +static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) { int si_code; -- cgit v1.2.3 From ae626f686351d220d5eab746dc3a19e90bb1de9a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:39 +0200 Subject: s390/mm,fault: replace WARN_ON_ONCE() with unreachable() do_secure_storage_access() contains a switch statements which handles all possible return values from get_fault_type(). Therefore remove the pointless default case error handling and replace it with unreachable(). Reviewed-by: Janosch Frank Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 4ab698988618..dd1dca10bd6b 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -618,8 +618,7 @@ void do_secure_storage_access(struct pt_regs *regs) BUG(); break; default: - do_fault_error(regs, VM_FAULT_BADMAP); - WARN_ON_ONCE(1); + unreachable(); } } NOKPROBE_SYMBOL(do_secure_storage_access); -- cgit v1.2.3 From 9641613f48bbf8626355118de58f96eec5aebf49 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:40 +0200 Subject: s390/mm,fault: use get_fault_address() everywhere Use the get_fault_address() helper function instead of open-coding it at many locations. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index dd1dca10bd6b..ba2056b35910 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -176,7 +176,7 @@ static void dump_fault_info(struct pt_regs *regs) unsigned long asce; pr_alert("Failing address: %016lx TEID: %016lx\n", - regs->int_parm_long & __FAIL_ADDR_MASK, regs->int_parm_long); + get_fault_address(regs), regs->int_parm_long); pr_alert("Fault in "); switch (regs->int_parm_long & 3) { case 3: @@ -210,7 +210,7 @@ static void dump_fault_info(struct pt_regs *regs) unreachable(); } pr_cont("ASCE.\n"); - dump_pagetable(asce, regs->int_parm_long & __FAIL_ADDR_MASK); + dump_pagetable(asce, get_fault_address(regs)); } int show_unhandled_signals = 1; @@ -237,8 +237,7 @@ void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault) static void do_sigsegv(struct pt_regs *regs, int si_code) { report_user_fault(regs, SIGSEGV, 1); - force_sig_fault(SIGSEGV, si_code, - (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); + force_sig_fault(SIGSEGV, si_code, (void __user *)get_fault_address(regs)); } static void do_no_context(struct pt_regs *regs, vm_fault_t fault) @@ -280,8 +279,7 @@ static void do_low_address(struct pt_regs *regs) static void do_sigbus(struct pt_regs *regs) { - force_sig_fault(SIGBUS, BUS_ADRERR, - (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)get_fault_address(regs)); } static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) @@ -547,7 +545,7 @@ NOKPROBE_SYMBOL(do_dat_exception); void do_secure_storage_access(struct pt_regs *regs) { - unsigned long addr = regs->int_parm_long & __FAIL_ADDR_MASK; + unsigned long addr = get_fault_address(regs); struct vm_area_struct *vma; struct mm_struct *mm; struct page *page; @@ -625,8 +623,8 @@ NOKPROBE_SYMBOL(do_secure_storage_access); void do_non_secure_storage_access(struct pt_regs *regs) { - unsigned long gaddr = regs->int_parm_long & __FAIL_ADDR_MASK; struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; + unsigned long gaddr = get_fault_address(regs); if (get_fault_type(regs) != GMAP_FAULT) { do_fault_error(regs, VM_FAULT_BADMAP); @@ -640,8 +638,8 @@ NOKPROBE_SYMBOL(do_non_secure_storage_access); void do_secure_storage_violation(struct pt_regs *regs) { - unsigned long gaddr = regs->int_parm_long & __FAIL_ADDR_MASK; struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; + unsigned long gaddr = get_fault_address(regs); /* * If the VM has been rebooted, its address space might still contain -- cgit v1.2.3 From 4416d2ed816699b8e7491f1e8e8b15f69685af57 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:41 +0200 Subject: s390/mm,fault: use static key for store indication Generate slightly better code by using a static key to implement store indication. This allows to get rid of a memory access on the hot path. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index ba2056b35910..5466a30d1ec4 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -60,12 +61,12 @@ enum fault_type { GMAP_FAULT, }; -static unsigned long store_indication __read_mostly; +static DEFINE_STATIC_KEY_FALSE(have_store_indication); static int __init fault_init(void) { if (test_facility(75)) - store_indication = 0xc00; + static_branch_enable(&have_store_indication); return 0; } early_initcall(fault_init); @@ -104,11 +105,13 @@ static unsigned long get_fault_address(struct pt_regs *regs) return trans_exc_code & __FAIL_ADDR_MASK; } -static bool fault_is_write(struct pt_regs *regs) +static __always_inline bool fault_is_write(struct pt_regs *regs) { unsigned long trans_exc_code = regs->int_parm_long; - return (trans_exc_code & store_indication) == 0x400; + if (static_branch_likely(&have_store_indication)) + return (trans_exc_code & 0xc00) == 0x400; + return false; } static int bad_address(void *p) -- cgit v1.2.3 From 44ae766353af6eb3007308b8565f39cd8d0dd19a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:42 +0200 Subject: s390/mm: move translation-exception identification structure to fault.h Move translation-exception identification structure to new fault.h header file, change it to a union, and change existing kvm code accordingly. The new union will be used by subsequent patches. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/fault.h | 28 +++++++++++++++++++++++++++ arch/s390/kvm/gaccess.c | 45 ++++++++++++++----------------------------- 2 files changed, 42 insertions(+), 31 deletions(-) create mode 100644 arch/s390/include/asm/fault.h diff --git a/arch/s390/include/asm/fault.h b/arch/s390/include/asm/fault.h new file mode 100644 index 000000000000..d326f56603d6 --- /dev/null +++ b/arch/s390/include/asm/fault.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright IBM Corp. 1999, 2023 + */ +#ifndef _ASM_S390_FAULT_H +#define _ASM_S390_FAULT_H + +union teid { + unsigned long val; + struct { + unsigned long addr : 52; /* Translation-exception Address */ + unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */ + unsigned long : 2; + unsigned long b56 : 1; + unsigned long : 3; + unsigned long b60 : 1; + unsigned long b61 : 1; + unsigned long as : 2; /* ASCE Identifier */ + }; +}; + +enum { + TEID_FSI_UNKNOWN = 0, /* Unknown whether fetch or store */ + TEID_FSI_STORE = 1, /* Exception was due to store operation */ + TEID_FSI_FETCH = 2 /* Exception was due to fetch operation */ +}; + +#endif /* _ASM_S390_FAULT_H */ diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 6d6bc19b37dc..8d7763a13671 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -11,7 +11,7 @@ #include #include #include - +#include #include #include "kvm-s390.h" #include "gaccess.h" @@ -466,23 +466,6 @@ static int ar_translation(struct kvm_vcpu *vcpu, union asce *asce, u8 ar, return 0; } -struct trans_exc_code_bits { - unsigned long addr : 52; /* Translation-exception Address */ - unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */ - unsigned long : 2; - unsigned long b56 : 1; - unsigned long : 3; - unsigned long b60 : 1; - unsigned long b61 : 1; - unsigned long as : 2; /* ASCE Identifier */ -}; - -enum { - FSI_UNKNOWN = 0, /* Unknown whether fetch or store */ - FSI_STORE = 1, /* Exception was due to store operation */ - FSI_FETCH = 2 /* Exception was due to fetch operation */ -}; - enum prot_type { PROT_TYPE_LA = 0, PROT_TYPE_KEYC = 1, @@ -497,11 +480,11 @@ static int trans_exc_ending(struct kvm_vcpu *vcpu, int code, unsigned long gva, enum gacc_mode mode, enum prot_type prot, bool terminate) { struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm; - struct trans_exc_code_bits *tec; + union teid *teid; memset(pgm, 0, sizeof(*pgm)); pgm->code = code; - tec = (struct trans_exc_code_bits *)&pgm->trans_exc_code; + teid = (union teid *)&pgm->trans_exc_code; switch (code) { case PGM_PROTECTION: @@ -511,25 +494,25 @@ static int trans_exc_ending(struct kvm_vcpu *vcpu, int code, unsigned long gva, WARN_ON_ONCE(1); break; case PROT_TYPE_IEP: - tec->b61 = 1; + teid->b61 = 1; fallthrough; case PROT_TYPE_LA: - tec->b56 = 1; + teid->b56 = 1; break; case PROT_TYPE_KEYC: - tec->b60 = 1; + teid->b60 = 1; break; case PROT_TYPE_ALC: - tec->b60 = 1; + teid->b60 = 1; fallthrough; case PROT_TYPE_DAT: - tec->b61 = 1; + teid->b61 = 1; break; } if (terminate) { - tec->b56 = 0; - tec->b60 = 0; - tec->b61 = 0; + teid->b56 = 0; + teid->b60 = 0; + teid->b61 = 0; } fallthrough; case PGM_ASCE_TYPE: @@ -543,9 +526,9 @@ static int trans_exc_ending(struct kvm_vcpu *vcpu, int code, unsigned long gva, * exc_access_id has to be set to 0 for some instructions. Both * cases have to be handled by the caller. */ - tec->addr = gva >> PAGE_SHIFT; - tec->fsi = mode == GACC_STORE ? FSI_STORE : FSI_FETCH; - tec->as = psw_bits(vcpu->arch.sie_block->gpsw).as; + teid->addr = gva >> PAGE_SHIFT; + teid->fsi = mode == GACC_STORE ? TEID_FSI_STORE : TEID_FSI_FETCH; + teid->as = psw_bits(vcpu->arch.sie_block->gpsw).as; fallthrough; case PGM_ALEN_TRANSLATION: case PGM_ALE_SEQUENCE: -- cgit v1.2.3 From 5be05c35e72f7492105987e0231afab46b69935e Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:43 +0200 Subject: s390/mm,fault: improve readability by using teid union Get rid of some magic numbers, and use the teid union and also some ptrace PSW defines to improve readability. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 49 +++++++++++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 5466a30d1ec4..bcaac0f939df 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #include #include @@ -43,8 +45,6 @@ #include #include "../kernel/entry.h" -#define __FAIL_ADDR_MASK -4096L - /* * Allocate private vm_fault_reason from top. * Please make sure it won't collide with vm_fault_reason. @@ -76,11 +76,9 @@ early_initcall(fault_init); */ static enum fault_type get_fault_type(struct pt_regs *regs) { - unsigned long trans_exc_code; + union teid teid = { .val = regs->int_parm_long }; - trans_exc_code = regs->int_parm_long & 3; - /* Primary address space */ - if (likely(trans_exc_code == 0)) { + if (likely(teid.as == PSW_BITS_AS_PRIMARY)) { if (user_mode(regs)) return USER_FAULT; if (!IS_ENABLED(CONFIG_PGSTE)) @@ -89,10 +87,10 @@ static enum fault_type get_fault_type(struct pt_regs *regs) return GMAP_FAULT; return KERNEL_FAULT; } - if (trans_exc_code == 2) + if (teid.as == PSW_BITS_AS_SECONDARY) return USER_FAULT; /* Access register mode, not used in the kernel */ - if (trans_exc_code == 1) + if (teid.as == PSW_BITS_AS_ACCREG) return USER_FAULT; /* Home space -> access via kernel ASCE */ return KERNEL_FAULT; @@ -100,17 +98,17 @@ static enum fault_type get_fault_type(struct pt_regs *regs) static unsigned long get_fault_address(struct pt_regs *regs) { - unsigned long trans_exc_code = regs->int_parm_long; + union teid teid = { .val = regs->int_parm_long }; - return trans_exc_code & __FAIL_ADDR_MASK; + return teid.addr * PAGE_SIZE; } static __always_inline bool fault_is_write(struct pt_regs *regs) { - unsigned long trans_exc_code = regs->int_parm_long; + union teid teid = { .val = regs->int_parm_long }; if (static_branch_likely(&have_store_indication)) - return (trans_exc_code & 0xc00) == 0x400; + return teid.fsi == TEID_FSI_STORE; return false; } @@ -176,22 +174,23 @@ bad: static void dump_fault_info(struct pt_regs *regs) { + union teid teid = { .val = regs->int_parm_long }; unsigned long asce; pr_alert("Failing address: %016lx TEID: %016lx\n", - get_fault_address(regs), regs->int_parm_long); + get_fault_address(regs), teid.val); pr_alert("Fault in "); - switch (regs->int_parm_long & 3) { - case 3: + switch (teid.as) { + case PSW_BITS_AS_HOME: pr_cont("home space "); break; - case 2: + case PSW_BITS_AS_SECONDARY: pr_cont("secondary space "); break; - case 1: + case PSW_BITS_AS_ACCREG: pr_cont("access register "); break; - case 0: + case PSW_BITS_AS_PRIMARY: pr_cont("primary space "); break; } @@ -497,11 +496,10 @@ out: void do_protection_exception(struct pt_regs *regs) { - unsigned long trans_exc_code; + union teid teid = { .val = regs->int_parm_long }; vm_fault_t fault; int access; - trans_exc_code = regs->int_parm_long; /* * Protection exceptions are suppressing, decrement psw address. * The exception to this rule are aborted transactions, for these @@ -514,13 +512,12 @@ void do_protection_exception(struct pt_regs *regs) * as a special case because the translation exception code * field is not guaranteed to contain valid data in this case. */ - if (unlikely(!(trans_exc_code & 4))) { + if (unlikely(!teid.b61)) { do_low_address(regs); return; } - if (unlikely(MACHINE_HAS_NX && (trans_exc_code & 0x80))) { - regs->int_parm_long = (trans_exc_code & ~PAGE_MASK) | - (regs->psw.addr & PAGE_MASK); + if (unlikely(MACHINE_HAS_NX && teid.b56)) { + regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); access = VM_EXEC; fault = VM_FAULT_BADACCESS; } else { @@ -548,6 +545,7 @@ NOKPROBE_SYMBOL(do_dat_exception); void do_secure_storage_access(struct pt_regs *regs) { + union teid teid = { .val = regs->int_parm_long }; unsigned long addr = get_fault_address(regs); struct vm_area_struct *vma; struct mm_struct *mm; @@ -561,8 +559,7 @@ void do_secure_storage_access(struct pt_regs *regs) * process. Bit 61 is not reliable without the misc UV feature, * therefore this needs to be checked too. */ - if (uv_has_feature(BIT_UV_FEAT_MISC) && - !test_bit_inv(61, ®s->int_parm_long)) { + if (uv_has_feature(BIT_UV_FEAT_MISC) && !teid.b61) { /* * When this happens, userspace did something that it * was not supposed to do, e.g. branching into secure -- cgit v1.2.3 From f67c2da9f1c2d56e3adab342fc39c00c0c5ec790 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:44 +0200 Subject: s390/mm,fault: use get_kernel_nofault() to dereference in dump_pagetable() The page table dumper uses get_kernel_nofault() to test if dereferencing page table entries is possible. Use the result, which is the required page table entry, instead of throwing it away and dereferencing a second time without any safe guard. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 45 +++++++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index bcaac0f939df..c37e28fbfbef 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -112,59 +112,52 @@ static __always_inline bool fault_is_write(struct pt_regs *regs) return false; } -static int bad_address(void *p) -{ - unsigned long dummy; - - return get_kernel_nofault(dummy, (unsigned long *)p); -} - static void dump_pagetable(unsigned long asce, unsigned long address) { - unsigned long *table = __va(asce & _ASCE_ORIGIN); + unsigned long entry, *table = __va(asce & _ASCE_ORIGIN); pr_alert("AS:%016lx ", asce); switch (asce & _ASCE_TYPE_MASK) { case _ASCE_TYPE_REGION1: table += (address & _REGION1_INDEX) >> _REGION1_SHIFT; - if (bad_address(table)) + if (get_kernel_nofault(entry, table)) goto bad; - pr_cont("R1:%016lx ", *table); - if (*table & _REGION_ENTRY_INVALID) + pr_cont("R1:%016lx ", entry); + if (entry & _REGION_ENTRY_INVALID) goto out; - table = __va(*table & _REGION_ENTRY_ORIGIN); + table = __va(entry & _REGION_ENTRY_ORIGIN); fallthrough; case _ASCE_TYPE_REGION2: table += (address & _REGION2_INDEX) >> _REGION2_SHIFT; - if (bad_address(table)) + if (get_kernel_nofault(entry, table)) goto bad; - pr_cont("R2:%016lx ", *table); - if (*table & _REGION_ENTRY_INVALID) + pr_cont("R2:%016lx ", entry); + if (entry & _REGION_ENTRY_INVALID) goto out; - table = __va(*table & _REGION_ENTRY_ORIGIN); + table = __va(entry & _REGION_ENTRY_ORIGIN); fallthrough; case _ASCE_TYPE_REGION3: table += (address & _REGION3_INDEX) >> _REGION3_SHIFT; - if (bad_address(table)) + if (get_kernel_nofault(entry, table)) goto bad; - pr_cont("R3:%016lx ", *table); - if (*table & (_REGION_ENTRY_INVALID | _REGION3_ENTRY_LARGE)) + pr_cont("R3:%016lx ", entry); + if (entry & (_REGION_ENTRY_INVALID | _REGION3_ENTRY_LARGE)) goto out; - table = __va(*table & _REGION_ENTRY_ORIGIN); + table = __va(entry & _REGION_ENTRY_ORIGIN); fallthrough; case _ASCE_TYPE_SEGMENT: table += (address & _SEGMENT_INDEX) >> _SEGMENT_SHIFT; - if (bad_address(table)) + if (get_kernel_nofault(entry, table)) goto bad; - pr_cont("S:%016lx ", *table); - if (*table & (_SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_LARGE)) + pr_cont("S:%016lx ", entry); + if (entry & (_SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_LARGE)) goto out; - table = __va(*table & _SEGMENT_ENTRY_ORIGIN); + table = __va(entry & _SEGMENT_ENTRY_ORIGIN); } table += (address & _PAGE_INDEX) >> _PAGE_SHIFT; - if (bad_address(table)) + if (get_kernel_nofault(entry, table)) goto bad; - pr_cont("P:%016lx ", *table); + pr_cont("P:%016lx ", entry); out: pr_cont("\n"); return; -- cgit v1.2.3 From cca12b427d43bb57be207d634e1d44e9792d1cb4 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:45 +0200 Subject: s390/mm,fault: remove VM_FAULT_PFAULT Handling of VM_FAULT_PFAULT and VM_FAULT_BADCONTEXT is nearly identical; the only difference is within do_no_context() where however the fault_type (KERNEL_FAULT vs GMAP_FAULT) makes sure that both types will be handled differently. Therefore it is possible to get rid of VM_FAULT_PFAULT and use VM_FAULT_BADCONTEXT instead. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index c37e28fbfbef..ee8a40917592 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -53,7 +53,6 @@ #define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000) #define VM_FAULT_BADACCESS ((__force vm_fault_t)0x20000000) #define VM_FAULT_SIGNAL ((__force vm_fault_t)0x10000000) -#define VM_FAULT_PFAULT ((__force vm_fault_t)0x8000000) enum fault_type { KERNEL_FAULT, @@ -294,7 +293,6 @@ static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) } fallthrough; case VM_FAULT_BADCONTEXT: - case VM_FAULT_PFAULT: do_no_context(regs, fault); break; case VM_FAULT_SIGNAL: @@ -459,7 +457,7 @@ retry: * mmap_lock has not been released */ current->thread.gmap_pfault = 1; - fault = VM_FAULT_PFAULT; + fault = VM_FAULT_BADCONTEXT; goto out_up; } flags &= ~FAULT_FLAG_RETRY_NOWAIT; -- cgit v1.2.3 From 5db06565cad67442e4677e7c0da7f7dad1eccb03 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:46 +0200 Subject: s390/mm,fault: get rid of do_low_address() There is only one caller of do_low_address(). Given that this code is quite special just get rid of do_low_address, and add it to do_protection_exception() in order to make the code a bit more readable. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index ee8a40917592..87bfcf4d3082 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -257,20 +257,6 @@ static void do_no_context(struct pt_regs *regs, vm_fault_t fault) die(regs, "Oops"); } -static void do_low_address(struct pt_regs *regs) -{ - /* - * Low-address protection hit in kernel mode means - * NULL pointer write access in kernel mode. - */ - if (regs->psw.mask & PSW_MASK_PSTATE) { - /* Low-address protection hit in user mode: 'cannot happen' */ - die(regs, "Low-address protection"); - } - - do_no_context(regs, VM_FAULT_BADACCESS); -} - static void do_sigbus(struct pt_regs *regs) { force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)get_fault_address(regs)); @@ -504,8 +490,15 @@ void do_protection_exception(struct pt_regs *regs) * field is not guaranteed to contain valid data in this case. */ if (unlikely(!teid.b61)) { - do_low_address(regs); - return; + if (user_mode(regs)) { + /* Low-address protection in user mode: cannot happen */ + die(regs, "Low-address protection"); + } + /* + * Low-address protection in kernel mode means + * NULL pointer write access in kernel mode. + */ + return do_no_context(regs, VM_FAULT_BADACCESS); } if (unlikely(MACHINE_HAS_NX && teid.b56)) { regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); -- cgit v1.2.3 From 64ea33fb09f8fdcc9eb9f253906886a5801bb48a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:47 +0200 Subject: s390/mm,fault: call do_fault_error() only from do_exception() Remove duplicated fault error handling and handle it only once within do_exception(). Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 87bfcf4d3082..9ed979690665 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -321,7 +321,7 @@ static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) * 11 Page translation -> Not present (nullification) * 3b Region third trans. -> Not present (nullification) */ -static inline vm_fault_t do_exception(struct pt_regs *regs, int access) +static void do_exception(struct pt_regs *regs, int access) { struct vm_area_struct *vma; struct task_struct *tsk; @@ -340,7 +340,7 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) */ clear_thread_flag(TIF_PER_TRAP); if (kprobe_page_fault(regs, 14)) - return 0; + return; mm = tsk->mm; address = get_fault_address(regs); is_write = fault_is_write(regs); @@ -468,14 +468,13 @@ out_gmap: out_up: mmap_read_unlock(mm); out: - return fault; + if (unlikely(fault)) + do_fault_error(regs, fault); } void do_protection_exception(struct pt_regs *regs) { union teid teid = { .val = regs->int_parm_long }; - vm_fault_t fault; - int access; /* * Protection exceptions are suppressing, decrement psw address. @@ -502,26 +501,16 @@ void do_protection_exception(struct pt_regs *regs) } if (unlikely(MACHINE_HAS_NX && teid.b56)) { regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); - access = VM_EXEC; - fault = VM_FAULT_BADACCESS; - } else { - access = VM_WRITE; - fault = do_exception(regs, access); + do_fault_error(regs, VM_FAULT_BADACCESS); + return; } - if (unlikely(fault)) - do_fault_error(regs, fault); + do_exception(regs, VM_WRITE); } NOKPROBE_SYMBOL(do_protection_exception); void do_dat_exception(struct pt_regs *regs) { - vm_fault_t fault; - int access; - - access = VM_ACCESS_FLAGS; - fault = do_exception(regs, access); - if (unlikely(fault)) - do_fault_error(regs, fault); + do_exception(regs, VM_ACCESS_FLAGS); } NOKPROBE_SYMBOL(do_dat_exception); -- cgit v1.2.3 From 0f1a14e0348eb48e93e139af569944349b725f3f Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:48 +0200 Subject: s390/mm,fault: simplify kfence fault handling do_no_context() can be simplified by removing its fault parameter, which is only used to decide if kfence_handle_page_fault() should be called. If the fault happened within the kernel space it is ok to always check if this happened on a page which was unmapped because of the kfence feature. Limiting the check to the VM_FAULT_BADCONTEXT case doesn't add any value. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 9ed979690665..e0dbc231828c 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -234,7 +234,7 @@ static void do_sigsegv(struct pt_regs *regs, int si_code) force_sig_fault(SIGSEGV, si_code, (void __user *)get_fault_address(regs)); } -static void do_no_context(struct pt_regs *regs, vm_fault_t fault) +static void do_no_context(struct pt_regs *regs) { enum fault_type fault_type; unsigned long address; @@ -243,7 +243,7 @@ static void do_no_context(struct pt_regs *regs, vm_fault_t fault) if (fixup_exception(regs)) return; fault_type = get_fault_type(regs); - if ((fault_type == KERNEL_FAULT) && (fault == VM_FAULT_BADCONTEXT)) { + if (fault_type == KERNEL_FAULT) { address = get_fault_address(regs); is_write = fault_is_write(regs); if (kfence_handle_page_fault(address, is_write, regs)) @@ -279,28 +279,28 @@ static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) } fallthrough; case VM_FAULT_BADCONTEXT: - do_no_context(regs, fault); + do_no_context(regs); break; case VM_FAULT_SIGNAL: if (!user_mode(regs)) - do_no_context(regs, fault); + do_no_context(regs); break; default: /* fault & VM_FAULT_ERROR */ if (fault & VM_FAULT_OOM) { if (!user_mode(regs)) - do_no_context(regs, fault); + do_no_context(regs); else pagefault_out_of_memory(); } else if (fault & VM_FAULT_SIGSEGV) { /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) - do_no_context(regs, fault); + do_no_context(regs); else do_sigsegv(regs, SEGV_MAPERR); } else if (fault & VM_FAULT_SIGBUS) { /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) - do_no_context(regs, fault); + do_no_context(regs); else do_sigbus(regs); } else { @@ -497,7 +497,7 @@ void do_protection_exception(struct pt_regs *regs) * Low-address protection in kernel mode means * NULL pointer write access in kernel mode. */ - return do_no_context(regs, VM_FAULT_BADACCESS); + return do_no_context(regs); } if (unlikely(MACHINE_HAS_NX && teid.b56)) { regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); -- cgit v1.2.3 From 0f86ac4ba71329936c44ef71222f1c50284a8928 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:49 +0200 Subject: s390/mm,fault: remove VM_FAULT_BADCONTEXT Remove VM_FAULT_BADCONTEXT and instead call do_no_context() via wrappers. This adds two new wrappers similar to what x86 has: handle_fault_error() and handle_fault_error_nolock(). Both of them simply call do_no_context(), while handle_fault_error() also unlocks mmap lock, which avoids adding lots of mmap_read_unlock() calls with this and subsequent patches. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index e0dbc231828c..05c908051e93 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -49,7 +49,6 @@ * Allocate private vm_fault_reason from top. * Please make sure it won't collide with vm_fault_reason. */ -#define VM_FAULT_BADCONTEXT ((__force vm_fault_t)0x80000000) #define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000) #define VM_FAULT_BADACCESS ((__force vm_fault_t)0x20000000) #define VM_FAULT_SIGNAL ((__force vm_fault_t)0x10000000) @@ -257,6 +256,19 @@ static void do_no_context(struct pt_regs *regs) die(regs, "Oops"); } +static inline void handle_fault_error_nolock(struct pt_regs *regs) +{ + do_no_context(regs); +} + +static void handle_fault_error(struct pt_regs *regs) +{ + struct mm_struct *mm = current->mm; + + mmap_read_unlock(mm); + handle_fault_error_nolock(regs); +} + static void do_sigbus(struct pt_regs *regs) { force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)get_fault_address(regs)); @@ -277,8 +289,6 @@ static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) do_sigsegv(regs, si_code); break; } - fallthrough; - case VM_FAULT_BADCONTEXT: do_no_context(regs); break; case VM_FAULT_SIGNAL: @@ -344,15 +354,14 @@ static void do_exception(struct pt_regs *regs, int access) mm = tsk->mm; address = get_fault_address(regs); is_write = fault_is_write(regs); - fault = VM_FAULT_BADCONTEXT; type = get_fault_type(regs); switch (type) { case KERNEL_FAULT: - goto out; + return handle_fault_error_nolock(regs); case USER_FAULT: case GMAP_FAULT: if (faulthandler_disabled() || !mm) - goto out; + return handle_fault_error_nolock(regs); break; } perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); @@ -443,8 +452,7 @@ retry: * mmap_lock has not been released */ current->thread.gmap_pfault = 1; - fault = VM_FAULT_BADCONTEXT; - goto out_up; + return handle_fault_error(regs); } flags &= ~FAULT_FLAG_RETRY_NOWAIT; flags |= FAULT_FLAG_TRIED; -- cgit v1.2.3 From b61a0922b6dc7dfa6cef5c6d8ab1036826c43b8d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:50 +0200 Subject: s390/mm,fault: remove VM_FAULT_SIGNAL Remove VM_FAULT_SIGNAL and open-code it at the only two locations where it is used. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 05c908051e93..7dd965d153d5 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -51,7 +51,6 @@ */ #define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000) #define VM_FAULT_BADACCESS ((__force vm_fault_t)0x20000000) -#define VM_FAULT_SIGNAL ((__force vm_fault_t)0x10000000) enum fault_type { KERNEL_FAULT, @@ -291,10 +290,6 @@ static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) } do_no_context(regs); break; - case VM_FAULT_SIGNAL: - if (!user_mode(regs)) - do_no_context(regs); - break; default: /* fault & VM_FAULT_ERROR */ if (fault & VM_FAULT_OOM) { if (!user_mode(regs)) @@ -393,8 +388,9 @@ static void do_exception(struct pt_regs *regs, int access) count_vm_vma_lock_event(VMA_LOCK_RETRY); /* Quick path to respond to signals */ if (fault_signal_pending(fault, regs)) { - fault = VM_FAULT_SIGNAL; - goto out; + if (!user_mode(regs)) + handle_fault_error_nolock(regs); + return; } lock_mmap: mmap_read_lock(mm); @@ -429,10 +425,11 @@ retry: goto out_up; fault = handle_mm_fault(vma, address, flags, regs); if (fault_signal_pending(fault, regs)) { - fault = VM_FAULT_SIGNAL; if (flags & FAULT_FLAG_RETRY_NOWAIT) - goto out_up; - goto out; + mmap_read_unlock(mm); + if (!user_mode(regs)) + handle_fault_error_nolock(regs); + return; } /* The fault is fully completed (including releasing mmap lock) */ if (fault & VM_FAULT_COMPLETED) { -- cgit v1.2.3 From 7c194d84a9ce662426b2ecb59da54bb80c6b1d91 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:51 +0200 Subject: s390/mm,fault: remove VM_FAULT_BADMAP and VM_FAULT_BADACCESS Remove the last two private vm_fault reasons: VM_FAULT_BADMAP and VM_FAULT_BADACCESS. In order to achieve this add an si_code parameter to do_no_context() and it's wrappers and directly call the wrappers instead of relying on do_fault_error() handling. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 136 +++++++++++++++++++-------------------------------- 1 file changed, 51 insertions(+), 85 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 7dd965d153d5..bee22e91e14b 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -45,13 +45,6 @@ #include #include "../kernel/entry.h" -/* - * Allocate private vm_fault_reason from top. - * Please make sure it won't collide with vm_fault_reason. - */ -#define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000) -#define VM_FAULT_BADACCESS ((__force vm_fault_t)0x20000000) - enum fault_type { KERNEL_FAULT, USER_FAULT, @@ -232,12 +225,17 @@ static void do_sigsegv(struct pt_regs *regs, int si_code) force_sig_fault(SIGSEGV, si_code, (void __user *)get_fault_address(regs)); } -static void do_no_context(struct pt_regs *regs) +static void do_no_context(struct pt_regs *regs, int si_code) { enum fault_type fault_type; unsigned long address; bool is_write; + if (user_mode(regs)) { + if (WARN_ON_ONCE(!si_code)) + si_code = SEGV_MAPERR; + return do_sigsegv(regs, si_code); + } if (fixup_exception(regs)) return; fault_type = get_fault_type(regs); @@ -255,17 +253,17 @@ static void do_no_context(struct pt_regs *regs) die(regs, "Oops"); } -static inline void handle_fault_error_nolock(struct pt_regs *regs) +static inline void handle_fault_error_nolock(struct pt_regs *regs, int si_code) { - do_no_context(regs); + do_no_context(regs, si_code); } -static void handle_fault_error(struct pt_regs *regs) +static void handle_fault_error(struct pt_regs *regs, int si_code) { struct mm_struct *mm = current->mm; mmap_read_unlock(mm); - handle_fault_error_nolock(regs); + handle_fault_error_nolock(regs, si_code); } static void do_sigbus(struct pt_regs *regs) @@ -275,43 +273,26 @@ static void do_sigbus(struct pt_regs *regs) static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) { - int si_code; - - switch (fault) { - case VM_FAULT_BADACCESS: - case VM_FAULT_BADMAP: - /* Bad memory access. Check if it is kernel or user space. */ - if (user_mode(regs)) { - /* User mode accesses just cause a SIGSEGV */ - si_code = (fault == VM_FAULT_BADMAP) ? - SEGV_MAPERR : SEGV_ACCERR; - do_sigsegv(regs, si_code); - break; - } - do_no_context(regs); - break; - default: /* fault & VM_FAULT_ERROR */ - if (fault & VM_FAULT_OOM) { - if (!user_mode(regs)) - do_no_context(regs); - else - pagefault_out_of_memory(); - } else if (fault & VM_FAULT_SIGSEGV) { - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - do_no_context(regs); - else - do_sigsegv(regs, SEGV_MAPERR); - } else if (fault & VM_FAULT_SIGBUS) { - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - do_no_context(regs); - else - do_sigbus(regs); - } else { - BUG(); - } - break; + /* fault & VM_FAULT_ERROR */ + if (fault & VM_FAULT_OOM) { + if (!user_mode(regs)) + do_no_context(regs, 0); + else + pagefault_out_of_memory(); + } else if (fault & VM_FAULT_SIGSEGV) { + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) + do_no_context(regs, 0); + else + do_sigsegv(regs, SEGV_MAPERR); + } else if (fault & VM_FAULT_SIGBUS) { + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) + do_no_context(regs, 0); + else + do_sigbus(regs); + } else { + BUG(); } } @@ -352,11 +333,11 @@ static void do_exception(struct pt_regs *regs, int access) type = get_fault_type(regs); switch (type) { case KERNEL_FAULT: - return handle_fault_error_nolock(regs); + return handle_fault_error_nolock(regs, 0); case USER_FAULT: case GMAP_FAULT: if (faulthandler_disabled() || !mm) - return handle_fault_error_nolock(regs); + return handle_fault_error_nolock(regs, 0); break; } perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); @@ -389,7 +370,7 @@ static void do_exception(struct pt_regs *regs, int access) /* Quick path to respond to signals */ if (fault_signal_pending(fault, regs)) { if (!user_mode(regs)) - handle_fault_error_nolock(regs); + handle_fault_error_nolock(regs, 0); return; } lock_mmap: @@ -401,34 +382,30 @@ lock_mmap: current->thread.gmap_write_flag = !!(flags & FAULT_FLAG_WRITE); current->thread.gmap_int_code = regs->int_code & 0xffff; address = __gmap_translate(gmap, address); - if (address == -EFAULT) { - fault = VM_FAULT_BADMAP; - goto out_up; - } + if (address == -EFAULT) + return handle_fault_error(regs, SEGV_MAPERR); if (gmap->pfault_enabled) flags |= FAULT_FLAG_RETRY_NOWAIT; } retry: - fault = VM_FAULT_BADMAP; vma = find_vma(mm, address); if (!vma) - goto out_up; + return handle_fault_error(regs, SEGV_MAPERR); if (unlikely(vma->vm_start > address)) { if (!(vma->vm_flags & VM_GROWSDOWN)) - goto out_up; + return handle_fault_error(regs, SEGV_MAPERR); vma = expand_stack(mm, address); if (!vma) - goto out; + return handle_fault_error_nolock(regs, SEGV_MAPERR); } - fault = VM_FAULT_BADACCESS; if (unlikely(!(vma->vm_flags & access))) - goto out_up; + return handle_fault_error(regs, SEGV_ACCERR); fault = handle_mm_fault(vma, address, flags, regs); if (fault_signal_pending(fault, regs)) { if (flags & FAULT_FLAG_RETRY_NOWAIT) mmap_read_unlock(mm); if (!user_mode(regs)) - handle_fault_error_nolock(regs); + handle_fault_error_nolock(regs, 0); return; } /* The fault is fully completed (including releasing mmap lock) */ @@ -449,7 +426,7 @@ retry: * mmap_lock has not been released */ current->thread.gmap_pfault = 1; - return handle_fault_error(regs); + return handle_fault_error(regs, 0); } flags &= ~FAULT_FLAG_RETRY_NOWAIT; flags |= FAULT_FLAG_TRIED; @@ -460,10 +437,8 @@ out_gmap: if (IS_ENABLED(CONFIG_PGSTE) && gmap) { address = __gmap_link(gmap, current->thread.gmap_addr, address); - if (address == -EFAULT) { - fault = VM_FAULT_BADMAP; - goto out_up; - } + if (address == -EFAULT) + return handle_fault_error(regs, SEGV_MAPERR); if (address == -ENOMEM) { fault = VM_FAULT_OOM; goto out_up; @@ -502,12 +477,11 @@ void do_protection_exception(struct pt_regs *regs) * Low-address protection in kernel mode means * NULL pointer write access in kernel mode. */ - return do_no_context(regs); + return do_no_context(regs, 0); } if (unlikely(MACHINE_HAS_NX && teid.b56)) { regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); - do_fault_error(regs, VM_FAULT_BADACCESS); - return; + return handle_fault_error_nolock(regs, SEGV_ACCERR); } do_exception(regs, VM_WRITE); } @@ -560,20 +534,15 @@ void do_secure_storage_access(struct pt_regs *regs) mmap_read_lock(mm); addr = __gmap_translate(gmap, addr); mmap_read_unlock(mm); - if (IS_ERR_VALUE(addr)) { - do_fault_error(regs, VM_FAULT_BADMAP); - break; - } + if (IS_ERR_VALUE(addr)) + return handle_fault_error_nolock(regs, SEGV_MAPERR); fallthrough; case USER_FAULT: mm = current->mm; mmap_read_lock(mm); vma = find_vma(mm, addr); - if (!vma) { - mmap_read_unlock(mm); - do_fault_error(regs, VM_FAULT_BADMAP); - break; - } + if (!vma) + return handle_fault_error(regs, SEGV_MAPERR); page = follow_page(vma, addr, FOLL_WRITE | FOLL_GET); if (IS_ERR_OR_NULL(page)) { mmap_read_unlock(mm); @@ -604,11 +573,8 @@ void do_non_secure_storage_access(struct pt_regs *regs) struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; unsigned long gaddr = get_fault_address(regs); - if (get_fault_type(regs) != GMAP_FAULT) { - do_fault_error(regs, VM_FAULT_BADMAP); - WARN_ON_ONCE(1); - return; - } + if (WARN_ON_ONCE(get_fault_type(regs) != GMAP_FAULT)) + return handle_fault_error_nolock(regs, SEGV_MAPERR); if (gmap_convert_to_secure(gmap, gaddr) == -EINVAL) send_sig(SIGSEGV, current, 0); } -- cgit v1.2.3 From b20c8216c1e0d5a5a5c9f40df0cf9bbc795e84f1 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:52 +0200 Subject: s390/mm,fault: move VM_FAULT_ERROR handling to do_exception() Get rid of do_fault_error() and move its contents to do_exception(), which makes do_exception(). With removing do_fault_error() it is also possible to get rid of the handle_fault_error_nolock() wrapper. Instead rename do_no_context() to handle_fault_error_nolock(). In result the whole fault handling looks much more like on other architectures. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 81 ++++++++++++++++++++++------------------------------ 1 file changed, 34 insertions(+), 47 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index bee22e91e14b..249aefcf7c4e 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -225,7 +225,7 @@ static void do_sigsegv(struct pt_regs *regs, int si_code) force_sig_fault(SIGSEGV, si_code, (void __user *)get_fault_address(regs)); } -static void do_no_context(struct pt_regs *regs, int si_code) +static void handle_fault_error_nolock(struct pt_regs *regs, int si_code) { enum fault_type fault_type; unsigned long address; @@ -253,11 +253,6 @@ static void do_no_context(struct pt_regs *regs, int si_code) die(regs, "Oops"); } -static inline void handle_fault_error_nolock(struct pt_regs *regs, int si_code) -{ - do_no_context(regs, si_code); -} - static void handle_fault_error(struct pt_regs *regs, int si_code) { struct mm_struct *mm = current->mm; @@ -271,31 +266,6 @@ static void do_sigbus(struct pt_regs *regs) force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)get_fault_address(regs)); } -static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) -{ - /* fault & VM_FAULT_ERROR */ - if (fault & VM_FAULT_OOM) { - if (!user_mode(regs)) - do_no_context(regs, 0); - else - pagefault_out_of_memory(); - } else if (fault & VM_FAULT_SIGSEGV) { - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - do_no_context(regs, 0); - else - do_sigsegv(regs, SEGV_MAPERR); - } else if (fault & VM_FAULT_SIGBUS) { - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - do_no_context(regs, 0); - else - do_sigbus(regs); - } else { - BUG(); - } -} - /* * This routine handles page faults. It determines the address, * and the problem, and then passes it off to one of the appropriate @@ -362,9 +332,9 @@ static void do_exception(struct pt_regs *regs, int access) vma_end_read(vma); if (!(fault & VM_FAULT_RETRY)) { count_vm_vma_lock_event(VMA_LOCK_SUCCESS); - if (likely(!(fault & VM_FAULT_ERROR))) - fault = 0; - goto out; + if (unlikely(fault & VM_FAULT_ERROR)) + goto error; + return; } count_vm_vma_lock_event(VMA_LOCK_RETRY); /* Quick path to respond to signals */ @@ -412,13 +382,14 @@ retry: if (fault & VM_FAULT_COMPLETED) { if (gmap) { mmap_read_lock(mm); - goto out_gmap; + goto gmap; } - fault = 0; - goto out; + return; + } + if (unlikely(fault & VM_FAULT_ERROR)) { + mmap_read_unlock(mm); + goto error; } - if (unlikely(fault & VM_FAULT_ERROR)) - goto out_up; if (fault & VM_FAULT_RETRY) { if (IS_ENABLED(CONFIG_PGSTE) && gmap && (flags & FAULT_FLAG_RETRY_NOWAIT)) { /* @@ -433,7 +404,7 @@ retry: mmap_read_lock(mm); goto retry; } -out_gmap: +gmap: if (IS_ENABLED(CONFIG_PGSTE) && gmap) { address = __gmap_link(gmap, current->thread.gmap_addr, address); @@ -441,15 +412,31 @@ out_gmap: return handle_fault_error(regs, SEGV_MAPERR); if (address == -ENOMEM) { fault = VM_FAULT_OOM; - goto out_up; + mmap_read_unlock(mm); + goto error; } } - fault = 0; -out_up: mmap_read_unlock(mm); -out: - if (unlikely(fault)) - do_fault_error(regs, fault); + return; +error: + if (fault & VM_FAULT_OOM) { + if (!user_mode(regs)) + handle_fault_error_nolock(regs, 0); + else + pagefault_out_of_memory(); + } else if (fault & VM_FAULT_SIGSEGV) { + if (!user_mode(regs)) + handle_fault_error_nolock(regs, 0); + else + do_sigsegv(regs, SEGV_MAPERR); + } else if (fault & VM_FAULT_SIGBUS) { + if (!user_mode(regs)) + handle_fault_error_nolock(regs, 0); + else + do_sigbus(regs); + } else { + BUG(); + } } void do_protection_exception(struct pt_regs *regs) @@ -477,7 +464,7 @@ void do_protection_exception(struct pt_regs *regs) * Low-address protection in kernel mode means * NULL pointer write access in kernel mode. */ - return do_no_context(regs, 0); + return handle_fault_error_nolock(regs, 0); } if (unlikely(MACHINE_HAS_NX && teid.b56)) { regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); -- cgit v1.2.3 From c8b5d574fcea49e9974b7658d5337f93d1921f8c Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 18 Oct 2023 12:33:05 +0200 Subject: s390/diag: add missing virt_to_phys() translation to diag224() Diagnose 224 expects a physical address, but all users pass a virtual address. Translate the address to fix this. Reported-by: Mete Durlu Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/diag.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index f9f06cd8fcee..92fdc35f028c 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -245,6 +245,7 @@ EXPORT_SYMBOL(diag8c); int diag224(void *ptr) { + unsigned long addr = __pa(ptr); int rc = -EOPNOTSUPP; diag_stat_inc(DIAG_STAT_X224); @@ -253,7 +254,7 @@ int diag224(void *ptr) "0: lhi %0,0x0\n" "1:\n" EX_TABLE(0b,1b) - : "+d" (rc) :"d" (0), "d" (ptr) : "memory"); + : "+d" (rc) :"d" (0), "d" (addr) : "memory"); return rc; } EXPORT_SYMBOL(diag224); -- cgit v1.2.3 From 16ba44826a04834d3eeeda4b731c2ea3481062b7 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 17 Oct 2023 21:07:03 +0200 Subject: s390/cmma: fix initial kernel address space page table walk If the cmma no-dat feature is available the kernel page tables are walked to identify and mark all pages which are used for address translation (all region, segment, and page tables). In a subsequent loop all other pages are marked as "no-dat" pages with the ESSA instruction. This information is visible to the hypervisor, so that the hypervisor can optimize purging of guest TLB entries. The initial loop however does not cover the complete kernel address space. This can result in pages being marked as not being used for dynamic address translation, even though they are. In turn guest TLB entries incorrectly may not be purged. Fix this by adjusting the end address of the kernel address range being walked. Cc: Reviewed-by: Claudio Imbrenda Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/page-states.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/arch/s390/mm/page-states.c b/arch/s390/mm/page-states.c index 1e2ea706aa22..c3db3c0e75c0 100644 --- a/arch/s390/mm/page-states.c +++ b/arch/s390/mm/page-states.c @@ -151,15 +151,22 @@ static void mark_kernel_p4d(pgd_t *pgd, unsigned long addr, unsigned long end) static void mark_kernel_pgd(void) { - unsigned long addr, next; + unsigned long addr, next, max_addr; struct page *page; pgd_t *pgd; int i; addr = 0; + /* + * Figure out maximum virtual address accessible with the + * kernel ASCE. This is required to keep the page table walker + * from accessing non-existent entries. + */ + max_addr = (S390_lowcore.kernel_asce.val & _ASCE_TYPE_MASK) >> 2; + max_addr = 1UL << (max_addr * 11 + 31); pgd = pgd_offset_k(addr); do { - next = pgd_addr_end(addr, MODULES_END); + next = pgd_addr_end(addr, max_addr); if (pgd_none(*pgd)) continue; if (!pgd_folded(*pgd)) { @@ -168,7 +175,7 @@ static void mark_kernel_pgd(void) set_bit(PG_arch_1, &page[i].flags); } mark_kernel_p4d(pgd, addr, next); - } while (pgd++, addr = next, addr != MODULES_END); + } while (pgd++, addr = next, addr != max_addr); } void __init cmma_init_nodat(void) -- cgit v1.2.3 From 09cda0a400519b1541591c506e54c9c48e3101bf Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 17 Oct 2023 21:07:04 +0200 Subject: s390/mm: add missing arch_set_page_dat() call to vmem_crst_alloc() If the cmma no-dat feature is available all pages that are not used for dynamic address translation are marked as "no-dat" with the ESSA instruction. This information is visible to the hypervisor, so that the hypervisor can optimize purging of guest TLB entries. This also means that pages which are used for dynamic address translation must not be marked as "no-dat", since the hypervisor may then incorrectly not purge guest TLB entries. Region and segment tables allocated via vmem_crst_alloc() are incorrectly marked as "no-dat", as soon as slab_is_available() returns true. Such tables are allocated e.g. when kernel page tables are split, memory is hotplugged, or a DCSS segment is loaded. Fix this by adding the missing arch_set_page_dat() call. Cc: Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/vmem.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 86384586eb20..2e8a1064f103 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -46,8 +47,11 @@ void *vmem_crst_alloc(unsigned long val) unsigned long *table; table = vmem_alloc_pages(CRST_ALLOC_ORDER); - if (table) - crst_table_init(table, val); + if (!table) + return NULL; + crst_table_init(table, val); + if (slab_is_available()) + arch_set_page_dat(virt_to_page(table), CRST_ALLOC_ORDER); return table; } -- cgit v1.2.3 From 1954da4a2b621a3328a63382cae7e5f5e2af502c Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 20 Oct 2023 17:26:50 +0200 Subject: s390/mm: add missing arch_set_page_dat() call to gmap allocations If the cmma no-dat feature is available all pages that are not used for dynamic address translation are marked as "no-dat" with the ESSA instruction. This information is visible to the hypervisor, so that the hypervisor can optimize purging of guest TLB entries. This also means that pages which are used for dynamic address translation must not be marked as "no-dat", since the hypervisor may then incorrectly not purge guest TLB entries. Region, segment, and page tables allocated within the gmap code are incorrectly marked as "no-dat", since an explicit call to arch_set_page_dat() is missing, which would remove the "no-dat" mark. In order to fix this add a new gmap_alloc_crst() function which should be used to allocate region and segment tables, and which also calls arch_set_page_dat(). Also add the arch_set_page_dat() call to page_table_alloc_pgste(). Cc: Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/gmap.c | 24 ++++++++++++++++++------ arch/s390/mm/pgalloc.c | 1 + 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 906a7bfc2a78..20786f6883b2 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -21,10 +21,22 @@ #include #include +#include #include #define GMAP_SHADOW_FAKE_TABLE 1ULL +static struct page *gmap_alloc_crst(void) +{ + struct page *page; + + page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + if (!page) + return NULL; + arch_set_page_dat(page, CRST_ALLOC_ORDER); + return page; +} + /** * gmap_alloc - allocate and initialize a guest address space * @limit: maximum address of the gmap address space @@ -67,7 +79,7 @@ static struct gmap *gmap_alloc(unsigned long limit) spin_lock_init(&gmap->guest_table_lock); spin_lock_init(&gmap->shadow_lock); refcount_set(&gmap->ref_count, 1); - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) goto out_free; page->index = 0; @@ -308,7 +320,7 @@ static int gmap_alloc_table(struct gmap *gmap, unsigned long *table, unsigned long *new; /* since we dont free the gmap table until gmap_free we can unlock */ - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) return -ENOMEM; new = page_to_virt(page); @@ -1759,7 +1771,7 @@ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, BUG_ON(!gmap_is_shadow(sg)); /* Allocate a shadow region second table */ - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) return -ENOMEM; page->index = r2t & _REGION_ENTRY_ORIGIN; @@ -1843,7 +1855,7 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, BUG_ON(!gmap_is_shadow(sg)); /* Allocate a shadow region second table */ - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) return -ENOMEM; page->index = r3t & _REGION_ENTRY_ORIGIN; @@ -1927,7 +1939,7 @@ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE)); /* Allocate a shadow segment table */ - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) return -ENOMEM; page->index = sgt & _REGION_ENTRY_ORIGIN; @@ -2855,7 +2867,7 @@ int s390_replace_asce(struct gmap *gmap) if ((gmap->asce & _ASCE_TYPE_MASK) == _ASCE_TYPE_SEGMENT) return -EINVAL; - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) return -ENOMEM; page->index = 0; diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index a6b71bc71896..a7e4b6e11cc5 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -146,6 +146,7 @@ struct page *page_table_alloc_pgste(struct mm_struct *mm) ptdesc = pagetable_alloc(GFP_KERNEL, 0); if (ptdesc) { table = (u64 *)ptdesc_to_virt(ptdesc); + arch_set_page_dat(virt_to_page(table), 0); memset64(table, _PAGE_INVALID, PTRS_PER_PTE); memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE); } -- cgit v1.2.3 From a13e8bdf0f788552aa37e1f9a6fe5fcca4095d2b Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Tue, 10 Oct 2023 16:14:22 +0200 Subject: s390/pai_crypto: use PERF_ATTACH_TASK define for per task detection Use define PERF_ATTACH_TASK bit in event->attach_state to determine system wide invocation or per task invocation in event initialization. This bit is set in common code and before calling PMU device driver specific event initialization. It is set once and never changes. It is save to use and also in sync with other PMU specific code. No functional change. Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_pai_crypto.c | 2 +- arch/s390/kernel/perf_pai_ext.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 19d53edf50d2..85d7398ac279 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -269,7 +269,7 @@ static int paicrypt_event_init(struct perf_event *event) a->config > PAI_CRYPTO_BASE + paicrypt_cnt) return -EINVAL; /* Allow only CPU wide operation, no process context for now. */ - if (event->hw.target || event->cpu == -1) + if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1) return -ENOENT; /* Allow only CRYPTO_ALL for sampling. */ if (a->sample_period && a->config != PAI_CRYPTO_BASE) diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index e91c77155f08..edceff2143b9 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -248,7 +248,7 @@ static int paiext_event_init(struct perf_event *event) if (rc) return rc; /* Allow only CPU wide operation, no process context for now. */ - if (event->hw.target || event->cpu == -1) + if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1) return -ENOENT; /* Allow only event NNPA_ALL for sampling. */ if (a->sample_period && a->config != PAI_NNPA_BASE) -- cgit v1.2.3 From b286997e83dcf7b498329a66a8a22fc8a5bf50f0 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Wed, 11 Oct 2023 12:09:30 +0200 Subject: s390/pai: initialize event count once at initialization Event count value is initialized and set to zero in function paicrypt_start(). This function is called once per CPU when an event is started on that CPU. This leads to event count value being set to zero as many times as there are online CPUs. This is not necessary. The event count value is bound to the event and it is sufficient to initialize the event counter once at event creation time. This is done when the event structure is dynamicly allocated with __GFP_ZERO flag. This sets member count to zero. Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_pai_crypto.c | 1 - arch/s390/kernel/perf_pai_ext.c | 1 - 2 files changed, 2 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 85d7398ac279..35940234da75 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -322,7 +322,6 @@ static void paicrypt_start(struct perf_event *event, int flags) if (!event->hw.last_tag) { event->hw.last_tag = 1; sum = paicrypt_getall(event); /* Get current value */ - local64_set(&event->count, 0); local64_set(&event->hw.prev_count, sum); } } diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index edceff2143b9..8ba0f1a3a39d 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -326,7 +326,6 @@ static void paiext_start(struct perf_event *event, int flags) event->hw.last_tag = 1; sum = paiext_getall(event); /* Get current value */ local64_set(&event->hw.prev_count, sum); - local64_set(&event->count, 0); } static int paiext_add(struct perf_event *event, int flags) -- cgit v1.2.3 From aecd5a37b5ef4de4f6402dc079672e4243cc4c13 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Wed, 18 Oct 2023 11:53:52 +0200 Subject: s390/pai_crypto: remove per-cpu variable assignement in event initialization Function paicrypt_event_init() initializes the PMU device driver specific details for an event. It is called once per event creation. The function paicrypt_event_init() is not necessarily executed on that CPU the event will be used for. When an event is activated, function paicrypt_start() is used to start the event on that CPU. The per CPU data structure struct paicrypt_map has a pointer to the event which is active for a particular CPU. This pointer is set in function paicrypt_start() to point to the currently installed event. There is no need to also set this pointer in function paicrypt_event_init() where is might be assigned to the wrong CPU. Therefore remove this assignment in paicrypt_event_init(). Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_pai_crypto.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 35940234da75..77fd24e6cbb6 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -285,7 +285,6 @@ static int paicrypt_event_init(struct perf_event *event) * are active at the same time. */ event->hw.last_tag = 0; - cpump->event = event; event->destroy = paicrypt_event_destroy; if (a->sample_period) { -- cgit v1.2.3 From e3f4170ccf20ebe2985b8e166184dd2d54220b60 Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Fri, 29 Sep 2023 19:12:04 +0200 Subject: s390/sclp: handle default case in sclp memory notifier When new memory notifier types are added in common code and unimplemented on s390, return success instead of EINVAL return code. This prevents breakage when new memory notifier types are added. Reviewed-by: Heiko Carstens Signed-off-by: Sumanth Korikkar Signed-off-by: Vasily Gorbik --- drivers/s390/char/sclp_cmd.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index b73edf0cd725..11c428f4c7cf 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -353,7 +353,6 @@ static int sclp_mem_notifier(struct notifier_block *nb, sclp_mem_change_state(start, size, 0); break; default: - rc = -EINVAL; break; } mutex_unlock(&sclp_mem_mutex); -- cgit v1.2.3 From 44d93045247661acbd50b1629e62f415f2747577 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 24 Oct 2023 10:15:19 +0200 Subject: s390/cmma: fix detection of DAT pages If the cmma no-dat feature is available the kernel page tables are walked to identify and mark all pages which are used for address translation (all region, segment, and page tables). In a subsequent loop all other pages are marked as "no-dat" pages with the ESSA instruction. This information is visible to the hypervisor, so that the hypervisor can optimize purging of guest TLB entries. The initial loop however is incorrect: only the first three of the four pages which belong to segment and region tables will be marked as being used for DAT. The last page is incorrectly marked as no-dat. This can result in incorrect guest TLB flushes. Fix this by simply marking all four pages. Cc: Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/page-states.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/mm/page-states.c b/arch/s390/mm/page-states.c index c3db3c0e75c0..20c0b160efee 100644 --- a/arch/s390/mm/page-states.c +++ b/arch/s390/mm/page-states.c @@ -121,7 +121,7 @@ static void mark_kernel_pud(p4d_t *p4d, unsigned long addr, unsigned long end) continue; if (!pud_folded(*pud)) { page = phys_to_page(pud_val(*pud)); - for (i = 0; i < 3; i++) + for (i = 0; i < 4; i++) set_bit(PG_arch_1, &page[i].flags); } mark_kernel_pmd(pud, addr, next); @@ -142,7 +142,7 @@ static void mark_kernel_p4d(pgd_t *pgd, unsigned long addr, unsigned long end) continue; if (!p4d_folded(*p4d)) { page = phys_to_page(p4d_val(*p4d)); - for (i = 0; i < 3; i++) + for (i = 0; i < 4; i++) set_bit(PG_arch_1, &page[i].flags); } mark_kernel_pud(p4d, addr, next); @@ -171,7 +171,7 @@ static void mark_kernel_pgd(void) continue; if (!pgd_folded(*pgd)) { page = phys_to_page(pgd_val(*pgd)); - for (i = 0; i < 3; i++) + for (i = 0; i < 4; i++) set_bit(PG_arch_1, &page[i].flags); } mark_kernel_p4d(pgd, addr, next); -- cgit v1.2.3 From 84bb41d5df48868055d159d9247b80927f1f70f9 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 24 Oct 2023 10:15:20 +0200 Subject: s390/cmma: fix handling of swapper_pg_dir and invalid_pg_dir If the cmma no-dat feature is available the kernel page tables are walked to identify and mark all pages which are used for address translation (all region, segment, and page tables). In a subsequent loop all other pages are marked as "no-dat" pages with the ESSA instruction. This information is visible to the hypervisor, so that the hypervisor can optimize purging of guest TLB entries. All pages used for swapper_pg_dir and invalid_pg_dir are incorrectly marked as no-dat, which in turn can result in incorrect guest TLB flushes. Fix this by marking those pages correctly as being used for DAT. Cc: Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/page-states.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/s390/mm/page-states.c b/arch/s390/mm/page-states.c index 20c0b160efee..a31acb2c4ef2 100644 --- a/arch/s390/mm/page-states.c +++ b/arch/s390/mm/page-states.c @@ -188,6 +188,12 @@ void __init cmma_init_nodat(void) return; /* Mark pages used in kernel page tables */ mark_kernel_pgd(); + page = virt_to_page(&swapper_pg_dir); + for (i = 0; i < 4; i++) + set_bit(PG_arch_1, &page[i].flags); + page = virt_to_page(&invalid_pg_dir); + for (i = 0; i < 4; i++) + set_bit(PG_arch_1, &page[i].flags); /* Set all kernel pages not used for page tables to stable/no-dat */ for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { -- cgit v1.2.3 From f139a7a2f88190ef28162d3e485324bab8a4e12e Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Mon, 31 Jul 2023 17:07:02 +0200 Subject: s390: delete the unused store_prefix() function Its last usage was deleted in commit 4df29d2b9024 ("s390/smp: rework absolute lowcore access"). Reviewed-by: Alexander Gordeev Signed-off-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/lowcore.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 3366431dcad5..5dc1b6345006 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -222,12 +222,4 @@ static inline void set_prefix(__u32 address) asm volatile("spx %0" : : "Q" (address) : "memory"); } -static inline __u32 store_prefix(void) -{ - __u32 address; - - asm volatile("stpx %0" : "=Q" (address)); - return address; -} - #endif /* _ASM_S390_LOWCORE_H */ -- cgit v1.2.3 From f48781d220ee6bb59816383237bf0abd1c65c493 Mon Sep 17 00:00:00 2001 From: Peter Oberparleiter Date: Fri, 18 Aug 2023 15:11:18 +0200 Subject: s390/cio: export CMG value as decimal Change format of the "cmg" sysfs attribute from hex to decimal to make it easier to consume. Note that this should not break any existing users since only values 2 and 3 are currently exported. Also the main user already assumes decimal notation [1]. [1] https://sourceforge.net/p/sblim/gather/ci/master/tree/plugin/metriczCH.c Reviewed-by: Vineeth Vijayan Signed-off-by: Peter Oberparleiter Signed-off-by: Vasily Gorbik --- drivers/s390/cio/chp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/cio/chp.c b/drivers/s390/cio/chp.c index 5440f285f349..675d7ed82356 100644 --- a/drivers/s390/cio/chp.c +++ b/drivers/s390/cio/chp.c @@ -334,7 +334,7 @@ static ssize_t chp_cmg_show(struct device *dev, struct device_attribute *attr, return 0; if (chp->cmg == -1) /* channel measurements not available */ return sprintf(buf, "unknown\n"); - return sprintf(buf, "%x\n", chp->cmg); + return sprintf(buf, "%d\n", chp->cmg); } static DEVICE_ATTR(cmg, 0444, chp_cmg_show, NULL); -- cgit v1.2.3 From e78002aa9a5a9eaf18bc29ccaef7f405616fd68e Mon Sep 17 00:00:00 2001 From: Peter Oberparleiter Date: Fri, 18 Aug 2023 15:14:02 +0200 Subject: s390/cio: fix virtual vs physical address confusion Fix virtual vs physical address confusion (which currently are the same). Reviewed-by: Vineeth Vijayan Signed-off-by: Peter Oberparleiter Signed-off-by: Vasily Gorbik --- drivers/s390/cio/chsc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index 0abd77f4b664..cc6dc5076ffc 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -881,8 +881,8 @@ int __chsc_do_secm(struct channel_subsystem *css, int enable) secm_area->request.code = 0x0016; secm_area->key = PAGE_DEFAULT_KEY >> 4; - secm_area->cub_addr1 = (u64)(unsigned long)css->cub_addr1; - secm_area->cub_addr2 = (u64)(unsigned long)css->cub_addr2; + secm_area->cub_addr1 = virt_to_phys(css->cub_addr1); + secm_area->cub_addr2 = virt_to_phys(css->cub_addr2); secm_area->operation_code = enable ? 0 : 1; -- cgit v1.2.3 From e37988bcd1fddb726cf08e4f04a1b8dc4a2f80aa Mon Sep 17 00:00:00 2001 From: Justin Stitt Date: Mon, 23 Oct 2023 19:14:49 +0000 Subject: s390/sclp: replace deprecated strncpy with strtomem Let's move away from using strncpy() as it is deprecated [1]. Instead use strtomem() as `e.id` is already marked as nonstring: | char id[4] __nonstring; We don't need strtomem_pad() because `e` is already memset to 0 -- rendering any additional NUL-padding useless. Link: https://www.kernel.org/doc/html/latest/process/deprecated.html#strncpy-on-nul-terminated-strings [1] Link: https://github.com/KSPP/linux/issues/90 Cc: linux-hardening@vger.kernel.org Signed-off-by: Justin Stitt Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20231023-strncpy-drivers-s390-char-sclp-c-v1-1-eaeef80522bb@google.com Signed-off-by: Vasily Gorbik --- drivers/s390/char/sclp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index ba9b202c5dee..d53ee34d398f 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -81,7 +81,7 @@ static inline void sclp_trace(int prio, char *id, u32 a, u64 b, bool err) struct sclp_trace_entry e; memset(&e, 0, sizeof(e)); - strncpy(e.id, id, sizeof(e.id)); + strtomem(e.id, id); e.a = a; e.b = b; debug_event(&sclp_debug, prio, &e, sizeof(e)); -- cgit v1.2.3 From 991a211aa99f468cd291a97b8dcb448ebc77f6c4 Mon Sep 17 00:00:00 2001 From: Justin Stitt Date: Mon, 23 Oct 2023 19:24:38 +0000 Subject: s390/cio: replace deprecated strncpy with strscpy strncpy() is deprecated for use on NUL-terminated destination strings [1] and as such we should prefer more robust and less ambiguous string interfaces. We expect both `params` and `id` to be NUL-terminated based on their usage with format strings: format_node_data(iuparams, iunodeid, &lir->incident_node); format_node_data(auparams, aunodeid, &lir->attached_node); switch (lir->iq.class) { case LIR_IQ_CLASS_DEGRADED: pr_warn("Link degraded: RS=%02x RSID=%04x IC=%02x " "IUPARAMS=%s IUNODEID=%s AUPARAMS=%s AUNODEID=%s\n", sei_area->rs, sei_area->rsid, lir->ic, iuparams, iunodeid, auparams, aunodeid); NUL-padding is not required as both `params` and `id` have been memset to 0: memset(params, 0, PARAMS_LEN); memset(id, 0, NODEID_LEN); Considering the above, a suitable replacement is `strscpy` [2] due to the fact that it guarantees NUL-termination on the destination buffer without unnecessarily NUL-padding. Note that there's no overread bugs in the current implementation as the string literal "n/a" has a size much smaller than PARAMS_LEN or NODEID_LEN. Nonetheless, let's favor strscpy(). Link: https://www.kernel.org/doc/html/latest/process/deprecated.html#strncpy-on-nul-terminated-strings [1] Link: https://manpages.debian.org/testing/linux-manual-4.8/strscpy.9.en.html [2] Link: https://github.com/KSPP/linux/issues/90 Cc: linux-hardening@vger.kernel.org Signed-off-by: Justin Stitt Reviewed-by: Vineeth Vijayan Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20231023-strncpy-drivers-s390-cio-chsc-c-v1-1-8b76a7b83260@google.com Signed-off-by: Vasily Gorbik --- drivers/s390/cio/chsc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index cc6dc5076ffc..f8b04ce61556 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -393,8 +393,8 @@ static void format_node_data(char *params, char *id, struct node_descriptor *nd) memset(id, 0, NODEID_LEN); if (nd->validity != ND_VALIDITY_VALID) { - strncpy(params, "n/a", PARAMS_LEN - 1); - strncpy(id, "n/a", NODEID_LEN - 1); + strscpy(params, "n/a", PARAMS_LEN); + strscpy(id, "n/a", NODEID_LEN); return; } -- cgit v1.2.3