Merge tag 's390-6.15-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux

Pull s390 updates from Vasily Gorbik:

 - Add sorting of mcount locations at build time

 - Rework uaccess functions with C exception handling to shorten inline
   assembly size and enable full inlining. This yields near-optimal code
   for small constant copies with a ~40kb kernel size increase

 - Add support for a configurable STRICT_MM_TYPECHECKS which allows to
   generate better code, but also allows to have type checking for debug
   builds

 - Optimize get_lowcore() for common callers with alternatives that
   nearly revert to the pre-relocated lowcore code, while also slightly
   reducing syscall entry and exit time

 - Convert MACHINE_HAS_* checks for single facility tests into cpu_has_*
   style macros that call test_facility(), and for features with
   additional conditions, add a new ALT_TYPE_FEATURE alternative to
   provide a static branch via alternative patching. Also, move machine
   feature detection to the decompressor for early patching and add
   debugging functionality to easily show which alternatives are patched

 - Add exception table support to early boot / startup code to get rid
   of the open coded exception handling

 - Use asm_inline for all inline assemblies with EX_TABLE or ALTERNATIVE
   to ensure correct inlining and unrolling decisions

 - Remove 2k page table leftovers now that s390 has been switched to
   always allocate 4k page tables

 - Split kfence pool into 4k mappings in arch_kfence_init_pool() and
   remove the architecture-specific kfence_split_mapping()

 - Use READ_ONCE_NOCHECK() in regs_get_kernel_stack_nth() to silence
   spurious KASAN warnings from opportunistic ftrace argument tracing

 - Force __atomic_add_const() variants on s390 to always return void,
   ensuring compile errors for improper usage

 - Remove s390's ioremap_wt() and pgprot_writethrough() due to
   mismatched semantics and lack of known users, relying on asm-generic
   fallbacks

 - Signal eventfd in vfio-ap to notify userspace when the guest AP
   configuration changes, including during mdev removal

 - Convert mdev_types from an array to a pointer in vfio-ccw and vfio-ap
   drivers to avoid fake flex array confusion

 - Cleanup trap code

 - Remove references to the outdated linux390@de.ibm.com address

 - Other various small fixes and improvements all over the code

* tag 's390-6.15-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux: (78 commits)
  s390: Use inline qualifier for all EX_TABLE and ALTERNATIVE inline assemblies
  s390/kfence: Split kfence pool into 4k mappings in arch_kfence_init_pool()
  s390/ptrace: Avoid KASAN false positives in regs_get_kernel_stack_nth()
  s390/boot: Ignore vmlinux.map
  s390/sysctl: Remove "vm/allocate_pgste" sysctl
  s390: Remove 2k vs 4k page table leftovers
  s390/tlb: Use mm_has_pgste() instead of mm_alloc_pgste()
  s390/lowcore: Use lghi instead llilh to clear register
  s390/syscall: Merge __do_syscall() and do_syscall()
  s390/spinlock: Implement SPINLOCK_LOCKVAL with inline assembly
  s390/smp: Implement raw_smp_processor_id() with inline assembly
  s390/current: Implement current with inline assembly
  s390/lowcore: Use inline qualifier for get_lowcore() inline assembly
  s390: Move s390 sysctls into their own file under arch/s390
  s390/syscall: Simplify syscall_get_arguments()
  s390/vfio-ap: Notify userspace that guest's AP config changed when mdev removed
  s390: Remove ioremap_wt() and pgprot_writethrough()
  s390/mm: Add configurable STRICT_MM_TYPECHECKS
  s390/mm: Convert pgste_val() into function
  s390/mm: Convert pgprot_val() into function
  ...

35 files changed, 395 insertions, 315 deletions

diff --git a/arch/s390/include/asm/abs_lowcore.h b/arch/s390/include/asm/abs_lowcore.h
index 004d17ea05cf..317c07c09ae4 100644
--- a/arch/s390/include/asm/abs_lowcore.h
+++ b/arch/s390/include/asm/abs_lowcore.h
@@ -25,11 +25,4 @@ static inline void put_abs_lowcore(struct lowcore *lc)
 	put_cpu();
 }
 
-extern int relocate_lowcore;
-
-static inline int have_relocated_lowcore(void)
-{
-	return relocate_lowcore;
-}
-
 #endif /* _ASM_S390_ABS_LOWCORE_H */
diff --git a/arch/s390/include/asm/alternative.h b/arch/s390/include/asm/alternative.h
index 73e781b56bfe..c7bf60a541e9 100644
--- a/arch/s390/include/asm/alternative.h
+++ b/arch/s390/include/asm/alternative.h
@@ -32,8 +32,8 @@
 #define ALT_CTX_ALL		(ALT_CTX_EARLY | ALT_CTX_LATE)
 
 #define ALT_TYPE_FACILITY	0
-#define ALT_TYPE_SPEC		1
-#define ALT_TYPE_LOWCORE	2
+#define ALT_TYPE_FEATURE	1
+#define ALT_TYPE_SPEC		2
 
 #define ALT_DATA_SHIFT		0
 #define ALT_TYPE_SHIFT		20
@@ -43,13 +43,14 @@
 					 ALT_TYPE_FACILITY << ALT_TYPE_SHIFT	| \
 					 (facility) << ALT_DATA_SHIFT)
 
+#define ALT_FEATURE(feature)		(ALT_CTX_EARLY << ALT_CTX_SHIFT		| \
+					 ALT_TYPE_FEATURE << ALT_TYPE_SHIFT	| \
+					 (feature) << ALT_DATA_SHIFT)
+
 #define ALT_SPEC(facility)		(ALT_CTX_LATE << ALT_CTX_SHIFT		| \
 					 ALT_TYPE_SPEC << ALT_TYPE_SHIFT	| \
 					 (facility) << ALT_DATA_SHIFT)
 
-#define ALT_LOWCORE			(ALT_CTX_EARLY << ALT_CTX_SHIFT		| \
-					 ALT_TYPE_LOWCORE << ALT_TYPE_SHIFT)
-
 #ifndef __ASSEMBLY__
 
 #include <linux/types.h>
diff --git a/arch/s390/include/asm/appldata.h b/arch/s390/include/asm/appldata.h
index a92ebbc7aa7a..99b2902c10fd 100644
--- a/arch/s390/include/asm/appldata.h
+++ b/arch/s390/include/asm/appldata.h
@@ -9,6 +9,7 @@
 #define _ASM_S390_APPLDATA_H
 
 #include <linux/io.h>
+#include <asm/machine.h>
 #include <asm/diag.h>
 
 #define APPLDATA_START_INTERVAL_REC	0x80
@@ -48,7 +49,7 @@ static inline int appldata_asm(struct appldata_parameter_list *parm_list,
 {
 	int ry;
 
-	if (!MACHINE_IS_VM)
+	if (!machine_is_vm())
 		return -EOPNOTSUPP;
 	parm_list->diag = 0xdc;
 	parm_list->function = fn;
diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h
index 2e829c16fd8a..d23ea0c94e4e 100644
--- a/arch/s390/include/asm/asm-extable.h
+++ b/arch/s390/include/asm/asm-extable.h
@@ -14,6 +14,8 @@
 #define EX_TYPE_UA_LOAD_REGPAIR	6
 #define EX_TYPE_ZEROPAD		7
 #define EX_TYPE_FPC		8
+#define EX_TYPE_UA_MVCOS_TO	9
+#define EX_TYPE_UA_MVCOS_FROM	10
 
 #define EX_DATA_REG_ERR_SHIFT	0
 #define EX_DATA_REG_ERR		GENMASK(3, 0)
@@ -84,4 +86,10 @@
 #define EX_TABLE_FPC(_fault, _target)					\
 	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0)
 
+#define EX_TABLE_UA_MVCOS_TO(_fault, _target)				\
+	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_MVCOS_TO, __stringify(%%r0), __stringify(%%r0), 0)
+
+#define EX_TABLE_UA_MVCOS_FROM(_fault, _target)				\
+	__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_MVCOS_FROM, __stringify(%%r0), __stringify(%%r0), 0)
+
 #endif /* __ASM_EXTABLE_H */
diff --git a/arch/s390/include/asm/atomic_ops.h b/arch/s390/include/asm/atomic_ops.h
index 585678bbcd7a..21c26d842832 100644
--- a/arch/s390/include/asm/atomic_ops.h
+++ b/arch/s390/include/asm/atomic_ops.h
@@ -163,10 +163,10 @@ __ATOMIC64_OPS(__atomic64_xor, "xgr")
 
 #undef __ATOMIC64_OPS
 
-#define __atomic_add_const(val, ptr)		__atomic_add(val, ptr)
-#define __atomic_add_const_barrier(val, ptr)	__atomic_add(val, ptr)
-#define __atomic64_add_const(val, ptr)		__atomic64_add(val, ptr)
-#define __atomic64_add_const_barrier(val, ptr)	__atomic64_add(val, ptr)
+#define __atomic_add_const(val, ptr)		((void)__atomic_add(val, ptr))
+#define __atomic_add_const_barrier(val, ptr)	((void)__atomic_add(val, ptr))
+#define __atomic64_add_const(val, ptr)		((void)__atomic64_add(val, ptr))
+#define __atomic64_add_const_barrier(val, ptr)	((void)__atomic64_add(val, ptr))
 
 #endif /* MARCH_HAS_Z196_FEATURES */
 
diff --git a/arch/s390/include/asm/cpu_mf.h b/arch/s390/include/asm/cpu_mf.h
index e1a279e0d6a6..1798fbd59068 100644
--- a/arch/s390/include/asm/cpu_mf.h
+++ b/arch/s390/include/asm/cpu_mf.h
@@ -171,7 +171,7 @@ static inline int qctri(struct cpumf_ctr_info *info)
 {
 	int rc = -EINVAL;
 
-	asm volatile (
+	asm_inline volatile (
 		"0:	qctri	%1\n"
 		"1:	lhi	%0,0\n"
 		"2:\n"
@@ -185,7 +185,7 @@ static inline int lcctl(u64 ctl)
 {
 	int cc;
 
-	asm volatile (
+	asm_inline volatile (
 		"	lcctl	%[ctl]\n"
 		CC_IPM(cc)
 		: CC_OUT(cc, cc)
@@ -200,7 +200,7 @@ static inline int __ecctr(u64 ctr, u64 *content)
 	u64 _content;
 	int cc;
 
-	asm volatile (
+	asm_inline volatile (
 		"	ecctr	%[_content],%[ctr]\n"
 		CC_IPM(cc)
 		: CC_OUT(cc, cc), [_content] "=d" (_content)
diff --git a/arch/s390/include/asm/cpufeature.h b/arch/s390/include/asm/cpufeature.h
index 931204613753..e08169bd63a5 100644
--- a/arch/s390/include/asm/cpufeature.h
+++ b/arch/s390/include/asm/cpufeature.h
@@ -9,6 +9,8 @@
 #ifndef __ASM_S390_CPUFEATURE_H
 #define __ASM_S390_CPUFEATURE_H
 
+#include <asm/facility.h>
+
 enum {
 	S390_CPU_FEATURE_MSA,
 	S390_CPU_FEATURE_VXRS,
@@ -20,4 +22,16 @@ enum {
 
 int cpu_have_feature(unsigned int nr);
 
+#define cpu_has_bear()		test_facility(193)
+#define cpu_has_edat1()		test_facility(8)
+#define cpu_has_edat2()		test_facility(78)
+#define cpu_has_gs()		test_facility(133)
+#define cpu_has_idte()		test_facility(3)
+#define cpu_has_nx()		test_facility(130)
+#define cpu_has_rdp()		test_facility(194)
+#define cpu_has_seq_insn()	test_facility(85)
+#define cpu_has_tlb_lc()	test_facility(51)
+#define cpu_has_topology()	test_facility(11)
+#define cpu_has_vx()		test_facility(129)
+
 #endif /* __ASM_S390_CPUFEATURE_H */
diff --git a/arch/s390/include/asm/current.h b/arch/s390/include/asm/current.h
index d03a922c641e..f9529f7cf62c 100644
--- a/arch/s390/include/asm/current.h
+++ b/arch/s390/include/asm/current.h
@@ -11,9 +11,25 @@
 #define _S390_CURRENT_H
 
 #include <asm/lowcore.h>
+#include <asm/machine.h>
 
 struct task_struct;
 
-#define current ((struct task_struct *const)get_lowcore()->current_task)
+static __always_inline struct task_struct *get_current(void)
+{
+	unsigned long ptr, lc_current;
+
+	lc_current = offsetof(struct lowcore, current_task);
+	asm_inline(
+		ALTERNATIVE("	lg	%[ptr],%[offzero](%%r0)\n",
+			    "	lg	%[ptr],%[offalt](%%r0)\n",
+			    ALT_FEATURE(MFEATURE_LOWCORE))
+		: [ptr] "=d" (ptr)
+		: [offzero] "i" (lc_current),
+		  [offalt] "i" (lc_current + LOWCORE_ALT_ADDRESS));
+	return (struct task_struct *)ptr;
+}
+
+#define current get_current()
 
 #endif /* !(_S390_CURRENT_H) */
diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h
index 5790630e31f0..8db8db3b1018 100644
--- a/arch/s390/include/asm/diag.h
+++ b/arch/s390/include/asm/diag.h
@@ -66,7 +66,7 @@ static inline void diag10_range(unsigned long start_pfn, unsigned long num_pfn)
 	end_addr = pfn_to_phys(start_pfn + num_pfn - 1);
 
 	diag_stat_inc(DIAG_STAT_X010);
-	asm volatile(
+	asm_inline volatile(
 		"0:	diag	%0,%1,0x10\n"
 		"1:	nopr	%%r7\n"
 		EX_TABLE(0b, 1b)
diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h
index 8f2c23cc52b6..a03df312081e 100644
--- a/arch/s390/include/asm/elf.h
+++ b/arch/s390/include/asm/elf.h
@@ -158,9 +158,6 @@ enum {
 #define ELF_DATA	ELFDATA2MSB
 #define ELF_ARCH	EM_S390
 
-/* s390 specific phdr types */
-#define PT_S390_PGSTE	0x70000000
-
 /*
  * ELF register definitions..
  */
@@ -191,35 +188,6 @@ typedef s390_compat_regs compat_elf_gregset_t;
 	 && (x)->e_ident[EI_CLASS] == ELF_CLASS)
 #define compat_start_thread	start_thread31
 
-struct arch_elf_state {
-	int rc;
-};
-
-#define INIT_ARCH_ELF_STATE { .rc = 0 }
-
-#define arch_check_elf(ehdr, interp, interp_ehdr, state) (0)
-#ifdef CONFIG_PGSTE
-#define arch_elf_pt_proc(ehdr, phdr, elf, interp, state)	\
-({								\
-	struct arch_elf_state *_state = state;			\
-	if ((phdr)->p_type == PT_S390_PGSTE &&			\
-	    !page_table_allocate_pgste &&			\
-	    !test_thread_flag(TIF_PGSTE) &&			\
-	    !current->mm->context.alloc_pgste) {		\
-		set_thread_flag(TIF_PGSTE);			\
-		set_pt_regs_flag(task_pt_regs(current),		\
-				 PIF_EXECVE_PGSTE_RESTART);	\
-		_state->rc = -EAGAIN;				\
-	}							\
-	_state->rc;						\
-})
-#else
-#define arch_elf_pt_proc(ehdr, phdr, elf, interp, state)	\
-({								\
-	(state)->rc;						\
-})
-#endif
-
 /* For SVR4/S390 the function pointer to be registered with `atexit` is
    passed in R14. */
 #define ELF_PLAT_INIT(_r, load_addr) \
diff --git a/arch/s390/include/asm/fpu.h b/arch/s390/include/asm/fpu.h
index c84cb33913e2..960c6c67ad6c 100644
--- a/arch/s390/include/asm/fpu.h
+++ b/arch/s390/include/asm/fpu.h
@@ -44,6 +44,7 @@
 #ifndef _ASM_S390_FPU_H
 #define _ASM_S390_FPU_H
 
+#include <linux/cpufeature.h>
 #include <linux/processor.h>
 #include <linux/preempt.h>
 #include <linux/string.h>
@@ -51,12 +52,6 @@
 #include <asm/sigcontext.h>
 #include <asm/fpu-types.h>
 #include <asm/fpu-insn.h>
-#include <asm/facility.h>
-
-static inline bool cpu_has_vx(void)
-{
-	return likely(test_facility(129));
-}
 
 enum {
 	KERNEL_FPC_BIT = 0,
diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h
index 663e87220e89..931fcc413598 100644
--- a/arch/s390/include/asm/hugetlb.h
+++ b/arch/s390/include/asm/hugetlb.h
@@ -9,12 +9,13 @@
 #ifndef _ASM_S390_HUGETLB_H
 #define _ASM_S390_HUGETLB_H
 
+#include <linux/cpufeature.h>
 #include <linux/pgtable.h>
 #include <linux/swap.h>
 #include <linux/swapops.h>
 #include <asm/page.h>
 
-#define hugepages_supported()			(MACHINE_HAS_EDAT1)
+#define hugepages_supported()	cpu_has_edat1()
 
 #define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
diff --git a/arch/s390/include/asm/io.h b/arch/s390/include/asm/io.h
index fc9933a743d6..251e0372ccbd 100644
--- a/arch/s390/include/asm/io.h
+++ b/arch/s390/include/asm/io.h
@@ -34,8 +34,6 @@ void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr);
 
 #define ioremap_wc(addr, size)  \
 	ioremap_prot((addr), (size), pgprot_val(pgprot_writecombine(PAGE_KERNEL)))
-#define ioremap_wt(addr, size)  \
-	ioremap_prot((addr), (size), pgprot_val(pgprot_writethrough(PAGE_KERNEL)))
 
 static inline void __iomem *ioport_map(unsigned long port, unsigned int nr)
 {
diff --git a/arch/s390/include/asm/kfence.h b/arch/s390/include/asm/kfence.h
index e47fd8cbe701..e95e35eb8a3f 100644
--- a/arch/s390/include/asm/kfence.h
+++ b/arch/s390/include/asm/kfence.h
@@ -12,27 +12,16 @@ void __kernel_map_pages(struct page *page, int numpages, int enable);
 
 static __always_inline bool arch_kfence_init_pool(void)
 {
-	return true;
-}
-
-#define arch_kfence_test_address(addr) ((addr) & PAGE_MASK)
-
-/*
- * Do not split kfence pool to 4k mapping with arch_kfence_init_pool(),
- * but earlier where page table allocations still happen with memblock.
- * Reason is that arch_kfence_init_pool() gets called when the system
- * is still in a limbo state - disabling and enabling bottom halves is
- * not yet allowed, but that is what our page_table_alloc() would do.
- */
-static __always_inline void kfence_split_mapping(void)
-{
 #ifdef CONFIG_KFENCE
 	unsigned long pool_pages = KFENCE_POOL_SIZE >> PAGE_SHIFT;
 
 	set_memory_4k((unsigned long)__kfence_pool, pool_pages);
 #endif
+	return true;
 }
 
+#define arch_kfence_test_address(addr) ((addr) & PAGE_MASK)
+
 static inline bool kfence_protect_page(unsigned long addr, bool protect)
 {
 	__kernel_map_pages(virt_to_page((void *)addr), 1, !protect);
diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
index 42a092fa1029..1a31f1f93ed3 100644
--- a/arch/s390/include/asm/lowcore.h
+++ b/arch/s390/include/asm/lowcore.h
@@ -10,6 +10,7 @@
 #define _ASM_S390_LOWCORE_H
 
 #include <linux/types.h>
+#include <asm/machine.h>
 #include <asm/ptrace.h>
 #include <asm/ctlreg.h>
 #include <asm/cpu.h>
@@ -126,7 +127,7 @@ struct lowcore {
 	__u64	int_clock;			/* 0x0318 */
 	__u8	pad_0x0320[0x0328-0x0320];	/* 0x0320 */
 	__u64	clock_comparator;		/* 0x0328 */
-	__u64	boot_clock[2];			/* 0x0330 */
+	__u8	pad_0x0330[0x0340-0x0330];	/* 0x0330 */
 
 	/* Current process. */
 	__u64	current_task;			/* 0x0340 */
@@ -222,9 +223,12 @@ static __always_inline struct lowcore *get_lowcore(void)
 
 	if (__is_defined(__DECOMPRESSOR))
 		return NULL;
-	asm(ALTERNATIVE("llilh %[lc],0", "llilh %[lc],%[alt]", ALT_LOWCORE)
-	    : [lc] "=d" (lc)
-	    : [alt] "i" (LOWCORE_ALT_ADDRESS >> 16));
+	asm_inline(
+		ALTERNATIVE("	lghi	%[lc],0",
+			    "	llilh	%[lc],%[alt]",
+			    ALT_FEATURE(MFEATURE_LOWCORE))
+		: [lc] "=d" (lc)
+		: [alt] "i" (LOWCORE_ALT_ADDRESS >> 16));
 	return lc;
 }
 
@@ -238,15 +242,15 @@ static inline void set_prefix(__u32 address)
 #else /* __ASSEMBLY__ */
 
 .macro GET_LC reg
-	ALTERNATIVE "llilh	\reg,0",					\
+	ALTERNATIVE "lghi	\reg,0",					\
 		__stringify(llilh	\reg, LOWCORE_ALT_ADDRESS >> 16),	\
-		ALT_LOWCORE
+		ALT_FEATURE(MFEATURE_LOWCORE)
 .endm
 
 .macro STMG_LC start, end, savearea
 	ALTERNATIVE "stmg	\start, \end, \savearea",				\
 		__stringify(stmg	\start, \end, LOWCORE_ALT_ADDRESS + \savearea),	\
-		ALT_LOWCORE
+		ALT_FEATURE(MFEATURE_LOWCORE)
 .endm
 
 #endif /* __ASSEMBLY__ */
diff --git a/arch/s390/include/asm/machine.h b/arch/s390/include/asm/machine.h
new file mode 100644
index 000000000000..54478caa5237
--- /dev/null
+++ b/arch/s390/include/asm/machine.h
@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright IBM Corp. 2024
+ */
+
+#ifndef __ASM_S390_MACHINE_H
+#define __ASM_S390_MACHINE_H
+
+#include <linux/const.h>
+
+#define MFEATURE_LOWCORE	0
+#define MFEATURE_PCI_MIO	1
+#define MFEATURE_SCC		2
+#define MFEATURE_TLB_GUEST	3
+#define MFEATURE_TX		4
+#define MFEATURE_ESOP		5
+#define MFEATURE_DIAG9C		6
+#define MFEATURE_VM		7
+#define MFEATURE_KVM		8
+#define MFEATURE_LPAR		9
+
+#ifndef __ASSEMBLY__
+
+#include <linux/bitops.h>
+#include <asm/alternative.h>
+
+extern unsigned long machine_features[1];
+
+#define MAX_MFEATURE_BIT (sizeof(machine_features) * BITS_PER_BYTE)
+
+static inline void __set_machine_feature(unsigned int nr, unsigned long *mfeatures)
+{
+	if (nr >= MAX_MFEATURE_BIT)
+		return;
+	__set_bit(nr, mfeatures);
+}
+
+static inline void set_machine_feature(unsigned int nr)
+{
+	__set_machine_feature(nr, machine_features);
+}
+
+static inline void __clear_machine_feature(unsigned int nr, unsigned long *mfeatures)
+{
+	if (nr >= MAX_MFEATURE_BIT)
+		return;
+	__clear_bit(nr, mfeatures);
+}
+
+static inline void clear_machine_feature(unsigned int nr)
+{
+	__clear_machine_feature(nr, machine_features);
+}
+
+static bool __test_machine_feature(unsigned int nr, unsigned long *mfeatures)
+{
+	if (nr >= MAX_MFEATURE_BIT)
+		return false;
+	return test_bit(nr, mfeatures);
+}
+
+static bool test_machine_feature(unsigned int nr)
+{
+	return __test_machine_feature(nr, machine_features);
+}
+
+static __always_inline bool __test_machine_feature_constant(unsigned int nr)
+{
+	asm goto(
+		ALTERNATIVE("brcl 15,%l[l_no]", "brcl 0,0", ALT_FEATURE(%[nr]))
+		:
+		: [nr] "i" (nr)
+		:
+		: l_no);
+	return true;
+l_no:
+	return false;
+}
+
+#define DEFINE_MACHINE_HAS_FEATURE(name, feature)				\
+static __always_inline bool machine_has_##name(void)				\
+{										\
+	if (!__is_defined(__DECOMPRESSOR) && __builtin_constant_p(feature))	\
+		return __test_machine_feature_constant(feature);		\
+	return test_machine_feature(feature);					\
+}
+
+DEFINE_MACHINE_HAS_FEATURE(relocated_lowcore, MFEATURE_LOWCORE)
+DEFINE_MACHINE_HAS_FEATURE(scc, MFEATURE_SCC)
+DEFINE_MACHINE_HAS_FEATURE(tlb_guest, MFEATURE_TLB_GUEST)
+DEFINE_MACHINE_HAS_FEATURE(tx, MFEATURE_TX)
+DEFINE_MACHINE_HAS_FEATURE(esop, MFEATURE_ESOP)
+DEFINE_MACHINE_HAS_FEATURE(diag9c, MFEATURE_DIAG9C)
+DEFINE_MACHINE_HAS_FEATURE(vm, MFEATURE_VM)
+DEFINE_MACHINE_HAS_FEATURE(kvm, MFEATURE_KVM)
+DEFINE_MACHINE_HAS_FEATURE(lpar, MFEATURE_LPAR)
+
+#define machine_is_vm	machine_has_vm
+#define machine_is_kvm	machine_has_kvm
+#define machine_is_lpar	machine_has_lpar
+
+#endif /* __ASSEMBLY__ */
+#endif /* __ASM_S390_MACHINE_H */
diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h
index 4c2dc7abc285..f07e49b419ab 100644
--- a/arch/s390/include/asm/mmu.h
+++ b/arch/s390/include/asm/mmu.h
@@ -22,10 +22,7 @@ typedef struct {
 	 * The following bitfields need a down_write on the mm
 	 * semaphore when they are written to. As they are only
 	 * written once, they can be read without a lock.
-	 *
-	 * The mmu context allocates 4K page tables.
 	 */
-	unsigned int alloc_pgste:1;
 	/* The mmu context uses extended page tables. */
 	unsigned int has_pgste:1;
 	/* The mmu context uses storage keys. */
diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h
index d56eb0a1f37b..88f84beebb9e 100644
--- a/arch/s390/include/asm/mmu_context.h
+++ b/arch/s390/include/asm/mmu_context.h
@@ -29,9 +29,6 @@ static inline int init_new_context(struct task_struct *tsk,
 	mm->context.gmap_asce = 0;
 	mm->context.flush_mm = 0;
 #ifdef CONFIG_PGSTE
-	mm->context.alloc_pgste = page_table_allocate_pgste ||
-		test_thread_flag(TIF_PGSTE) ||
-		(current->mm && current->mm->context.alloc_pgste);
 	mm->context.has_pgste = 0;
 	mm->context.uses_skeys = 0;
 	mm->context.uses_cmm = 0;
diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h
index 1ff145f7b52b..4e5dbabdf202 100644
--- a/arch/s390/include/asm/page.h
+++ b/arch/s390/include/asm/page.h
@@ -71,9 +71,11 @@ static inline void copy_page(void *to, void *from)
 #define vma_alloc_zeroed_movable_folio(vma, vaddr) \
 	vma_alloc_folio(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, 0, vma, vaddr)
 
-/*
- * These are used to make use of C type-checking..
- */
+#ifdef CONFIG_STRICT_MM_TYPECHECKS
+#define STRICT_MM_TYPECHECKS
+#endif
+
+#ifdef STRICT_MM_TYPECHECKS
 
 typedef struct { unsigned long pgprot; } pgprot_t;
 typedef struct { unsigned long pgste; } pgste_t;
@@ -82,43 +84,48 @@ typedef struct { unsigned long pmd; } pmd_t;
 typedef struct { unsigned long pud; } pud_t;
 typedef struct { unsigned long p4d; } p4d_t;
 typedef struct { unsigned long pgd; } pgd_t;
-typedef pte_t *pgtable_t;
 
-#define pgprot_val(x)	((x).pgprot)
-#define pgste_val(x)	((x).pgste)
-
-static inline unsigned long pte_val(pte_t pte)
-{
-	return pte.pte;
+#define DEFINE_PGVAL_FUNC(name)						\
+static __always_inline unsigned long name ## _val(name ## _t name)	\
+{									\
+	return name.name;						\
 }
 
-static inline unsigned long pmd_val(pmd_t pmd)
-{
-	return pmd.pmd;
-}
+#else /* STRICT_MM_TYPECHECKS */
 
-static inline unsigned long pud_val(pud_t pud)
-{
-	return pud.pud;
-}
+typedef unsigned long pgprot_t;
+typedef unsigned long pgste_t;
+typedef unsigned long pte_t;
+typedef unsigned long pmd_t;
+typedef unsigned long pud_t;
+typedef unsigned long p4d_t;
+typedef unsigned long pgd_t;
 
-static inline unsigned long p4d_val(p4d_t p4d)
-{
-	return p4d.p4d;
+#define DEFINE_PGVAL_FUNC(name)						\
+static __always_inline unsigned long name ## _val(name ## _t name)	\
+{									\
+	return name;							\
 }
 
-static inline unsigned long pgd_val(pgd_t pgd)
-{
-	return pgd.pgd;
-}
+#endif /* STRICT_MM_TYPECHECKS */
+
+DEFINE_PGVAL_FUNC(pgprot)
+DEFINE_PGVAL_FUNC(pgste)
+DEFINE_PGVAL_FUNC(pte)
+DEFINE_PGVAL_FUNC(pmd)
+DEFINE_PGVAL_FUNC(pud)
+DEFINE_PGVAL_FUNC(p4d)
+DEFINE_PGVAL_FUNC(pgd)
+
+typedef pte_t *pgtable_t;
 
+#define __pgprot(x)	((pgprot_t) { (x) } )
 #define __pgste(x)	((pgste_t) { (x) } )
 #define __pte(x)        ((pte_t) { (x) } )
 #define __pmd(x)        ((pmd_t) { (x) } )
 #define __pud(x)	((pud_t) { (x) } )
 #define __p4d(x)	((p4d_t) { (x) } )
 #define __pgd(x)        ((pgd_t) { (x) } )
-#define __pgprot(x)     ((pgprot_t) { (x) } )
 
 static inline void page_set_storage_key(unsigned long addr,
 					unsigned char skey, int mapped)
diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h
index b19b6ed2ab53..005497ffebda 100644
--- a/arch/s390/include/asm/pgalloc.h
+++ b/arch/s390/include/asm/pgalloc.h
@@ -26,7 +26,6 @@ unsigned long *page_table_alloc(struct mm_struct *);
 struct ptdesc *page_table_alloc_pgste(struct mm_struct *mm);
 void page_table_free(struct mm_struct *, unsigned long *);
 void page_table_free_pgste(struct ptdesc *ptdesc);
-extern int page_table_allocate_pgste;
 
 static inline void crst_table_init(unsigned long *crst, unsigned long entry)
 {
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 3ca5af4cfe43..f8a6b54986ec 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -14,6 +14,7 @@
 
 #include <linux/sched.h>
 #include <linux/mm_types.h>
+#include <linux/cpufeature.h>
 #include <linux/page-flags.h>
 #include <linux/radix-tree.h>
 #include <linux/atomic.h>
@@ -583,13 +584,14 @@ static inline int mm_is_protected(struct mm_struct *mm)
 	return 0;
 }
 
-static inline int mm_alloc_pgste(struct mm_struct *mm)
+static inline pgste_t clear_pgste_bit(pgste_t pgste, unsigned long mask)
 {
-#ifdef CONFIG_PGSTE
-	if (unlikely(mm->context.alloc_pgste))
-		return 1;
-#endif
-	return 0;
+	return __pgste(pgste_val(pgste) & ~mask);
+}
+
+static inline pgste_t set_pgste_bit(pgste_t pgste, unsigned long mask)
+{
+	return __pgste(pgste_val(pgste) | mask);
 }
 
 static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
@@ -1339,7 +1341,7 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
 	 * PTE does not have _PAGE_PROTECT set, to avoid unnecessary overhead.
 	 * A local RDP can be used to do the flush.
 	 */
-	if (MACHINE_HAS_RDP && !(pte_val(*ptep) & _PAGE_PROTECT))
+	if (cpu_has_rdp() && !(pte_val(*ptep) & _PAGE_PROTECT))
 		__ptep_rdp(address, ptep, 0, 0, 1);
 }
 #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault
@@ -1354,7 +1356,7 @@ static inline int ptep_set_access_flags(struct vm_area_struct *vma,
 {
 	if (pte_same(*ptep, entry))
 		return 0;
-	if (MACHINE_HAS_RDP && !mm_has_pgste(vma->vm_mm) && pte_allow_rdp(*ptep, entry))
+	if (cpu_has_rdp() && !mm_has_pgste(vma->vm_mm) && pte_allow_rdp(*ptep, entry))
 		ptep_reset_dat_prot(vma->vm_mm, addr, ptep, entry);
 	else
 		ptep_xchg_direct(vma->vm_mm, addr, ptep, entry);
@@ -1402,9 +1404,6 @@ void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr);
 #define pgprot_writecombine	pgprot_writecombine
 pgprot_t pgprot_writecombine(pgprot_t prot);
 
-#define pgprot_writethrough	pgprot_writethrough
-pgprot_t pgprot_writethrough(pgprot_t prot);
-
 #define PFN_PTE_SHIFT		PAGE_SHIFT
 
 /*
@@ -1890,7 +1889,7 @@ static inline int pmd_trans_huge(pmd_t pmd)
 #define has_transparent_hugepage has_transparent_hugepage
 static inline int has_transparent_hugepage(void)
 {
-	return MACHINE_HAS_EDAT1 ? 1 : 0;
+	return cpu_has_edat1() ? 1 : 0;
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 4f8d5592c298..d09a92db95f7 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -416,7 +416,11 @@ static __always_inline bool regs_irqs_disabled(struct pt_regs *regs)
 
 static __always_inline void bpon(void)
 {
-	asm volatile(ALTERNATIVE("nop", ".insn	rrf,0xb2e80000,0,0,13,0", ALT_SPEC(82)));
+	asm_inline volatile(
+		ALTERNATIVE("	nop\n",
+			    "	.insn	rrf,0xb2e80000,0,0,13,0\n",
+			    ALT_SPEC(82))
+		);
 }
 
 #endif /* __ASSEMBLY__ */
diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h
index 788bc4467445..c66f3fc6daaf 100644
--- a/arch/s390/include/asm/ptrace.h
+++ b/arch/s390/include/asm/ptrace.h
@@ -12,13 +12,11 @@
 #include <asm/tpi.h>
 
 #define PIF_SYSCALL			0	/* inside a system call */
-#define PIF_EXECVE_PGSTE_RESTART	1	/* restart execve for PGSTE binaries */
 #define PIF_SYSCALL_RET_SET		2	/* return value was set via ptrace */
 #define PIF_GUEST_FAULT			3	/* indicates program check in sie64a */
 #define PIF_FTRACE_FULL_REGS		4	/* all register contents valid (ftrace) */
 
 #define _PIF_SYSCALL			BIT(PIF_SYSCALL)
-#define _PIF_EXECVE_PGSTE_RESTART	BIT(PIF_EXECVE_PGSTE_RESTART)
 #define _PIF_SYSCALL_RET_SET		BIT(PIF_SYSCALL_RET_SET)
 #define _PIF_GUEST_FAULT		BIT(PIF_GUEST_FAULT)
 #define _PIF_FTRACE_FULL_REGS		BIT(PIF_FTRACE_FULL_REGS)
diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h
index 18f37dff03c9..1e62919bacf4 100644
--- a/arch/s390/include/asm/sclp.h
+++ b/arch/s390/include/asm/sclp.h
@@ -168,6 +168,7 @@ int sclp_early_read_storage_info(void);
 int sclp_early_get_core_info(struct sclp_core_info *info);
 void sclp_early_get_ipl_info(struct sclp_ipl_info *info);
 void sclp_early_detect(void);
+void sclp_early_detect_machine_features(void);
 void sclp_early_printk(const char *s);
 void __sclp_early_printk(const char *s, unsigned int len);
 void sclp_emergency_printk(const char *s);
diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h
index 70b920b32827..031e881b4d88 100644
--- a/arch/s390/include/asm/setup.h
+++ b/arch/s390/include/asm/setup.h
@@ -13,28 +13,6 @@
 #define PARMAREA		0x10400
 
 #define COMMAND_LINE_SIZE CONFIG_COMMAND_LINE_SIZE
-/*
- * Machine features detected in early.c
- */
-
-#define MACHINE_FLAG_VM		BIT(0)
-#define MACHINE_FLAG_KVM	BIT(1)
-#define MACHINE_FLAG_LPAR	BIT(2)
-#define MACHINE_FLAG_DIAG9C	BIT(3)
-#define MACHINE_FLAG_ESOP	BIT(4)
-#define MACHINE_FLAG_IDTE	BIT(5)
-#define MACHINE_FLAG_EDAT1	BIT(7)
-#define MACHINE_FLAG_EDAT2	BIT(8)
-#define MACHINE_FLAG_TOPOLOGY	BIT(10)
-#define MACHINE_FLAG_TE		BIT(11)
-#define MACHINE_FLAG_TLB_LC	BIT(12)
-#define MACHINE_FLAG_TLB_GUEST	BIT(14)
-#define MACHINE_FLAG_NX		BIT(15)
-#define MACHINE_FLAG_GS		BIT(16)
-#define MACHINE_FLAG_SCC	BIT(17)
-#define MACHINE_FLAG_PCI_MIO	BIT(18)
-#define MACHINE_FLAG_RDP	BIT(19)
-#define MACHINE_FLAG_SEQ_INSN	BIT(20)
 
 #define LPP_MAGIC		BIT(31)
 #define LPP_PID_MASK		_AC(0xffffffff, UL)
@@ -78,26 +56,6 @@ extern unsigned long max_mappable;
 /* The Write Back bit position in the physaddr is given by the SLPC PCI */
 extern unsigned long mio_wb_bit_mask;
 
-#define MACHINE_IS_VM		(get_lowcore()->machine_flags & MACHINE_FLAG_VM)
-#define MACHINE_IS_KVM		(get_lowcore()->machine_flags & MACHINE_FLAG_KVM)
-#define MACHINE_IS_LPAR		(get_lowcore()->machine_flags & MACHINE_FLAG_LPAR)
-
-#define MACHINE_HAS_DIAG9C	(get_lowcore()->machine_flags & MACHINE_FLAG_DIAG9C)
-#define MACHINE_HAS_ESOP	(get_lowcore()->machine_flags & MACHINE_FLAG_ESOP)
-#define MACHINE_HAS_IDTE	(get_lowcore()->machine_flags & MACHINE_FLAG_IDTE)
-#define MACHINE_HAS_EDAT1	(get_lowcore()->machine_flags & MACHINE_FLAG_EDAT1)
-#define MACHINE_HAS_EDAT2	(get_lowcore()->machine_flags & MACHINE_FLAG_EDAT2)
-#define MACHINE_HAS_TOPOLOGY	(get_lowcore()->machine_flags & MACHINE_FLAG_TOPOLOGY)
-#define MACHINE_HAS_TE		(get_lowcore()->machine_flags & MACHINE_FLAG_TE)
-#define MACHINE_HAS_TLB_LC	(get_lowcore()->machine_flags & MACHINE_FLAG_TLB_LC)
-#define MACHINE_HAS_TLB_GUEST	(get_lowcore()->machine_flags & MACHINE_FLAG_TLB_GUEST)
-#define MACHINE_HAS_NX		(get_lowcore()->machine_flags & MACHINE_FLAG_NX)
-#define MACHINE_HAS_GS		(get_lowcore()->machine_flags & MACHINE_FLAG_GS)
-#define MACHINE_HAS_SCC		(get_lowcore()->machine_flags & MACHINE_FLAG_SCC)
-#define MACHINE_HAS_PCI_MIO	(get_lowcore()->machine_flags & MACHINE_FLAG_PCI_MIO)
-#define MACHINE_HAS_RDP		(get_lowcore()->machine_flags & MACHINE_FLAG_RDP)
-#define MACHINE_HAS_SEQ_INSN	(get_lowcore()->machine_flags & MACHINE_FLAG_SEQ_INSN)
-
 /*
  * Console mode. Override with conmode=
  */
diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h
index 7feca96c48c6..03f4d01664f8 100644
--- a/arch/s390/include/asm/smp.h
+++ b/arch/s390/include/asm/smp.h
@@ -7,11 +7,29 @@
 #ifndef __ASM_SMP_H
 #define __ASM_SMP_H
 
-#include <asm/sigp.h>
-#include <asm/lowcore.h>
 #include <asm/processor.h>
+#include <asm/lowcore.h>
+#include <asm/machine.h>
+#include <asm/sigp.h>
+
+static __always_inline unsigned int raw_smp_processor_id(void)
+{
+	unsigned long lc_cpu_nr;
+	unsigned int cpu;
+
+	BUILD_BUG_ON(sizeof_field(struct lowcore, cpu_nr) != sizeof(cpu));
+	lc_cpu_nr = offsetof(struct lowcore, cpu_nr);
+	asm_inline(
+		ALTERNATIVE("   ly      %[cpu],%[offzero](%%r0)\n",
+			    "   ly      %[cpu],%[offalt](%%r0)\n",
+			    ALT_FEATURE(MFEATURE_LOWCORE))
+		: [cpu] "=d" (cpu)
+		: [offzero] "i" (lc_cpu_nr),
+		  [offalt] "i" (lc_cpu_nr + LOWCORE_ALT_ADDRESS),
+		  "m" (((struct lowcore *)0)->cpu_nr));
+	return cpu;
+}
 
-#define raw_smp_processor_id()	(get_lowcore()->cpu_nr)
 #define arch_scale_cpu_capacity smp_cpu_get_capacity
 
 extern struct mutex smp_cpu_state_mutex;
diff --git a/arch/s390/include/asm/spinlock.h b/arch/s390/include/asm/spinlock.h
index f87dd0a84855..f9935db9fd76 100644
--- a/arch/s390/include/asm/spinlock.h
+++ b/arch/s390/include/asm/spinlock.h
@@ -16,7 +16,23 @@
 #include <asm/processor.h>
 #include <asm/alternative.h>
 
-#define SPINLOCK_LOCKVAL (get_lowcore()->spinlock_lockval)
+static __always_inline unsigned int spinlock_lockval(void)
+{
+	unsigned long lc_lockval;
+	unsigned int lockval;
+
+	BUILD_BUG_ON(sizeof_field(struct lowcore, spinlock_lockval) != sizeof(lockval));
+	lc_lockval = offsetof(struct lowcore, spinlock_lockval);
+	asm_inline(
+		ALTERNATIVE("   ly      %[lockval],%[offzero](%%r0)\n",
+			    "   ly      %[lockval],%[offalt](%%r0)\n",
+			    ALT_FEATURE(MFEATURE_LOWCORE))
+		: [lockval] "=d" (lockval)
+		: [offzero] "i" (lc_lockval),
+		  [offalt] "i" (lc_lockval + LOWCORE_ALT_ADDRESS),
+		  "m" (((struct lowcore *)0)->spinlock_lockval));
+	return lockval;
+}
 
 extern int spin_retry;
 
@@ -60,7 +76,7 @@ static inline int arch_spin_trylock_once(arch_spinlock_t *lp)
 	int old = 0;
 
 	barrier();
-	return likely(arch_try_cmpxchg(&lp->lock, &old, SPINLOCK_LOCKVAL));
+	return likely(arch_try_cmpxchg(&lp->lock, &old, spinlock_lockval()));
 }
 
 static inline void arch_spin_lock(arch_spinlock_t *lp)
diff --git a/arch/s390/include/asm/syscall.h b/arch/s390/include/asm/syscall.h
index 27e3d804b311..0213ec800b57 100644
--- a/arch/s390/include/asm/syscall.h
+++ b/arch/s390/include/asm/syscall.h
@@ -65,15 +65,13 @@ static inline void syscall_get_arguments(struct task_struct *task,
 					 unsigned long *args)
 {
 	unsigned long mask = -1UL;
-	unsigned int n = 6;
 
 #ifdef CONFIG_COMPAT
 	if (test_tsk_thread_flag(task, TIF_31BIT))
 		mask = 0xffffffff;
 #endif
-	while (n-- > 0)
-		if (n > 0)
-			args[n] = regs->gprs[2 + n] & mask;
+	for (int i = 1; i < 6; i++)
+		args[i] = regs->gprs[2 + i] & mask;
 
 	args[0] = regs->orig_gpr2 & mask;
 }
diff --git a/arch/s390/include/asm/sysinfo.h b/arch/s390/include/asm/sysinfo.h
index edca5a751df4..9088c5267f35 100644
--- a/arch/s390/include/asm/sysinfo.h
+++ b/arch/s390/include/asm/sysinfo.h
@@ -11,8 +11,34 @@
 #ifndef __ASM_S390_SYSINFO_H
 #define __ASM_S390_SYSINFO_H
 
-#include <asm/bitsperlong.h>
 #include <linux/uuid.h>
+#include <asm/bitsperlong.h>
+#include <asm/asm.h>
+
+/*
+ * stsi - store system information
+ *
+ * Returns the current configuration level if function code 0 was specified.
+ * Otherwise returns 0 on success or a negative value on error.
+ */
+static inline int stsi(void *sysinfo, int fc, int sel1, int sel2)
+{
+	int r0 = (fc << 28) | sel1;
+	int cc;
+
+	asm volatile(
+		"	lr	%%r0,%[r0]\n"
+		"	lr	%%r1,%[r1]\n"
+		"	stsi	%[sysinfo]\n"
+		"	lr	%[r0],%%r0\n"
+		CC_IPM(cc)
+		: CC_OUT(cc, cc), [r0] "+d" (r0), [sysinfo] "=Q" (*(char *)sysinfo)
+		: [r1] "d" (sel2)
+		: CC_CLOBBER_LIST("0", "1", "memory"));
+	if (cc == 3)
+		return -EOPNOTSUPP;
+	return fc ? 0 : (unsigned int)r0 >> 28;
+}
 
 struct sysinfo_1_1_1 {
 	unsigned char p:1;
diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h
index c33f7144d1b9..1ad5e82c2f65 100644
--- a/arch/s390/include/asm/thread_info.h
+++ b/arch/s390/include/asm/thread_info.h
@@ -67,7 +67,6 @@ void arch_setup_new_exec(void);
 #define TIF_NEED_RESCHED_LAZY	3	/* lazy rescheduling needed */
 #define TIF_UPROBE		4	/* breakpointed or single-stepping */
 #define TIF_PATCH_PENDING	5	/* pending live patching update */
-#define TIF_PGSTE		6	/* New mm's will use 4K page tables */
 #define TIF_NOTIFY_SIGNAL	7	/* signal notifications exist */
 #define TIF_GUARDED_STORAGE	8	/* load guarded storage control block */
 #define TIF_ISOLATE_BP_GUEST	9	/* Run KVM guests with isolated BP */
@@ -89,7 +88,6 @@ void arch_setup_new_exec(void);
 #define _TIF_NEED_RESCHED_LAZY	BIT(TIF_NEED_RESCHED_LAZY)
 #define _TIF_UPROBE		BIT(TIF_UPROBE)
 #define _TIF_PATCH_PENDING	BIT(TIF_PATCH_PENDING)
-#define _TIF_PGSTE		BIT(TIF_PGSTE)
 #define _TIF_NOTIFY_SIGNAL	BIT(TIF_NOTIFY_SIGNAL)
 #define _TIF_GUARDED_STORAGE	BIT(TIF_GUARDED_STORAGE)
 #define _TIF_ISOLATE_BP_GUEST	BIT(TIF_ISOLATE_BP_GUEST)
diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h
index a9460bd6555b..bed8d0b5a282 100644
--- a/arch/s390/include/asm/timex.h
+++ b/arch/s390/include/asm/timex.h
@@ -13,6 +13,7 @@
 #include <linux/preempt.h>
 #include <linux/time64.h>
 #include <asm/lowcore.h>
+#include <asm/machine.h>
 #include <asm/asm.h>
 
 /* The value of the TOD clock for 1.1.1970. */
@@ -267,7 +268,7 @@ static __always_inline u128 eitod_to_ns(u128 todval)
  */
 static inline int tod_after(unsigned long a, unsigned long b)
 {
-	if (MACHINE_HAS_SCC)
+	if (machine_has_scc())
 		return (long) a > (long) b;
 	return a > b;
 }
@@ -281,7 +282,7 @@ static inline int tod_after(unsigned long a, unsigned long b)
  */
 static inline int tod_after_eq(unsigned long a, unsigned long b)
 {
-	if (MACHINE_HAS_SCC)
+	if (machine_has_scc())
 		return (long) a >= (long) b;
 	return a >= b;
 }
diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h
index 72655fd2d867..f20601995bb0 100644
--- a/arch/s390/include/asm/tlb.h
+++ b/arch/s390/include/asm/tlb.h
@@ -84,7 +84,7 @@ static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
 	tlb->mm->context.flush_mm = 1;
 	tlb->freed_tables = 1;
 	tlb->cleared_pmds = 1;
-	if (mm_alloc_pgste(tlb->mm))
+	if (mm_has_pgste(tlb->mm))
 		gmap_unlink(tlb->mm, (unsigned long *)pte, address);
 	tlb_remove_ptdesc(tlb, virt_to_ptdesc(pte));
 }
diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h
index 9dfd46dd03c6..75491baa2197 100644
--- a/arch/s390/include/asm/tlbflush.h
+++ b/arch/s390/include/asm/tlbflush.h
@@ -2,9 +2,11 @@
 #ifndef _S390_TLBFLUSH_H
 #define _S390_TLBFLUSH_H
 
+#include <linux/cpufeature.h>
 #include <linux/mm.h>
 #include <linux/sched.h>
 #include <asm/processor.h>
+#include <asm/machine.h>
 
 /*
  * Flush all TLB entries on the local CPU.
@@ -22,7 +24,7 @@ static inline void __tlb_flush_idte(unsigned long asce)
 	unsigned long opt;
 
 	opt = IDTE_PTOA;
-	if (MACHINE_HAS_TLB_GUEST)
+	if (machine_has_tlb_guest())
 		opt |= IDTE_GUEST_ASCE;
 	/* Global TLB flush for the mm */
 	asm volatile("idte 0,%1,%0" : : "a" (opt), "a" (asce) : "cc");
@@ -52,7 +54,7 @@ static inline void __tlb_flush_mm(struct mm_struct *mm)
 	cpumask_copy(mm_cpumask(mm), &mm->context.cpu_attach_mask);
 	barrier();
 	gmap_asce = READ_ONCE(mm->context.gmap_asce);
-	if (MACHINE_HAS_IDTE && gmap_asce != -1UL) {
+	if (cpu_has_idte() && gmap_asce != -1UL) {
 		if (gmap_asce)
 			__tlb_flush_idte(gmap_asce);
 		__tlb_flush_idte(mm->context.asce);
@@ -66,7 +68,7 @@ static inline void __tlb_flush_mm(struct mm_struct *mm)
 
 static inline void __tlb_flush_kernel(void)
 {
-	if (MACHINE_HAS_IDTE)
+	if (cpu_has_idte())
 		__tlb_flush_idte(init_mm.context.asce);
 	else
 		__tlb_flush_global();
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index f5920163ee97..8629d70ec38b 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -13,6 +13,7 @@
 /*
  * User space memory access functions
  */
+#include <linux/pgtable.h>
 #include <asm/asm-extable.h>
 #include <asm/processor.h>
 #include <asm/extable.h>
@@ -22,116 +23,69 @@
 
 void debug_user_asce(int exit);
 
-union oac {
-	unsigned int val;
-	struct {
-		struct {
-			unsigned short key : 4;
-			unsigned short	   : 4;
-			unsigned short as  : 2;
-			unsigned short	   : 4;
-			unsigned short k   : 1;
-			unsigned short a   : 1;
-		} oac1;
-		struct {
-			unsigned short key : 4;
-			unsigned short	   : 4;
-			unsigned short as  : 2;
-			unsigned short	   : 4;
-			unsigned short k   : 1;
-			unsigned short a   : 1;
-		} oac2;
-	};
-};
-
-static __always_inline __must_check unsigned long
-raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key)
-{
-	unsigned long rem;
-	union oac spec = {
-		.oac2.key = key,
-		.oac2.as = PSW_BITS_AS_SECONDARY,
-		.oac2.k = 1,
-		.oac2.a = 1,
-	};
-
-	asm_inline volatile(
-		"	lr	%%r0,%[spec]\n"
-		"0:	mvcos	0(%[to]),0(%[from]),%[size]\n"
-		"1:	jz	5f\n"
-		"	algr	%[size],%[val]\n"
-		"	slgr	%[from],%[val]\n"
-		"	slgr	%[to],%[val]\n"
-		"	j	0b\n"
-		"2:	la	%[rem],4095(%[from])\n" /* rem = from + 4095 */
-		"	nr	%[rem],%[val]\n"	/* rem = (from + 4095) & -4096 */
-		"	slgr	%[rem],%[from]\n"
-		"	clgr	%[size],%[rem]\n"	/* copy crosses next page boundary? */
-		"	jnh	6f\n"
-		"3:	mvcos	0(%[to]),0(%[from]),%[rem]\n"
-		"4:	slgr	%[size],%[rem]\n"
-		"	j	6f\n"
-		"5:	lghi	%[size],0\n"
-		"6:\n"
-		EX_TABLE(0b, 2b)
-		EX_TABLE(1b, 2b)
-		EX_TABLE(3b, 6b)
-		EX_TABLE(4b, 6b)
-		: [size] "+&a" (size), [from] "+&a" (from), [to] "+&a" (to), [rem] "=&a" (rem)
-		: [val] "a" (-4096UL), [spec] "d" (spec.val)
-		: "cc", "memory", "0");
-	return size;
-}
+#ifdef CONFIG_KMSAN
+#define uaccess_kmsan_or_inline noinline __maybe_unused __no_sanitize_memory
+#else
+#define uaccess_kmsan_or_inline __always_inline
+#endif
 
-static __always_inline __must_check unsigned long
-raw_copy_from_user(void *to, const void __user *from, unsigned long n)
-{
-	return raw_copy_from_user_key(to, from, n, 0);
-}
+#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_TO_USER
 
-static __always_inline __must_check unsigned long
-raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key)
+static uaccess_kmsan_or_inline __must_check unsigned long
+raw_copy_from_user(void *to, const void __user *from, unsigned long size)
 {
-	unsigned long rem;
-	union oac spec = {
-		.oac1.key = key,
-		.oac1.as = PSW_BITS_AS_SECONDARY,
-		.oac1.k = 1,
-		.oac1.a = 1,
-	};
-
-	asm_inline volatile(
-		"	lr	%%r0,%[spec]\n"
-		"0:	mvcos	0(%[to]),0(%[from]),%[size]\n"
-		"1:	jz	5f\n"
-		"	algr	%[size],%[val]\n"
-		"	slgr	%[to],%[val]\n"
-		"	slgr	%[from],%[val]\n"
-		"	j	0b\n"
-		"2:	la	%[rem],4095(%[to])\n"	/* rem = to + 4095 */
-		"	nr	%[rem],%[val]\n"	/* rem = (to + 4095) & -4096 */
-		"	slgr	%[rem],%[to]\n"
-		"	clgr	%[size],%[rem]\n"	/* copy crosses next page boundary? */
-		"	jnh	6f\n"
-		"3:	mvcos	0(%[to]),0(%[from]),%[rem]\n"
-		"4:	slgr	%[size],%[rem]\n"
-		"	j	6f\n"
-		"5:	lghi	%[size],0\n"
-		"6:\n"
-		EX_TABLE(0b, 2b)
-		EX_TABLE(1b, 2b)
-		EX_TABLE(3b, 6b)
-		EX_TABLE(4b, 6b)
-		: [size] "+&a" (size), [to] "+&a" (to), [from] "+&a" (from), [rem] "=&a" (rem)
-		: [val] "a" (-4096UL), [spec] "d" (spec.val)
-		: "cc", "memory", "0");
-	return size;
+	unsigned long osize;
+	int cc;
+
+	while (1) {
+		osize = size;
+		asm_inline volatile(
+			"	lhi	%%r0,%[spec]\n"
+			"0:	mvcos	%[to],%[from],%[size]\n"
+			"1:	nopr	%%r7\n"
+			CC_IPM(cc)
+			EX_TABLE_UA_MVCOS_FROM(0b, 0b)
+			EX_TABLE_UA_MVCOS_FROM(1b, 0b)
+			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to)
+			: [spec] "I" (0x81), [from] "Q" (*(const char __user *)from)
+			: CC_CLOBBER_LIST("memory", "0"));
+		if (__builtin_constant_p(osize) && osize <= 4096)
+			return osize - size;
+		if (likely(CC_TRANSFORM(cc) == 0))
+			return osize - size;
+		size -= 4096;
+		to += 4096;
+		from += 4096;
+	}
 }
 
-static __always_inline __must_check unsigned long
-raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+static uaccess_kmsan_or_inline __must_check unsigned long
+raw_copy_to_user(void __user *to, const void *from, unsigned long size)
 {
-	return raw_copy_to_user_key(to, from, n, 0);
+	unsigned long osize;
+	int cc;
+
+	while (1) {
+		osize = size;
+		asm_inline volatile(
+			"	llilh	%%r0,%[spec]\n"
+			"0:	mvcos	%[to],%[from],%[size]\n"
+			"1:	nopr	%%r7\n"
+			CC_IPM(cc)
+			EX_TABLE_UA_MVCOS_TO(0b, 0b)
+			EX_TABLE_UA_MVCOS_TO(1b, 0b)
+			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
+			: [spec] "I" (0x81), [from] "Q" (*(const char *)from)
+			: CC_CLOBBER_LIST("memory", "0"));
+		if (__builtin_constant_p(osize) && osize <= 4096)
+			return osize - size;
+		if (likely(CC_TRANSFORM(cc) == 0))
+			return osize - size;
+		size -= 4096;
+		to += 4096;
+		from += 4096;
+	}
 }
 
 unsigned long __must_check
@@ -158,12 +112,6 @@ copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned lo
 
 int __noreturn __put_user_bad(void);
 
-#ifdef CONFIG_KMSAN
-#define uaccess_kmsan_or_inline noinline __maybe_unused __no_sanitize_memory
-#else
-#define uaccess_kmsan_or_inline __always_inline
-#endif
-
 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 
 #define DEFINE_PUT_USER_NOINSTR(type)					\
@@ -199,7 +147,7 @@ __put_user_##type##_noinstr(unsigned type __user *to,			\
 {									\
 	int rc;								\
 									\
-	asm volatile(							\
+	asm_inline volatile(						\
 		"	llilh	%%r0,%[spec]\n"				\
 		"0:	mvcos	%[to],%[from],%[size]\n"		\
 		"1:	lhi	%[rc],0\n"				\
@@ -315,7 +263,7 @@ __get_user_##type##_noinstr(unsigned type *to,				\
 {									\
 	int rc;								\
 									\
-	asm volatile(							\
+	asm_inline volatile(						\
 		"	lhi	%%r0,%[spec]\n"				\
 		"0:	mvcos	%[to],%[from],%[size]\n"		\
 		"1:	lhi	%[rc],0\n"				\
@@ -415,12 +363,34 @@ long __must_check strncpy_from_user(char *dst, const char __user *src, long coun
 
 long __must_check strnlen_user(const char __user *src, long count);
 
-/*
- * Zero Userspace
- */
-unsigned long __must_check __clear_user(void __user *to, unsigned long size);
+static uaccess_kmsan_or_inline __must_check unsigned long
+__clear_user(void __user *to, unsigned long size)
+{
+	unsigned long osize;
+	int cc;
+
+	while (1) {
+		osize = size;
+		asm_inline volatile(
+			"	llilh	%%r0,%[spec]\n"
+			"0:	mvcos	%[to],%[from],%[size]\n"
+			"1:	nopr	%%r7\n"
+			CC_IPM(cc)
+			EX_TABLE_UA_MVCOS_TO(0b, 0b)
+			EX_TABLE_UA_MVCOS_TO(1b, 0b)
+			: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
+			: [spec] "I" (0x81), [from] "Q" (*(const char *)empty_zero_page)
+			: CC_CLOBBER_LIST("memory", "0"));
+		if (__builtin_constant_p(osize) && osize <= 4096)
+			return osize - size;
+		if (CC_TRANSFORM(cc) == 0)
+			return osize - size;
+		size -= 4096;
+		to += 4096;
+	}
+}
 
-static inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
+static __always_inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
 {
 	might_fault();
 	return __clear_user(to, n);
@@ -520,7 +490,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
 		_old = ((unsigned int)old & 0xff) << shift;
 		_new = ((unsigned int)new & 0xff) << shift;
 		mask = ~(0xff << shift);
-		asm volatile(
+		asm_inline volatile(
 			"	spka	0(%[key])\n"
 			"	sacf	256\n"
 			"	llill	%[count],%[max_loops]\n"
@@ -568,7 +538,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
 		_old = ((unsigned int)old & 0xffff) << shift;
 		_new = ((unsigned int)new & 0xffff) << shift;
 		mask = ~(0xffff << shift);
-		asm volatile(
+		asm_inline volatile(
 			"	spka	0(%[key])\n"
 			"	sacf	256\n"
 			"	llill	%[count],%[max_loops]\n"
@@ -610,7 +580,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
 	case 4:	{
 		unsigned int prev = old;
 
-		asm volatile(
+		asm_inline volatile(
 			"	spka	0(%[key])\n"
 			"	sacf	256\n"
 			"0:	cs	%[prev],%[new],%[address]\n"
@@ -631,7 +601,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
 	case 8: {
 		unsigned long prev = old;
 
-		asm volatile(
+		asm_inline volatile(
 			"	spka	0(%[key])\n"
 			"	sacf	256\n"
 			"0:	csg	%[prev],%[new],%[address]\n"
@@ -652,7 +622,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
 	case 16: {
 		__uint128_t prev = old;
 
-		asm volatile(
+		asm_inline volatile(
 			"	spka	0(%[key])\n"
 			"	sacf	256\n"
 			"0:	cdsg	%[prev],%[new],%[address]\n"
diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h
index 203acd6e431b..eaa19dee7699 100644
--- a/arch/s390/include/asm/word-at-a-time.h
+++ b/arch/s390/include/asm/word-at-a-time.h
@@ -52,7 +52,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr)
 {
 	unsigned long data;
 
-	asm volatile(
+	asm_inline volatile(
 		"0:	lg	%[data],0(%[addr])\n"
 		"1:	nopr	%%r7\n"
 		EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr])