diff options
Diffstat (limited to 'arch/s390')
101 files changed, 1894 insertions, 907 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index c6722112527d..a2dcef0aacc7 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -62,9 +62,6 @@ config PCI_QUIRKS config ARCH_SUPPORTS_UPROBES def_bool y -config DEBUG_RODATA - def_bool y - config S390 def_bool y select ARCH_HAS_DEVMEM_IS_ALLOWED @@ -72,7 +69,10 @@ config S390 select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_GIGANTIC_PAGE select ARCH_HAS_KCOV + select ARCH_HAS_SET_MEMORY select ARCH_HAS_SG_CHAIN + select ARCH_HAS_STRICT_KERNEL_RWX + select ARCH_HAS_STRICT_MODULE_RWX select ARCH_HAS_UBSAN_SANITIZE_ALL select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_INLINE_READ_LOCK @@ -134,9 +134,11 @@ config S390 select HAVE_EBPF_JIT if PACK_STACK && HAVE_MARCH_Z196_FEATURES select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL + select HAVE_COPY_THREAD_TLS select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_API_DEBUG select HAVE_DMA_CONTIGUOUS + select DMA_NOOP_OPS select HAVE_DYNAMIC_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS select HAVE_EFFICIENT_UNALIGNED_ACCESS diff --git a/arch/s390/Kconfig.debug b/arch/s390/Kconfig.debug index 26c5d5beb4be..ba5f878a295c 100644 --- a/arch/s390/Kconfig.debug +++ b/arch/s390/Kconfig.debug @@ -17,7 +17,7 @@ config S390_PTDUMP kernel. If in doubt, say "N" -config DEBUG_SET_MODULE_RONX +config EARLY_PRINTK def_bool y - depends on MODULES + endmenu diff --git a/arch/s390/appldata/appldata_os.c b/arch/s390/appldata/appldata_os.c index 69b23b25ac34..08b9e942a262 100644 --- a/arch/s390/appldata/appldata_os.c +++ b/arch/s390/appldata/appldata_os.c @@ -113,21 +113,21 @@ static void appldata_get_os_data(void *data) j = 0; for_each_online_cpu(i) { os_data->os_cpu[j].per_cpu_user = - cputime_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_USER]); + nsecs_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_USER]); os_data->os_cpu[j].per_cpu_nice = - cputime_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_NICE]); + nsecs_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_NICE]); os_data->os_cpu[j].per_cpu_system = - cputime_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM]); + nsecs_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM]); os_data->os_cpu[j].per_cpu_idle = - cputime_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_IDLE]); + nsecs_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_IDLE]); os_data->os_cpu[j].per_cpu_irq = - cputime_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_IRQ]); + nsecs_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_IRQ]); os_data->os_cpu[j].per_cpu_softirq = - cputime_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ]); + nsecs_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ]); os_data->os_cpu[j].per_cpu_iowait = - cputime_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_IOWAIT]); + nsecs_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_IOWAIT]); os_data->os_cpu[j].per_cpu_steal = - cputime_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_STEAL]); + nsecs_to_jiffies(kcpustat_cpu(i).cpustat[CPUTIME_STEAL]); os_data->os_cpu[j].cpu_id = i; j++; } diff --git a/arch/s390/boot/compressed/Makefile b/arch/s390/boot/compressed/Makefile index 6bd2c9022be3..f7e4c834ea24 100644 --- a/arch/s390/boot/compressed/Makefile +++ b/arch/s390/boot/compressed/Makefile @@ -19,7 +19,8 @@ KBUILD_CFLAGS += $(call cc-option,-ffreestanding) GCOV_PROFILE := n UBSAN_SANITIZE := n -OBJECTS := $(addprefix $(objtree)/arch/s390/kernel/, head.o sclp.o ebcdic.o als.o) +OBJECTS := $(addprefix $(objtree)/arch/s390/kernel/, head.o ebcdic.o als.o) +OBJECTS += $(objtree)/drivers/s390/char/sclp_early_core.o OBJECTS += $(obj)/head.o $(obj)/misc.o $(obj)/piggy.o LDFLAGS_vmlinux := --oformat $(LD_BFD) -e startup -T diff --git a/arch/s390/boot/compressed/misc.c b/arch/s390/boot/compressed/misc.c index 8515dd5a5663..fa95041fa9f6 100644 --- a/arch/s390/boot/compressed/misc.c +++ b/arch/s390/boot/compressed/misc.c @@ -66,7 +66,7 @@ static unsigned long free_mem_end_ptr; static int puts(const char *s) { - _sclp_print_early(s); + sclp_early_printk(s); return 0; } diff --git a/arch/s390/configs/default_defconfig b/arch/s390/configs/default_defconfig index e00975361fec..143b1e00b818 100644 --- a/arch/s390/configs/default_defconfig +++ b/arch/s390/configs/default_defconfig @@ -678,6 +678,7 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_ZCRYPT=m +CONFIG_PKEY=m CONFIG_CRYPTO_SHA1_S390=m CONFIG_CRYPTO_SHA256_S390=m CONFIG_CRYPTO_SHA512_S390=m diff --git a/arch/s390/configs/performance_defconfig b/arch/s390/configs/performance_defconfig index 2cf87343b590..2358bf33c5ef 100644 --- a/arch/s390/configs/performance_defconfig +++ b/arch/s390/configs/performance_defconfig @@ -628,6 +628,7 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_ZCRYPT=m +CONFIG_PKEY=m CONFIG_CRYPTO_SHA1_S390=m CONFIG_CRYPTO_SHA256_S390=m CONFIG_CRYPTO_SHA512_S390=m diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index d1033de4c4ee..402c530c6da5 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -6,7 +6,7 @@ obj-$(CONFIG_CRYPTO_SHA1_S390) += sha1_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA256_S390) += sha256_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA512_S390) += sha512_s390.o sha_common.o obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o -obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o +obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o paes_s390.o obj-$(CONFIG_S390_PRNG) += prng.o obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o obj-$(CONFIG_CRYPTO_CRC32_S390) += crc32-vx_s390.o diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 303d28eb03a2..591cbdf615af 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -28,6 +28,7 @@ #include <linux/cpufeature.h> #include <linux/init.h> #include <linux/spinlock.h> +#include <linux/fips.h> #include <crypto/xts.h> #include <asm/cpacf.h> @@ -501,6 +502,12 @@ static int xts_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, if (err) return err; + /* In fips mode only 128 bit or 256 bit keys are valid */ + if (fips_enabled && key_len != 32 && key_len != 64) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + /* Pick the correct function code based on the key length */ fc = (key_len == 32) ? CPACF_KM_XTS_128 : (key_len == 64) ? CPACF_KM_XTS_256 : 0; diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c index 8b83144206eb..0d296662bbf0 100644 --- a/arch/s390/crypto/des_s390.c +++ b/arch/s390/crypto/des_s390.c @@ -18,6 +18,7 @@ #include <linux/module.h> #include <linux/cpufeature.h> #include <linux/crypto.h> +#include <linux/fips.h> #include <crypto/algapi.h> #include <crypto/des.h> #include <asm/cpacf.h> @@ -221,6 +222,8 @@ static struct crypto_alg cbc_des_alg = { * same as DES. Implementers MUST reject keys that exhibit this * property. * + * In fips mode additinally check for all 3 keys are unique. + * */ static int des3_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key_len) @@ -234,6 +237,17 @@ static int des3_setkey(struct crypto_tfm *tfm, const u8 *key, tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } + + /* in fips mode, ensure k1 != k2 and k2 != k3 and k1 != k3 */ + if (fips_enabled && + !(crypto_memneq(key, &key[DES_KEY_SIZE], DES_KEY_SIZE) && + crypto_memneq(&key[DES_KEY_SIZE], &key[DES_KEY_SIZE * 2], + DES_KEY_SIZE) && + crypto_memneq(key, &key[DES_KEY_SIZE * 2], DES_KEY_SIZE))) { + tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; + return -EINVAL; + } + memcpy(ctx->key, key, key_len); return 0; } diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c new file mode 100644 index 000000000000..d69ea495c4d7 --- /dev/null +++ b/arch/s390/crypto/paes_s390.c @@ -0,0 +1,619 @@ +/* + * Cryptographic API. + * + * s390 implementation of the AES Cipher Algorithm with protected keys. + * + * s390 Version: + * Copyright IBM Corp. 2017 + * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> + * Harald Freudenberger <freude@de.ibm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License (version 2 only) + * as published by the Free Software Foundation. + * + */ + +#define KMSG_COMPONENT "paes_s390" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <linux/bug.h> +#include <linux/err.h> +#include <linux/module.h> +#include <linux/cpufeature.h> +#include <linux/init.h> +#include <linux/spinlock.h> +#include <crypto/xts.h> +#include <asm/cpacf.h> +#include <asm/pkey.h> + +static u8 *ctrblk; +static DEFINE_SPINLOCK(ctrblk_lock); + +static cpacf_mask_t km_functions, kmc_functions, kmctr_functions; + +struct s390_paes_ctx { + struct pkey_seckey sk; + struct pkey_protkey pk; + unsigned long fc; +}; + +struct s390_pxts_ctx { + struct pkey_seckey sk[2]; + struct pkey_protkey pk[2]; + unsigned long fc; +}; + +static inline int __paes_convert_key(struct pkey_seckey *sk, + struct pkey_protkey *pk) +{ + int i, ret; + + /* try three times in case of failure */ + for (i = 0; i < 3; i++) { + ret = pkey_skey2pkey(sk, pk); + if (ret == 0) + break; + } + + return ret; +} + +static int __paes_set_key(struct s390_paes_ctx *ctx) +{ + unsigned long fc; + + if (__paes_convert_key(&ctx->sk, &ctx->pk)) + return -EINVAL; + + /* Pick the correct function code based on the protected key type */ + fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PAES_128 : + (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KM_PAES_192 : + (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KM_PAES_256 : 0; + + /* Check if the function code is available */ + ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; + + return ctx->fc ? 0 : -EINVAL; +} + +static int ecb_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); + + if (key_len != SECKEYBLOBSIZE) + return -EINVAL; + + memcpy(ctx->sk.seckey, in_key, SECKEYBLOBSIZE); + if (__paes_set_key(ctx)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + return 0; +} + +static int ecb_paes_crypt(struct blkcipher_desc *desc, + unsigned long modifier, + struct blkcipher_walk *walk) +{ + struct s390_paes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int nbytes, n, k; + int ret; + + ret = blkcipher_walk_virt(desc, walk); + while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { + /* only use complete blocks */ + n = nbytes & ~(AES_BLOCK_SIZE - 1); + k = cpacf_km(ctx->fc | modifier, ctx->pk.protkey, + walk->dst.virt.addr, walk->src.virt.addr, n); + if (k) + ret = blkcipher_walk_done(desc, walk, nbytes - k); + if (k < n) { + if (__paes_set_key(ctx) != 0) + return blkcipher_walk_done(desc, walk, -EIO); + } + } + return ret; +} + +static int ecb_paes_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ecb_paes_crypt(desc, CPACF_ENCRYPT, &walk); +} + +static int ecb_paes_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ecb_paes_crypt(desc, CPACF_DECRYPT, &walk); +} + +static struct crypto_alg ecb_paes_alg = { + .cra_name = "ecb(paes)", + .cra_driver_name = "ecb-paes-s390", + .cra_priority = 400, /* combo: aes + ecb */ + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s390_paes_ctx), + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(ecb_paes_alg.cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SECKEYBLOBSIZE, + .max_keysize = SECKEYBLOBSIZE, + .setkey = ecb_paes_set_key, + .encrypt = ecb_paes_encrypt, + .decrypt = ecb_paes_decrypt, + } + } +}; + +static int __cbc_paes_set_key(struct s390_paes_ctx *ctx) +{ + unsigned long fc; + + if (__paes_convert_key(&ctx->sk, &ctx->pk)) + return -EINVAL; + + /* Pick the correct function code based on the protected key type */ + fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMC_PAES_128 : + (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMC_PAES_192 : + (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? CPACF_KMC_PAES_256 : 0; + + /* Check if the function code is available */ + ctx->fc = (fc && cpacf_test_func(&kmc_functions, fc)) ? fc : 0; + + return ctx->fc ? 0 : -EINVAL; +} + +static int cbc_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); + + memcpy(ctx->sk.seckey, in_key, SECKEYBLOBSIZE); + if (__cbc_paes_set_key(ctx)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + return 0; +} + +static int cbc_paes_crypt(struct blkcipher_desc *desc, unsigned long modifier, + struct blkcipher_walk *walk) +{ + struct s390_paes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int nbytes, n, k; + int ret; + struct { + u8 iv[AES_BLOCK_SIZE]; + u8 key[MAXPROTKEYSIZE]; + } param; + + ret = blkcipher_walk_virt(desc, walk); + memcpy(param.iv, walk->iv, AES_BLOCK_SIZE); + memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); + while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { + /* only use complete blocks */ + n = nbytes & ~(AES_BLOCK_SIZE - 1); + k = cpacf_kmc(ctx->fc | modifier, ¶m, + walk->dst.virt.addr, walk->src.virt.addr, n); + if (k) + ret = blkcipher_walk_done(desc, walk, nbytes - k); + if (n < k) { + if (__cbc_paes_set_key(ctx) != 0) + return blkcipher_walk_done(desc, walk, -EIO); + memcpy(param.key, ctx->pk.protkey, MAXPROTKEYSIZE); + } + } + memcpy(walk->iv, param.iv, AES_BLOCK_SIZE); + return ret; +} + +static int cbc_paes_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return cbc_paes_crypt(desc, 0, &walk); +} + +static int cbc_paes_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return cbc_paes_crypt(desc, CPACF_DECRYPT, &walk); +} + +static struct crypto_alg cbc_paes_alg = { + .cra_name = "cbc(paes)", + .cra_driver_name = "cbc-paes-s390", + .cra_priority = 400, /* combo: aes + cbc */ + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s390_paes_ctx), + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(cbc_paes_alg.cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SECKEYBLOBSIZE, + .max_keysize = SECKEYBLOBSIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = cbc_paes_set_key, + .encrypt = cbc_paes_encrypt, + .decrypt = cbc_paes_decrypt, + } + } +}; + +static int __xts_paes_set_key(struct s390_pxts_ctx *ctx) +{ + unsigned long fc; + + if (__paes_convert_key(&ctx->sk[0], &ctx->pk[0]) || + __paes_convert_key(&ctx->sk[1], &ctx->pk[1])) + return -EINVAL; + + if (ctx->pk[0].type != ctx->pk[1].type) + return -EINVAL; + + /* Pick the correct function code based on the protected key type */ + fc = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? CPACF_KM_PXTS_128 : + (ctx->pk[0].type == PKEY_KEYTYPE_AES_256) ? + CPACF_KM_PXTS_256 : 0; + + /* Check if the function code is available */ + ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0; + + return ctx->fc ? 0 : -EINVAL; +} + +static int xts_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct s390_pxts_ctx *ctx = crypto_tfm_ctx(tfm); + u8 ckey[2 * AES_MAX_KEY_SIZE]; + unsigned int ckey_len; + + memcpy(ctx->sk[0].seckey, in_key, SECKEYBLOBSIZE); + memcpy(ctx->sk[1].seckey, in_key + SECKEYBLOBSIZE, SECKEYBLOBSIZE); + if (__xts_paes_set_key(ctx)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + /* + * xts_check_key verifies the key length is not odd and makes + * sure that the two keys are not the same. This can be done + * on the two protected keys as well + */ + ckey_len = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? + AES_KEYSIZE_128 : AES_KEYSIZE_256; + memcpy(ckey, ctx->pk[0].protkey, ckey_len); + memcpy(ckey + ckey_len, ctx->pk[1].protkey, ckey_len); + return xts_check_key(tfm, ckey, 2*ckey_len); +} + +static int xts_paes_crypt(struct blkcipher_desc *desc, unsigned long modifier, + struct blkcipher_walk *walk) +{ + struct s390_pxts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int keylen, offset, nbytes, n, k; + int ret; + struct { + u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */ + u8 tweak[16]; + u8 block[16]; + u8 bit[16]; + u8 xts[16]; + } pcc_param; + struct { + u8 key[MAXPROTKEYSIZE]; /* key + verification pattern */ + u8 init[16]; + } xts_param; + + ret = blkcipher_walk_virt(desc, walk); + keylen = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 48 : 64; + offset = (ctx->pk[0].type == PKEY_KEYTYPE_AES_128) ? 16 : 0; +retry: + memset(&pcc_param, 0, sizeof(pcc_param)); + memcpy(pcc_param.tweak, walk->iv, sizeof(pcc_param.tweak)); + memcpy(pcc_param.key + offset, ctx->pk[1].protkey, keylen); + cpacf_pcc(ctx->fc, pcc_param.key + offset); + + memcpy(xts_param.key + offset, ctx->pk[0].protkey, keylen); + memcpy(xts_param.init, pcc_param.xts, 16); + + while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { + /* only use complete blocks */ + n = nbytes & ~(AES_BLOCK_SIZE - 1); + k = cpacf_km(ctx->fc | modifier, xts_param.key + offset, + walk->dst.virt.addr, walk->src.virt.addr, n); + if (k) + ret = blkcipher_walk_done(desc, walk, nbytes - k); + if (k < n) { + if (__xts_paes_set_key(ctx) != 0) + return blkcipher_walk_done(desc, walk, -EIO); + goto retry; + } + } + return ret; +} + +static int xts_paes_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return xts_paes_crypt(desc, 0, &walk); +} + +static int xts_paes_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return xts_paes_crypt(desc, CPACF_DECRYPT, &walk); +} + +static struct crypto_alg xts_paes_alg = { + .cra_name = "xts(paes)", + .cra_driver_name = "xts-paes-s390", + .cra_priority = 400, /* combo: aes + xts */ + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s390_pxts_ctx), + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(xts_paes_alg.cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = 2 * SECKEYBLOBSIZE, + .max_keysize = 2 * SECKEYBLOBSIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = xts_paes_set_key, + .encrypt = xts_paes_encrypt, + .decrypt = xts_paes_decrypt, + } + } +}; + +static int __ctr_paes_set_key(struct s390_paes_ctx *ctx) +{ + unsigned long fc; + + if (__paes_convert_key(&ctx->sk, &ctx->pk)) + return -EINVAL; + + /* Pick the correct function code based on the protected key type */ + fc = (ctx->pk.type == PKEY_KEYTYPE_AES_128) ? CPACF_KMCTR_PAES_128 : + (ctx->pk.type == PKEY_KEYTYPE_AES_192) ? CPACF_KMCTR_PAES_192 : + (ctx->pk.type == PKEY_KEYTYPE_AES_256) ? + CPACF_KMCTR_PAES_256 : 0; + + /* Check if the function code is available */ + ctx->fc = (fc && cpacf_test_func(&kmctr_functions, fc)) ? fc : 0; + + return ctx->fc ? 0 : -EINVAL; +} + +static int ctr_paes_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct s390_paes_ctx *ctx = crypto_tfm_ctx(tfm); + + memcpy(ctx->sk.seckey, in_key, key_len); + if (__ctr_paes_set_key(ctx)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + return 0; +} + +static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) +{ + unsigned int i, n; + + /* only use complete blocks, max. PAGE_SIZE */ + memcpy(ctrptr, iv, AES_BLOCK_SIZE); + n = (nbytes > PAGE_SIZE) ? PAGE_SIZE : nbytes & ~(AES_BLOCK_SIZE - 1); + for (i = (n / AES_BLOCK_SIZE) - 1; i > 0; i--) { + memcpy(ctrptr + AES_BLOCK_SIZE, ctrptr, AES_BLOCK_SIZE); + crypto_inc(ctrptr + AES_BLOCK_SIZE, AES_BLOCK_SIZE); + ctrptr += AES_BLOCK_SIZE; + } + return n; +} + +static int ctr_paes_crypt(struct blkcipher_desc *desc, unsigned long modifier, + struct blkcipher_walk *walk) +{ + struct s390_paes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + u8 buf[AES_BLOCK_SIZE], *ctrptr; + unsigned int nbytes, n, k; + int ret, locked; + + locked = spin_trylock(&ctrblk_lock); + + ret = blkcipher_walk_virt_block(desc, walk, AES_BLOCK_SIZE); + while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { + n = AES_BLOCK_SIZE; + if (nbytes >= 2*AES_BLOCK_SIZE && locked) + n = __ctrblk_init(ctrblk, walk->iv, nbytes); + ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; + k = cpacf_kmctr(ctx->fc | modifier, ctx->pk.protkey, + walk->dst.virt.addr, walk->src.virt.addr, + n, ctrptr); + if (k) { + if (ctrptr == ctrblk) + memcpy(walk->iv, ctrptr + k - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + crypto_inc(walk->iv, AES_BLOCK_SIZE); + ret = blkcipher_walk_done(desc, walk, nbytes - n); + } + if (k < n) { + if (__ctr_paes_set_key(ctx) != 0) + return blkcipher_walk_done(desc, walk, -EIO); + } + } + if (locked) + spin_unlock(&ctrblk_lock); + /* + * final block may be < AES_BLOCK_SIZE, copy only nbytes + */ + if (nbytes) { + while (1) { + if (cpacf_kmctr(ctx->fc | modifier, + ctx->pk.protkey, buf, + walk->src.virt.addr, AES_BLOCK_SIZE, + walk->iv) == AES_BLOCK_SIZE) + break; + if (__ctr_paes_set_key(ctx) != 0) + return blkcipher_walk_done(desc, walk, -EIO); + } + memcpy(walk->dst.virt.addr, buf, nbytes); + crypto_inc(walk->iv, AES_BLOCK_SIZE); + ret = blkcipher_walk_done(desc, walk, 0); + } + + return ret; +} + +static int ctr_paes_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ctr_paes_crypt(desc, 0, &walk); +} + +static int ctr_paes_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return ctr_paes_crypt(desc, CPACF_DECRYPT, &walk); +} + +static struct crypto_alg ctr_paes_alg = { + .cra_name = "ctr(paes)", + .cra_driver_name = "ctr-paes-s390", + .cra_priority = 400, /* combo: aes + ctr */ + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct s390_paes_ctx), + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(ctr_paes_alg.cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SECKEYBLOBSIZE, + .max_keysize = SECKEYBLOBSIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ctr_paes_set_key, + .encrypt = ctr_paes_encrypt, + .decrypt = ctr_paes_decrypt, + } + } +}; + +static inline void __crypto_unregister_alg(struct crypto_alg *alg) +{ + if (!list_empty(&alg->cra_list)) + crypto_unregister_alg(alg); +} + +static void paes_s390_fini(void) +{ + if (ctrblk) + free_page((unsigned long) ctrblk); + __crypto_unregister_alg(&ctr_paes_alg); + __crypto_unregister_alg(&xts_paes_alg); + __crypto_unregister_alg(&cbc_paes_alg); + __crypto_unregister_alg(&ecb_paes_alg); +} + +static int __init paes_s390_init(void) +{ + int ret; + + /* Query available functions for KM, KMC and KMCTR */ + cpacf_query(CPACF_KM, &km_functions); + cpacf_query(CPACF_KMC, &kmc_functions); + cpacf_query(CPACF_KMCTR, &kmctr_functions); + + if (cpacf_test_func(&km_functions, CPACF_KM_PAES_128) || + cpacf_test_func(&km_functions, CPACF_KM_PAES_192) || + cpacf_test_func(&km_functions, CPACF_KM_PAES_256)) { + ret = crypto_register_alg(&ecb_paes_alg); + if (ret) + goto out_err; + } + + if (cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_128) || + cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_192) || + cpacf_test_func(&kmc_functions, CPACF_KMC_PAES_256)) { + ret = crypto_register_alg(&cbc_paes_alg); + if (ret) + goto out_err; + } + + if (cpacf_test_func(&km_functions, CPACF_KM_PXTS_128) || + cpacf_test_func(&km_functions, CPACF_KM_PXTS_256)) { + ret = crypto_register_alg(&xts_paes_alg); + if (ret) + goto out_err; + } + + if (cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_128) || + cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_192) || + cpacf_test_func(&kmctr_functions, CPACF_KMCTR_PAES_256)) { + ret = crypto_register_alg(&ctr_paes_alg); + if (ret) + goto out_err; + ctrblk = (u8 *) __get_free_page(GFP_KERNEL); + if (!ctrblk) { + ret = -ENOMEM; + goto out_err; + } + } + + return 0; +out_err: + paes_s390_fini(); + return ret; +} + +module_init(paes_s390_init); +module_exit(paes_s390_fini); + +MODULE_ALIAS_CRYPTO("aes-all"); + +MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm with protected keys"); +MODULE_LICENSE("GPL"); diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c index daf9bb063aaa..85b7f5efe06a 100644 --- a/arch/s390/crypto/prng.c +++ b/arch/s390/crypto/prng.c @@ -110,22 +110,30 @@ static const u8 initial_parm_block[32] __initconst = { /*** helper functions ***/ +/* + * generate_entropy: + * This algorithm produces 64 bytes of entropy data based on 1024 + * individual stckf() invocations assuming that each stckf() value + * contributes 0.25 bits of entropy. So the caller gets 256 bit + * entropy per 64 byte or 4 bits entropy per byte. + */ static int generate_entropy(u8 *ebuf, size_t nbytes) { int n, ret = 0; - u8 *pg, *h, hash[32]; + u8 *pg, *h, hash[64]; - pg = (u8 *) __get_free_page(GFP_KERNEL); + /* allocate 2 pages */ + pg = (u8 *) __get_free_pages(GFP_KERNEL, 1); if (!pg) { prng_errorflag = PRNG_GEN_ENTROPY_FAILED; return -ENOMEM; } while (nbytes) { - /* fill page with urandom bytes */ - get_random_bytes(pg, PAGE_SIZE); - /* exor page with stckf values */ - for (n = 0; n < PAGE_SIZE / sizeof(u64); n++) { + /* fill pages with urandom bytes */ + get_random_bytes(pg, 2*PAGE_SIZE); + /* exor pages with 1024 stckf values */ + for (n = 0; n < 2 * PAGE_SIZE / sizeof(u64); n++) { u64 *p = ((u64 *)pg) + n; *p ^= get_tod_clock_fast(); } @@ -134,8 +142,8 @@ static int generate_entropy(u8 *ebuf, size_t nbytes) h = hash; else h = ebuf; - /* generate sha256 from this page */ - cpacf_kimd(CPACF_KIMD_SHA_256, h, pg, PAGE_SIZE); + /* hash over the filled pages */ + cpacf_kimd(CPACF_KIMD_SHA_512, h, pg, 2*PAGE_SIZE); if (n < sizeof(hash)) memcpy(ebuf, hash, n); ret += n; @@ -143,7 +151,7 @@ static int generate_entropy(u8 *ebuf, size_t nbytes) nbytes -= n; } - free_page((unsigned long)pg); + free_pages((unsigned long)pg, 1); return ret; } @@ -334,7 +342,7 @@ static int __init prng_sha512_selftest(void) static int __init prng_sha512_instantiate(void) { int ret, datalen; - u8 seed[64]; + u8 seed[64 + 32 + 16]; pr_debug("prng runs in SHA-512 mode " "with chunksize=%d and reseed_limit=%u\n", @@ -357,12 +365,12 @@ static int __init prng_sha512_instantiate(void) if (ret) goto outfree; - /* generate initial seed bytestring, first 48 bytes of entropy */ - ret = generate_entropy(seed, 48); - if (ret != 48) + /* generate initial seed bytestring, with 256 + 128 bits entropy */ + ret = generate_entropy(seed, 64 + 32); + if (ret != 64 + 32) goto outfree; /* followed by 16 bytes of unique nonce */ - get_tod_clock_ext(seed + 48); + get_tod_clock_ext(seed + 64 + 32); /* initial seed of the ppno drng */ cpacf_ppno(CPACF_PPNO_SHA512_DRNG_SEED, @@ -395,9 +403,9 @@ static void prng_sha512_deinstantiate(void) static int prng_sha512_reseed(void) { int ret; - u8 seed[32]; + u8 seed[64]; - /* generate 32 bytes of fresh entropy */ + /* fetch 256 bits of fresh entropy */ ret = generate_entropy(seed, sizeof(seed)); if (ret != sizeof(seed)) return ret; diff --git a/arch/s390/defconfig b/arch/s390/defconfig index d00e368fb5e6..68bfd09f1b02 100644 --- a/arch/s390/defconfig +++ b/arch/s390/defconfig @@ -229,6 +229,7 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_ZCRYPT=m +CONFIG_PKEY=m CONFIG_CRYPTO_SHA1_S390=m CONFIG_CRYPTO_SHA256_S390=m CONFIG_CRYPTO_SHA512_S390=m diff --git a/arch/s390/include/asm/cacheflush.h b/arch/s390/include/asm/cacheflush.h index 58fae7d098cf..0499334f9473 100644 --- a/arch/s390/include/asm/cacheflush.h +++ b/arch/s390/include/asm/cacheflush.h @@ -4,9 +4,31 @@ /* Caches aren't brain-dead on the s390. */ #include <asm-generic/cacheflush.h> -int set_memory_ro(unsigned long addr, int numpages); -int set_memory_rw(unsigned long addr, int numpages); -int set_memory_nx(unsigned long addr, int numpages); -int set_memory_x(unsigned long addr, int numpages); +#define SET_MEMORY_RO 1UL +#define SET_MEMORY_RW 2UL +#define SET_MEMORY_NX 4UL +#define SET_MEMORY_X 8UL + +int __set_memory(unsigned long addr, int numpages, unsigned long flags); + +static inline int set_memory_ro(unsigned long addr, int numpages) +{ + return __set_memory(addr, numpages, SET_MEMORY_RO); +} + +static inline int set_memory_rw(unsigned long addr, int numpages) +{ + return __set_memory(addr, numpages, SET_MEMORY_RW); +} + +static inline int set_memory_nx(unsigned long addr, int numpages) +{ + return __set_memory(addr, numpages, SET_MEMORY_NX); +} + +static inline int set_memory_x(unsigned long addr, int numpages) +{ + return __set_memory(addr, numpages, SET_MEMORY_X); +} #endif /* _S390_CACHEFLUSH_H */ diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h index 2c680db7e5c1..e2dfbf280d12 100644 --- a/arch/s390/include/asm/cpacf.h +++ b/arch/s390/include/asm/cpacf.h @@ -28,8 +28,9 @@ #define CPACF_PPNO 0xb93c /* MSA5 */ /* - * Decryption modifier bit + * En/decryption modifier bits */ +#define CPACF_ENCRYPT 0x00 #define CPACF_DECRYPT 0x80 /* @@ -42,8 +43,13 @@ #define CPACF_KM_AES_128 0x12 #define CPACF_KM_AES_192 0x13 #define CPACF_KM_AES_256 0x14 +#define CPACF_KM_PAES_128 0x1a +#define CPACF_KM_PAES_192 0x1b +#define CPACF_KM_PAES_256 0x1c #define CPACF_KM_XTS_128 0x32 #define CPACF_KM_XTS_256 0x34 +#define CPACF_KM_PXTS_128 0x3a +#define CPACF_KM_PXTS_256 0x3c /* * Function codes for the KMC (CIPHER MESSAGE WITH CHAINING) @@ -56,6 +62,9 @@ #define CPACF_KMC_AES_128 0x12 #define CPACF_KMC_AES_192 0x13 #define CPACF_KMC_AES_256 0x14 +#define CPACF_KMC_PAES_128 0x1a +#define CPACF_KMC_PAES_192 0x1b +#define CPACF_KMC_PAES_256 0x1c #define CPACF_KMC_PRNG 0x43 /* @@ -69,6 +78,9 @@ #define CPACF_KMCTR_AES_128 0x12 #define CPACF_KMCTR_AES_192 0x13 #define CPACF_KMCTR_AES_256 0x14 +#define CPACF_KMCTR_PAES_128 0x1a +#define CPACF_KMCTR_PAES_192 0x1b +#define CPACF_KMCTR_PAES_256 0x1c /* * Function codes for the KIMD (COMPUTE INTERMEDIATE MESSAGE DIGEST) @@ -99,6 +111,18 @@ #define CPACF_KMAC_TDEA_192 0x03 /* + * Function codes for the PCKMO (PERFORM CRYPTOGRAPHIC KEY MANAGEMENT) + * instruction + */ +#define CPACF_PCKMO_QUERY 0x00 +#define CPACF_PCKMO_ENC_DES_KEY 0x01 +#define CPACF_PCKMO_ENC_TDES_128_KEY 0x02 +#define CPACF_PCKMO_ENC_TDES_192_KEY 0x03 +#define CPACF_PCKMO_ENC_AES_128_KEY 0x12 +#define CPACF_PCKMO_ENC_AES_192_KEY 0x13 +#define CPACF_PCKMO_ENC_AES_256_KEY 0x14 + +/* * Function codes for the PPNO (PERFORM PSEUDORANDOM NUMBER OPERATION) * instruction */ @@ -397,4 +421,24 @@ static inline void cpacf_pcc(unsigned long func, void *param) : "cc", "memory"); } +/** + * cpacf_pckmo() - executes the PCKMO (PERFORM CRYPTOGRAPHIC KEY + * MANAGEMENT) instruction + * @func: the function code passed to PCKMO; see CPACF_PCKMO_xxx defines + * @param: address of parameter block; see POP for details on each func + * + * Returns 0. + */ +static inline void cpacf_pckmo(long func, void *param) +{ + register unsigned long r0 asm("0") = (unsigned long) func; + register unsigned long r1 asm("1") = (unsigned long) param; + + asm volatile( + " .insn rre,%[opc] << 16,0,0\n" /* PCKMO opcode */ + : + : [fc] "d" (r0), [pba] "a" (r1), [opc] "i" (CPACF_PCKMO) + : "cc", "memory"); +} + #endif /* _ASM_S390_CPACF_H */ diff --git a/arch/s390/include/asm/cpu_mf.h b/arch/s390/include/asm/cpu_mf.h index 428c41239a49..d1e0707310fd 100644 --- a/arch/s390/include/asm/cpu_mf.h +++ b/arch/s390/include/asm/cpu_mf.h @@ -199,14 +199,15 @@ static inline int ecctr(u64 ctr, u64 *val) /* Store CPU counter multiple for the MT utilization counter set */ static inline int stcctm5(u64 num, u64 *val) { - typedef struct { u64 _[num]; } addrtype; int cc; asm volatile ( " .insn rsy,0xeb0000000017,%2,5,%1\n" " ipm %0\n" " srl %0,28\n" - : "=d" (cc), "=Q" (*(addrtype *) val) : "d" (num) : "cc"); + : "=d" (cc) + : "Q" (*val), "d" (num) + : "cc", "memory"); return cc; } diff --git a/arch/s390/include/asm/cputime.h b/arch/s390/include/asm/cputime.h index 221b454c734a..d1c407ddf703 100644 --- a/arch/s390/include/asm/cputime.h +++ b/arch/s390/include/asm/cputime.h @@ -25,33 +25,6 @@ static inline unsigned long __div(unsigned long long n, unsigned long base) return n / base; } -#define cputime_one_jiffy jiffies_to_cputime(1) - -/* - * Convert cputime to jiffies and back. - */ -static inline unsigned long cputime_to_jiffies(const cputime_t cputime) -{ - return __div((__force unsigned long long) cputime, CPUTIME_PER_SEC / HZ); -} - -static inline cputime_t jiffies_to_cputime(const unsigned int jif) -{ - return (__force cputime_t)(jif * (CPUTIME_PER_SEC / HZ)); -} - -static inline u64 cputime64_to_jiffies64(cputime64_t cputime) -{ - unsigned long long jif = (__force unsigned long long) cputime; - do_div(jif, CPUTIME_PER_SEC / HZ); - return jif; -} - -static inline cputime64_t jiffies64_to_cputime64(const u64 jif) -{ - return (__force cputime64_t)(jif * (CPUTIME_PER_SEC / HZ)); -} - /* * Convert cputime to microseconds and back. */ @@ -60,88 +33,8 @@ static inline unsigned int cputime_to_usecs(const cputime_t cputime) return (__force unsigned long long) cputime >> 12; } -static inline cputime_t usecs_to_cputime(const unsigned int m) -{ - return (__force cputime_t)(m * CPUTIME_PER_USEC); -} - -#define usecs_to_cputime64(m) usecs_to_cputime(m) - -/* - * Convert cputime to milliseconds and back. - */ -static inline unsigned int cputime_to_secs(const cputime_t cputime) -{ - return __div((__force unsigned long long) cputime, CPUTIME_PER_SEC / 2) >> 1; -} - -static inline cputime_t secs_to_cputime(const unsigned int s) -{ - return (__force cputime_t)(s * CPUTIME_PER_SEC); -} - -/* - * Convert cputime to timespec and back. - */ -static inline cputime_t timespec_to_cputime(const struct timespec *value) -{ - unsigned long long ret = value->tv_sec * CPUTIME_PER_SEC; - return (__force cputime_t)(ret + __div(value->tv_nsec * CPUTIME_PER_USEC, NSEC_PER_USEC)); -} - -static inline void cputime_to_timespec(const cputime_t cputime, - struct timespec *value) -{ - unsigned long long __cputime = (__force unsigned long long) cputime; - value->tv_nsec = (__cputime % CPUTIME_PER_SEC) * NSEC_PER_USEC / CPUTIME_PER_USEC; - value->tv_sec = __cputime / CPUTIME_PER_SEC; -} - -/* - * Convert cputime to timeval and back. - * Since cputime and timeval have the same resolution (microseconds) - * this is easy. - */ -static inline cputime_t timeval_to_cputime(const struct timeval *value) -{ - unsigned long long ret = value->tv_sec * CPUTIME_PER_SEC; - return (__force cputime_t)(ret + value->tv_usec * CPUTIME_PER_USEC); -} - -static inline void cputime_to_timeval(const cputime_t cputime, - struct timeval *value) -{ - unsigned long long __cputime = (__force unsigned long long) cputime; - value->tv_usec = (__cputime % CPUTIME_PER_SEC) / CPUTIME_PER_USEC; - value->tv_sec = __cputime / CPUTIME_PER_SEC; -} - -/* - * Convert cputime to clock and back. - */ -static inline clock_t cputime_to_clock_t(cputime_t cputime) -{ - unsigned long long clock = (__force unsigned long long) cputime; - do_div(clock, CPUTIME_PER_SEC / USER_HZ); - return clock; -} - -static inline cputime_t clock_t_to_cputime(unsigned long x) -{ - return (__force cputime_t)(x * (CPUTIME_PER_SEC / USER_HZ)); -} - -/* - * Convert cputime64 to clock. - */ -static inline clock_t cputime64_to_clock_t(cputime64_t cputime) -{ - unsigned long long clock = (__force unsigned long long) cputime; - do_div(clock, CPUTIME_PER_SEC / USER_HZ); - return clock; -} -cputime64_t arch_cpu_idle_time(int cpu); +u64 arch_cpu_idle_time(int cpu); #define arch_idle_time(cpu) arch_cpu_idle_time(cpu) diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctl_reg.h index 8e136b88cdf4..d0441ad2a990 100644 --- a/arch/s390/include/asm/ctl_reg.h +++ b/arch/s390/include/asm/ctl_reg.h @@ -9,7 +9,7 @@ #include <linux/bug.h> -#define __ctl_load(array, low, high) { \ +#define __ctl_load(array, low, high) do { \ typedef struct { char _[sizeof(array)]; } addrtype; \ \ BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ @@ -18,9 +18,9 @@ : \ : "Q" (*(addrtype *)(&array)), "i" (low), "i" (high) \ : "memory"); \ -} +} while (0) -#define __ctl_store(array, low, high) { \ +#define __ctl_store(array, low, high) do { \ typedef struct { char _[sizeof(array)]; } addrtype; \ \ BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ @@ -28,7 +28,7 @@ " stctg %1,%2,%0\n" \ : "=Q" (*(addrtype *)(&array)) \ : "i" (low), "i" (high)); \ -} +} while (0) static inline void __ctl_set_bit(unsigned int cr, unsigned int bit) { @@ -62,7 +62,9 @@ union ctlreg0 { unsigned long : 4; unsigned long afp : 1; /* AFP-register control */ unsigned long vx : 1; /* Vector enablement control */ - unsigned long : 17; + unsigned long : 7; + unsigned long sssm : 1; /* Service signal subclass mask */ + unsigned long : 9; }; }; diff --git a/arch/s390/include/asm/device.h b/arch/s390/include/asm/device.h index 4a9f35e0973f..5203fc87f080 100644 --- a/arch/s390/include/asm/device.h +++ b/arch/s390/include/asm/device.h @@ -4,7 +4,6 @@ * This file is released under the GPLv2 */ struct dev_archdata { - struct dma_map_ops *dma_ops; }; struct pdev_archdata { diff --git a/arch/s390/include/asm/dma-mapping.h b/arch/s390/include/asm/dma-mapping.h index ffaba07f50ab..3108b8dbe266 100644 --- a/arch/s390/include/asm/dma-mapping.h +++ b/arch/s390/include/asm/dma-mapping.h @@ -10,12 +10,10 @@ #define DMA_ERROR_CODE (~(dma_addr_t) 0x0) -extern struct dma_map_ops s390_pci_dma_ops; +extern const struct dma_map_ops s390_pci_dma_ops; -static inline struct dma_map_ops *get_dma_ops(struct device *dev) +static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) { - if (dev && dev->archdata.dma_ops) - return dev->archdata.dma_ops; return &dma_noop_ops; } diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h index f4381e1fb19e..83aaefed2a7b 100644 --- a/arch/s390/include/asm/elf.h +++ b/arch/s390/include/asm/elf.h @@ -103,6 +103,8 @@ #define HWCAP_S390_HIGH_GPRS 512 #define HWCAP_S390_TE 1024 #define HWCAP_S390_VXRS 2048 +#define HWCAP_S390_VXRS_BCD 4096 +#define HWCAP_S390_VXRS_EXT 8192 /* Internal bits, not exposed via elf */ #define HWCAP_INT_SIE 1UL diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index 4c7fac75090e..cd546a245c68 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -14,7 +14,7 @@ #define is_hugepage_only_range(mm, addr, len) 0 #define hugetlb_free_pgd_range free_pgd_range -#define hugepages_supported() (MACHINE_HAS_HPAGE) +#define hugepages_supported() (MACHINE_HAS_EDAT1) void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); diff --git a/arch/s390/include/asm/kprobes.h b/arch/s390/include/asm/kprobes.h index 591e5a5279b0..84c0f9086483 100644 --- a/arch/s390/include/asm/kprobes.h +++ b/arch/s390/include/asm/kprobes.h @@ -27,6 +27,11 @@ * 2005-Dec Used as a template for s390 by Mike Grundy * <grundym@us.ibm.com> */ +#include <asm-generic/kprobes.h> + +#define BREAKPOINT_INSTRUCTION 0x0002 + +#ifdef CONFIG_KPROBES #include <linux/types.h> #include <linux/ptrace.h> #include <linux/percpu.h> @@ -37,7 +42,6 @@ struct pt_regs; struct kprobe; typedef u16 kprobe_opcode_t; -#define BREAKPOINT_INSTRUCTION 0x0002 /* Maximum instruction size is 3 (16bit) halfwords: */ #define MAX_INSN_SIZE 0x0003 @@ -91,4 +95,5 @@ int probe_is_insn_relative_long(u16 *insn); #define flush_insn_slot(p) do { } while (0) +#endif /* CONFIG_KPROBES */ #endif /* _ASM_S390_KPROBES_H */ diff --git a/arch/s390/include/asm/livepatch.h b/arch/s390/include/asm/livepatch.h index 2c1213785892..6de5c6cb0061 100644 --- a/arch/s390/include/asm/livepatch.h +++ b/arch/s390/include/asm/livepatch.h @@ -17,7 +17,7 @@ #ifndef ASM_LIVEPATCH_H #define ASM_LIVEPATCH_H -#include <linux/module.h> +#include <asm/ptrace.h> static inline int klp_check_compiler_support(void) { diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 9bfad2ad6312..61261e0e95c0 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -85,53 +85,56 @@ struct lowcore { __u64 mcck_enter_timer; /* 0x02c0 */ __u64 exit_timer; /* 0x02c8 */ __u64 user_timer; /* 0x02d0 */ - __u64 system_timer; /* 0x02d8 */ - __u64 steal_timer; /* 0x02e0 */ - __u64 last_update_timer; /* 0x02e8 */ - __u64 last_update_clock; /* 0x02f0 */ - __u64 int_clock; /* 0x02f8 */ - __u64 mcck_clock; /* 0x0300 */ - __u64 clock_comparator; /* 0x0308 */ + __u64 guest_timer; /* 0x02d8 */ + __u64 system_timer; /* 0x02e0 */ + __u64 hardirq_timer; /* 0x02e8 */ + __u64 softirq_timer; /* 0x02f0 */ + __u64 steal_timer; /* 0x02f8 */ + __u64 last_update_timer; /* 0x0300 */ + __u64 last_update_clock; /* 0x0308 */ + __u64 int_clock; /* 0x0310 */ + __u64 mcck_clock; /* 0x0318 */ + __u64 clock_comparator; /* 0x0320 */ /* Current process. */ - __u64 current_task; /* 0x0310 */ - __u8 pad_0x318[0x320-0x318]; /* 0x0318 */ - __u64 kernel_stack; /* 0x0320 */ + __u64 current_task; /* 0x0328 */ + __u8 pad_0x318[0x320-0x318]; /* 0x0330 */ + __u64 kernel_stack; /* 0x0338 */ /* Interrupt, panic and restart stack. */ - __u64 async_stack; /* 0x0328 */ - __u64 panic_stack; /* 0x0330 */ - __u64 restart_stack; /* 0x0338 */ + __u64 async_stack; /* 0x0340 */ + __u64 panic_stack; /* 0x0348 */ + __u64 restart_stack; /* 0x0350 */ /* Restart function and parameter. */ - __u64 restart_fn; /* 0x0340 */ - __u64 restart_data; /* 0x0348 */ - __u64 restart_source; /* 0x0350 */ + __u64 restart_fn; /* 0x0358 */ + __u64 restart_data; /* 0x0360 */ + __u64 restart_source; /* 0x0368 */ /* Address space pointer. */ - __u64 kernel_asce; /* 0x0358 */ - __u64 user_asce; /* 0x0360 */ + __u64 kernel_asce; /* 0x0370 */ + __u64 user_asce; /* 0x0378 */ /* * The lpp and current_pid fields form a * 64-bit value that is set as program * parameter with the LPP instruction. */ - __u32 lpp; /* 0x0368 */ - __u32 current_pid; /* 0x036c */ + __u32 lpp; /* 0x0380 */ + __u32 current_pid; /* 0x0384 */ /* SMP info area */ - __u32 cpu_nr; /* 0x0370 */ - __u32 softirq_pending; /* 0x0374 */ - __u64 percpu_offset; /* 0x0378 */ - __u64 vdso_per_cpu_data; /* 0x0380 */ - __u64 machine_flags; /* 0x0388 */ - __u32 preempt_count; /* 0x0390 */ - __u8 pad_0x0394[0x0398-0x0394]; /* 0x0394 */ - __u64 gmap; /* 0x0398 */ - __u32 spinlock_lockval; /* 0x03a0 */ - __u32 fpu_flags; /* 0x03a4 */ - __u8 pad_0x03a8[0x0400-0x03a8]; /* 0x03a8 */ + __u32 cpu_nr; /* 0x0388 */ + __u32 softirq_pending; /* 0x038c */ + __u64 percpu_offset; /* 0x0390 */ + __u64 vdso_per_cpu_data; /* 0x0398 */ + __u64 machine_flags; /* 0x03a0 */ + __u32 preempt_count; /* 0x03a8 */ + __u8 pad_0x03ac[0x03b0-0x03ac]; /* 0x03ac */ + __u64 gmap; /* 0x03b0 */ + __u32 spinlock_lockval; /* 0x03b8 */ + __u32 fpu_flags; /* 0x03bc */ + __u8 pad_0x03c0[0x0400-0x03c0]; /* 0x03c0 */ /* Per cpu primary space access list */ __u32 paste[16]; /* 0x0400 */ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 67f7a991c929..9b828c073176 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -63,7 +63,7 @@ static inline void set_user_asce(struct mm_struct *mm) S390_lowcore.user_asce = mm->context.asce; if (current->thread.mm_segment.ar4) __ctl_load(S390_lowcore.user_asce, 7, 7); - set_cpu_flag(CIF_ASCE); + set_cpu_flag(CIF_ASCE_PRIMARY); } static inline void clear_user_asce(void) @@ -81,7 +81,7 @@ static inline void load_kernel_asce(void) __ctl_store(asce, 1, 1); if (asce != S390_lowcore.kernel_asce) __ctl_load(S390_lowcore.kernel_asce, 1, 1); - set_cpu_flag(CIF_ASCE); + set_cpu_flag(CIF_ASCE_PRIMARY); } static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, diff --git a/arch/s390/include/asm/pci_clp.h b/arch/s390/include/asm/pci_clp.h index d6f1b1d94352..938b8cc19fc6 100644 --- a/arch/s390/include/asm/pci_clp.h +++ b/arch/s390/include/asm/pci_clp.h @@ -85,7 +85,7 @@ struct clp_rsp_query_pci { u32 fid; /* pci function id */ u8 bar_size[PCI_BAR_COUNT]; u16 pchid; - u32 bar[PCI_BAR_COUNT]; + __le32 bar[PCI_BAR_COUNT]; u8 pfip[CLP_PFIP_NR_SEGMENTS]; /* pci function internal path */ u32 : 16; u8 fmb_len; diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 0362cd5fa187..7ed1972b1920 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -200,6 +200,7 @@ static inline int is_module_addr(void *addr) */ /* Hardware bits in the page table entry */ +#define _PAGE_NOEXEC 0x100 /* HW no-execute bit */ #define _PAGE_PROTECT 0x200 /* HW read-only bit */ #define _PAGE_INVALID 0x400 /* HW invalid bit */ #define _PAGE_LARGE 0x800 /* Bit to mark a large pte */ @@ -277,6 +278,7 @@ static inline int is_module_addr(void *addr) /* Bits in the region table entry */ #define _REGION_ENTRY_ORIGIN ~0xfffUL/* region/segment table origin */ #define _REGION_ENTRY_PROTECT 0x200 /* region protection bit */ +#define _REGION_ENTRY_NOEXEC 0x100 /* region no-execute bit */ #define _REGION_ENTRY_OFFSET 0xc0 /* region table offset */ #define _REGION_ENTRY_INVALID 0x20 /* invalid region table entry */ #define _REGION_ENTRY_TYPE_MASK 0x0c /* region/segment table type mask */ @@ -316,6 +318,7 @@ static inline int is_module_addr(void *addr) #define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address */ #define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* segment table origin */ #define _SEGMENT_ENTRY_PROTECT 0x200 /* page protection bit */ +#define _SEGMENT_ENTRY_NOEXEC 0x100 /* region no-execute bit */ #define _SEGMENT_ENTRY_INVALID 0x20 /* invalid segment table entry */ #define _SEGMENT_ENTRY (0) @@ -385,17 +388,23 @@ static inline int is_module_addr(void *addr) * Page protection definitions. */ #define PAGE_NONE __pgprot(_PAGE_PRESENT | _PAGE_INVALID | _PAGE_PROTECT) -#define PAGE_READ __pgprot(_PAGE_PRESENT | _PAGE_READ | \ +#define PAGE_RO __pgprot(_PAGE_PRESENT | _PAGE_READ | \ + _PAGE_NOEXEC | _PAGE_INVALID | _PAGE_PROTECT) +#define PAGE_RX __pgprot(_PAGE_PRESENT | _PAGE_READ | \ _PAGE_INVALID | _PAGE_PROTECT) -#define PAGE_WRITE __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ +#define PAGE_RW __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ + _PAGE_NOEXEC | _PAGE_INVALID | _PAGE_PROTECT) +#define PAGE_RWX __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ _PAGE_INVALID | _PAGE_PROTECT) #define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ - _PAGE_YOUNG | _PAGE_DIRTY) + _PAGE_YOUNG | _PAGE_DIRTY | _PAGE_NOEXEC) #define PAGE_KERNEL __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ - _PAGE_YOUNG | _PAGE_DIRTY) + _PAGE_YOUNG | _PAGE_DIRTY | _PAGE_NOEXEC) #define PAGE_KERNEL_RO __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_YOUNG | \ - _PAGE_PROTECT) + _PAGE_PROTECT | _PAGE_NOEXEC) +#define PAGE_KERNEL_EXEC __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ + _PAGE_YOUNG | _PAGE_DIRTY) /* * On s390 the page table entry has an invalid bit and a read-only bit. @@ -404,43 +413,51 @@ static inline int is_module_addr(void *addr) */ /*xwr*/ #define __P000 PAGE_NONE -#define __P001 PAGE_READ -#define __P010 PAGE_READ -#define __P011 PAGE_READ -#define __P100 PAGE_READ -#define __P101 PAGE_READ -#define __P110 PAGE_READ -#define __P111 PAGE_READ +#define __P001 PAGE_RO +#define __P010 PAGE_RO +#define __P011 PAGE_RO +#define __P100 PAGE_RX +#define __P101 PAGE_RX +#define __P110 PAGE_RX +#define __P111 PAGE_RX #define __S000 PAGE_NONE -#define __S001 PAGE_READ -#define __S010 PAGE_WRITE -#define __S011 PAGE_WRITE -#define __S100 PAGE_READ -#define __S101 PAGE_READ -#define __S110 PAGE_WRITE -#define __S111 PAGE_WRITE +#define __S001 PAGE_RO +#define __S010 PAGE_RW +#define __S011 PAGE_RW +#define __S100 PAGE_RX +#define __S101 PAGE_RX +#define __S110 PAGE_RWX +#define __S111 PAGE_RWX /* * Segment entry (large page) protection definitions. */ #define SEGMENT_NONE __pgprot(_SEGMENT_ENTRY_INVALID | \ _SEGMENT_ENTRY_PROTECT) -#define SEGMENT_READ __pgprot(_SEGMENT_ENTRY_PROTECT | \ +#define SEGMENT_RO __pgprot(_SEGMENT_ENTRY_PROTECT | \ + _SEGMENT_ENTRY_READ | \ + _SEGMENT_ENTRY_NOEXEC) +#define SEGMENT_RX __pgprot(_SEGMENT_ENTRY_PROTECT | \ _SEGMENT_ENTRY_READ) -#define SEGMENT_WRITE __pgprot(_SEGMENT_ENTRY_READ | \ +#define SEGMENT_RW __pgprot(_SEGMENT_ENTRY_READ | \ + _SEGMENT_ENTRY_WRITE | \ + _SEGMENT_ENTRY_NOEXEC) +#define SEGMENT_RWX __pgprot(_SEGMENT_ENTRY_READ | \ _SEGMENT_ENTRY_WRITE) #define SEGMENT_KERNEL __pgprot(_SEGMENT_ENTRY | \ _SEGMENT_ENTRY_LARGE | \ _SEGMENT_ENTRY_READ | \ _SEGMENT_ENTRY_WRITE | \ _SEGMENT_ENTRY_YOUNG | \ - _SEGMENT_ENTRY_DIRTY) + _SEGMENT_ENTRY_DIRTY | \ + _SEGMENT_ENTRY_NOEXEC) #define SEGMENT_KERNEL_RO __pgprot(_SEGMENT_ENTRY | \ _SEGMENT_ENTRY_LARGE | \ _SEGMENT_ENTRY_READ | \ _SEGMENT_ENTRY_YOUNG | \ - _SEGMENT_ENTRY_PROTECT) + _SEGMENT_ENTRY_PROTECT | \ + _SEGMENT_ENTRY_NOEXEC) /* * Region3 entry (large page) protection definitions. @@ -451,12 +468,14 @@ static inline int is_module_addr(void *addr) _REGION3_ENTRY_READ | \ _REGION3_ENTRY_WRITE | \ _REGION3_ENTRY_YOUNG | \ - _REGION3_ENTRY_DIRTY) + _REGION3_ENTRY_DIRTY | \ + _REGION_ENTRY_NOEXEC) #define REGION3_KERNEL_RO __pgprot(_REGION_ENTRY_TYPE_R3 | \ _REGION3_ENTRY_LARGE | \ _REGION3_ENTRY_READ | \ _REGION3_ENTRY_YOUNG | \ - _REGION_ENTRY_PROTECT) + _REGION_ENTRY_PROTECT | \ + _REGION_ENTRY_NOEXEC) static inline int mm_has_pgste(struct mm_struct *mm) { @@ -621,12 +640,12 @@ static inline int pud_bad(pud_t pud) static inline int pmd_present(pmd_t pmd) { - return pmd_val(pmd) != _SEGMENT_ENTRY_INVALID; + return pmd_val(pmd) != _SEGMENT_ENTRY_EMPTY; } static inline int pmd_none(pmd_t pmd) { - return pmd_val(pmd) == _SEGMENT_ENTRY_INVALID; + return pmd_val(pmd) == _SEGMENT_ENTRY_EMPTY; } static inline unsigned long pmd_pfn(pmd_t pmd) @@ -784,7 +803,7 @@ static inline void pud_clear(pud_t *pud) static inline void pmd_clear(pmd_t *pmdp) { - pmd_val(*pmdp) = _SEGMENT_ENTRY_INVALID; + pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY; } static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) @@ -801,14 +820,14 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) pte_val(pte) &= _PAGE_CHG_MASK; pte_val(pte) |= pgprot_val(newprot); /* - * newprot for PAGE_NONE, PAGE_READ and PAGE_WRITE has the - * invalid bit set, clear it again for readable, young pages + * newprot for PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX + * has the invalid bit set, clear it again for readable, young pages */ if ((pte_val(pte) & _PAGE_YOUNG) && (pte_val(pte) & _PAGE_READ)) pte_val(pte) &= ~_PAGE_INVALID; /* - * newprot for PAGE_READ and PAGE_WRITE has the page protection - * bit set, clear it again for writable, dirty pages + * newprot for PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX has the page + * protection bit set, clear it again for writable, dirty pages */ if ((pte_val(pte) & _PAGE_DIRTY) && (pte_val(pte) & _PAGE_WRITE)) pte_val(pte) &= ~_PAGE_PROTECT; @@ -1029,6 +1048,8 @@ int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t entry) { + if (!MACHINE_HAS_NX) + pte_val(entry) &= ~_PAGE_NOEXEC; if (mm_has_pgste(mm)) ptep_set_pte_at(mm, addr, ptep, entry); else @@ -1173,14 +1194,18 @@ static inline pud_t pud_mkdirty(pud_t pud) static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot) { /* - * pgprot is PAGE_NONE, PAGE_READ, or PAGE_WRITE (see __Pxxx / __Sxxx) - * Convert to segment table entry format. + * pgprot is PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW or PAGE_RWX + * (see __Pxxx / __Sxxx). Convert to segment table entry format. */ if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE)) return pgprot_val(SEGMENT_NONE); - if (pgprot_val(pgprot) == pgprot_val(PAGE_READ)) - return pgprot_val(SEGMENT_READ); - return pgprot_val(SEGMENT_WRITE); + if (pgprot_val(pgprot) == pgprot_val(PAGE_RO)) + return pgprot_val(SEGMENT_RO); + if (pgprot_val(pgprot) == pgprot_val(PAGE_RX)) + return pgprot_val(SEGMENT_RX); + if (pgprot_val(pgprot) == pgprot_val(PAGE_RW)) + return pgprot_val(SEGMENT_RW); + return pgprot_val(SEGMENT_RWX); } static inline pmd_t pmd_mkyoung(pmd_t pmd) @@ -1315,6 +1340,8 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t entry) { + if (!MACHINE_HAS_NX) + pmd_val(entry) &= ~_SEGMENT_ENTRY_NOEXEC; *pmdp = entry; } @@ -1330,7 +1357,7 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd) static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { - return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID)); + return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); } #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL @@ -1340,10 +1367,10 @@ static inline pmd_t pmdp_huge_get_and_clear_full(struct mm_struct *mm, { if (full) { pmd_t pmd = *pmdp; - *pmdp = __pmd(_SEGMENT_ENTRY_INVALID); + *pmdp = __pmd(_SEGMENT_ENTRY_EMPTY); return pmd; } - return pmdp_xchg_lazy(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID)); + return pmdp_xchg_lazy(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); } #define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH @@ -1357,7 +1384,7 @@ static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, static inline void pmdp_invalidate(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { - pmdp_xchg_direct(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID)); + pmdp_xchg_direct(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); } #define __HAVE_ARCH_PMDP_SET_WRPROTECT @@ -1389,7 +1416,7 @@ static inline int pmd_trans_huge(pmd_t pmd) #define has_transparent_hugepage has_transparent_hugepage static inline int has_transparent_hugepage(void) { - return MACHINE_HAS_HPAGE ? 1 : 0; + return MACHINE_HAS_EDAT1 ? 1 : 0; } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ diff --git a/arch/s390/include/asm/pkey.h b/arch/s390/include/asm/pkey.h new file mode 100644 index 000000000000..b48aef4188f6 --- /dev/null +++ b/arch/s390/include/asm/pkey.h @@ -0,0 +1,90 @@ +/* + * Kernelspace interface to the pkey device driver + * + * Copyright IBM Corp. 2016 + * + * Author: Harald Freudenberger <freude@de.ibm.com> + * + */ + +#ifndef _KAPI_PKEY_H +#define _KAPI_PKEY_H + +#include <linux/ioctl.h> +#include <linux/types.h> +#include <uapi/asm/pkey.h> + +/* + * Generate (AES) random secure key. + * @param cardnr may be -1 (use default card) + * @param domain may be -1 (use default domain) + * @param keytype one of the PKEY_KEYTYPE values + * @param seckey pointer to buffer receiving the secure key + * @return 0 on success, negative errno value on failure + */ +int pkey_genseckey(__u16 cardnr, __u16 domain, + __u32 keytype, struct pkey_seckey *seckey); + +/* + * Generate (AES) secure key with given key value. + * @param cardnr may be -1 (use default card) + * @param domain may be -1 (use default domain) + * @param keytype one of the PKEY_KEYTYPE values + * @param clrkey pointer to buffer with clear key data + * @param seckey pointer to buffer receiving the secure key + * @return 0 on success, negative errno value on failure + */ +int pkey_clr2seckey(__u16 cardnr, __u16 domain, __u32 keytype, + const struct pkey_clrkey *clrkey, + struct pkey_seckey *seckey); + +/* + * Derive (AES) proteced key from the (AES) secure key blob. + * @param cardnr may be -1 (use default card) + * @param domain may be -1 (use default domain) + * @param seckey pointer to buffer with the input secure key + * @param protkey pointer to buffer receiving the protected key and + * additional info (type, length) + * @return 0 on success, negative errno value on failure + */ +int pkey_sec2protkey(__u16 cardnr, __u16 domain, + const struct pkey_seckey *seckey, + struct pkey_protkey *protkey); + +/* + * Derive (AES) protected key from a given clear key value. + * @param keytype one of the PKEY_KEYTYPE values + * @param clrkey pointer to buffer with clear key data + * @param protkey pointer to buffer receiving the protected key and + * additional info (type, length) + * @return 0 on success, negative errno value on failure + */ +int pkey_clr2protkey(__u32 keytype, + const struct pkey_clrkey *clrkey, + struct pkey_protkey *protkey); + +/* + * Search for a matching crypto card based on the Master Key + * Verification Pattern provided inside a secure key. + * @param seckey pointer to buffer with the input secure key + * @param cardnr pointer to cardnr, receives the card number on success + * @param domain pointer to domain, receives the domain number on success + * @param verify if set, always verify by fetching verification pattern + * from card + * @return 0 on success, negative errno value on failure. If no card could be + * found, -ENODEV is returned. + */ +int pkey_findcard(const struct pkey_seckey *seckey, + __u16 *cardnr, __u16 *domain, int verify); + +/* + * Find card and transform secure key to protected key. + * @param seckey pointer to buffer with the input secure key + * @param protkey pointer to buffer receiving the protected key and + * additional info (type, length) + * @return 0 on success, negative errno value on failure + */ +int pkey_skey2pkey(const struct pkey_seckey *seckey, + struct pkey_protkey *protkey); + +#endif /* _KAPI_PKEY_H */ diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 6bca916a5ba0..e4988710aa86 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -14,14 +14,16 @@ #include <linux/const.h> #define CIF_MCCK_PENDING 0 /* machine check handling is pending */ -#define CIF_ASCE 1 /* user asce needs fixup / uaccess */ -#define CIF_NOHZ_DELAY 2 /* delay HZ disable for a tick */ -#define CIF_FPU 3 /* restore FPU registers */ -#define CIF_IGNORE_IRQ 4 /* ignore interrupt (for udelay) */ -#define CIF_ENABLED_WAIT 5 /* in enabled wait state */ +#define CIF_ASCE_PRIMARY 1 /* primary asce needs fixup / uaccess */ +#define CIF_ASCE_SECONDARY 2 /* secondary asce needs fixup / uaccess */ +#define CIF_NOHZ_DELAY 3 /* delay HZ disable for a tick */ +#define CIF_FPU 4 /* restore FPU registers */ +#define CIF_IGNORE_IRQ 5 /* ignore interrupt (for udelay) */ +#define CIF_ENABLED_WAIT 6 /* in enabled wait state */ #define _CIF_MCCK_PENDING _BITUL(CIF_MCCK_PENDING) -#define _CIF_ASCE _BITUL(CIF_ASCE) +#define _CIF_ASCE_PRIMARY _BITUL(CIF_ASCE_PRIMARY) +#define _CIF_ASCE_SECONDARY _BITUL(CIF_ASCE_SECONDARY) #define _CIF_NOHZ_DELAY _BITUL(CIF_NOHZ_DELAY) #define _CIF_FPU _BITUL(CIF_FPU) #define _CIF_IGNORE_IRQ _BITUL(CIF_IGNORE_IRQ) @@ -89,7 +91,8 @@ extern void execve_tail(void); * User space process size: 2GB for 31 bit, 4TB or 8PT for 64 bit. */ -#define TASK_SIZE_OF(tsk) ((tsk)->mm->context.asce_limit) +#define TASK_SIZE_OF(tsk) ((tsk)->mm ? \ + (tsk)->mm->context.asce_limit : TASK_MAX_SIZE) #define TASK_UNMAPPED_BASE (test_thread_flag(TIF_31BIT) ? \ (1UL << 30) : (1UL << 41)) #define TASK_SIZE TASK_SIZE_OF(current) @@ -111,7 +114,10 @@ struct thread_struct { unsigned int acrs[NUM_ACRS]; unsigned long ksp; /* kernel stack pointer */ unsigned long user_timer; /* task cputime in user space */ + unsigned long guest_timer; /* task cputime in kvm guest */ unsigned long system_timer; /* task cputime in kernel space */ + unsigned long hardirq_timer; /* task cputime in hardirq context */ + unsigned long softirq_timer; /* task cputime in softirq context */ unsigned long sys_call_table; /* system call table address */ mm_segment_t mm_segment; unsigned long gmap_addr; /* address of last gmap fault. */ @@ -197,10 +203,12 @@ struct stack_frame { struct task_struct; struct mm_struct; struct seq_file; +struct pt_regs; typedef int (*dump_trace_func_t)(void *data, unsigned long address, int reliable); void dump_trace(dump_trace_func_t func, void *data, struct task_struct *task, unsigned long sp); +void show_registers(struct pt_regs *regs); void show_cacheinfo(struct seq_file *m); @@ -358,12 +366,12 @@ extern void (*s390_base_ext_handler_fn)(void); extern int memcpy_real(void *, void *, size_t); extern void memcpy_absolute(void *, void *, size_t); -#define mem_assign_absolute(dest, val) { \ +#define mem_assign_absolute(dest, val) do { \ __typeof__(dest) __tmp = (val); \ \ BUILD_BUG_ON(sizeof(__tmp) != sizeof(val)); \ memcpy_absolute(&(dest), &__tmp, sizeof(__tmp)); \ -} +} while (0) #endif /* __ASSEMBLY__ */ diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 8db92a5b3bf1..ace3bd315438 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -101,7 +101,12 @@ struct zpci_report_error_header { u8 data[0]; /* Subsequent Data passed verbatim to SCLP ET 24 */ } __packed; -int _sclp_get_core_info_early(struct sclp_core_info *info); +int sclp_early_get_core_info(struct sclp_core_info *info); +void sclp_early_get_ipl_info(struct sclp_ipl_info *info); +void sclp_early_detect(void); +void sclp_early_printk(const char *s); +void __sclp_early_printk(const char *s, unsigned int len); + int _sclp_get_core_info(struct sclp_core_info *info); int sclp_core_configure(u8 core); int sclp_core_deconfigure(u8 core); @@ -110,20 +115,17 @@ int sclp_sdias_copy(void *dest, int blk_num, int nr_blks); int sclp_chp_configure(struct chp_id chpid); int sclp_chp_deconfigure(struct chp_id chpid); int sclp_chp_read_info(struct sclp_chp_info *info); -void sclp_get_ipl_info(struct sclp_ipl_info *info); int sclp_pci_configure(u32 fid); int sclp_pci_deconfigure(u32 fid); int sclp_pci_report(struct zpci_report_error_header *report, u32 fh, u32 fid); int memcpy_hsa_kernel(void *dest, unsigned long src, size_t count); int memcpy_hsa_user(void __user *dest, unsigned long src, size_t count); -void sclp_early_detect(void); -void _sclp_print_early(const char *); void sclp_ocf_cpc_name_copy(char *dst); static inline int sclp_get_core_info(struct sclp_core_info *info, int early) { if (early) - return _sclp_get_core_info_early(info); + return sclp_early_get_core_info(info); return _sclp_get_core_info(info); } diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index 5e8d57e1cc5e..30bdb5a027f3 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -30,6 +30,7 @@ #define MACHINE_FLAG_TLB_LC _BITUL(12) #define MACHINE_FLAG_VX _BITUL(13) #define MACHINE_FLAG_CAD _BITUL(14) +#define MACHINE_FLAG_NX _BITUL(15) #define LPP_MAGIC _BITUL(31) #define LPP_PFAULT_PID_MASK _AC(0xffffffff, UL) @@ -58,9 +59,6 @@ extern void detect_memory_memblock(void); #define MACHINE_HAS_DIAG9C (S390_lowcore.machine_flags & MACHINE_FLAG_DIAG9C) #define MACHINE_HAS_ESOP (S390_lowcore.machine_flags & MACHINE_FLAG_ESOP) -#define MACHINE_HAS_PFMF MACHINE_HAS_EDAT1 -#define MACHINE_HAS_HPAGE MACHINE_HAS_EDAT1 - #define MACHINE_HAS_IDTE (S390_lowcore.machine_flags & MACHINE_FLAG_IDTE) #define MACHINE_HAS_DIAG44 (S390_lowcore.machine_flags & MACHINE_FLAG_DIAG44) #define MACHINE_HAS_EDAT1 (S390_lowcore.machine_flags & MACHINE_FLAG_EDAT1) @@ -71,6 +69,7 @@ extern void detect_memory_memblock(void); #define MACHINE_HAS_TLB_LC (S390_lowcore.machine_flags & MACHINE_FLAG_TLB_LC) #define MACHINE_HAS_VX (S390_lowcore.machine_flags & MACHINE_FLAG_VX) #define MACHINE_HAS_CAD (S390_lowcore.machine_flags & MACHINE_FLAG_CAD) +#define MACHINE_HAS_NX (S390_lowcore.machine_flags & MACHINE_FLAG_NX) /* * Console mode. Override with conmode= diff --git a/arch/s390/include/asm/spinlock.h b/arch/s390/include/asm/spinlock.h index 7ecd8902a5c3..ffc45048ea7d 100644 --- a/arch/s390/include/asm/spinlock.h +++ b/arch/s390/include/asm/spinlock.h @@ -63,7 +63,7 @@ static inline int arch_spin_value_unlocked(arch_spinlock_t lock) static inline int arch_spin_is_locked(arch_spinlock_t *lp) { - return ACCESS_ONCE(lp->lock) != 0; + return READ_ONCE(lp->lock) != 0; } static inline int arch_spin_trylock_once(arch_spinlock_t *lp) diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h index de8298800722..354344dcc198 100644 --- a/arch/s390/include/asm/timex.h +++ b/arch/s390/include/asm/timex.h @@ -178,14 +178,6 @@ int get_phys_clock(unsigned long long *clock); void init_cpu_timer(void); unsigned long long monotonic_clock(void); -void tod_to_timeval(__u64 todval, struct timespec64 *xt); - -static inline -void stck_to_timespec64(unsigned long long stck, struct timespec64 *ts) -{ - tod_to_timeval(stck - TOD_UNIX_EPOCH, ts); -} - extern u64 sched_clock_base_cc; /** diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index f82b04e85a21..136932ff4250 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -14,6 +14,7 @@ */ #include <linux/sched.h> #include <linux/errno.h> +#include <asm/processor.h> #include <asm/ctl_reg.h> #define VERIFY_READ 0 @@ -36,18 +37,20 @@ #define get_ds() (KERNEL_DS) #define get_fs() (current->thread.mm_segment) +#define segment_eq(a,b) ((a).ar4 == (b).ar4) -#define set_fs(x) \ -{ \ - unsigned long __pto; \ - current->thread.mm_segment = (x); \ - __pto = current->thread.mm_segment.ar4 ? \ - S390_lowcore.user_asce : S390_lowcore.kernel_asce; \ - __ctl_load(__pto, 7, 7); \ +static inline void set_fs(mm_segment_t fs) +{ + current->thread.mm_segment = fs; + if (segment_eq(fs, KERNEL_DS)) { + set_cpu_flag(CIF_ASCE_SECONDARY); + __ctl_load(S390_lowcore.kernel_asce, 7, 7); + } else { + clear_cpu_flag(CIF_ASCE_SECONDARY); + __ctl_load(S390_lowcore.user_asce, 7, 7); + } } -#define segment_eq(a,b) ((a).ar4 == (b).ar4) - static inline int __range_ok(unsigned long addr, unsigned long size) { return 1; @@ -177,7 +180,7 @@ static inline int __put_user_fn(void *x, void __user *ptr, unsigned long size) (unsigned long *)x, size, spec); break; - }; + } return rc; } @@ -207,7 +210,7 @@ static inline int __get_user_fn(void *x, const void __user *ptr, unsigned long s (unsigned long __user *)ptr, size, spec); break; - }; + } return rc; } diff --git a/arch/s390/include/uapi/asm/Kbuild b/arch/s390/include/uapi/asm/Kbuild index bf736e764cb4..6848ba5c1454 100644 --- a/arch/s390/include/uapi/asm/Kbuild +++ b/arch/s390/include/uapi/asm/Kbuild @@ -24,6 +24,7 @@ header-y += mman.h header-y += monwriter.h header-y += msgbuf.h header-y += param.h +header-y += pkey.h header-y += poll.h header-y += posix_types.h header-y += ptrace.h diff --git a/arch/s390/include/uapi/asm/pkey.h b/arch/s390/include/uapi/asm/pkey.h new file mode 100644 index 000000000000..ed7f19c27ce5 --- /dev/null +++ b/arch/s390/include/uapi/asm/pkey.h @@ -0,0 +1,112 @@ +/* + * Userspace interface to the pkey device driver + * + * Copyright IBM Corp. 2017 + * + * Author: Harald Freudenberger <freude@de.ibm.com> + * + */ + +#ifndef _UAPI_PKEY_H +#define _UAPI_PKEY_H + +#include <linux/ioctl.h> +#include <linux/types.h> + +/* + * Ioctl calls supported by the pkey device driver + */ + +#define PKEY_IOCTL_MAGIC 'p' + +#define SECKEYBLOBSIZE 64 /* secure key blob size is always 64 bytes */ +#define MAXPROTKEYSIZE 64 /* a protected key blob may be up to 64 bytes */ +#define MAXCLRKEYSIZE 32 /* a clear key value may be up to 32 bytes */ + +/* defines for the type field within the pkey_protkey struct */ +#define PKEY_KEYTYPE_AES_128 1 +#define PKEY_KEYTYPE_AES_192 2 +#define PKEY_KEYTYPE_AES_256 3 + +/* Struct to hold a secure key blob */ +struct pkey_seckey { + __u8 seckey[SECKEYBLOBSIZE]; /* the secure key blob */ +}; + +/* Struct to hold protected key and length info */ +struct pkey_protkey { + __u32 type; /* key type, one of the PKEY_KEYTYPE values */ + __u32 len; /* bytes actually stored in protkey[] */ + __u8 protkey[MAXPROTKEYSIZE]; /* the protected key blob */ +}; + +/* Struct to hold a clear key value */ +struct pkey_clrkey { + __u8 clrkey[MAXCLRKEYSIZE]; /* 16, 24, or 32 byte clear key value */ +}; + +/* + * Generate secure key + */ +struct pkey_genseck { + __u16 cardnr; /* in: card to use or FFFF for any */ + __u16 domain; /* in: domain or FFFF for any */ + __u32 keytype; /* in: key type to generate */ + struct pkey_seckey seckey; /* out: the secure key blob */ +}; +#define PKEY_GENSECK _IOWR(PKEY_IOCTL_MAGIC, 0x01, struct pkey_genseck) + +/* + * Construct secure key from clear key value + */ +struct pkey_clr2seck { + __u16 cardnr; /* in: card to use or FFFF for any */ + __u16 domain; /* in: domain or FFFF for any */ + __u32 keytype; /* in: key type to generate */ + struct pkey_clrkey clrkey; /* in: the clear key value */ + struct pkey_seckey seckey; /* out: the secure key blob */ +}; +#define PKEY_CLR2SECK _IOWR(PKEY_IOCTL_MAGIC, 0x02, struct pkey_clr2seck) + +/* + * Fabricate protected key from a secure key + */ +struct pkey_sec2protk { + __u16 cardnr; /* in: card to use or FFFF for any */ + __u16 domain; /* in: domain or FFFF for any */ + struct pkey_seckey seckey; /* in: the secure key blob */ + struct pkey_protkey protkey; /* out: the protected key */ +}; +#define PKEY_SEC2PROTK _IOWR(PKEY_IOCTL_MAGIC, 0x03, struct pkey_sec2protk) + +/* + * Fabricate protected key from an clear key value + */ +struct pkey_clr2protk { + __u32 keytype; /* in: key type to generate */ + struct pkey_clrkey clrkey; /* in: the clear key value */ + struct pkey_protkey protkey; /* out: the protected key */ +}; +#define PKEY_CLR2PROTK _IOWR(PKEY_IOCTL_MAGIC, 0x04, struct pkey_clr2protk) + +/* + * Search for matching crypto card based on the Master Key + * Verification Pattern provided inside a secure key. + */ +struct pkey_findcard { + struct pkey_seckey seckey; /* in: the secure key blob */ + __u16 cardnr; /* out: card number */ + __u16 domain; /* out: domain number */ +}; +#define PKEY_FINDCARD _IOWR(PKEY_IOCTL_MAGIC, 0x05, struct pkey_findcard) + +/* + * Combined together: findcard + sec2prot + */ +struct pkey_skey2pkey { + struct pkey_seckey seckey; /* in: the secure key blob */ + struct pkey_protkey protkey; /* out: the protected key */ +}; +#define PKEY_SKEY2PKEY _IOWR(PKEY_IOCTL_MAGIC, 0x06, struct pkey_skey2pkey) + +#endif /* _UAPI_PKEY_H */ diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 36b5101c8606..060ce548fe8b 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -10,31 +10,25 @@ CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) # Do not trace early setup code CFLAGS_REMOVE_als.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_early.o = $(CC_FLAGS_FTRACE) -CFLAGS_REMOVE_sclp.o = $(CC_FLAGS_FTRACE) endif GCOV_PROFILE_als.o := n GCOV_PROFILE_early.o := n -GCOV_PROFILE_sclp.o := n KCOV_INSTRUMENT_als.o := n KCOV_INSTRUMENT_early.o := n -KCOV_INSTRUMENT_sclp.o := n UBSAN_SANITIZE_als.o := n UBSAN_SANITIZE_early.o := n -UBSAN_SANITIZE_sclp.o := n # -# Use -march=z900 for sclp.c and als.c to be able to print an error +# Use -march=z900 for als.c to be able to print an error # message if the kernel is started on a machine which is too old # ifneq ($(CC_FLAGS_MARCH),-march=z900) CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH) CFLAGS_als.o += -march=z900 -CFLAGS_REMOVE_sclp.o += $(CC_FLAGS_MARCH) -CFLAGS_sclp.o += -march=z900 AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH) AFLAGS_head.o += -march=z900 endif @@ -61,7 +55,7 @@ CFLAGS_sysinfo.o += -w obj-y := traps.o time.o process.o base.o early.o setup.o idle.o vtime.o obj-y += processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o -obj-y += debug.o irq.o ipl.o dis.o diag.o sclp.o vdso.o als.o +obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o als.o obj-y += sysinfo.o jump_label.o lgr.o os_info.o machine_kexec.o pgm_check.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o obj-y += entry.o reipl.o relocate_kernel.o @@ -76,7 +70,7 @@ obj-$(CONFIG_AUDIT) += audit.o compat-obj-$(CONFIG_AUDIT) += compat_audit.o obj-$(CONFIG_COMPAT) += compat_linux.o compat_signal.o obj-$(CONFIG_COMPAT) += compat_wrapper.o $(compat-obj-y) - +obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o diff --git a/arch/s390/kernel/als.c b/arch/s390/kernel/als.c index a16e9d1bf9e3..14769eb52a33 100644 --- a/arch/s390/kernel/als.c +++ b/arch/s390/kernel/als.c @@ -41,7 +41,8 @@ static void __init print_machine_type(void) get_cpu_id(&id); u16_to_hex(type_str, id.machine); strcat(mach_str, type_str); - _sclp_print_early(mach_str); + strcat(mach_str, "\n"); + sclp_early_printk(mach_str); } static void __init u16_to_decimal(char *str, u16 val) @@ -79,7 +80,8 @@ static void __init print_missing_facilities(void) * z/VM adds a four character prefix. */ if (strlen(als_str) > 70) { - _sclp_print_early(als_str); + strcat(als_str, "\n"); + sclp_early_printk(als_str); *als_str = '\0'; } u16_to_decimal(val_str, i * BITS_PER_LONG + j); @@ -87,13 +89,14 @@ static void __init print_missing_facilities(void) first = 0; } } - _sclp_print_early(als_str); - _sclp_print_early("See Principles of Operations for facility bits"); + strcat(als_str, "\n"); + sclp_early_printk(als_str); + sclp_early_printk("See Principles of Operations for facility bits\n"); } static void __init facility_mismatch(void) { - _sclp_print_early("The Linux kernel requires more recent processor hardware"); + sclp_early_printk("The Linux kernel requires more recent processor hardware\n"); print_machine_type(); print_missing_facilities(); disabled_wait(0x8badcccc); diff --git a/arch/s390/kernel/compat_linux.c b/arch/s390/kernel/compat_linux.c index 96df4547377a..a3d14161abcb 100644 --- a/arch/s390/kernel/compat_linux.c +++ b/arch/s390/kernel/compat_linux.c @@ -28,7 +28,6 @@ #include <linux/shm.h> #include <linux/uio.h> #include <linux/quota.h> -#include <linux/module.h> #include <linux/poll.h> #include <linux/personality.h> #include <linux/stat.h> diff --git a/arch/s390/kernel/cpcmd.c b/arch/s390/kernel/cpcmd.c index 7f48e568ac64..9f0e4a2785f7 100644 --- a/arch/s390/kernel/cpcmd.c +++ b/arch/s390/kernel/cpcmd.c @@ -9,7 +9,7 @@ #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/stddef.h> diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index f9293bfefb7f..dd1d5c62c374 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -8,7 +8,8 @@ #include <linux/crash_dump.h> #include <asm/lowcore.h> #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/init.h> +#include <linux/mm.h> #include <linux/gfp.h> #include <linux/slab.h> #include <linux/bootmem.h> @@ -31,6 +32,7 @@ static struct memblock_type oldmem_type = { .max = 1, .total_size = 0, .regions = &oldmem_region, + .name = "oldmem", }; struct save_area { @@ -329,7 +331,11 @@ static void *nt_init_name(void *buf, Elf64_Word type, void *desc, int d_len, static inline void *nt_init(void *buf, Elf64_Word type, void *desc, int d_len) { - return nt_init_name(buf, type, desc, d_len, KEXEC_CORE_NOTE_NAME); + const char *note_name = "LINUX"; + + if (type == NT_PRPSINFO || type == NT_PRSTATUS || type == NT_PRFPREG) + note_name = KEXEC_CORE_NOTE_NAME; + return nt_init_name(buf, type, desc, d_len, note_name); } /* diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index 79f8ae933520..530226b6cb19 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -20,7 +20,7 @@ #include <linux/string.h> #include <linux/sysctl.h> #include <linux/uaccess.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/fs.h> #include <linux/debugfs.h> @@ -866,7 +866,7 @@ static inline void debug_finish_entry(debug_info_t * id, debug_entry_t* active, int level, int exception) { - active->id.stck = get_tod_clock_fast(); + active->id.stck = get_tod_clock_fast() - sched_clock_base_cc; active->id.fields.cpuid = smp_processor_id(); active->caller = __builtin_return_address(0); active->id.fields.exception = exception; @@ -1455,23 +1455,24 @@ int debug_dflt_header_fn(debug_info_t * id, struct debug_view *view, int area, debug_entry_t * entry, char *out_buf) { - struct timespec64 time_spec; + unsigned long sec, usec; char *except_str; unsigned long caller; int rc = 0; unsigned int level; level = entry->id.fields.level; - stck_to_timespec64(entry->id.stck, &time_spec); + sec = (entry->id.stck >> 12) + (sched_clock_base_cc >> 12); + sec = sec - (TOD_UNIX_EPOCH >> 12); + usec = do_div(sec, USEC_PER_SEC); if (entry->id.fields.exception) except_str = "*"; else except_str = "-"; caller = (unsigned long) entry->caller; - rc += sprintf(out_buf, "%02i %011lld:%06lu %1u %1s %02i %p ", - area, (long long)time_spec.tv_sec, - time_spec.tv_nsec / 1000, level, except_str, + rc += sprintf(out_buf, "%02i %011ld:%06lu %1u %1s %02i %p ", + area, sec, usec, level, except_str, entry->id.fields.cpuid, (void *)caller); return rc; } diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index a97354c8c667..ac6abcd3fe6a 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -5,7 +5,8 @@ * Author(s): Michael Holzheu <holzheu@de.ibm.com> */ -#include <linux/module.h> +#include <linux/export.h> +#include <linux/init.h> #include <linux/cpu.h> #include <linux/seq_file.h> #include <linux/debugfs.h> diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index 9f017cf417f6..f7e82302a71e 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -16,7 +16,7 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/delay.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/kallsyms.h> #include <linux/reboot.h> #include <linux/kprobes.h> diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 324f1c147a41..4e65c79cc5f2 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -354,6 +354,10 @@ static __init void detect_machine_facilities(void) S390_lowcore.machine_flags |= MACHINE_FLAG_VX; __ctl_set_bit(0, 17); } + if (test_facility(130)) { + S390_lowcore.machine_flags |= MACHINE_FLAG_NX; + __ctl_set_bit(0, 20); + } } static inline void save_vector_registers(void) @@ -364,6 +368,18 @@ static inline void save_vector_registers(void) #endif } +static int __init topology_setup(char *str) +{ + bool enabled; + int rc; + + rc = kstrtobool(str, &enabled); + if (!rc && !enabled) + S390_lowcore.machine_flags &= ~MACHINE_HAS_TOPOLOGY; + return rc; +} +early_param("topology", topology_setup); + static int __init disable_vector_extension(char *str) { S390_lowcore.machine_flags &= ~MACHINE_FLAG_VX; @@ -372,6 +388,21 @@ static int __init disable_vector_extension(char *str) } early_param("novx", disable_vector_extension); +static int __init noexec_setup(char *str) +{ + bool enabled; + int rc; + + rc = kstrtobool(str, &enabled); + if (!rc && !enabled) { + /* Disable no-execute support */ + S390_lowcore.machine_flags &= ~MACHINE_FLAG_NX; + __ctl_clear_bit(0, 20); + } + return rc; +} +early_param("noexec", noexec_setup); + static int __init cad_setup(char *str) { int val; diff --git a/arch/s390/kernel/early_printk.c b/arch/s390/kernel/early_printk.c new file mode 100644 index 000000000000..819cb15c67e8 --- /dev/null +++ b/arch/s390/kernel/early_printk.c @@ -0,0 +1,35 @@ +/* + * Copyright IBM Corp. 2017 + */ + +#include <linux/console.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <asm/sclp.h> + +static void sclp_early_write(struct console *con, const char *s, unsigned int len) +{ + __sclp_early_printk(s, len); +} + +static struct console sclp_early_console = { + .name = "earlysclp", + .write = sclp_early_write, + .flags = CON_PRINTBUFFER | CON_BOOT, + .index = -1, +}; + +static int __init setup_early_printk(char *buf) +{ + if (early_console) + return 0; + /* Accept only "earlyprintk" and "earlyprintk=sclp" */ + if (buf && strncmp(buf, "sclp", 4)) + return 0; + if (!sclp.has_linemode && !sclp.has_vt220) + return 0; + early_console = &sclp_early_console; + register_console(early_console); + return 0; +} +early_param("earlyprintk", setup_early_printk); diff --git a/arch/s390/kernel/ebcdic.c b/arch/s390/kernel/ebcdic.c index b971c6be6298..1d5392b36ad8 100644 --- a/arch/s390/kernel/ebcdic.c +++ b/arch/s390/kernel/ebcdic.c @@ -8,8 +8,8 @@ * Martin Peschke <peschke@fh-brandenburg.de> */ -#include <linux/module.h> -#include <asm/types.h> +#include <linux/types.h> +#include <linux/export.h> #include <asm/ebcdic.h> /* diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 97298c58b2be..dff2152350a7 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -50,7 +50,8 @@ _TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ _TIF_UPROBE) _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ _TIF_SYSCALL_TRACEPOINT) -_CIF_WORK = (_CIF_MCCK_PENDING | _CIF_ASCE | _CIF_FPU) +_CIF_WORK = (_CIF_MCCK_PENDING | _CIF_ASCE_PRIMARY | \ + _CIF_ASCE_SECONDARY | _CIF_FPU) _PIF_WORK = (_PIF_PER_TRAP) #define BASED(name) name-cleanup_critical(%r13) @@ -103,8 +104,7 @@ _PIF_WORK = (_PIF_PER_TRAP) CHECK_STACK 1<<STACK_SHIFT,\savearea aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) j 3f -1: LAST_BREAK %r14 - UPDATE_VTIME %r14,%r15,\timer +1: UPDATE_VTIME %r14,%r15,\timer 2: lg %r15,__LC_ASYNC_STACK # load async stack 3: la %r11,STACK_FRAME_OVERHEAD(%r15) .endm @@ -121,18 +121,6 @@ _PIF_WORK = (_PIF_PER_TRAP) mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer .endm - .macro LAST_BREAK scratch - srag \scratch,%r10,23 -#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES - jz .+10 - stg %r10,__TASK_thread+__THREAD_last_break(%r12) -#else - jz .+14 - lghi \scratch,__TASK_thread - stg %r10,__THREAD_last_break(\scratch,%r12) -#endif - .endm - .macro REENABLE_IRQS stg %r8,__LC_RETURN_PSW ni __LC_RETURN_PSW,0xbf @@ -278,15 +266,14 @@ ENTRY(system_call) stpt __LC_SYNC_ENTER_TIMER .Lsysc_stmg: stmg %r8,%r15,__LC_SAVE_AREA_SYNC - lg %r10,__LC_LAST_BREAK lg %r12,__LC_CURRENT + lghi %r13,__TASK_thread lghi %r14,_PIF_SYSCALL .Lsysc_per: lg %r15,__LC_KERNEL_STACK la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs - LAST_BREAK %r13 .Lsysc_vtime: - UPDATE_VTIME %r10,%r13,__LC_SYNC_ENTER_TIMER + UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER stmg %r0,%r7,__PT_R0(%r11) mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC mvc __PT_PSW(16,%r11),__LC_SVC_OLD_PSW @@ -294,12 +281,7 @@ ENTRY(system_call) stg %r14,__PT_FLAGS(%r11) .Lsysc_do_svc: # load address of system call table -#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES - lg %r10,__TASK_thread+__THREAD_sysc_table(%r12) -#else - lghi %r13,__TASK_thread lg %r10,__THREAD_sysc_table(%r13,%r12) -#endif llgh %r8,__PT_INT_CODE+2(%r11) slag %r8,%r8,2 # shift and test for svc 0 jnz .Lsysc_nr_ok @@ -358,8 +340,8 @@ ENTRY(system_call) jo .Lsysc_notify_resume TSTMSK __LC_CPU_FLAGS,_CIF_FPU jo .Lsysc_vxrs - TSTMSK __LC_CPU_FLAGS,_CIF_ASCE - jo .Lsysc_uaccess + TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) + jnz .Lsysc_asce j .Lsysc_return # beware of critical section cleanup # @@ -377,12 +359,15 @@ ENTRY(system_call) jg s390_handle_mcck # TIF bit will be cleared by handler # -# _CIF_ASCE is set, load user space asce +# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce # -.Lsysc_uaccess: - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE +.Lsysc_asce: + ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY lctlg %c1,%c1,__LC_USER_ASCE # load primary asce - j .Lsysc_return + TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_SECONDARY + jz .Lsysc_return + larl %r14,.Lsysc_return + jg set_fs_fixup # # CIF_FPU is set, restore floating-point controls and floating-point registers. @@ -399,13 +384,11 @@ ENTRY(system_call) brasl %r14,do_signal TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL jno .Lsysc_return +.Lsysc_do_syscall: + lghi %r13,__TASK_thread lmg %r2,%r7,__PT_R2(%r11) # load svc arguments - lghi %r8,0 # svc 0 returns -ENOSYS - llgh %r1,__PT_INT_CODE+2(%r11) # load new svc number - cghi %r1,NR_syscalls - jnl .Lsysc_nr_ok # invalid svc number -> do svc 0 - slag %r8,%r1,2 - j .Lsysc_nr_ok # restart svc + lghi %r1,0 # svc 0 returns -ENOSYS + j .Lsysc_do_svc # # _TIF_NOTIFY_RESUME is set, call do_notify_resume @@ -508,8 +491,7 @@ ENTRY(pgm_check_handler) 1: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) j 3f -2: LAST_BREAK %r14 - UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER +2: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER lg %r15,__LC_KERNEL_STACK lgr %r14,%r12 aghi %r14,__TASK_thread # pointer to thread_struct @@ -518,6 +500,7 @@ ENTRY(pgm_check_handler) jz 3f mvc __THREAD_trap_tdb(256,%r14),0(%r13) 3: la %r11,STACK_FRAME_OVERHEAD(%r15) + stg %r10,__THREAD_last_break(%r14) stmg %r0,%r7,__PT_R0(%r11) mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC stmg %r8,%r9,__PT_PSW(%r11) @@ -547,6 +530,8 @@ ENTRY(pgm_check_handler) LOCKDEP_SYS_EXIT tm __PT_PSW+1(%r11),0x01 # returning to user ? jno .Lsysc_restore + TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL + jo .Lsysc_do_syscall j .Lsysc_tif # @@ -564,6 +549,7 @@ ENTRY(pgm_check_handler) # .Lpgm_svcper: mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW + lghi %r13,__TASK_thread larl %r14,.Lsysc_per stg %r14,__LC_RETURN_PSW+8 lghi %r14,_PIF_SYSCALL | _PIF_PER_TRAP @@ -576,7 +562,6 @@ ENTRY(io_int_handler) STCK __LC_INT_CLOCK stpt __LC_ASYNC_ENTER_TIMER stmg %r8,%r15,__LC_SAVE_AREA_ASYNC - lg %r10,__LC_LAST_BREAK lg %r12,__LC_CURRENT larl %r13,cleanup_critical lmg %r8,%r9,__LC_IO_OLD_PSW @@ -680,8 +665,8 @@ ENTRY(io_int_handler) jo .Lio_notify_resume TSTMSK __LC_CPU_FLAGS,_CIF_FPU jo .Lio_vxrs - TSTMSK __LC_CPU_FLAGS,_CIF_ASCE - jo .Lio_uaccess + TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) + jnz .Lio_asce j .Lio_return # beware of critical section cleanup # @@ -694,12 +679,15 @@ ENTRY(io_int_handler) j .Lio_return # -# _CIF_ASCE is set, load user space asce +# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce # -.Lio_uaccess: - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE +.Lio_asce: + ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY lctlg %c1,%c1,__LC_USER_ASCE # load primary asce - j .Lio_return + TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_SECONDARY + jz .Lio_return + larl %r14,.Lio_return + jg set_fs_fixup # # CIF_FPU is set, restore floating-point controls and floating-point registers. @@ -750,7 +738,6 @@ ENTRY(ext_int_handler) STCK __LC_INT_CLOCK stpt __LC_ASYNC_ENTER_TIMER stmg %r8,%r15,__LC_SAVE_AREA_ASYNC - lg %r10,__LC_LAST_BREAK lg %r12,__LC_CURRENT larl %r13,cleanup_critical lmg %r8,%r9,__LC_EXT_OLD_PSW @@ -893,7 +880,6 @@ ENTRY(mcck_int_handler) la %r1,4095 # revalidate r1 spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # revalidate cpu timer lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs - lg %r10,__LC_LAST_BREAK lg %r12,__LC_CURRENT larl %r13,cleanup_critical lmg %r8,%r9,__LC_MCK_OLD_PSW @@ -1088,9 +1074,10 @@ cleanup_critical: 0: # check if base register setup + TIF bit load has been done clg %r9,BASED(.Lcleanup_system_call_insn+16) jhe 0f - # set up saved registers r10 and r12 - stg %r10,16(%r11) # r10 last break - stg %r12,32(%r11) # r12 task struct pointer + # set up saved register r12 task struct pointer + stg %r12,32(%r11) + # set up saved register r13 __TASK_thread offset + mvc 40(8,%r11),BASED(.Lcleanup_system_call_const) 0: # check if the user time update has been done clg %r9,BASED(.Lcleanup_system_call_insn+24) jh 0f @@ -1107,14 +1094,7 @@ cleanup_critical: stg %r15,__LC_SYSTEM_TIMER 0: # update accounting time stamp mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER - # do LAST_BREAK - lg %r9,16(%r11) - srag %r9,%r9,23 - jz 0f - lgr %r9,%r12 - aghi %r9,__TASK_thread - mvc __THREAD_last_break(8,%r9),16(%r11) -0: # set up saved register r11 + # set up saved register r11 lg %r15,__LC_KERNEL_STACK la %r9,STACK_FRAME_OVERHEAD(%r15) stg %r9,24(%r11) # r11 pt_regs pointer @@ -1136,6 +1116,8 @@ cleanup_critical: .quad .Lsysc_per .quad .Lsysc_vtime+36 .quad .Lsysc_vtime+42 +.Lcleanup_system_call_const: + .quad __TASK_thread .Lcleanup_sysc_tif: larl %r9,.Lsysc_tif diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index e79f030dd276..33f901865326 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -80,5 +80,6 @@ long sys_s390_pci_mmio_read(unsigned long, void __user *, size_t); DECLARE_PER_CPU(u64, mt_cycles[8]); void verify_facilities(void); +void set_fs_fixup(void); #endif /* _ENTRY_H */ diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 7a55c29b0b33..fb07a70820af 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -12,7 +12,7 @@ #include <linux/notifier.h> #include <linux/init.h> #include <linux/cpu.h> -#include <asm/cputime.h> +#include <linux/cputime.h> #include <asm/nmi.h> #include <asm/smp.h> #include "entry.h" @@ -43,7 +43,7 @@ void enabled_wait(void) idle->clock_idle_enter = idle->clock_idle_exit = 0ULL; idle->idle_time += idle_time; idle->idle_count++; - account_idle_time(idle_time); + account_idle_time(cputime_to_nsecs(idle_time)); write_seqcount_end(&idle->seqcount); } NOKPROBE_SYMBOL(enabled_wait); @@ -57,8 +57,8 @@ static ssize_t show_idle_count(struct device *dev, do { seq = read_seqcount_begin(&idle->seqcount); - idle_count = ACCESS_ONCE(idle->idle_count); - if (ACCESS_ONCE(idle->clock_idle_enter)) + idle_count = READ_ONCE(idle->idle_count); + if (READ_ONCE(idle->clock_idle_enter)) idle_count++; } while (read_seqcount_retry(&idle->seqcount, seq)); return sprintf(buf, "%llu\n", idle_count); @@ -75,16 +75,16 @@ static ssize_t show_idle_time(struct device *dev, do { now = get_tod_clock(); seq = read_seqcount_begin(&idle->seqcount); - idle_time = ACCESS_ONCE(idle->idle_time); - idle_enter = ACCESS_ONCE(idle->clock_idle_enter); - idle_exit = ACCESS_ONCE(idle->clock_idle_exit); + idle_time = READ_ONCE(idle->idle_time); + idle_enter = READ_ONCE(idle->clock_idle_enter); + idle_exit = READ_ONCE(idle->clock_idle_exit); } while (read_seqcount_retry(&idle->seqcount, seq)); idle_time += idle_enter ? ((idle_exit ? : now) - idle_enter) : 0; return sprintf(buf, "%llu\n", idle_time >> 12); } DEVICE_ATTR(idle_time_us, 0444, show_idle_time, NULL); -cputime64_t arch_cpu_idle_time(int cpu) +u64 arch_cpu_idle_time(int cpu) { struct s390_idle_data *idle = &per_cpu(s390_idle, cpu); unsigned long long now, idle_enter, idle_exit; @@ -93,10 +93,11 @@ cputime64_t arch_cpu_idle_time(int cpu) do { now = get_tod_clock(); seq = read_seqcount_begin(&idle->seqcount); - idle_enter = ACCESS_ONCE(idle->clock_idle_enter); - idle_exit = ACCESS_ONCE(idle->clock_idle_exit); + idle_enter = READ_ONCE(idle->clock_idle_enter); + idle_exit = READ_ONCE(idle->clock_idle_exit); } while (read_seqcount_retry(&idle->seqcount, seq)); - return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0; + + return cputime_to_nsecs(idle_enter ? ((idle_exit ?: now) - idle_enter) : 0); } void arch_cpu_idle_enter(void) diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index ff3364a067ff..b67dafb7b7cf 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -8,7 +8,8 @@ */ #include <linux/types.h> -#include <linux/module.h> +#include <linux/export.h> +#include <linux/init.h> #include <linux/device.h> #include <linux/delay.h> #include <linux/reboot.h> @@ -1546,7 +1547,8 @@ static void dump_reipl_run(struct shutdown_trigger *trigger) unsigned long ipib = (unsigned long) reipl_block_actual; unsigned int csum; - csum = csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); + csum = (__force unsigned int) + csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); mem_assign_absolute(S390_lowcore.ipib, ipib); mem_assign_absolute(S390_lowcore.ipib_checksum, csum); dump_run(trigger); @@ -1863,7 +1865,7 @@ static int __init s390_ipl_init(void) { char str[8] = {0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40}; - sclp_get_ipl_info(&sclp_ipl_info); + sclp_early_get_ipl_info(&sclp_ipl_info); /* * Fix loadparm: There are systems where the (SCSI) LOADPARM * returned by read SCP info is invalid (contains EBCDIC blanks) diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index ef60f4177331..6dca93b29bed 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -12,11 +12,12 @@ #include <linux/seq_file.h> #include <linux/proc_fs.h> #include <linux/profile.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/kernel.h> #include <linux/ftrace.h> #include <linux/errno.h> #include <linux/slab.h> +#include <linux/init.h> #include <linux/cpu.h> #include <linux/irq.h> #include <asm/irq_regs.h> diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c index 083b05f5f5ab..6aa630a8d24f 100644 --- a/arch/s390/kernel/jump_label.c +++ b/arch/s390/kernel/jump_label.c @@ -4,7 +4,6 @@ * Copyright IBM Corp. 2011 * Author(s): Jan Glauber <jang@linux.vnet.ibm.com> */ -#include <linux/module.h> #include <linux/uaccess.h> #include <linux/stop_machine.h> #include <linux/jump_label.h> diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index 84e0557b16fe..76f9eda1d7c0 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -45,11 +45,17 @@ DEFINE_INSN_CACHE_OPS(dmainsn); static void *alloc_dmainsn_page(void) { - return (void *)__get_free_page(GFP_KERNEL | GFP_DMA); + void *page; + + page = (void *) __get_free_page(GFP_KERNEL | GFP_DMA); + if (page) + set_memory_x((unsigned long) page, 1); + return page; } static void free_dmainsn_page(void *page) { + set_memory_nx((unsigned long) page, 1); free_page((unsigned long)page); } diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index fbc07891f9e7..1a27f307a920 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -45,7 +45,8 @@ void *module_alloc(unsigned long size) if (PAGE_ALIGN(size) > MODULES_LEN) return NULL; return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, + GFP_KERNEL, PAGE_KERNEL_EXEC, + 0, NUMA_NO_NODE, __builtin_return_address(0)); } diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 9862196b4b89..80c093e0c6f1 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -13,7 +13,7 @@ #include <linux/errno.h> #include <linux/hardirq.h> #include <linux/time.h> -#include <linux/module.h> +#include <linux/export.h> #include <asm/lowcore.h> #include <asm/smp.h> #include <asm/stp.h> @@ -116,6 +116,19 @@ static int notrace s390_validate_registers(union mci mci, int umode) s390_handle_damage(); kill_task = 1; } + /* Validate control registers */ + if (!mci.cr) { + /* + * Control registers have unknown contents. + * Can't recover and therefore stopping machine. + */ + s390_handle_damage(); + } else { + asm volatile( + " lctlg 0,15,0(%0)\n" + " ptlb\n" + : : "a" (&S390_lowcore.cregs_save_area) : "memory"); + } if (!mci.fp) { /* * Floating point registers can't be restored. If the @@ -208,18 +221,6 @@ static int notrace s390_validate_registers(union mci mci, int umode) */ kill_task = 1; } - /* Validate control registers */ - if (!mci.cr) { - /* - * Control registers have unknown contents. - * Can't recover and therefore stopping machine. - */ - s390_handle_damage(); - } else { - asm volatile( - " lctlg 0,15,0(%0)" - : : "a" (&S390_lowcore.cregs_save_area) : "memory"); - } /* * We don't even try to validate the TOD register, since we simply * can't write something sensible into that register. diff --git a/arch/s390/kernel/os_info.c b/arch/s390/kernel/os_info.c index 87f05e475ae8..753ba63182b9 100644 --- a/arch/s390/kernel/os_info.c +++ b/arch/s390/kernel/os_info.c @@ -26,7 +26,7 @@ static struct os_info os_info __page_aligned_data; u32 os_info_csum(struct os_info *os_info) { int size = sizeof(*os_info) - offsetof(struct os_info, version_major); - return csum_partial(&os_info->version_major, size, 0); + return (__force u32)csum_partial(&os_info->version_major, size, 0); } /* @@ -46,7 +46,7 @@ void os_info_entry_add(int nr, void *ptr, u64 size) { os_info.entry[nr].addr = (u64)(unsigned long)ptr; os_info.entry[nr].size = size; - os_info.entry[nr].csum = csum_partial(ptr, size, 0); + os_info.entry[nr].csum = (__force u32)csum_partial(ptr, size, 0); os_info.csum = os_info_csum(&os_info); } @@ -93,7 +93,7 @@ static void os_info_old_alloc(int nr, int align) msg = "copy failed"; goto fail_free; } - csum = csum_partial(buf_align, size, 0); + csum = (__force u32)csum_partial(buf_align, size, 0); if (csum != os_info_old->entry[nr].csum) { msg = "checksum failed"; goto fail_free; diff --git a/arch/s390/kernel/perf_cpum_cf_events.c b/arch/s390/kernel/perf_cpum_cf_events.c index 4554a4bae39e..c343ac2cf6c5 100644 --- a/arch/s390/kernel/perf_cpum_cf_events.c +++ b/arch/s390/kernel/perf_cpum_cf_events.c @@ -309,7 +309,7 @@ __init const struct attribute_group **cpumf_cf_event_group(void) default: model = NULL; break; - }; + } if (!model) goto out; diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 400d14f0b9f5..54281660582c 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -23,7 +23,7 @@ #include <linux/compat.h> #include <linux/kprobes.h> #include <linux/random.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/init_task.h> #include <asm/io.h> #include <asm/processor.h> @@ -100,8 +100,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) return 0; } -int copy_thread(unsigned long clone_flags, unsigned long new_stackp, - unsigned long arg, struct task_struct *p) +int copy_thread_tls(unsigned long clone_flags, unsigned long new_stackp, + unsigned long arg, struct task_struct *p, unsigned long tls) { struct fake_frame { @@ -156,7 +156,6 @@ int copy_thread(unsigned long clone_flags, unsigned long new_stackp, /* Set a new TLS ? */ if (clone_flags & CLONE_SETTLS) { - unsigned long tls = frame->childregs.gprs[6]; if (is_compat_task()) { p->thread.acrs[0] = (unsigned int)tls; } else { @@ -234,3 +233,16 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) ret = PAGE_ALIGN(mm->brk + brk_rnd()); return (ret > mm->brk) ? ret : mm->brk; } + +void set_fs_fixup(void) +{ + struct pt_regs *regs = current_pt_regs(); + static bool warned; + + set_fs(USER_DS); + if (warned) + return; + WARN(1, "Unbalanced set_fs - int code: 0x%x\n", regs->int_code); + show_registers(regs); + warned = true; +} diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c index 9e60ef144d03..bc2b60dcb178 100644 --- a/arch/s390/kernel/processor.c +++ b/arch/s390/kernel/processor.c @@ -32,7 +32,7 @@ static bool machine_has_cpu_mhz; void __init cpu_detect_mhz_feature(void) { if (test_facility(34) && __ecag(ECAG_CPU_ATTRIBUTE, 0) != -1UL) - machine_has_cpu_mhz = 1; + machine_has_cpu_mhz = true; } static void update_cpu_mhz(void *arg) @@ -73,7 +73,7 @@ void cpu_init(void) get_cpu_id(id); if (machine_has_cpu_mhz) update_cpu_mhz(NULL); - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; BUG_ON(current->mm); enter_lazy_tlb(&init_mm, current); @@ -92,7 +92,7 @@ static void show_cpu_summary(struct seq_file *m, void *v) { static const char *hwcap_str[] = { "esan3", "zarch", "stfle", "msa", "ldisp", "eimm", "dfp", - "edat", "etf3eh", "highgprs", "te", "vx" + "edat", "etf3eh", "highgprs", "te", "vx", "vxd", "vxe" }; static const char * const int_hwcap_str[] = { "sie" diff --git a/arch/s390/kernel/sclp.c b/arch/s390/kernel/sclp.c deleted file mode 100644 index f08af675f36f..000000000000 --- a/arch/s390/kernel/sclp.c +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright IBM Corp. 2015 - * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> - */ -#include <linux/kernel.h> -#include <asm/ebcdic.h> -#include <asm/irq.h> -#include <asm/lowcore.h> -#include <asm/processor.h> -#include <asm/sclp.h> - -#define EVTYP_VT220MSG_MASK 0x00000040 -#define EVTYP_MSG_MASK 0x40000000 - -static char _sclp_work_area[4096] __aligned(PAGE_SIZE) __section(data); -static bool have_vt220 __section(data); -static bool have_linemode __section(data); - -static void _sclp_wait_int(void) -{ - unsigned long cr0, cr0_new, psw_mask, addr; - psw_t psw_ext_save, psw_wait; - - __ctl_store(cr0, 0, 0); - cr0_new = cr0 | 0x200; - __ctl_load(cr0_new, 0, 0); - - psw_ext_save = S390_lowcore.external_new_psw; - psw_mask = __extract_psw(); - S390_lowcore.external_new_psw.mask = psw_mask; - psw_wait.mask = psw_mask | PSW_MASK_EXT | PSW_MASK_WAIT; - S390_lowcore.ext_int_code = 0; - - do { - asm volatile( - " larl %[addr],0f\n" - " stg %[addr],%[psw_wait_addr]\n" - " stg %[addr],%[psw_ext_addr]\n" - " lpswe %[psw_wait]\n" - "0:\n" - : [addr] "=&d" (addr), - [psw_wait_addr] "=Q" (psw_wait.addr), - [psw_ext_addr] "=Q" (S390_lowcore.external_new_psw.addr) - : [psw_wait] "Q" (psw_wait) - : "cc", "memory"); - } while (S390_lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG); - - __ctl_load(cr0, 0, 0); - S390_lowcore.external_new_psw = psw_ext_save; -} - -static int _sclp_servc(unsigned int cmd, char *sccb) -{ - unsigned int cc; - - do { - asm volatile( - " .insn rre,0xb2200000,%1,%2\n" - " ipm %0\n" - : "=d" (cc) : "d" (cmd), "a" (sccb) - : "cc", "memory"); - cc >>= 28; - if (cc == 3) - return -EINVAL; - _sclp_wait_int(); - } while (cc != 0); - return (*(unsigned short *)(sccb + 6) == 0x20) ? 0 : -EIO; -} - -static int _sclp_setup(int disable) -{ - static unsigned char init_sccb[] = { - 0x00, 0x1c, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x04, - 0x80, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 - }; - unsigned int *masks; - int rc; - - memcpy(_sclp_work_area, init_sccb, 28); - masks = (unsigned int *)(_sclp_work_area + 12); - if (disable) - memset(masks, 0, 16); - /* SCLP write mask */ - rc = _sclp_servc(0x00780005, _sclp_work_area); - if (rc) - return rc; - have_vt220 = masks[2] & EVTYP_VT220MSG_MASK; - have_linemode = masks[2] & EVTYP_MSG_MASK; - return 0; -} - -/* Output multi-line text using SCLP Message interface. */ -static void _sclp_print_lm(const char *str) -{ - static unsigned char write_head[] = { - /* sccb header */ - 0x00, 0x52, /* 0 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 2 */ - /* evbuf */ - 0x00, 0x4a, /* 8 */ - 0x02, 0x00, 0x00, 0x00, /* 10 */ - /* mdb */ - 0x00, 0x44, /* 14 */ - 0x00, 0x01, /* 16 */ - 0xd4, 0xc4, 0xc2, 0x40, /* 18 */ - 0x00, 0x00, 0x00, 0x01, /* 22 */ - /* go */ - 0x00, 0x38, /* 26 */ - 0x00, 0x01, /* 28 */ - 0x00, 0x00, 0x00, 0x00, /* 30 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 34 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 42 */ - 0x00, 0x00, 0x00, 0x00, /* 50 */ - 0x00, 0x00, /* 54 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 56 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 64 */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 72 */ - 0x00, 0x00, /* 80 */ - }; - static unsigned char write_mto[] = { - /* mto */ - 0x00, 0x0a, /* 0 */ - 0x00, 0x04, /* 2 */ - 0x10, 0x00, /* 4 */ - 0x00, 0x00, 0x00, 0x00 /* 6 */ - }; - unsigned char *ptr, ch; - unsigned int count; - - memcpy(_sclp_work_area, write_head, sizeof(write_head)); - ptr = _sclp_work_area + sizeof(write_head); - do { - memcpy(ptr, write_mto, sizeof(write_mto)); - for (count = sizeof(write_mto); (ch = *str++) != 0; count++) { - if (ch == 0x0a) - break; - ptr[count] = _ascebc[ch]; - } - /* Update length fields in mto, mdb, evbuf and sccb */ - *(unsigned short *) ptr = count; - *(unsigned short *)(_sclp_work_area + 14) += count; - *(unsigned short *)(_sclp_work_area + 8) += count; - *(unsigned short *)(_sclp_work_area + 0) += count; - ptr += count; - } while (ch != 0); - - /* SCLP write data */ - _sclp_servc(0x00760005, _sclp_work_area); -} - -/* Output multi-line text (plus a newline) using SCLP VT220 - * interface. - */ -static void _sclp_print_vt220(const char *str) -{ - static unsigned char const write_head[] = { - /* sccb header */ - 0x00, 0x0e, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - /* evbuf header */ - 0x00, 0x06, - 0x1a, 0x00, 0x00, 0x00, - }; - size_t len = strlen(str); - - if (sizeof(write_head) + len >= sizeof(_sclp_work_area)) - len = sizeof(_sclp_work_area) - sizeof(write_head) - 1; - - memcpy(_sclp_work_area, write_head, sizeof(write_head)); - memcpy(_sclp_work_area + sizeof(write_head), str, len); - _sclp_work_area[sizeof(write_head) + len] = '\n'; - - /* Update length fields in evbuf and sccb headers */ - *(unsigned short *)(_sclp_work_area + 8) += len + 1; - *(unsigned short *)(_sclp_work_area + 0) += len + 1; - - /* SCLP write data */ - (void)_sclp_servc(0x00760005, _sclp_work_area); -} - -/* Output one or more lines of text on the SCLP console (VT220 and / - * or line-mode). All lines get terminated; no need for a trailing LF. - */ -void _sclp_print_early(const char *str) -{ - if (_sclp_setup(0) != 0) - return; - if (have_linemode) - _sclp_print_lm(str); - if (have_vt220) - _sclp_print_vt220(str); - _sclp_setup(1); -} diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 865a48871ca4..e4d811f17971 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -636,6 +636,8 @@ static void __init reserve_crashkernel(void) static void __init reserve_initrd(void) { #ifdef CONFIG_BLK_DEV_INITRD + if (!INITRD_START || !INITRD_SIZE) + return; initrd_start = INITRD_START; initrd_end = initrd_start + INITRD_SIZE; memblock_reserve(INITRD_START, INITRD_SIZE); @@ -747,7 +749,7 @@ static int __init setup_hwcaps(void) /* * Huge page support HWCAP_S390_HPAGE is bit 7. */ - if (MACHINE_HAS_HPAGE) + if (MACHINE_HAS_EDAT1) elf_hwcap |= HWCAP_S390_HPAGE; /* @@ -767,8 +769,14 @@ static int __init setup_hwcaps(void) * can be disabled with the "novx" parameter. Use MACHINE_HAS_VX * instead of facility bit 129. */ - if (MACHINE_HAS_VX) + if (MACHINE_HAS_VX) { elf_hwcap |= HWCAP_S390_VXRS; + if (test_facility(134)) + elf_hwcap |= HWCAP_S390_VXRS_EXT; + if (test_facility(135)) + elf_hwcap |= HWCAP_S390_VXRS_BCD; + } + get_cpu_id(&cpu_id); add_device_randomness(&cpu_id, sizeof(cpu_id)); switch (cpu_id.machine) { @@ -820,10 +828,10 @@ static void __init setup_randomness(void) { struct sysinfo_3_2_2 *vmms; - vmms = (struct sysinfo_3_2_2 *) alloc_page(GFP_KERNEL); - if (vmms && stsi(vmms, 3, 2, 2) == 0 && vmms->count) - add_device_randomness(&vmms, vmms->count); - free_page((unsigned long) vmms); + vmms = (struct sysinfo_3_2_2 *) memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (stsi(vmms, 3, 2, 2) == 0 && vmms->count) + add_device_randomness(&vmms->vm, sizeof(vmms->vm[0]) * vmms->count); + memblock_free((unsigned long) vmms, PAGE_SIZE); } /* diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index e49f61aadaf9..d0a74d7ce433 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -20,7 +20,7 @@ #include <linux/workqueue.h> #include <linux/bootmem.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/mm.h> #include <linux/err.h> diff --git a/arch/s390/kernel/stacktrace.c b/arch/s390/kernel/stacktrace.c index 355db9db8210..0085b2d8ed7d 100644 --- a/arch/s390/kernel/stacktrace.c +++ b/arch/s390/kernel/stacktrace.c @@ -8,7 +8,7 @@ #include <linux/sched.h> #include <linux/stacktrace.h> #include <linux/kallsyms.h> -#include <linux/module.h> +#include <linux/export.h> static int __save_address(void *data, unsigned long address, int nosched) { diff --git a/arch/s390/kernel/swsusp.S b/arch/s390/kernel/swsusp.S index 1ff21f05d7dd..6bebc935e9c2 100644 --- a/arch/s390/kernel/swsusp.S +++ b/arch/s390/kernel/swsusp.S @@ -196,7 +196,7 @@ pgm_check_entry: larl %r15,init_thread_union ahi %r15,1<<(PAGE_SHIFT+THREAD_SIZE_ORDER) larl %r2,.Lpanic_string - larl %r3,_sclp_print_early + larl %r3,sclp_early_printk lghi %r1,0 sam31 sigp %r1,%r0,SIGP_SET_ARCHITECTURE @@ -273,7 +273,7 @@ restore_registers: .Ldisabled_wait_31: .long 0x000a0000,0x00000000 .Lpanic_string: - .asciz "Resume not possible because suspend CPU is no longer available" + .asciz "Resume not possible because suspend CPU is no longer available\n" .align 8 .Lrestart_diag308_psw: .long 0x00080000,0x80000000 diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index 24021c1e3ecb..12b6b138e354 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -10,7 +10,7 @@ #include <linux/seq_file.h> #include <linux/init.h> #include <linux/delay.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/slab.h> #include <asm/ebcdic.h> #include <asm/sysinfo.h> diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 52949df88529..de66abb479c9 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -16,7 +16,7 @@ #include <linux/kernel_stat.h> #include <linux/errno.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/sched.h> #include <linux/kernel.h> #include <linux/param.h> @@ -110,7 +110,7 @@ unsigned long long monotonic_clock(void) } EXPORT_SYMBOL(monotonic_clock); -void tod_to_timeval(__u64 todval, struct timespec64 *xt) +static void tod_to_timeval(__u64 todval, struct timespec64 *xt) { unsigned long long sec; @@ -120,7 +120,6 @@ void tod_to_timeval(__u64 todval, struct timespec64 *xt) todval -= (sec * 1000000) << 12; xt->tv_nsec = ((todval * 1000) >> 12); } -EXPORT_SYMBOL(tod_to_timeval); void clock_comparator_work(void) { @@ -492,7 +491,7 @@ static void __init stp_reset(void) pr_warn("The real or virtual hardware system does not provide an STP interface\n"); free_page((unsigned long) stp_page); stp_page = NULL; - stp_online = 0; + stp_online = false; } } diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index 93dcbae1e98d..2cd5f4f1013c 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -38,7 +38,6 @@ static void set_topology_timer(void); static void topology_work_fn(struct work_struct *work); static struct sysinfo_15_1_x *tl_info; -static bool topology_enabled = true; static DECLARE_WORK(topology_work, topology_work_fn); /* @@ -59,7 +58,7 @@ static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu) cpumask_t mask; cpumask_copy(&mask, cpumask_of(cpu)); - if (!topology_enabled || !MACHINE_HAS_TOPOLOGY) + if (!MACHINE_HAS_TOPOLOGY) return mask; for (; info; info = info->next) { if (cpumask_test_cpu(cpu, &info->mask)) @@ -74,7 +73,7 @@ static cpumask_t cpu_thread_map(unsigned int cpu) int i; cpumask_copy(&mask, cpumask_of(cpu)); - if (!topology_enabled || !MACHINE_HAS_TOPOLOGY) + if (!MACHINE_HAS_TOPOLOGY) return mask; cpu -= cpu % (smp_cpu_mtid + 1); for (i = 0; i <= smp_cpu_mtid; i++) @@ -428,12 +427,6 @@ static const struct cpumask *cpu_drawer_mask(int cpu) return &cpu_topology[cpu].drawer_mask; } -static int __init early_parse_topology(char *p) -{ - return kstrtobool(p, &topology_enabled); -} -early_param("topology", early_parse_topology); - static struct sched_domain_topology_level s390_topology[] = { { cpu_thread_mask, cpu_smt_flags, SD_INIT_NAME(SMT) }, { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) }, @@ -461,18 +454,16 @@ static void __init alloc_masks(struct sysinfo_15_1_x *info, void __init topology_init_early(void) { struct sysinfo_15_1_x *info; - int i; set_sched_topology(s390_topology); if (!MACHINE_HAS_TOPOLOGY) goto out; - tl_info = memblock_virt_alloc(sizeof(*tl_info), PAGE_SIZE); + tl_info = memblock_virt_alloc(PAGE_SIZE, PAGE_SIZE); info = tl_info; store_topology(info); - pr_info("The CPU configuration topology of the machine is:"); - for (i = 0; i < TOPOLOGY_NR_MAG; i++) - printk(KERN_CONT " %d", info->mag[i]); - printk(KERN_CONT " / %d\n", info->mnest); + pr_info("The CPU configuration topology of the machine is: %d %d %d %d %d %d / %d\n", + info->mag[0], info->mag[1], info->mag[2], info->mag[3], + info->mag[4], info->mag[5], info->mnest); alloc_masks(info, &socket_info, 1); alloc_masks(info, &book_info, 2); alloc_masks(info, &drawer_info, 3); diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index 5904abf6b1ae..10516ae3b55e 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -9,7 +9,7 @@ * as published by the Free Software Foundation. */ -#include <linux/module.h> +#include <linux/init.h> #include <linux/errno.h> #include <linux/sched.h> #include <linux/kernel.h> diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 3667d20e997f..5ccf95396251 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -44,6 +44,7 @@ SECTIONS *(.gnu.warning) } :text = 0x0700 + . = ALIGN(PAGE_SIZE); _etext = .; /* End of text section */ NOTES :text :note @@ -79,7 +80,13 @@ SECTIONS . = ALIGN(PAGE_SIZE); /* Init code and data */ __init_begin = .; - INIT_TEXT_SECTION(PAGE_SIZE) + . = ALIGN(PAGE_SIZE); + .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) { + VMLINUX_SYMBOL(_sinittext) = . ; + INIT_TEXT + . = ALIGN(PAGE_SIZE); + VMLINUX_SYMBOL(_einittext) = . ; + } /* * .exit.text is discarded at runtime, not link time, diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 1b5c5ee9fc1b..31bd96e81167 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -6,13 +6,13 @@ */ #include <linux/kernel_stat.h> +#include <linux/cputime.h> #include <linux/export.h> #include <linux/kernel.h> #include <linux/timex.h> #include <linux/types.h> #include <linux/time.h> -#include <asm/cputime.h> #include <asm/vtimer.h> #include <asm/vtime.h> #include <asm/cpu_mf.h> @@ -90,14 +90,41 @@ static void update_mt_scaling(void) __this_cpu_write(mt_scaling_jiffies, jiffies_64); } +static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new) +{ + u64 delta; + + delta = new - *tsk_vtime; + *tsk_vtime = new; + return delta; +} + + +static inline u64 scale_vtime(u64 vtime) +{ + u64 mult = __this_cpu_read(mt_scaling_mult); + u64 div = __this_cpu_read(mt_scaling_div); + + if (smp_cpu_mtid) + return vtime * mult / div; + return vtime; +} + +static void account_system_index_scaled(struct task_struct *p, + cputime_t cputime, cputime_t scaled, + enum cpu_usage_stat index) +{ + p->stimescaled += cputime_to_nsecs(scaled); + account_system_index_time(p, cputime_to_nsecs(cputime), index); +} + /* * Update process times based on virtual cpu times stored by entry.S * to the lowcore fields user_timer, system_timer & steal_clock. */ static int do_account_vtime(struct task_struct *tsk) { - u64 timer, clock, user, system, steal; - u64 user_scaled, system_scaled; + u64 timer, clock, user, guest, system, hardirq, softirq, steal; timer = S390_lowcore.last_update_timer; clock = S390_lowcore.last_update_clock; @@ -110,53 +137,76 @@ static int do_account_vtime(struct task_struct *tsk) #endif : "=m" (S390_lowcore.last_update_timer), "=m" (S390_lowcore.last_update_clock)); - S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; - S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock; + clock = S390_lowcore.last_update_clock - clock; + timer -= S390_lowcore.last_update_timer; + + if (hardirq_count()) + S390_lowcore.hardirq_timer += timer; + else + S390_lowcore.system_timer += timer; /* Update MT utilization calculation */ if (smp_cpu_mtid && time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies))) update_mt_scaling(); - user = S390_lowcore.user_timer - tsk->thread.user_timer; - S390_lowcore.steal_timer -= user; - tsk->thread.user_timer = S390_lowcore.user_timer; - - system = S390_lowcore.system_timer - tsk->thread.system_timer; - S390_lowcore.steal_timer -= system; - tsk->thread.system_timer = S390_lowcore.system_timer; - - user_scaled = user; - system_scaled = system; - /* Do MT utilization scaling */ - if (smp_cpu_mtid) { - u64 mult = __this_cpu_read(mt_scaling_mult); - u64 div = __this_cpu_read(mt_scaling_div); + /* Calculate cputime delta */ + user = update_tsk_timer(&tsk->thread.user_timer, + READ_ONCE(S390_lowcore.user_timer)); + guest = update_tsk_timer(&tsk->thread.guest_timer, + READ_ONCE(S390_lowcore.guest_timer)); + system = update_tsk_timer(&tsk->thread.system_timer, + READ_ONCE(S390_lowcore.system_timer)); + hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, + READ_ONCE(S390_lowcore.hardirq_timer)); + softirq = update_tsk_timer(&tsk->thread.softirq_timer, + READ_ONCE(S390_lowcore.softirq_timer)); + S390_lowcore.steal_timer += + clock - user - guest - system - hardirq - softirq; + + /* Push account value */ + if (user) { + account_user_time(tsk, cputime_to_nsecs(user)); + tsk->utimescaled += cputime_to_nsecs(scale_vtime(user)); + } - user_scaled = (user_scaled * mult) / div; - system_scaled = (system_scaled * mult) / div; + if (guest) { + account_guest_time(tsk, cputime_to_nsecs(guest)); + tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest)); } - account_user_time(tsk, user); - tsk->utimescaled += user_scaled; - account_system_time(tsk, 0, system); - tsk->stimescaled += system_scaled; + + if (system) + account_system_index_scaled(tsk, system, scale_vtime(system), + CPUTIME_SYSTEM); + if (hardirq) + account_system_index_scaled(tsk, hardirq, scale_vtime(hardirq), + CPUTIME_IRQ); + if (softirq) + account_system_index_scaled(tsk, softirq, scale_vtime(softirq), + CPUTIME_SOFTIRQ); steal = S390_lowcore.steal_timer; if ((s64) steal > 0) { S390_lowcore.steal_timer = 0; - account_steal_time(steal); + account_steal_time(cputime_to_nsecs(steal)); } - return virt_timer_forward(user + system); + return virt_timer_forward(user + guest + system + hardirq + softirq); } void vtime_task_switch(struct task_struct *prev) { do_account_vtime(prev); prev->thread.user_timer = S390_lowcore.user_timer; + prev->thread.guest_timer = S390_lowcore.guest_timer; prev->thread.system_timer = S390_lowcore.system_timer; + prev->thread.hardirq_timer = S390_lowcore.hardirq_timer; + prev->thread.softirq_timer = S390_lowcore.softirq_timer; S390_lowcore.user_timer = current->thread.user_timer; + S390_lowcore.guest_timer = current->thread.guest_timer; S390_lowcore.system_timer = current->thread.system_timer; + S390_lowcore.hardirq_timer = current->thread.hardirq_timer; + S390_lowcore.softirq_timer = current->thread.softirq_timer; } /* @@ -164,7 +214,7 @@ void vtime_task_switch(struct task_struct *prev) * accounting system time in order to correctly compute * the stolen time accounting. */ -void vtime_account_user(struct task_struct *tsk) +void vtime_flush(struct task_struct *tsk) { if (do_account_vtime(tsk)) virt_timer_expire(); @@ -176,32 +226,22 @@ void vtime_account_user(struct task_struct *tsk) */ void vtime_account_irq_enter(struct task_struct *tsk) { - u64 timer, system, system_scaled; + u64 timer; timer = S390_lowcore.last_update_timer; S390_lowcore.last_update_timer = get_vtimer(); - S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; - - /* Update MT utilization calculation */ - if (smp_cpu_mtid && - time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies))) - update_mt_scaling(); - - system = S390_lowcore.system_timer - tsk->thread.system_timer; - S390_lowcore.steal_timer -= system; - tsk->thread.system_timer = S390_lowcore.system_timer; - system_scaled = system; - /* Do MT utilization scaling */ - if (smp_cpu_mtid) { - u64 mult = __this_cpu_read(mt_scaling_mult); - u64 div = __this_cpu_read(mt_scaling_div); - - system_scaled = (system_scaled * mult) / div; - } - account_system_time(tsk, 0, system); - tsk->stimescaled += system_scaled; - - virt_timer_forward(system); + timer -= S390_lowcore.last_update_timer; + + if ((tsk->flags & PF_VCPU) && (irq_count() == 0)) + S390_lowcore.guest_timer += timer; + else if (hardirq_count()) + S390_lowcore.hardirq_timer += timer; + else if (in_serving_softirq()) + S390_lowcore.softirq_timer += timer; + else + S390_lowcore.system_timer += timer; + + virt_timer_forward(timer); } EXPORT_SYMBOL_GPL(vtime_account_irq_enter); @@ -310,7 +350,7 @@ static void __add_vtimer(struct vtimer_list *timer, int periodic) } /* - * add_virt_timer - add an oneshot virtual CPU timer + * add_virt_timer - add a oneshot virtual CPU timer */ void add_virt_timer(struct vtimer_list *timer) { diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 4aa8a7e2a1da..4492c9363178 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -373,7 +373,7 @@ void ipte_unlock(struct kvm_vcpu *vcpu) ipte_unlock_simple(vcpu); } -static int ar_translation(struct kvm_vcpu *vcpu, union asce *asce, ar_t ar, +static int ar_translation(struct kvm_vcpu *vcpu, union asce *asce, u8 ar, enum gacc_mode mode) { union alet alet; @@ -465,7 +465,9 @@ static int ar_translation(struct kvm_vcpu *vcpu, union asce *asce, ar_t ar, struct trans_exc_code_bits { unsigned long addr : 52; /* Translation-exception Address */ unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */ - unsigned long : 6; + unsigned long : 2; + unsigned long b56 : 1; + unsigned long : 3; unsigned long b60 : 1; unsigned long b61 : 1; unsigned long as : 2; /* ASCE Identifier */ @@ -485,7 +487,7 @@ enum prot_type { }; static int trans_exc(struct kvm_vcpu *vcpu, int code, unsigned long gva, - ar_t ar, enum gacc_mode mode, enum prot_type prot) + u8 ar, enum gacc_mode mode, enum prot_type prot) { struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm; struct trans_exc_code_bits *tec; @@ -497,14 +499,18 @@ static int trans_exc(struct kvm_vcpu *vcpu, int code, unsigned long gva, switch (code) { case PGM_PROTECTION: switch (prot) { + case PROT_TYPE_LA: + tec->b56 = 1; + break; + case PROT_TYPE_KEYC: + tec->b60 = 1; + break; case PROT_TYPE_ALC: tec->b60 = 1; /* FALL THROUGH */ case PROT_TYPE_DAT: tec->b61 = 1; break; - default: /* LA and KEYC set b61 to 0, other params undefined */ - return code; } /* FALL THROUGH */ case PGM_ASCE_TYPE: @@ -539,7 +545,7 @@ static int trans_exc(struct kvm_vcpu *vcpu, int code, unsigned long gva, } static int get_vcpu_asce(struct kvm_vcpu *vcpu, union asce *asce, - unsigned long ga, ar_t ar, enum gacc_mode mode) + unsigned long ga, u8 ar, enum gacc_mode mode) { int rc; struct psw_bits psw = psw_bits(vcpu->arch.sie_block->gpsw); @@ -771,7 +777,7 @@ static int low_address_protection_enabled(struct kvm_vcpu *vcpu, return 1; } -static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, +static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, unsigned long *pages, unsigned long nr_pages, const union asce asce, enum gacc_mode mode) { @@ -803,7 +809,7 @@ static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, return 0; } -int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data, +int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, void *data, unsigned long len, enum gacc_mode mode) { psw_t *psw = &vcpu->arch.sie_block->gpsw; @@ -877,7 +883,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, * Note: The IPTE lock is not taken during this function, so the caller * has to take care of this. */ -int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar, +int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, u8 ar, unsigned long *gpa, enum gacc_mode mode) { psw_t *psw = &vcpu->arch.sie_block->gpsw; @@ -910,7 +916,7 @@ int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar, /** * check_gva_range - test a range of guest virtual addresses for accessibility */ -int check_gva_range(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar, +int check_gva_range(struct kvm_vcpu *vcpu, unsigned long gva, u8 ar, unsigned long length, enum gacc_mode mode) { unsigned long gpa; diff --git a/arch/s390/kvm/gaccess.h b/arch/s390/kvm/gaccess.h index 8756569ad938..7ce47fd36f28 100644 --- a/arch/s390/kvm/gaccess.h +++ b/arch/s390/kvm/gaccess.h @@ -162,11 +162,11 @@ enum gacc_mode { }; int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, - ar_t ar, unsigned long *gpa, enum gacc_mode mode); -int check_gva_range(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar, + u8 ar, unsigned long *gpa, enum gacc_mode mode); +int check_gva_range(struct kvm_vcpu *vcpu, unsigned long gva, u8 ar, unsigned long length, enum gacc_mode mode); -int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data, +int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, void *data, unsigned long len, enum gacc_mode mode); int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, @@ -218,7 +218,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, * if data has been changed in guest space in case of an exception. */ static inline __must_check -int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data, +int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, void *data, unsigned long len) { return access_guest(vcpu, ga, ar, data, len, GACC_STORE); @@ -238,7 +238,7 @@ int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data, * data will be copied from guest space to kernel space. */ static inline __must_check -int read_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data, +int read_guest(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, void *data, unsigned long len) { return access_guest(vcpu, ga, ar, data, len, GACC_FETCH); @@ -247,10 +247,11 @@ int read_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data, /** * read_guest_instr - copy instruction data from guest space to kernel space * @vcpu: virtual cpu + * @ga: guest address * @data: destination address in kernel space * @len: number of bytes to copy * - * Copy @len bytes from the current psw address (guest space) to @data (kernel + * Copy @len bytes from the given address (guest space) to @data (kernel * space). * * The behaviour of read_guest_instr is identical to read_guest, except that @@ -258,10 +259,10 @@ int read_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data, * address-space mode. */ static inline __must_check -int read_guest_instr(struct kvm_vcpu *vcpu, void *data, unsigned long len) +int read_guest_instr(struct kvm_vcpu *vcpu, unsigned long ga, void *data, + unsigned long len) { - return access_guest(vcpu, vcpu->arch.sie_block->gpsw.addr, 0, data, len, - GACC_IFETCH); + return access_guest(vcpu, ga, 0, data, len, GACC_IFETCH); } /** diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/guestdbg.c index d7c6a7f53ced..23d9a4e12da1 100644 --- a/arch/s390/kvm/guestdbg.c +++ b/arch/s390/kvm/guestdbg.c @@ -388,14 +388,13 @@ void kvm_s390_prepare_debug_exit(struct kvm_vcpu *vcpu) #define per_write_wp_event(code) \ (code & (PER_CODE_STORE | PER_CODE_STORE_REAL)) -static int debug_exit_required(struct kvm_vcpu *vcpu) +static int debug_exit_required(struct kvm_vcpu *vcpu, u8 perc, + unsigned long peraddr) { - u8 perc = vcpu->arch.sie_block->perc; struct kvm_debug_exit_arch *debug_exit = &vcpu->run->debug.arch; struct kvm_hw_wp_info_arch *wp_info = NULL; struct kvm_hw_bp_info_arch *bp_info = NULL; unsigned long addr = vcpu->arch.sie_block->gpsw.addr; - unsigned long peraddr = vcpu->arch.sie_block->peraddr; if (guestdbg_hw_bp_enabled(vcpu)) { if (per_write_wp_event(perc) && @@ -437,36 +436,118 @@ exit_required: return 1; } +static int per_fetched_addr(struct kvm_vcpu *vcpu, unsigned long *addr) +{ + u8 exec_ilen = 0; + u16 opcode[3]; + int rc; + + if (vcpu->arch.sie_block->icptcode == ICPT_PROGI) { + /* PER address references the fetched or the execute instr */ + *addr = vcpu->arch.sie_block->peraddr; + /* + * Manually detect if we have an EXECUTE instruction. As + * instructions are always 2 byte aligned we can read the + * first two bytes unconditionally + */ + rc = read_guest_instr(vcpu, *addr, &opcode, 2); + if (rc) + return rc; + if (opcode[0] >> 8 == 0x44) + exec_ilen = 4; + if ((opcode[0] & 0xff0f) == 0xc600) + exec_ilen = 6; + } else { + /* instr was suppressed, calculate the responsible instr */ + *addr = __rewind_psw(vcpu->arch.sie_block->gpsw, + kvm_s390_get_ilen(vcpu)); + if (vcpu->arch.sie_block->icptstatus & 0x01) { + exec_ilen = (vcpu->arch.sie_block->icptstatus & 0x60) >> 4; + if (!exec_ilen) + exec_ilen = 4; + } + } + + if (exec_ilen) { + /* read the complete EXECUTE instr to detect the fetched addr */ + rc = read_guest_instr(vcpu, *addr, &opcode, exec_ilen); + if (rc) + return rc; + if (exec_ilen == 6) { + /* EXECUTE RELATIVE LONG - RIL-b format */ + s32 rl = *((s32 *) (opcode + 1)); + + /* rl is a _signed_ 32 bit value specifying halfwords */ + *addr += (u64)(s64) rl * 2; + } else { + /* EXECUTE - RX-a format */ + u32 base = (opcode[1] & 0xf000) >> 12; + u32 disp = opcode[1] & 0x0fff; + u32 index = opcode[0] & 0x000f; + + *addr = base ? vcpu->run->s.regs.gprs[base] : 0; + *addr += index ? vcpu->run->s.regs.gprs[index] : 0; + *addr += disp; + } + *addr = kvm_s390_logical_to_effective(vcpu, *addr); + } + return 0; +} + #define guest_per_enabled(vcpu) \ (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PER) int kvm_s390_handle_per_ifetch_icpt(struct kvm_vcpu *vcpu) { + const u64 cr10 = vcpu->arch.sie_block->gcr[10]; + const u64 cr11 = vcpu->arch.sie_block->gcr[11]; const u8 ilen = kvm_s390_get_ilen(vcpu); struct kvm_s390_pgm_info pgm_info = { .code = PGM_PER, .per_code = PER_CODE_IFETCH, .per_address = __rewind_psw(vcpu->arch.sie_block->gpsw, ilen), }; + unsigned long fetched_addr; + int rc; /* * The PSW points to the next instruction, therefore the intercepted * instruction generated a PER i-fetch event. PER address therefore * points at the previous PSW address (could be an EXECUTE function). */ - return kvm_s390_inject_prog_irq(vcpu, &pgm_info); + if (!guestdbg_enabled(vcpu)) + return kvm_s390_inject_prog_irq(vcpu, &pgm_info); + + if (debug_exit_required(vcpu, pgm_info.per_code, pgm_info.per_address)) + vcpu->guest_debug |= KVM_GUESTDBG_EXIT_PENDING; + + if (!guest_per_enabled(vcpu) || + !(vcpu->arch.sie_block->gcr[9] & PER_EVENT_IFETCH)) + return 0; + + rc = per_fetched_addr(vcpu, &fetched_addr); + if (rc < 0) + return rc; + if (rc) + /* instruction-fetching exceptions */ + return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); + + if (in_addr_range(fetched_addr, cr10, cr11)) + return kvm_s390_inject_prog_irq(vcpu, &pgm_info); + return 0; } -static void filter_guest_per_event(struct kvm_vcpu *vcpu) +static int filter_guest_per_event(struct kvm_vcpu *vcpu) { const u8 perc = vcpu->arch.sie_block->perc; - u64 peraddr = vcpu->arch.sie_block->peraddr; u64 addr = vcpu->arch.sie_block->gpsw.addr; u64 cr9 = vcpu->arch.sie_block->gcr[9]; u64 cr10 = vcpu->arch.sie_block->gcr[10]; u64 cr11 = vcpu->arch.sie_block->gcr[11]; /* filter all events, demanded by the guest */ u8 guest_perc = perc & (cr9 >> 24) & PER_CODE_MASK; + unsigned long fetched_addr; + int rc; if (!guest_per_enabled(vcpu)) guest_perc = 0; @@ -478,9 +559,17 @@ static void filter_guest_per_event(struct kvm_vcpu *vcpu) guest_perc &= ~PER_CODE_BRANCH; /* filter "instruction-fetching" events */ - if (guest_perc & PER_CODE_IFETCH && - !in_addr_range(peraddr, cr10, cr11)) - guest_perc &= ~PER_CODE_IFETCH; + if (guest_perc & PER_CODE_IFETCH) { + rc = per_fetched_addr(vcpu, &fetched_addr); + if (rc < 0) + return rc; + /* + * Don't inject an irq on exceptions. This would make handling + * on icpt code 8 very complex (as PSW was already rewound). + */ + if (rc || !in_addr_range(fetched_addr, cr10, cr11)) + guest_perc &= ~PER_CODE_IFETCH; + } /* All other PER events will be given to the guest */ /* TODO: Check altered address/address space */ @@ -489,6 +578,7 @@ static void filter_guest_per_event(struct kvm_vcpu *vcpu) if (!guest_perc) vcpu->arch.sie_block->iprcc &= ~PGM_PER; + return 0; } #define pssec(vcpu) (vcpu->arch.sie_block->gcr[1] & _ASCE_SPACE_SWITCH) @@ -496,14 +586,17 @@ static void filter_guest_per_event(struct kvm_vcpu *vcpu) #define old_ssec(vcpu) ((vcpu->arch.sie_block->tecmc >> 31) & 0x1) #define old_as_is_home(vcpu) !(vcpu->arch.sie_block->tecmc & 0xffff) -void kvm_s390_handle_per_event(struct kvm_vcpu *vcpu) +int kvm_s390_handle_per_event(struct kvm_vcpu *vcpu) { - int new_as; + int rc, new_as; - if (debug_exit_required(vcpu)) + if (debug_exit_required(vcpu, vcpu->arch.sie_block->perc, + vcpu->arch.sie_block->peraddr)) vcpu->guest_debug |= KVM_GUESTDBG_EXIT_PENDING; - filter_guest_per_event(vcpu); + rc = filter_guest_per_event(vcpu); + if (rc) + return rc; /* * Only RP, SAC, SACF, PT, PTI, PR, PC instructions can trigger @@ -532,4 +625,5 @@ void kvm_s390_handle_per_event(struct kvm_vcpu *vcpu) (pssec(vcpu) || old_ssec(vcpu))) vcpu->arch.sie_block->iprcc = PGM_SPACE_SWITCH; } + return 0; } diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c index 7a27eebab28a..59920f96ebc0 100644 --- a/arch/s390/kvm/intercept.c +++ b/arch/s390/kvm/intercept.c @@ -238,7 +238,9 @@ static int handle_prog(struct kvm_vcpu *vcpu) vcpu->stat.exit_program_interruption++; if (guestdbg_enabled(vcpu) && per_event(vcpu)) { - kvm_s390_handle_per_event(vcpu); + rc = kvm_s390_handle_per_event(vcpu); + if (rc) + return rc; /* the interrupt might have been filtered out completely */ if (vcpu->arch.sie_block->iprcc == 0) return 0; @@ -359,6 +361,9 @@ static int handle_partial_execution(struct kvm_vcpu *vcpu) static int handle_operexc(struct kvm_vcpu *vcpu) { + psw_t oldpsw, newpsw; + int rc; + vcpu->stat.exit_operation_exception++; trace_kvm_s390_handle_operexc(vcpu, vcpu->arch.sie_block->ipa, vcpu->arch.sie_block->ipb); @@ -369,6 +374,24 @@ static int handle_operexc(struct kvm_vcpu *vcpu) if (vcpu->arch.sie_block->ipa == 0 && vcpu->kvm->arch.user_instr0) return -EOPNOTSUPP; + rc = read_guest_lc(vcpu, __LC_PGM_NEW_PSW, &newpsw, sizeof(psw_t)); + if (rc) + return rc; + /* + * Avoid endless loops of operation exceptions, if the pgm new + * PSW will cause a new operation exception. + * The heuristic checks if the pgm new psw is within 6 bytes before + * the faulting psw address (with same DAT, AS settings) and the + * new psw is not a wait psw and the fault was not triggered by + * problem state. + */ + oldpsw = vcpu->arch.sie_block->gpsw; + if (oldpsw.addr - newpsw.addr <= 6 && + !(newpsw.mask & PSW_MASK_WAIT) && + !(oldpsw.mask & PSW_MASK_PSTATE) && + (newpsw.mask & PSW_MASK_ASC) == (oldpsw.mask & PSW_MASK_ASC) && + (newpsw.mask & PSW_MASK_DAT) == (oldpsw.mask & PSW_MASK_DAT)) + return -EOPNOTSUPP; return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 6484a250021e..f5694838234d 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -23,6 +23,7 @@ #include <linux/kvm_host.h> #include <linux/mman.h> #include <linux/module.h> +#include <linux/moduleparam.h> #include <linux/random.h> #include <linux/slab.h> #include <linux/timer.h> @@ -217,7 +218,7 @@ static void allow_cpu_feat(unsigned long nr) static inline int plo_test_bit(unsigned char nr) { register unsigned long r0 asm("0") = (unsigned long) nr | 0x100; - int cc = 3; /* subfunction not available */ + int cc; asm volatile( /* Parameter registers are ignored for "test bit" */ @@ -370,6 +371,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_S390_IRQCHIP: case KVM_CAP_VM_ATTRIBUTES: case KVM_CAP_MP_STATE: + case KVM_CAP_IMMEDIATE_EXIT: case KVM_CAP_S390_INJECT_IRQ: case KVM_CAP_S390_USER_SIGP: case KVM_CAP_S390_USER_STSI: @@ -442,6 +444,9 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot; int is_dirty = 0; + if (kvm_is_ucontrol(kvm)) + return -EINVAL; + mutex_lock(&kvm->slots_lock); r = -EINVAL; @@ -505,6 +510,14 @@ static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) } else if (MACHINE_HAS_VX) { set_kvm_facility(kvm->arch.model.fac_mask, 129); set_kvm_facility(kvm->arch.model.fac_list, 129); + if (test_facility(134)) { + set_kvm_facility(kvm->arch.model.fac_mask, 134); + set_kvm_facility(kvm->arch.model.fac_list, 134); + } + if (test_facility(135)) { + set_kvm_facility(kvm->arch.model.fac_mask, 135); + set_kvm_facility(kvm->arch.model.fac_list, 135); + } r = 0; } else r = -EINVAL; @@ -821,6 +834,13 @@ static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr) } memcpy(kvm->arch.model.fac_list, proc->fac_list, S390_ARCH_FAC_LIST_SIZE_BYTE); + VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx", + kvm->arch.model.ibc, + kvm->arch.model.cpuid); + VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx", + kvm->arch.model.fac_list[0], + kvm->arch.model.fac_list[1], + kvm->arch.model.fac_list[2]); } else ret = -EFAULT; kfree(proc); @@ -894,6 +914,13 @@ static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr) proc->ibc = kvm->arch.model.ibc; memcpy(&proc->fac_list, kvm->arch.model.fac_list, S390_ARCH_FAC_LIST_SIZE_BYTE); + VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx", + kvm->arch.model.ibc, + kvm->arch.model.cpuid); + VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx", + kvm->arch.model.fac_list[0], + kvm->arch.model.fac_list[1], + kvm->arch.model.fac_list[2]); if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc))) ret = -EFAULT; kfree(proc); @@ -917,6 +944,17 @@ static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr) S390_ARCH_FAC_LIST_SIZE_BYTE); memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list, sizeof(S390_lowcore.stfle_fac_list)); + VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx", + kvm->arch.model.ibc, + kvm->arch.model.cpuid); + VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx", + mach->fac_mask[0], + mach->fac_mask[1], + mach->fac_mask[2]); + VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx", + mach->fac_list[0], + mach->fac_list[1], + mach->fac_list[2]); if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach))) ret = -EFAULT; kfree(mach); @@ -1938,6 +1976,8 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi) vcpu->arch.sie_block->ecb2 |= 0x08; + if (test_kvm_facility(vcpu->kvm, 130)) + vcpu->arch.sie_block->ecb2 |= 0x20; vcpu->arch.sie_block->eca = 0x1002000U; if (sclp.has_cei) vcpu->arch.sie_block->eca |= 0x80000000U; @@ -2578,7 +2618,7 @@ static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu) * to look up the current opcode to get the length of the instruction * to be able to forward the PSW. */ - rc = read_guest_instr(vcpu, &opcode, 1); + rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1); ilen = insn_length(opcode); if (rc < 0) { return rc; @@ -2760,6 +2800,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) int rc; sigset_t sigsaved; + if (kvm_run->immediate_exit) + return -EINTR; + if (guestdbg_exit_pending(vcpu)) { kvm_s390_prepare_debug_exit(vcpu); return 0; diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index 3a4e97f1a9e6..af9fa91a0c91 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -86,9 +86,7 @@ static inline void kvm_s390_set_prefix(struct kvm_vcpu *vcpu, u32 prefix) kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); } -typedef u8 __bitwise ar_t; - -static inline u64 kvm_s390_get_base_disp_s(struct kvm_vcpu *vcpu, ar_t *ar) +static inline u64 kvm_s390_get_base_disp_s(struct kvm_vcpu *vcpu, u8 *ar) { u32 base2 = vcpu->arch.sie_block->ipb >> 28; u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16); @@ -101,7 +99,7 @@ static inline u64 kvm_s390_get_base_disp_s(struct kvm_vcpu *vcpu, ar_t *ar) static inline void kvm_s390_get_base_disp_sse(struct kvm_vcpu *vcpu, u64 *address1, u64 *address2, - ar_t *ar_b1, ar_t *ar_b2) + u8 *ar_b1, u8 *ar_b2) { u32 base1 = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28; u32 disp1 = (vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16; @@ -125,7 +123,7 @@ static inline void kvm_s390_get_regs_rre(struct kvm_vcpu *vcpu, int *r1, int *r2 *r2 = (vcpu->arch.sie_block->ipb & 0x000f0000) >> 16; } -static inline u64 kvm_s390_get_base_disp_rsy(struct kvm_vcpu *vcpu, ar_t *ar) +static inline u64 kvm_s390_get_base_disp_rsy(struct kvm_vcpu *vcpu, u8 *ar) { u32 base2 = vcpu->arch.sie_block->ipb >> 28; u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16) + @@ -140,7 +138,7 @@ static inline u64 kvm_s390_get_base_disp_rsy(struct kvm_vcpu *vcpu, ar_t *ar) return (base2 ? vcpu->run->s.regs.gprs[base2] : 0) + (long)(int)disp2; } -static inline u64 kvm_s390_get_base_disp_rs(struct kvm_vcpu *vcpu, ar_t *ar) +static inline u64 kvm_s390_get_base_disp_rs(struct kvm_vcpu *vcpu, u8 *ar) { u32 base2 = vcpu->arch.sie_block->ipb >> 28; u32 disp2 = ((vcpu->arch.sie_block->ipb & 0x0fff0000) >> 16); @@ -379,7 +377,7 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, void kvm_s390_clear_bp_data(struct kvm_vcpu *vcpu); void kvm_s390_prepare_debug_exit(struct kvm_vcpu *vcpu); int kvm_s390_handle_per_ifetch_icpt(struct kvm_vcpu *vcpu); -void kvm_s390_handle_per_event(struct kvm_vcpu *vcpu); +int kvm_s390_handle_per_event(struct kvm_vcpu *vcpu); /* support for Basic/Extended SCA handling */ static inline union ipte_control *kvm_s390_get_ipte_control(struct kvm *kvm) diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index e18435355c16..fb4b494cde9b 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -54,7 +54,7 @@ int kvm_s390_handle_aa(struct kvm_vcpu *vcpu) static int handle_set_clock(struct kvm_vcpu *vcpu) { int rc; - ar_t ar; + u8 ar; u64 op2, val; if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) @@ -79,7 +79,7 @@ static int handle_set_prefix(struct kvm_vcpu *vcpu) u64 operand2; u32 address; int rc; - ar_t ar; + u8 ar; vcpu->stat.instruction_spx++; @@ -117,7 +117,7 @@ static int handle_store_prefix(struct kvm_vcpu *vcpu) u64 operand2; u32 address; int rc; - ar_t ar; + u8 ar; vcpu->stat.instruction_stpx++; @@ -147,7 +147,7 @@ static int handle_store_cpu_address(struct kvm_vcpu *vcpu) u16 vcpu_id = vcpu->vcpu_id; u64 ga; int rc; - ar_t ar; + u8 ar; vcpu->stat.instruction_stap++; @@ -311,7 +311,7 @@ static int handle_sske(struct kvm_vcpu *vcpu) if (rc < 0) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); start += PAGE_SIZE; - }; + } if (m3 & (SSKE_MC | SSKE_MR)) { if (m3 & SSKE_MB) { @@ -380,7 +380,7 @@ static int handle_tpi(struct kvm_vcpu *vcpu) u32 tpi_data[3]; int rc; u64 addr; - ar_t ar; + u8 ar; addr = kvm_s390_get_base_disp_s(vcpu, &ar); if (addr & 3) @@ -548,7 +548,7 @@ int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu) psw_compat_t new_psw; u64 addr; int rc; - ar_t ar; + u8 ar; if (gpsw->mask & PSW_MASK_PSTATE) return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); @@ -575,7 +575,7 @@ static int handle_lpswe(struct kvm_vcpu *vcpu) psw_t new_psw; u64 addr; int rc; - ar_t ar; + u8 ar; if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); @@ -597,7 +597,7 @@ static int handle_stidp(struct kvm_vcpu *vcpu) u64 stidp_data = vcpu->kvm->arch.model.cpuid; u64 operand2; int rc; - ar_t ar; + u8 ar; vcpu->stat.instruction_stidp++; @@ -644,7 +644,7 @@ static void handle_stsi_3_2_2(struct kvm_vcpu *vcpu, struct sysinfo_3_2_2 *mem) ASCEBC(mem->vm[0].cpi, 16); } -static void insert_stsi_usr_data(struct kvm_vcpu *vcpu, u64 addr, ar_t ar, +static void insert_stsi_usr_data(struct kvm_vcpu *vcpu, u64 addr, u8 ar, u8 fc, u8 sel1, u16 sel2) { vcpu->run->exit_reason = KVM_EXIT_S390_STSI; @@ -663,7 +663,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu) unsigned long mem = 0; u64 operand2; int rc = 0; - ar_t ar; + u8 ar; vcpu->stat.instruction_stsi++; VCPU_EVENT(vcpu, 3, "STSI: fc: %u sel1: %u sel2: %u", fc, sel1, sel2); @@ -970,7 +970,7 @@ int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu) int reg, rc, nr_regs; u32 ctl_array[16]; u64 ga; - ar_t ar; + u8 ar; vcpu->stat.instruction_lctl++; @@ -1009,7 +1009,7 @@ int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu) int reg, rc, nr_regs; u32 ctl_array[16]; u64 ga; - ar_t ar; + u8 ar; vcpu->stat.instruction_stctl++; @@ -1043,7 +1043,7 @@ static int handle_lctlg(struct kvm_vcpu *vcpu) int reg, rc, nr_regs; u64 ctl_array[16]; u64 ga; - ar_t ar; + u8 ar; vcpu->stat.instruction_lctlg++; @@ -1081,7 +1081,7 @@ static int handle_stctg(struct kvm_vcpu *vcpu) int reg, rc, nr_regs; u64 ctl_array[16]; u64 ga; - ar_t ar; + u8 ar; vcpu->stat.instruction_stctg++; @@ -1132,7 +1132,7 @@ static int handle_tprot(struct kvm_vcpu *vcpu) unsigned long hva, gpa; int ret = 0, cc = 0; bool writable; - ar_t ar; + u8 ar; vcpu->stat.instruction_tprot++; diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index d8673e243f13..38556e395915 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -324,6 +324,9 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) /* Run-time-Instrumentation */ if (test_kvm_facility(vcpu->kvm, 64)) scb_s->ecb3 |= scb_o->ecb3 & 0x01U; + /* Instruction Execution Prevention */ + if (test_kvm_facility(vcpu->kvm, 130)) + scb_s->ecb2 |= scb_o->ecb2 & 0x20U; if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_SIIF)) scb_s->eca |= scb_o->eca & 0x00000001U; if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_IB)) @@ -899,7 +902,7 @@ static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) if (rc || scb_s->icptcode || signal_pending(current) || kvm_s390_vcpu_has_irq(vcpu, 0)) break; - }; + } if (rc == -EFAULT) { /* diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c index 501dcd4ca4a0..92e90e40b6fb 100644 --- a/arch/s390/lib/delay.c +++ b/arch/s390/lib/delay.c @@ -9,7 +9,7 @@ #include <linux/sched.h> #include <linux/delay.h> #include <linux/timex.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/irqflags.h> #include <linux/interrupt.h> #include <linux/irq.h> diff --git a/arch/s390/lib/mem.S b/arch/s390/lib/mem.S index 7422a706f310..7ff79a4ff00c 100644 --- a/arch/s390/lib/mem.S +++ b/arch/s390/lib/mem.S @@ -14,31 +14,29 @@ ENTRY(memmove) ltgr %r4,%r4 lgr %r1,%r2 bzr %r14 + aghi %r4,-1 clgr %r2,%r3 jnh .Lmemmove_forward - la %r5,0(%r4,%r3) + la %r5,1(%r4,%r3) clgr %r2,%r5 jl .Lmemmove_reverse .Lmemmove_forward: - aghi %r4,-1 srlg %r0,%r4,8 ltgr %r0,%r0 - jz .Lmemmove_rest -.Lmemmove_loop: + jz .Lmemmove_forward_remainder +.Lmemmove_forward_loop: mvc 0(256,%r1),0(%r3) la %r1,256(%r1) la %r3,256(%r3) - brctg %r0,.Lmemmove_loop -.Lmemmove_rest: + brctg %r0,.Lmemmove_forward_loop +.Lmemmove_forward_remainder: larl %r5,.Lmemmove_mvc ex %r4,0(%r5) br %r14 .Lmemmove_reverse: - aghi %r4,-1 -.Lmemmove_reverse_loop: ic %r0,0(%r4,%r3) stc %r0,0(%r4,%r1) - brctg %r4,.Lmemmove_reverse_loop + brctg %r4,.Lmemmove_reverse ic %r0,0(%r4,%r3) stc %r0,0(%r4,%r1) br %r14 @@ -70,12 +68,12 @@ ENTRY(memset) srlg %r3,%r4,8 ltgr %r3,%r3 lgr %r1,%r2 - jz .Lmemset_clear_rest + jz .Lmemset_clear_remainder .Lmemset_clear_loop: xc 0(256,%r1),0(%r1) la %r1,256(%r1) brctg %r3,.Lmemset_clear_loop -.Lmemset_clear_rest: +.Lmemset_clear_remainder: larl %r3,.Lmemset_xc ex %r4,0(%r3) br %r14 @@ -87,12 +85,12 @@ ENTRY(memset) aghi %r4,-2 srlg %r3,%r4,8 ltgr %r3,%r3 - jz .Lmemset_fill_rest + jz .Lmemset_fill_remainder .Lmemset_fill_loop: mvc 1(256,%r1),0(%r1) la %r1,256(%r1) brctg %r3,.Lmemset_fill_loop -.Lmemset_fill_rest: +.Lmemset_fill_remainder: larl %r3,.Lmemset_mvc ex %r4,0(%r3) br %r14 @@ -115,7 +113,7 @@ ENTRY(memcpy) ltgr %r5,%r5 lgr %r1,%r2 jnz .Lmemcpy_loop -.Lmemcpy_rest: +.Lmemcpy_remainder: larl %r5,.Lmemcpy_mvc ex %r4,0(%r5) br %r14 @@ -124,7 +122,7 @@ ENTRY(memcpy) la %r1,256(%r1) la %r3,256(%r3) brctg %r5,.Lmemcpy_loop - j .Lmemcpy_rest + j .Lmemcpy_remainder .Lmemcpy_mvc: mvc 0(1,%r1),0(%r3) EXPORT_SYMBOL(memcpy) diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index e48a48ec24bc..ba427eb6f14c 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -6,7 +6,7 @@ */ #include <linux/types.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/spinlock.h> #include <linux/init.h> #include <linux/smp.h> @@ -133,7 +133,7 @@ int arch_spin_trylock_retry(arch_spinlock_t *lp) int count; for (count = spin_retry; count > 0; count--) { - owner = ACCESS_ONCE(lp->lock); + owner = READ_ONCE(lp->lock); /* Try to get the lock if it is free. */ if (!owner) { if (_raw_compare_and_swap(&lp->lock, 0, cpu)) diff --git a/arch/s390/lib/string.c b/arch/s390/lib/string.c index f71d9f655970..4ee27339c792 100644 --- a/arch/s390/lib/string.c +++ b/arch/s390/lib/string.c @@ -9,7 +9,8 @@ #define IN_ARCH_STRING_C 1 #include <linux/types.h> -#include <linux/module.h> +#include <linux/string.h> +#include <linux/export.h> /* * Helper functions to find the end of a string diff --git a/arch/s390/lib/xor.c b/arch/s390/lib/xor.c index 7d94e3ec34a9..b4fd05c36151 100644 --- a/arch/s390/lib/xor.c +++ b/arch/s390/lib/xor.c @@ -6,7 +6,7 @@ */ #include <linux/types.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/raid/xor.h> static void xor_xc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) diff --git a/arch/s390/mm/cmm.c b/arch/s390/mm/cmm.c index 79ddd580d605..829c63dbc81a 100644 --- a/arch/s390/mm/cmm.c +++ b/arch/s390/mm/cmm.c @@ -10,6 +10,7 @@ #include <linux/fs.h> #include <linux/init.h> #include <linux/module.h> +#include <linux/moduleparam.h> #include <linux/gfp.h> #include <linux/sched.h> #include <linux/sysctl.h> diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index 861880df12c7..1b553d847140 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -1,6 +1,6 @@ #include <linux/seq_file.h> #include <linux/debugfs.h> -#include <linux/module.h> +#include <linux/sched.h> #include <linux/mm.h> #include <asm/sections.h> #include <asm/pgtable.h> @@ -49,8 +49,8 @@ static void print_prot(struct seq_file *m, unsigned int pr, int level) seq_printf(m, "I\n"); return; } - seq_printf(m, "%s", pr & _PAGE_PROTECT ? "RO " : "RW "); - seq_putc(m, '\n'); + seq_puts(m, (pr & _PAGE_PROTECT) ? "RO " : "RW "); + seq_puts(m, (pr & _PAGE_NOEXEC) ? "NX\n" : "X\n"); } static void note_page(struct seq_file *m, struct pg_state *st, @@ -117,7 +117,8 @@ static void walk_pte_level(struct seq_file *m, struct pg_state *st, for (i = 0; i < PTRS_PER_PTE && addr < max_addr; i++) { st->current_address = addr; pte = pte_offset_kernel(pmd, addr); - prot = pte_val(*pte) & (_PAGE_PROTECT | _PAGE_INVALID); + prot = pte_val(*pte) & + (_PAGE_PROTECT | _PAGE_INVALID | _PAGE_NOEXEC); note_page(m, st, prot, 4); addr += PAGE_SIZE; } @@ -135,7 +136,9 @@ static void walk_pmd_level(struct seq_file *m, struct pg_state *st, pmd = pmd_offset(pud, addr); if (!pmd_none(*pmd)) { if (pmd_large(*pmd)) { - prot = pmd_val(*pmd) & _SEGMENT_ENTRY_PROTECT; + prot = pmd_val(*pmd) & + (_SEGMENT_ENTRY_PROTECT | + _SEGMENT_ENTRY_NOEXEC); note_page(m, st, prot, 3); } else walk_pte_level(m, st, pmd, addr); @@ -157,7 +160,9 @@ static void walk_pud_level(struct seq_file *m, struct pg_state *st, pud = pud_offset(pgd, addr); if (!pud_none(*pud)) if (pud_large(*pud)) { - prot = pud_val(*pud) & _REGION_ENTRY_PROTECT; + prot = pud_val(*pud) & + (_REGION_ENTRY_PROTECT | + _REGION_ENTRY_NOEXEC); note_page(m, st, prot, 2); } else walk_pmd_level(m, st, pud, addr); @@ -183,6 +188,7 @@ static void walk_pgd_level(struct seq_file *m) else note_page(m, &st, _PAGE_INVALID, 1); addr += PGDIR_SIZE; + cond_resched(); } /* Flush out the last page */ st.current_address = max_addr; diff --git a/arch/s390/mm/extmem.c b/arch/s390/mm/extmem.c index 362237203144..9f118629b55f 100644 --- a/arch/s390/mm/extmem.c +++ b/arch/s390/mm/extmem.c @@ -14,7 +14,7 @@ #include <linux/spinlock.h> #include <linux/list.h> #include <linux/slab.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/bootmem.h> #include <linux/ctype.h> #include <linux/ioport.h> @@ -154,7 +154,7 @@ dcss_mkname(char *name, char *dcss_name) if (name[i] == '\0') break; dcss_name[i] = toupper(name[i]); - }; + } for (; i < 8; i++) dcss_name[i] = ' '; ASCEBC(dcss_name, 8); diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index d1faae5cdd12..bb5560eb2435 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -311,12 +311,34 @@ static noinline void do_sigbus(struct pt_regs *regs) force_sig_info(SIGBUS, &si, tsk); } -static noinline void do_fault_error(struct pt_regs *regs, int fault) +static noinline int signal_return(struct pt_regs *regs) +{ + u16 instruction; + int rc; + + rc = __get_user(instruction, (u16 __user *) regs->psw.addr); + if (rc) + return rc; + if (instruction == 0x0a77) { + set_pt_regs_flag(regs, PIF_SYSCALL); + regs->int_code = 0x00040077; + return 0; + } else if (instruction == 0x0aad) { + set_pt_regs_flag(regs, PIF_SYSCALL); + regs->int_code = 0x000400ad; + return 0; + } + return -EACCES; +} + +static noinline void do_fault_error(struct pt_regs *regs, int access, int fault) { int si_code; switch (fault) { case VM_FAULT_BADACCESS: + if (access == VM_EXEC && signal_return(regs) == 0) + break; case VM_FAULT_BADMAP: /* Bad memory access. Check if it is kernel or user space. */ if (user_mode(regs)) { @@ -324,7 +346,7 @@ static noinline void do_fault_error(struct pt_regs *regs, int fault) si_code = (fault == VM_FAULT_BADMAP) ? SEGV_MAPERR : SEGV_ACCERR; do_sigsegv(regs, si_code); - return; + break; } case VM_FAULT_BADCONTEXT: case VM_FAULT_PFAULT: @@ -525,7 +547,7 @@ out: void do_protection_exception(struct pt_regs *regs) { unsigned long trans_exc_code; - int fault; + int access, fault; trans_exc_code = regs->int_parm_long; /* @@ -544,9 +566,17 @@ void do_protection_exception(struct pt_regs *regs) do_low_address(regs); return; } - fault = do_exception(regs, VM_WRITE); + if (unlikely(MACHINE_HAS_NX && (trans_exc_code & 0x80))) { + regs->int_parm_long = (trans_exc_code & ~PAGE_MASK) | + (regs->psw.addr & PAGE_MASK); + access = VM_EXEC; + fault = VM_FAULT_BADACCESS; + } else { + access = VM_WRITE; + fault = do_exception(regs, access); + } if (unlikely(fault)) - do_fault_error(regs, fault); + do_fault_error(regs, access, fault); } NOKPROBE_SYMBOL(do_protection_exception); @@ -557,7 +587,7 @@ void do_dat_exception(struct pt_regs *regs) access = VM_READ | VM_EXEC | VM_WRITE; fault = do_exception(regs, access); if (unlikely(fault)) - do_fault_error(regs, fault); + do_fault_error(regs, access, fault); } NOKPROBE_SYMBOL(do_dat_exception); diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index ec1f0dedb948..a07b1ec1391d 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -359,8 +359,8 @@ static int __gmap_unlink_by_vmaddr(struct gmap *gmap, unsigned long vmaddr) spin_lock(&gmap->guest_table_lock); entry = radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT); if (entry) { - flush = (*entry != _SEGMENT_ENTRY_INVALID); - *entry = _SEGMENT_ENTRY_INVALID; + flush = (*entry != _SEGMENT_ENTRY_EMPTY); + *entry = _SEGMENT_ENTRY_EMPTY; } spin_unlock(&gmap->guest_table_lock); return flush; @@ -589,7 +589,7 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) return rc; ptl = pmd_lock(mm, pmd); spin_lock(&gmap->guest_table_lock); - if (*table == _SEGMENT_ENTRY_INVALID) { + if (*table == _SEGMENT_ENTRY_EMPTY) { rc = radix_tree_insert(&gmap->host_to_guest, vmaddr >> PMD_SHIFT, table); if (!rc) @@ -687,7 +687,7 @@ void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to) /* Find vma in the parent mm */ vma = find_vma(gmap->mm, vmaddr); size = min(to - gaddr, PMD_SIZE - (gaddr & ~PMD_MASK)); - zap_page_range(vma, vmaddr, size, NULL); + zap_page_range(vma, vmaddr, size); } up_read(&gmap->mm->mmap_sem); } diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index 4a0c5bce3552..9b4050caa4e9 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -59,8 +59,10 @@ static inline unsigned long __pte_to_rste(pte_t pte) rste |= move_set_bit(pte_val(pte), _PAGE_SOFT_DIRTY, _SEGMENT_ENTRY_SOFT_DIRTY); #endif + rste |= move_set_bit(pte_val(pte), _PAGE_NOEXEC, + _SEGMENT_ENTRY_NOEXEC); } else - rste = _SEGMENT_ENTRY_INVALID; + rste = _SEGMENT_ENTRY_EMPTY; return rste; } @@ -113,6 +115,8 @@ static inline pte_t __rste_to_pte(unsigned long rste) pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_SOFT_DIRTY, _PAGE_DIRTY); #endif + pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_NOEXEC, + _PAGE_NOEXEC); } else pte_val(pte) = _PAGE_INVALID; return pte; @@ -121,7 +125,11 @@ static inline pte_t __rste_to_pte(unsigned long rste) void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { - unsigned long rste = __pte_to_rste(pte); + unsigned long rste; + + rste = __pte_to_rste(pte); + if (!MACHINE_HAS_NX) + rste &= ~_SEGMENT_ENTRY_NOEXEC; /* Set correct table type for 2G hugepages */ if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index b67454ad8408..ee5066718b21 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -137,6 +137,9 @@ void __init mem_init(void) void free_initmem(void) { + __set_memory((unsigned long) _sinittext, + (_einittext - _sinittext) >> PAGE_SHIFT, + SET_MEMORY_RW | SET_MEMORY_NX); free_initmem_default(POISON_FREE_INITMEM); } @@ -148,6 +151,15 @@ void __init free_initrd_mem(unsigned long start, unsigned long end) } #endif +unsigned long memory_block_size_bytes(void) +{ + /* + * Make sure the memory block size is always greater + * or equal than the memory increment size. + */ + return max_t(unsigned long, MIN_MEMORY_BLOCK_SIZE, sclp.rzm); +} + #ifdef CONFIG_MEMORY_HOTPLUG int arch_add_memory(int nid, u64 start, u64 size, bool for_device) { @@ -191,15 +203,6 @@ int arch_add_memory(int nid, u64 start, u64 size, bool for_device) return rc; } -unsigned long memory_block_size_bytes(void) -{ - /* - * Make sure the memory block size is always greater - * or equal than the memory increment size. - */ - return max_t(unsigned long, MIN_MEMORY_BLOCK_SIZE, sclp.rzm); -} - #ifdef CONFIG_MEMORY_HOTREMOVE int arch_remove_memory(u64 start, u64 size) { diff --git a/arch/s390/mm/mem_detect.c b/arch/s390/mm/mem_detect.c index d612cc3eec6a..e58dca05b962 100644 --- a/arch/s390/mm/mem_detect.c +++ b/arch/s390/mm/mem_detect.c @@ -5,7 +5,6 @@ */ #include <linux/kernel.h> -#include <linux/module.h> #include <linux/memblock.h> #include <linux/init.h> #include <linux/debugfs.h> @@ -19,6 +18,8 @@ static inline void memblock_physmem_add(phys_addr_t start, phys_addr_t size) { + memblock_dbg("memblock_physmem_add: [%#016llx-%#016llx]\n", + start, start + size - 1); memblock_add_range(&memblock.memory, start, size, 0, 0); memblock_add_range(&memblock.physmem, start, size, 0, 0); } @@ -39,7 +40,8 @@ void __init detect_memory_memblock(void) memblock_set_bottom_up(true); do { size = 0; - type = tprot(addr); + /* assume lowcore is writable */ + type = addr ? tprot(addr) : CHUNK_READ_WRITE; do { size += rzm; if (max_physmem_end && addr + size >= max_physmem_end) @@ -55,4 +57,5 @@ void __init detect_memory_memblock(void) memblock_set_bottom_up(false); if (!max_physmem_end) max_physmem_end = memblock_end_of_DRAM(); + memblock_dump_all(); } diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index eb9df2822da1..7ae1282d5be9 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -26,11 +26,11 @@ #include <linux/personality.h> #include <linux/mm.h> #include <linux/mman.h> -#include <linux/module.h> #include <linux/random.h> #include <linux/compat.h> #include <linux/security.h> #include <asm/pgalloc.h> +#include <asm/elf.h> static unsigned long stack_maxrandom_size(void) { diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 44f150312a16..fc5dc33bb141 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -3,7 +3,6 @@ * Author(s): Jan Glauber <jang@linux.vnet.ibm.com> */ #include <linux/hugetlb.h> -#include <linux/module.h> #include <linux/mm.h> #include <asm/cacheflush.h> #include <asm/facility.h> @@ -81,24 +80,24 @@ static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr, } } -struct cpa { - unsigned int set_ro : 1; - unsigned int clear_ro : 1; -}; - static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, - struct cpa cpa) + unsigned long flags) { pte_t *ptep, new; ptep = pte_offset(pmdp, addr); do { - if (pte_none(*ptep)) + new = *ptep; + if (pte_none(new)) return -EINVAL; - if (cpa.set_ro) - new = pte_wrprotect(*ptep); - else if (cpa.clear_ro) - new = pte_mkwrite(pte_mkdirty(*ptep)); + if (flags & SET_MEMORY_RO) + new = pte_wrprotect(new); + else if (flags & SET_MEMORY_RW) + new = pte_mkwrite(pte_mkdirty(new)); + if ((flags & SET_MEMORY_NX) && MACHINE_HAS_NX) + pte_val(new) |= _PAGE_NOEXEC; + else if (flags & SET_MEMORY_X) + pte_val(new) &= ~_PAGE_NOEXEC; pgt_set((unsigned long *)ptep, pte_val(new), addr, CRDTE_DTT_PAGE); ptep++; addr += PAGE_SIZE; @@ -112,14 +111,17 @@ static int split_pmd_page(pmd_t *pmdp, unsigned long addr) unsigned long pte_addr, prot; pte_t *pt_dir, *ptep; pmd_t new; - int i, ro; + int i, ro, nx; pt_dir = vmem_pte_alloc(); if (!pt_dir) return -ENOMEM; pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT; ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT); + nx = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_NOEXEC); prot = pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL); + if (!nx) + prot &= ~_PAGE_NOEXEC; ptep = pt_dir; for (i = 0; i < PTRS_PER_PTE; i++) { pte_val(*ptep) = pte_addr | prot; @@ -133,19 +135,24 @@ static int split_pmd_page(pmd_t *pmdp, unsigned long addr) return 0; } -static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, struct cpa cpa) +static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, + unsigned long flags) { - pmd_t new; - - if (cpa.set_ro) - new = pmd_wrprotect(*pmdp); - else if (cpa.clear_ro) - new = pmd_mkwrite(pmd_mkdirty(*pmdp)); + pmd_t new = *pmdp; + + if (flags & SET_MEMORY_RO) + new = pmd_wrprotect(new); + else if (flags & SET_MEMORY_RW) + new = pmd_mkwrite(pmd_mkdirty(new)); + if ((flags & SET_MEMORY_NX) && MACHINE_HAS_NX) + pmd_val(new) |= _SEGMENT_ENTRY_NOEXEC; + else if (flags & SET_MEMORY_X) + pmd_val(new) &= ~_SEGMENT_ENTRY_NOEXEC; pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); } static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, - struct cpa cpa) + unsigned long flags) { unsigned long next; pmd_t *pmdp; @@ -163,9 +170,9 @@ static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, return rc; continue; } - modify_pmd_page(pmdp, addr, cpa); + modify_pmd_page(pmdp, addr, flags); } else { - rc = walk_pte_level(pmdp, addr, next, cpa); + rc = walk_pte_level(pmdp, addr, next, flags); if (rc) return rc; } @@ -181,14 +188,17 @@ static int split_pud_page(pud_t *pudp, unsigned long addr) unsigned long pmd_addr, prot; pmd_t *pm_dir, *pmdp; pud_t new; - int i, ro; + int i, ro, nx; pm_dir = vmem_pmd_alloc(); if (!pm_dir) return -ENOMEM; pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT; ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT); + nx = !!(pud_val(*pudp) & _REGION_ENTRY_NOEXEC); prot = pgprot_val(ro ? SEGMENT_KERNEL_RO : SEGMENT_KERNEL); + if (!nx) + prot &= ~_SEGMENT_ENTRY_NOEXEC; pmdp = pm_dir; for (i = 0; i < PTRS_PER_PMD; i++) { pmd_val(*pmdp) = pmd_addr | prot; @@ -202,19 +212,24 @@ static int split_pud_page(pud_t *pudp, unsigned long addr) return 0; } -static void modify_pud_page(pud_t *pudp, unsigned long addr, struct cpa cpa) +static void modify_pud_page(pud_t *pudp, unsigned long addr, + unsigned long flags) { - pud_t new; - - if (cpa.set_ro) - new = pud_wrprotect(*pudp); - else if (cpa.clear_ro) - new = pud_mkwrite(pud_mkdirty(*pudp)); + pud_t new = *pudp; + + if (flags & SET_MEMORY_RO) + new = pud_wrprotect(new); + else if (flags & SET_MEMORY_RW) + new = pud_mkwrite(pud_mkdirty(new)); + if ((flags & SET_MEMORY_NX) && MACHINE_HAS_NX) + pud_val(new) |= _REGION_ENTRY_NOEXEC; + else if (flags & SET_MEMORY_X) + pud_val(new) &= ~_REGION_ENTRY_NOEXEC; pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); } static int walk_pud_level(pgd_t *pgd, unsigned long addr, unsigned long end, - struct cpa cpa) + unsigned long flags) { unsigned long next; pud_t *pudp; @@ -232,9 +247,9 @@ static int walk_pud_level(pgd_t *pgd, unsigned long addr, unsigned long end, break; continue; } - modify_pud_page(pudp, addr, cpa); + modify_pud_page(pudp, addr, flags); } else { - rc = walk_pmd_level(pudp, addr, next, cpa); + rc = walk_pmd_level(pudp, addr, next, flags); } pudp++; addr = next; @@ -246,7 +261,7 @@ static int walk_pud_level(pgd_t *pgd, unsigned long addr, unsigned long end, static DEFINE_MUTEX(cpa_mutex); static int change_page_attr(unsigned long addr, unsigned long end, - struct cpa cpa) + unsigned long flags) { unsigned long next; int rc = -EINVAL; @@ -262,7 +277,7 @@ static int change_page_attr(unsigned long addr, unsigned long end, if (pgd_none(*pgdp)) break; next = pgd_addr_end(addr, end); - rc = walk_pud_level(pgdp, addr, next, cpa); + rc = walk_pud_level(pgdp, addr, next, flags); if (rc) break; cond_resched(); @@ -271,35 +286,10 @@ static int change_page_attr(unsigned long addr, unsigned long end, return rc; } -int set_memory_ro(unsigned long addr, int numpages) +int __set_memory(unsigned long addr, int numpages, unsigned long flags) { - struct cpa cpa = { - .set_ro = 1, - }; - addr &= PAGE_MASK; - return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa); -} - -int set_memory_rw(unsigned long addr, int numpages) -{ - struct cpa cpa = { - .clear_ro = 1, - }; - - addr &= PAGE_MASK; - return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa); -} - -/* not possible */ -int set_memory_nx(unsigned long addr, int numpages) -{ - return 0; -} - -int set_memory_x(unsigned long addr, int numpages) -{ - return 0; + return change_page_attr(addr, addr + numpages * PAGE_SIZE, flags); } #ifdef CONFIG_DEBUG_PAGEALLOC @@ -339,7 +329,7 @@ void __kernel_map_pages(struct page *page, int numpages, int enable) nr = min(numpages - i, nr); if (enable) { for (j = 0; j < nr; j++) { - pte_val(*pte) = address | pgprot_val(PAGE_KERNEL); + pte_val(*pte) &= ~_PAGE_INVALID; address += PAGE_SIZE; pte++; } diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index d56ef26d4681..b48dc5f1900b 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -275,6 +275,8 @@ void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, { pgste_t pgste; + if (!MACHINE_HAS_NX) + pte_val(pte) &= ~_PAGE_NOEXEC; if (mm_has_pgste(mm)) { pgste = pgste_get(ptep); pgste_set_key(ptep, pgste, pte, mm); @@ -742,7 +744,7 @@ int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) pgste_set_unlock(ptep, new); pte_unmap_unlock(ptep, ptl); - return 0; + return cc; } EXPORT_SYMBOL(reset_guest_reference_bit); diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 45becc8a44ec..60d38993f232 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -6,7 +6,7 @@ #include <linux/bootmem.h> #include <linux/pfn.h> #include <linux/mm.h> -#include <linux/module.h> +#include <linux/init.h> #include <linux/list.h> #include <linux/hugetlb.h> #include <linux/slab.h> @@ -79,6 +79,7 @@ pte_t __ref *vmem_pte_alloc(void) */ static int vmem_add_mem(unsigned long start, unsigned long size) { + unsigned long pgt_prot, sgt_prot, r3_prot; unsigned long pages4k, pages1m, pages2g; unsigned long end = start + size; unsigned long address = start; @@ -88,6 +89,14 @@ static int vmem_add_mem(unsigned long start, unsigned long size) pte_t *pt_dir; int ret = -ENOMEM; + pgt_prot = pgprot_val(PAGE_KERNEL); + sgt_prot = pgprot_val(SEGMENT_KERNEL); + r3_prot = pgprot_val(REGION3_KERNEL); + if (!MACHINE_HAS_NX) { + pgt_prot &= ~_PAGE_NOEXEC; + sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC; + r3_prot &= ~_REGION_ENTRY_NOEXEC; + } pages4k = pages1m = pages2g = 0; while (address < end) { pg_dir = pgd_offset_k(address); @@ -101,7 +110,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size) if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address && !(address & ~PUD_MASK) && (address + PUD_SIZE <= end) && !debug_pagealloc_enabled()) { - pud_val(*pu_dir) = address | pgprot_val(REGION3_KERNEL); + pud_val(*pu_dir) = address | r3_prot; address += PUD_SIZE; pages2g++; continue; @@ -116,7 +125,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size) if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address && !(address & ~PMD_MASK) && (address + PMD_SIZE <= end) && !debug_pagealloc_enabled()) { - pmd_val(*pm_dir) = address | pgprot_val(SEGMENT_KERNEL); + pmd_val(*pm_dir) = address | sgt_prot; address += PMD_SIZE; pages1m++; continue; @@ -129,7 +138,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size) } pt_dir = pte_offset_kernel(pm_dir, address); - pte_val(*pt_dir) = address | pgprot_val(PAGE_KERNEL); + pte_val(*pt_dir) = address | pgt_prot; address += PAGE_SIZE; pages4k++; } @@ -200,6 +209,7 @@ static void vmem_remove_range(unsigned long start, unsigned long size) */ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) { + unsigned long pgt_prot, sgt_prot; unsigned long address = start; pgd_t *pg_dir; pud_t *pu_dir; @@ -207,6 +217,12 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) pte_t *pt_dir; int ret = -ENOMEM; + pgt_prot = pgprot_val(PAGE_KERNEL); + sgt_prot = pgprot_val(SEGMENT_KERNEL); + if (!MACHINE_HAS_NX) { + pgt_prot &= ~_PAGE_NOEXEC; + sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC; + } for (address = start; address < end;) { pg_dir = pgd_offset_k(address); if (pgd_none(*pg_dir)) { @@ -238,8 +254,7 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) new_page = vmemmap_alloc_block(PMD_SIZE, node); if (!new_page) goto out; - pmd_val(*pm_dir) = __pa(new_page) | - _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE; + pmd_val(*pm_dir) = __pa(new_page) | sgt_prot; address = (address + PMD_SIZE) & PMD_MASK; continue; } @@ -259,8 +274,7 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) new_page = vmemmap_alloc_block(PAGE_SIZE, node); if (!new_page) goto out; - pte_val(*pt_dir) = - __pa(new_page) | pgprot_val(PAGE_KERNEL); + pte_val(*pt_dir) = __pa(new_page) | pgt_prot; } address += PAGE_SIZE; } @@ -372,13 +386,21 @@ out: */ void __init vmem_map_init(void) { - unsigned long size = _eshared - _stext; struct memblock_region *reg; for_each_memblock(memory, reg) vmem_add_mem(reg->base, reg->size); - set_memory_ro((unsigned long)_stext, size >> PAGE_SHIFT); - pr_info("Write protected kernel read-only data: %luk\n", size >> 10); + __set_memory((unsigned long) _stext, + (_etext - _stext) >> PAGE_SHIFT, + SET_MEMORY_RO | SET_MEMORY_X); + __set_memory((unsigned long) _etext, + (_eshared - _etext) >> PAGE_SHIFT, + SET_MEMORY_RO); + __set_memory((unsigned long) _sinittext, + (_einittext - _sinittext) >> PAGE_SHIFT, + SET_MEMORY_RO | SET_MEMORY_X); + pr_info("Write protected kernel read-only data: %luk\n", + (_eshared - _stext) >> 10); } /* diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 167b31b186c1..4ecf6d687509 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -1263,14 +1263,6 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp) } /* - * Classic BPF function stub. BPF programs will be converted into - * eBPF and then bpf_int_jit_compile() will be called. - */ -void bpf_jit_compile(struct bpf_prog *fp) -{ -} - -/* * Compile eBPF program "fp" */ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) @@ -1331,14 +1323,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) } if (bpf_jit_enable > 1) { bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf); - if (jit.prg_buf) - print_fn_code(jit.prg_buf, jit.size_prg); - } - if (jit.prg_buf) { - set_memory_ro((unsigned long)header, header->pages); - fp->bpf_func = (void *) jit.prg_buf; - fp->jited = 1; + print_fn_code(jit.prg_buf, jit.size_prg); } + bpf_jit_binary_lock_ro(header); + fp->bpf_func = (void *) jit.prg_buf; + fp->jited = 1; free_addrs: kfree(jit.addrs); out: @@ -1347,21 +1336,3 @@ out: tmp : orig_fp); return fp; } - -/* - * Free eBPF program - */ -void bpf_jit_free(struct bpf_prog *fp) -{ - unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; - struct bpf_binary_header *header = (void *)addr; - - if (!fp->jited) - goto free_filter; - - set_memory_rw(addr, header->pages); - bpf_jit_binary_free(header); - -free_filter: - bpf_prog_unlock_free(fp); -} diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 38e17d4d9884..364b9d824be3 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -224,8 +224,8 @@ static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) rc = zpci_load(&data, req, offset); if (!rc) { - data = data << ((8 - len) * 8); - data = le64_to_cpu(data); + data = le64_to_cpu((__force __le64) data); + data >>= (8 - len) * 8; *val = (u32) data; } else *val = 0xffffffff; @@ -238,8 +238,8 @@ static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) u64 data = val; int rc; - data = cpu_to_le64(data); - data = data >> ((8 - len) * 8); + data <<= (8 - len) * 8; + data = (__force u64) cpu_to_le64(data); rc = zpci_store(data, req, offset); return rc; } @@ -641,7 +641,7 @@ int pcibios_add_device(struct pci_dev *pdev) int i; pdev->dev.groups = zpci_attr_groups; - pdev->dev.archdata.dma_ops = &s390_pci_dma_ops; + pdev->dev.dma_ops = &s390_pci_dma_ops; zpci_map_resources(pdev); for (i = 0; i < PCI_BAR_COUNT; i++) { diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c index 1d7a9c71944a..9081a57fa340 100644 --- a/arch/s390/pci/pci_dma.c +++ b/arch/s390/pci/pci_dma.c @@ -650,7 +650,7 @@ static int __init dma_debug_do_init(void) } fs_initcall(dma_debug_do_init); -struct dma_map_ops s390_pci_dma_ops = { +const struct dma_map_ops s390_pci_dma_ops = { .alloc = s390_dma_alloc, .free = s390_dma_free, .map_sg = s390_dma_map_sg, diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c index 8cc53b1e6d03..0cf802de52a1 100644 --- a/arch/s390/tools/gen_facilities.c +++ b/arch/s390/tools/gen_facilities.c @@ -80,6 +80,8 @@ static struct facility_def facility_defs[] = { 76, /* msa extension 3 */ 77, /* msa extension 4 */ 78, /* enhanced-DAT 2 */ + 130, /* instruction-execution-protection */ + 131, /* enhanced-SOP 2 and side-effect */ -1 /* END */ } }, |