From e0d70bcb38d7eae5eb4da2db130a29d7007eac74 Mon Sep 17 00:00:00 2001 From: Bo YU Date: Tue, 30 Oct 2018 08:01:14 -0400 Subject: kobject: drop unnecessary cast "%llu" for u64 There is no searon for u64 var cast to unsigned long long type. Signed-off-by: Bo YU Signed-off-by: Greg Kroah-Hartman --- lib/kobject_uevent.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c index 63d0816ab23b..402765c3a9cb 100644 --- a/lib/kobject_uevent.c +++ b/lib/kobject_uevent.c @@ -579,7 +579,7 @@ int kobject_uevent_env(struct kobject *kobj, enum kobject_action action, mutex_lock(&uevent_sock_mutex); /* we will send an event, so request a new sequence number */ - retval = add_uevent_var(env, "SEQNUM=%llu", (unsigned long long)++uevent_seqnum); + retval = add_uevent_var(env, "SEQNUM=%llu", ++uevent_seqnum); if (retval) { mutex_unlock(&uevent_sock_mutex); goto exit; -- cgit v1.2.3 From 6be244dcd59b01245394b62a128901b1cfec468c Mon Sep 17 00:00:00 2001 From: Bo YU Date: Tue, 30 Oct 2018 08:01:15 -0400 Subject: kobject: Fix warnings in lib/kobject_uevent.c Add a blank after declaration. Signed-off-by: Bo YU Signed-off-by: Greg Kroah-Hartman --- lib/kobject_uevent.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c index 402765c3a9cb..27c6118afd1c 100644 --- a/lib/kobject_uevent.c +++ b/lib/kobject_uevent.c @@ -240,6 +240,7 @@ static int kobj_usermode_filter(struct kobject *kobj) ops = kobj_ns_ops(kobj); if (ops) { const void *init_ns, *ns; + ns = kobj->ktype->namespace(kobj); init_ns = ops->initial_ns(); return ns != init_ns; @@ -390,6 +391,7 @@ static int kobject_uevent_net_broadcast(struct kobject *kobj, ops = kobj_ns_ops(kobj); if (!ops && kobj->kset) { struct kobject *ksobj = &kobj->kset->kobj; + if (ksobj->parent != NULL) ops = kobj_ns_ops(ksobj->parent); } -- cgit v1.2.3 From d0894409d1711271cf5060e8ffe9e56359544d19 Mon Sep 17 00:00:00 2001 From: Paul Burton Date: Thu, 8 Nov 2018 23:44:56 +0000 Subject: lib/gcd: Remove use of CPU_NO_EFFICIENT_FFS macro The CPU_NO_EFFICIENT_FFS pre-processor macro is no longer used, with all architectures toggling the equivalent Kconfig symbol CONFIG_CPU_NO_EFFICIENT_FFS instead. Remove our check for the unused macro. Signed-off-by: Paul Burton Patchwork: https://patchwork.linux-mips.org/patch/21046/ Cc: Andrew Morton Cc: Zhaoxiu Zeng Cc: linux-mips@linux-mips.org Cc: linux-kernel@vger.kernel.org --- lib/gcd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gcd.c b/lib/gcd.c index 227dea924425..7948ab27f0a4 100644 --- a/lib/gcd.c +++ b/lib/gcd.c @@ -10,7 +10,7 @@ * has decent hardware division. */ -#if !defined(CONFIG_CPU_NO_EFFICIENT_FFS) && !defined(CPU_NO_EFFICIENT_FFS) +#if !defined(CONFIG_CPU_NO_EFFICIENT_FFS) /* If __ffs is available, the even/odd algorithm benchmarks slower. */ -- cgit v1.2.3 From 0a020d416d0af0b0c782e2a8363896e756e9121e Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 14 Nov 2018 08:22:28 +0000 Subject: lib: introduce initial implementation of object aggregation manager This lib tracks objects which could be of two types: 1) root object 2) nested object - with a "delta" which differentiates it from the associated root object The objects are tracked by a hashtable and reference-counted. User is responsible of implementing callbacks to create/destroy root entity related to each root object and callback to create/destroy nested object delta. Signed-off-by: Jiri Pirko Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- MAINTAINERS | 8 + include/linux/objagg.h | 46 +++ include/trace/events/objagg.h | 228 ++++++++++++ lib/Kconfig | 3 + lib/Kconfig.debug | 10 + lib/Makefile | 2 + lib/objagg.c | 501 +++++++++++++++++++++++++ lib/test_objagg.c | 835 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 1633 insertions(+) create mode 100644 include/linux/objagg.h create mode 100644 include/trace/events/objagg.h create mode 100644 lib/objagg.c create mode 100644 lib/test_objagg.c (limited to 'lib') diff --git a/MAINTAINERS b/MAINTAINERS index e110e327bf38..3bd775ba51ce 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10679,6 +10679,14 @@ L: linux-nfc@lists.01.org (moderated for non-subscribers) S: Supported F: drivers/nfc/nxp-nci +OBJAGG +M: Jiri Pirko +L: netdev@vger.kernel.org +S: Supported +F: lib/objagg.c +F: lib/test_objagg.c +F: include/linux/objagg.h + OBJTOOL M: Josh Poimboeuf M: Peter Zijlstra diff --git a/include/linux/objagg.h b/include/linux/objagg.h new file mode 100644 index 000000000000..34f38c186ea0 --- /dev/null +++ b/include/linux/objagg.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ + +#ifndef _OBJAGG_H +#define _OBJAGG_H + +struct objagg_ops { + size_t obj_size; + void * (*delta_create)(void *priv, void *parent_obj, void *obj); + void (*delta_destroy)(void *priv, void *delta_priv); + void * (*root_create)(void *priv, void *obj); + void (*root_destroy)(void *priv, void *root_priv); +}; + +struct objagg; +struct objagg_obj; + +const void *objagg_obj_root_priv(const struct objagg_obj *objagg_obj); +const void *objagg_obj_delta_priv(const struct objagg_obj *objagg_obj); +const void *objagg_obj_raw(const struct objagg_obj *objagg_obj); + +struct objagg_obj *objagg_obj_get(struct objagg *objagg, void *obj); +void objagg_obj_put(struct objagg *objagg, struct objagg_obj *objagg_obj); +struct objagg *objagg_create(const struct objagg_ops *ops, void *priv); +void objagg_destroy(struct objagg *objagg); + +struct objagg_obj_stats { + unsigned int user_count; + unsigned int delta_user_count; /* includes delta object users */ +}; + +struct objagg_obj_stats_info { + struct objagg_obj_stats stats; + struct objagg_obj *objagg_obj; /* associated object */ + bool is_root; +}; + +struct objagg_stats { + unsigned int stats_info_count; + struct objagg_obj_stats_info stats_info[]; +}; + +const struct objagg_stats *objagg_stats_get(struct objagg *objagg); +void objagg_stats_put(const struct objagg_stats *objagg_stats); + +#endif diff --git a/include/trace/events/objagg.h b/include/trace/events/objagg.h new file mode 100644 index 000000000000..fcec0fc9eb0c --- /dev/null +++ b/include/trace/events/objagg.h @@ -0,0 +1,228 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM objagg + +#if !defined(__TRACE_OBJAGG_H) || defined(TRACE_HEADER_MULTI_READ) +#define __TRACE_OBJAGG_H + +#include + +struct objagg; +struct objagg_obj; + +TRACE_EVENT(objagg_create, + TP_PROTO(const struct objagg *objagg), + + TP_ARGS(objagg), + + TP_STRUCT__entry( + __field(const void *, objagg) + ), + + TP_fast_assign( + __entry->objagg = objagg; + ), + + TP_printk("objagg %p", __entry->objagg) +); + +TRACE_EVENT(objagg_destroy, + TP_PROTO(const struct objagg *objagg), + + TP_ARGS(objagg), + + TP_STRUCT__entry( + __field(const void *, objagg) + ), + + TP_fast_assign( + __entry->objagg = objagg; + ), + + TP_printk("objagg %p", __entry->objagg) +); + +TRACE_EVENT(objagg_obj_create, + TP_PROTO(const struct objagg *objagg, + const struct objagg_obj *obj), + + TP_ARGS(objagg, obj), + + TP_STRUCT__entry( + __field(const void *, objagg) + __field(const void *, obj) + ), + + TP_fast_assign( + __entry->objagg = objagg; + __entry->obj = obj; + ), + + TP_printk("objagg %p, obj %p", __entry->objagg, __entry->obj) +); + +TRACE_EVENT(objagg_obj_destroy, + TP_PROTO(const struct objagg *objagg, + const struct objagg_obj *obj), + + TP_ARGS(objagg, obj), + + TP_STRUCT__entry( + __field(const void *, objagg) + __field(const void *, obj) + ), + + TP_fast_assign( + __entry->objagg = objagg; + __entry->obj = obj; + ), + + TP_printk("objagg %p, obj %p", __entry->objagg, __entry->obj) +); + +TRACE_EVENT(objagg_obj_get, + TP_PROTO(const struct objagg *objagg, + const struct objagg_obj *obj, + unsigned int refcount), + + TP_ARGS(objagg, obj, refcount), + + TP_STRUCT__entry( + __field(const void *, objagg) + __field(const void *, obj) + __field(unsigned int, refcount) + ), + + TP_fast_assign( + __entry->objagg = objagg; + __entry->obj = obj; + __entry->refcount = refcount; + ), + + TP_printk("objagg %p, obj %p, refcount %u", + __entry->objagg, __entry->obj, __entry->refcount) +); + +TRACE_EVENT(objagg_obj_put, + TP_PROTO(const struct objagg *objagg, + const struct objagg_obj *obj, + unsigned int refcount), + + TP_ARGS(objagg, obj, refcount), + + TP_STRUCT__entry( + __field(const void *, objagg) + __field(const void *, obj) + __field(unsigned int, refcount) + ), + + TP_fast_assign( + __entry->objagg = objagg; + __entry->obj = obj; + __entry->refcount = refcount; + ), + + TP_printk("objagg %p, obj %p, refcount %u", + __entry->objagg, __entry->obj, __entry->refcount) +); + +TRACE_EVENT(objagg_obj_parent_assign, + TP_PROTO(const struct objagg *objagg, + const struct objagg_obj *obj, + const struct objagg_obj *parent, + unsigned int parent_refcount), + + TP_ARGS(objagg, obj, parent, parent_refcount), + + TP_STRUCT__entry( + __field(const void *, objagg) + __field(const void *, obj) + __field(const void *, parent) + __field(unsigned int, parent_refcount) + ), + + TP_fast_assign( + __entry->objagg = objagg; + __entry->obj = obj; + __entry->parent = parent; + __entry->parent_refcount = parent_refcount; + ), + + TP_printk("objagg %p, obj %p, parent %p, parent_refcount %u", + __entry->objagg, __entry->obj, + __entry->parent, __entry->parent_refcount) +); + +TRACE_EVENT(objagg_obj_parent_unassign, + TP_PROTO(const struct objagg *objagg, + const struct objagg_obj *obj, + const struct objagg_obj *parent, + unsigned int parent_refcount), + + TP_ARGS(objagg, obj, parent, parent_refcount), + + TP_STRUCT__entry( + __field(const void *, objagg) + __field(const void *, obj) + __field(const void *, parent) + __field(unsigned int, parent_refcount) + ), + + TP_fast_assign( + __entry->objagg = objagg; + __entry->obj = obj; + __entry->parent = parent; + __entry->parent_refcount = parent_refcount; + ), + + TP_printk("objagg %p, obj %p, parent %p, parent_refcount %u", + __entry->objagg, __entry->obj, + __entry->parent, __entry->parent_refcount) +); + +TRACE_EVENT(objagg_obj_root_create, + TP_PROTO(const struct objagg *objagg, + const struct objagg_obj *obj), + + TP_ARGS(objagg, obj), + + TP_STRUCT__entry( + __field(const void *, objagg) + __field(const void *, obj) + ), + + TP_fast_assign( + __entry->objagg = objagg; + __entry->obj = obj; + ), + + TP_printk("objagg %p, obj %p", + __entry->objagg, __entry->obj) +); + +TRACE_EVENT(objagg_obj_root_destroy, + TP_PROTO(const struct objagg *objagg, + const struct objagg_obj *obj), + + TP_ARGS(objagg, obj), + + TP_STRUCT__entry( + __field(const void *, objagg) + __field(const void *, obj) + ), + + TP_fast_assign( + __entry->objagg = objagg; + __entry->obj = obj; + ), + + TP_printk("objagg %p, obj %p", + __entry->objagg, __entry->obj) +); + +#endif /* __TRACE_OBJAGG_H */ + +/* This part must be outside protection */ +#include diff --git a/lib/Kconfig b/lib/Kconfig index a9965f4af4dd..7dbbcfe9cd90 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -624,3 +624,6 @@ config GENERIC_LIB_CMPDI2 config GENERIC_LIB_UCMPDI2 bool + +config OBJAGG + tristate "objagg" if COMPILE_TEST diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 1af29b8224fd..b3c91b9e32f8 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1976,6 +1976,16 @@ config TEST_MEMCAT_P If unsure, say N. +config TEST_OBJAGG + tristate "Perform selftest on object aggreration manager" + default n + depends on OBJAGG + help + Enable this option to test object aggregation manager on boot + (or module load). + + If unsure, say N. + endif # RUNTIME_TESTING_MENU config MEMTEST diff --git a/lib/Makefile b/lib/Makefile index db06d1237898..f5262d30bfe6 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -75,6 +75,7 @@ obj-$(CONFIG_TEST_PARMAN) += test_parman.o obj-$(CONFIG_TEST_KMOD) += test_kmod.o obj-$(CONFIG_TEST_DEBUG_VIRTUAL) += test_debug_virtual.o obj-$(CONFIG_TEST_MEMCAT_P) += test_memcat_p.o +obj-$(CONFIG_TEST_OBJAGG) += test_objagg.o ifeq ($(CONFIG_DEBUG_KOBJECT),y) CFLAGS_kobject.o += -DDEBUG @@ -274,3 +275,4 @@ obj-$(CONFIG_GENERIC_LIB_LSHRDI3) += lshrdi3.o obj-$(CONFIG_GENERIC_LIB_MULDI3) += muldi3.o obj-$(CONFIG_GENERIC_LIB_CMPDI2) += cmpdi2.o obj-$(CONFIG_GENERIC_LIB_UCMPDI2) += ucmpdi2.o +obj-$(CONFIG_OBJAGG) += objagg.o diff --git a/lib/objagg.c b/lib/objagg.c new file mode 100644 index 000000000000..c9b457a91153 --- /dev/null +++ b/lib/objagg.c @@ -0,0 +1,501 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ + +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +struct objagg { + const struct objagg_ops *ops; + void *priv; + struct rhashtable obj_ht; + struct rhashtable_params ht_params; + struct list_head obj_list; + unsigned int obj_count; +}; + +struct objagg_obj { + struct rhash_head ht_node; /* member of objagg->obj_ht */ + struct list_head list; /* member of objagg->obj_list */ + struct objagg_obj *parent; /* if the object is nested, this + * holds pointer to parent, otherwise NULL + */ + union { + void *delta_priv; /* user delta private */ + void *root_priv; /* user root private */ + }; + unsigned int refcount; /* counts number of users of this object + * including nested objects + */ + struct objagg_obj_stats stats; + unsigned long obj[0]; +}; + +static unsigned int objagg_obj_ref_inc(struct objagg_obj *objagg_obj) +{ + return ++objagg_obj->refcount; +} + +static unsigned int objagg_obj_ref_dec(struct objagg_obj *objagg_obj) +{ + return --objagg_obj->refcount; +} + +static void objagg_obj_stats_inc(struct objagg_obj *objagg_obj) +{ + objagg_obj->stats.user_count++; + objagg_obj->stats.delta_user_count++; + if (objagg_obj->parent) + objagg_obj->parent->stats.delta_user_count++; +} + +static void objagg_obj_stats_dec(struct objagg_obj *objagg_obj) +{ + objagg_obj->stats.user_count--; + objagg_obj->stats.delta_user_count--; + if (objagg_obj->parent) + objagg_obj->parent->stats.delta_user_count--; +} + +static bool objagg_obj_is_root(const struct objagg_obj *objagg_obj) +{ + /* Nesting is not supported, so we can use ->parent + * to figure out if the object is root. + */ + return !objagg_obj->parent; +} + +/** + * objagg_obj_root_priv - obtains root private for an object + * @objagg_obj: objagg object instance + * + * Note: all locking must be provided by the caller. + * + * Either the object is root itself when the private is returned + * directly, or the parent is root and its private is returned + * instead. + * + * Returns a user private root pointer. + */ +const void *objagg_obj_root_priv(const struct objagg_obj *objagg_obj) +{ + if (objagg_obj_is_root(objagg_obj)) + return objagg_obj->root_priv; + WARN_ON(!objagg_obj_is_root(objagg_obj->parent)); + return objagg_obj->parent->root_priv; +} +EXPORT_SYMBOL(objagg_obj_root_priv); + +/** + * objagg_obj_delta_priv - obtains delta private for an object + * @objagg_obj: objagg object instance + * + * Note: all locking must be provided by the caller. + * + * Returns user private delta pointer or NULL in case the passed + * object is root. + */ +const void *objagg_obj_delta_priv(const struct objagg_obj *objagg_obj) +{ + if (objagg_obj_is_root(objagg_obj)) + return NULL; + return objagg_obj->delta_priv; +} +EXPORT_SYMBOL(objagg_obj_delta_priv); + +/** + * objagg_obj_raw - obtains object user private pointer + * @objagg_obj: objagg object instance + * + * Note: all locking must be provided by the caller. + * + * Returns user private pointer as was passed to objagg_obj_get() by "obj" arg. + */ +const void *objagg_obj_raw(const struct objagg_obj *objagg_obj) +{ + return objagg_obj->obj; +} +EXPORT_SYMBOL(objagg_obj_raw); + +static struct objagg_obj *objagg_obj_lookup(struct objagg *objagg, void *obj) +{ + return rhashtable_lookup_fast(&objagg->obj_ht, obj, objagg->ht_params); +} + +static int objagg_obj_parent_assign(struct objagg *objagg, + struct objagg_obj *objagg_obj, + struct objagg_obj *parent) +{ + void *delta_priv; + + delta_priv = objagg->ops->delta_create(objagg->priv, parent->obj, + objagg_obj->obj); + if (IS_ERR(delta_priv)) + return PTR_ERR(delta_priv); + + /* User returned a delta private, that means that + * our object can be aggregated into the parent. + */ + objagg_obj->parent = parent; + objagg_obj->delta_priv = delta_priv; + objagg_obj_ref_inc(objagg_obj->parent); + trace_objagg_obj_parent_assign(objagg, objagg_obj, + parent, + parent->refcount); + return 0; +} + +static int objagg_obj_parent_lookup_assign(struct objagg *objagg, + struct objagg_obj *objagg_obj) +{ + struct objagg_obj *objagg_obj_cur; + int err; + + list_for_each_entry(objagg_obj_cur, &objagg->obj_list, list) { + /* Nesting is not supported. In case the object + * is not root, it cannot be assigned as parent. + */ + if (!objagg_obj_is_root(objagg_obj_cur)) + continue; + err = objagg_obj_parent_assign(objagg, objagg_obj, + objagg_obj_cur); + if (!err) + return 0; + } + return -ENOENT; +} + +static void __objagg_obj_put(struct objagg *objagg, + struct objagg_obj *objagg_obj); + +static void objagg_obj_parent_unassign(struct objagg *objagg, + struct objagg_obj *objagg_obj) +{ + trace_objagg_obj_parent_unassign(objagg, objagg_obj, + objagg_obj->parent, + objagg_obj->parent->refcount); + objagg->ops->delta_destroy(objagg->priv, objagg_obj->delta_priv); + __objagg_obj_put(objagg, objagg_obj->parent); +} + +static int objagg_obj_root_create(struct objagg *objagg, + struct objagg_obj *objagg_obj) +{ + objagg_obj->root_priv = objagg->ops->root_create(objagg->priv, + objagg_obj->obj); + if (IS_ERR(objagg_obj->root_priv)) + return PTR_ERR(objagg_obj->root_priv); + + trace_objagg_obj_root_create(objagg, objagg_obj); + return 0; +} + +static void objagg_obj_root_destroy(struct objagg *objagg, + struct objagg_obj *objagg_obj) +{ + trace_objagg_obj_root_destroy(objagg, objagg_obj); + objagg->ops->root_destroy(objagg->priv, objagg_obj->root_priv); +} + +static int objagg_obj_init(struct objagg *objagg, + struct objagg_obj *objagg_obj) +{ + int err; + + /* Try to find if the object can be aggregated under an existing one. */ + err = objagg_obj_parent_lookup_assign(objagg, objagg_obj); + if (!err) + return 0; + /* If aggregation is not possible, make the object a root. */ + return objagg_obj_root_create(objagg, objagg_obj); +} + +static void objagg_obj_fini(struct objagg *objagg, + struct objagg_obj *objagg_obj) +{ + if (!objagg_obj_is_root(objagg_obj)) + objagg_obj_parent_unassign(objagg, objagg_obj); + else + objagg_obj_root_destroy(objagg, objagg_obj); +} + +static struct objagg_obj *objagg_obj_create(struct objagg *objagg, void *obj) +{ + struct objagg_obj *objagg_obj; + int err; + + objagg_obj = kzalloc(sizeof(*objagg_obj) + objagg->ops->obj_size, + GFP_KERNEL); + if (!objagg_obj) + return ERR_PTR(-ENOMEM); + objagg_obj_ref_inc(objagg_obj); + memcpy(objagg_obj->obj, obj, objagg->ops->obj_size); + + err = objagg_obj_init(objagg, objagg_obj); + if (err) + goto err_obj_init; + + err = rhashtable_insert_fast(&objagg->obj_ht, &objagg_obj->ht_node, + objagg->ht_params); + if (err) + goto err_ht_insert; + list_add(&objagg_obj->list, &objagg->obj_list); + objagg->obj_count++; + trace_objagg_obj_create(objagg, objagg_obj); + + return objagg_obj; + +err_ht_insert: + objagg_obj_fini(objagg, objagg_obj); +err_obj_init: + kfree(objagg_obj); + return ERR_PTR(err); +} + +static struct objagg_obj *__objagg_obj_get(struct objagg *objagg, void *obj) +{ + struct objagg_obj *objagg_obj; + + /* First, try to find the object exactly as user passed it, + * perhaps it is already in use. + */ + objagg_obj = objagg_obj_lookup(objagg, obj); + if (objagg_obj) { + objagg_obj_ref_inc(objagg_obj); + return objagg_obj; + } + + return objagg_obj_create(objagg, obj); +} + +/** + * objagg_obj_get - gets an object within objagg instance + * @objagg: objagg instance + * @obj: user-specific private object pointer + * + * Note: all locking must be provided by the caller. + * + * Size of the "obj" memory is specified in "objagg->ops". + * + * There are 3 main options this function wraps: + * 1) The object according to "obj" already exist. In that case + * the reference counter is incrementes and the object is returned. + * 2) The object does not exist, but it can be aggregated within + * another object. In that case, user ops->delta_create() is called + * to obtain delta data and a new object is created with returned + * user-delta private pointer. + * 3) The object does not exist and cannot be aggregated into + * any of the existing objects. In that case, user ops->root_create() + * is called to create the root and a new object is created with + * returned user-root private pointer. + * + * Returns a pointer to objagg object instance in case of success, + * otherwise it returns pointer error using ERR_PTR macro. + */ +struct objagg_obj *objagg_obj_get(struct objagg *objagg, void *obj) +{ + struct objagg_obj *objagg_obj; + + objagg_obj = __objagg_obj_get(objagg, obj); + if (IS_ERR(objagg_obj)) + return objagg_obj; + objagg_obj_stats_inc(objagg_obj); + trace_objagg_obj_get(objagg, objagg_obj, objagg_obj->refcount); + return objagg_obj; +} +EXPORT_SYMBOL(objagg_obj_get); + +static void objagg_obj_destroy(struct objagg *objagg, + struct objagg_obj *objagg_obj) +{ + trace_objagg_obj_destroy(objagg, objagg_obj); + --objagg->obj_count; + list_del(&objagg_obj->list); + rhashtable_remove_fast(&objagg->obj_ht, &objagg_obj->ht_node, + objagg->ht_params); + objagg_obj_fini(objagg, objagg_obj); + kfree(objagg_obj); +} + +static void __objagg_obj_put(struct objagg *objagg, + struct objagg_obj *objagg_obj) +{ + if (!objagg_obj_ref_dec(objagg_obj)) + objagg_obj_destroy(objagg, objagg_obj); +} + +/** + * objagg_obj_put - puts an object within objagg instance + * @objagg: objagg instance + * @objagg_obj: objagg object instance + * + * Note: all locking must be provided by the caller. + * + * Symmetric to objagg_obj_get(). + */ +void objagg_obj_put(struct objagg *objagg, struct objagg_obj *objagg_obj) +{ + trace_objagg_obj_put(objagg, objagg_obj, objagg_obj->refcount); + objagg_obj_stats_dec(objagg_obj); + __objagg_obj_put(objagg, objagg_obj); +} +EXPORT_SYMBOL(objagg_obj_put); + +/** + * objagg_create - creates a new objagg instance + * @ops: user-specific callbacks + * @priv: pointer to a private data passed to the ops + * + * Note: all locking must be provided by the caller. + * + * The purpose of the library is to provide an infrastructure to + * aggregate user-specified objects. Library does not care about the type + * of the object. User fills-up ops which take care of the specific + * user object manipulation. + * + * As a very stupid example, consider integer numbers. For example + * number 8 as a root object. That can aggregate number 9 with delta 1, + * number 10 with delta 2, etc. This example is implemented as + * a part of a testing module in test_objagg.c file. + * + * Each objagg instance contains multiple trees. Each tree node is + * represented by "an object". In the current implementation there can be + * only roots and leafs nodes. Leaf nodes are called deltas. + * But in general, this can be easily extended for intermediate nodes. + * In that extension, a delta would be associated with all non-root + * nodes. + * + * Returns a pointer to newly created objagg instance in case of success, + * otherwise it returns pointer error using ERR_PTR macro. + */ +struct objagg *objagg_create(const struct objagg_ops *ops, void *priv) +{ + struct objagg *objagg; + int err; + + if (WARN_ON(!ops || !ops->root_create || !ops->root_destroy || + !ops->delta_create || !ops->delta_destroy)) + return ERR_PTR(-EINVAL); + objagg = kzalloc(sizeof(*objagg), GFP_KERNEL); + if (!objagg) + return ERR_PTR(-ENOMEM); + objagg->ops = ops; + objagg->priv = priv; + INIT_LIST_HEAD(&objagg->obj_list); + + objagg->ht_params.key_len = ops->obj_size; + objagg->ht_params.key_offset = offsetof(struct objagg_obj, obj); + objagg->ht_params.head_offset = offsetof(struct objagg_obj, ht_node); + + err = rhashtable_init(&objagg->obj_ht, &objagg->ht_params); + if (err) + goto err_rhashtable_init; + + trace_objagg_create(objagg); + return objagg; + +err_rhashtable_init: + kfree(objagg); + return ERR_PTR(err); +} +EXPORT_SYMBOL(objagg_create); + +/** + * objagg_destroy - destroys a new objagg instance + * @objagg: objagg instance + * + * Note: all locking must be provided by the caller. + */ +void objagg_destroy(struct objagg *objagg) +{ + trace_objagg_destroy(objagg); + WARN_ON(!list_empty(&objagg->obj_list)); + rhashtable_destroy(&objagg->obj_ht); + kfree(objagg); +} +EXPORT_SYMBOL(objagg_destroy); + +static int objagg_stats_info_sort_cmp_func(const void *a, const void *b) +{ + const struct objagg_obj_stats_info *stats_info1 = a; + const struct objagg_obj_stats_info *stats_info2 = b; + + if (stats_info1->is_root != stats_info2->is_root) + return stats_info2->is_root - stats_info1->is_root; + if (stats_info1->stats.delta_user_count != + stats_info2->stats.delta_user_count) + return stats_info2->stats.delta_user_count - + stats_info1->stats.delta_user_count; + return stats_info2->stats.user_count - stats_info1->stats.user_count; +} + +/** + * objagg_stats_get - obtains stats of the objagg instance + * @objagg: objagg instance + * + * Note: all locking must be provided by the caller. + * + * The returned structure contains statistics of all object + * currently in use, ordered by following rules: + * 1) Root objects are always on lower indexes than the rest. + * 2) Objects with higher delta user count are always on lower + * indexes. + * 3) In case more objects have the same delta user count, + * the objects are ordered by user count. + * + * Returns a pointer to stats instance in case of success, + * otherwise it returns pointer error using ERR_PTR macro. + */ +const struct objagg_stats *objagg_stats_get(struct objagg *objagg) +{ + struct objagg_stats *objagg_stats; + struct objagg_obj *objagg_obj; + size_t alloc_size; + int i; + + alloc_size = sizeof(*objagg_stats) + + sizeof(objagg_stats->stats_info[0]) * objagg->obj_count; + objagg_stats = kzalloc(alloc_size, GFP_KERNEL); + if (!objagg_stats) + return ERR_PTR(-ENOMEM); + + i = 0; + list_for_each_entry(objagg_obj, &objagg->obj_list, list) { + memcpy(&objagg_stats->stats_info[i].stats, &objagg_obj->stats, + sizeof(objagg_stats->stats_info[0].stats)); + objagg_stats->stats_info[i].objagg_obj = objagg_obj; + objagg_stats->stats_info[i].is_root = + objagg_obj_is_root(objagg_obj); + i++; + } + objagg_stats->stats_info_count = i; + + sort(objagg_stats->stats_info, objagg_stats->stats_info_count, + sizeof(struct objagg_obj_stats_info), + objagg_stats_info_sort_cmp_func, NULL); + + return objagg_stats; +} +EXPORT_SYMBOL(objagg_stats_get); + +/** + * objagg_stats_puts - puts stats of the objagg instance + * @objagg_stats: objagg instance stats + * + * Note: all locking must be provided by the caller. + */ +void objagg_stats_put(const struct objagg_stats *objagg_stats) +{ + kfree(objagg_stats); +} +EXPORT_SYMBOL(objagg_stats_put); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Jiri Pirko "); +MODULE_DESCRIPTION("Object aggregation manager"); diff --git a/lib/test_objagg.c b/lib/test_objagg.c new file mode 100644 index 000000000000..aac5d8e8800c --- /dev/null +++ b/lib/test_objagg.c @@ -0,0 +1,835 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include + +struct tokey { + unsigned int id; +}; + +#define NUM_KEYS 32 + +static int key_id_index(unsigned int key_id) +{ + if (key_id >= NUM_KEYS) { + WARN_ON(1); + return 0; + } + return key_id; +} + +#define BUF_LEN 128 + +struct world { + unsigned int root_count; + unsigned int delta_count; + char next_root_buf[BUF_LEN]; + struct objagg_obj *objagg_objs[NUM_KEYS]; + unsigned int key_refs[NUM_KEYS]; +}; + +struct root { + struct tokey key; + char buf[BUF_LEN]; +}; + +struct delta { + unsigned int key_id_diff; +}; + +static struct objagg_obj *world_obj_get(struct world *world, + struct objagg *objagg, + unsigned int key_id) +{ + struct objagg_obj *objagg_obj; + struct tokey key; + int err; + + key.id = key_id; + objagg_obj = objagg_obj_get(objagg, &key); + if (IS_ERR(objagg_obj)) { + pr_err("Key %u: Failed to get object.\n", key_id); + return objagg_obj; + } + if (!world->key_refs[key_id_index(key_id)]) { + world->objagg_objs[key_id_index(key_id)] = objagg_obj; + } else if (world->objagg_objs[key_id_index(key_id)] != objagg_obj) { + pr_err("Key %u: God another object for the same key.\n", + key_id); + err = -EINVAL; + goto err_key_id_check; + } + world->key_refs[key_id_index(key_id)]++; + return objagg_obj; + +err_key_id_check: + objagg_obj_put(objagg, objagg_obj); + return ERR_PTR(err); +} + +static void world_obj_put(struct world *world, struct objagg *objagg, + unsigned int key_id) +{ + struct objagg_obj *objagg_obj; + + if (!world->key_refs[key_id_index(key_id)]) + return; + objagg_obj = world->objagg_objs[key_id_index(key_id)]; + objagg_obj_put(objagg, objagg_obj); + world->key_refs[key_id_index(key_id)]--; +} + +#define MAX_KEY_ID_DIFF 5 + +static void *delta_create(void *priv, void *parent_obj, void *obj) +{ + struct tokey *parent_key = parent_obj; + struct world *world = priv; + struct tokey *key = obj; + int diff = key->id - parent_key->id; + struct delta *delta; + + if (diff < 0 || diff > MAX_KEY_ID_DIFF) + return ERR_PTR(-EINVAL); + + delta = kzalloc(sizeof(*delta), GFP_KERNEL); + if (!delta) + return ERR_PTR(-ENOMEM); + delta->key_id_diff = diff; + world->delta_count++; + return delta; +} + +static void delta_destroy(void *priv, void *delta_priv) +{ + struct delta *delta = delta_priv; + struct world *world = priv; + + world->delta_count--; + kfree(delta); +} + +static void *root_create(void *priv, void *obj) +{ + struct world *world = priv; + struct tokey *key = obj; + struct root *root; + + root = kzalloc(sizeof(*root), GFP_KERNEL); + if (!root) + return ERR_PTR(-ENOMEM); + memcpy(&root->key, key, sizeof(root->key)); + memcpy(root->buf, world->next_root_buf, sizeof(root->buf)); + world->root_count++; + return root; +} + +static void root_destroy(void *priv, void *root_priv) +{ + struct root *root = root_priv; + struct world *world = priv; + + world->root_count--; + kfree(root); +} + +static int test_nodelta_obj_get(struct world *world, struct objagg *objagg, + unsigned int key_id, bool should_create_root) +{ + unsigned int orig_root_count = world->root_count; + struct objagg_obj *objagg_obj; + const struct root *root; + int err; + + if (should_create_root) + prandom_bytes(world->next_root_buf, + sizeof(world->next_root_buf)); + + objagg_obj = world_obj_get(world, objagg, key_id); + if (IS_ERR(objagg_obj)) { + pr_err("Key %u: Failed to get object.\n", key_id); + return PTR_ERR(objagg_obj); + } + if (should_create_root) { + if (world->root_count != orig_root_count + 1) { + pr_err("Key %u: Root was not created\n", key_id); + err = -EINVAL; + goto err_check_root_count; + } + } else { + if (world->root_count != orig_root_count) { + pr_err("Key %u: Root was incorrectly created\n", + key_id); + err = -EINVAL; + goto err_check_root_count; + } + } + root = objagg_obj_root_priv(objagg_obj); + if (root->key.id != key_id) { + pr_err("Key %u: Root has unexpected key id\n", key_id); + err = -EINVAL; + goto err_check_key_id; + } + if (should_create_root && + memcmp(world->next_root_buf, root->buf, sizeof(root->buf))) { + pr_err("Key %u: Buffer does not match the expected content\n", + key_id); + err = -EINVAL; + goto err_check_buf; + } + return 0; + +err_check_buf: +err_check_key_id: +err_check_root_count: + objagg_obj_put(objagg, objagg_obj); + return err; +} + +static int test_nodelta_obj_put(struct world *world, struct objagg *objagg, + unsigned int key_id, bool should_destroy_root) +{ + unsigned int orig_root_count = world->root_count; + + world_obj_put(world, objagg, key_id); + + if (should_destroy_root) { + if (world->root_count != orig_root_count - 1) { + pr_err("Key %u: Root was not destroyed\n", key_id); + return -EINVAL; + } + } else { + if (world->root_count != orig_root_count) { + pr_err("Key %u: Root was incorrectly destroyed\n", + key_id); + return -EINVAL; + } + } + return 0; +} + +static int check_stats_zero(struct objagg *objagg) +{ + const struct objagg_stats *stats; + int err = 0; + + stats = objagg_stats_get(objagg); + if (IS_ERR(stats)) + return PTR_ERR(stats); + + if (stats->stats_info_count != 0) { + pr_err("Stats: Object count is not zero while it should be\n"); + err = -EINVAL; + } + + objagg_stats_put(stats); + return err; +} + +static int check_stats_nodelta(struct objagg *objagg) +{ + const struct objagg_stats *stats; + int i; + int err; + + stats = objagg_stats_get(objagg); + if (IS_ERR(stats)) + return PTR_ERR(stats); + + if (stats->stats_info_count != NUM_KEYS) { + pr_err("Stats: Unexpected object count (%u expected, %u returned)\n", + NUM_KEYS, stats->stats_info_count); + err = -EINVAL; + goto stats_put; + } + + for (i = 0; i < stats->stats_info_count; i++) { + if (stats->stats_info[i].stats.user_count != 2) { + pr_err("Stats: incorrect user count\n"); + err = -EINVAL; + goto stats_put; + } + if (stats->stats_info[i].stats.delta_user_count != 2) { + pr_err("Stats: incorrect delta user count\n"); + err = -EINVAL; + goto stats_put; + } + } + err = 0; + +stats_put: + objagg_stats_put(stats); + return err; +} + +static void *delta_create_dummy(void *priv, void *parent_obj, void *obj) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static void delta_destroy_dummy(void *priv, void *delta_priv) +{ +} + +static const struct objagg_ops nodelta_ops = { + .obj_size = sizeof(struct tokey), + .delta_create = delta_create_dummy, + .delta_destroy = delta_destroy_dummy, + .root_create = root_create, + .root_destroy = root_destroy, +}; + +static int test_nodelta(void) +{ + struct world world = {}; + struct objagg *objagg; + int i; + int err; + + objagg = objagg_create(&nodelta_ops, &world); + if (IS_ERR(objagg)) + return PTR_ERR(objagg); + + err = check_stats_zero(objagg); + if (err) + goto err_stats_first_zero; + + /* First round of gets, the root objects should be created */ + for (i = 0; i < NUM_KEYS; i++) { + err = test_nodelta_obj_get(&world, objagg, i, true); + if (err) + goto err_obj_first_get; + } + + /* Do the second round of gets, all roots are already created, + * make sure that no new root is created + */ + for (i = 0; i < NUM_KEYS; i++) { + err = test_nodelta_obj_get(&world, objagg, i, false); + if (err) + goto err_obj_second_get; + } + + err = check_stats_nodelta(objagg); + if (err) + goto err_stats_nodelta; + + for (i = NUM_KEYS - 1; i >= 0; i--) { + err = test_nodelta_obj_put(&world, objagg, i, false); + if (err) + goto err_obj_first_put; + } + for (i = NUM_KEYS - 1; i >= 0; i--) { + err = test_nodelta_obj_put(&world, objagg, i, true); + if (err) + goto err_obj_second_put; + } + + err = check_stats_zero(objagg); + if (err) + goto err_stats_second_zero; + + objagg_destroy(objagg); + return 0; + +err_stats_nodelta: +err_obj_first_put: +err_obj_second_get: + for (i--; i >= 0; i--) + world_obj_put(&world, objagg, i); + + i = NUM_KEYS; +err_obj_first_get: +err_obj_second_put: + for (i--; i >= 0; i--) + world_obj_put(&world, objagg, i); +err_stats_first_zero: +err_stats_second_zero: + objagg_destroy(objagg); + return err; +} + +static const struct objagg_ops delta_ops = { + .obj_size = sizeof(struct tokey), + .delta_create = delta_create, + .delta_destroy = delta_destroy, + .root_create = root_create, + .root_destroy = root_destroy, +}; + +enum action { + ACTION_GET, + ACTION_PUT, +}; + +enum expect_delta { + EXPECT_DELTA_SAME, + EXPECT_DELTA_INC, + EXPECT_DELTA_DEC, +}; + +enum expect_root { + EXPECT_ROOT_SAME, + EXPECT_ROOT_INC, + EXPECT_ROOT_DEC, +}; + +struct expect_stats_info { + struct objagg_obj_stats stats; + bool is_root; + unsigned int key_id; +}; + +struct expect_stats { + unsigned int info_count; + struct expect_stats_info info[NUM_KEYS]; +}; + +struct action_item { + unsigned int key_id; + enum action action; + enum expect_delta expect_delta; + enum expect_root expect_root; + struct expect_stats expect_stats; +}; + +#define EXPECT_STATS(count, ...) \ +{ \ + .info_count = count, \ + .info = { __VA_ARGS__ } \ +} + +#define ROOT(key_id, user_count, delta_user_count) \ + {{user_count, delta_user_count}, true, key_id} + +#define DELTA(key_id, user_count) \ + {{user_count, user_count}, false, key_id} + +static const struct action_item action_items[] = { + { + 1, ACTION_GET, EXPECT_DELTA_SAME, EXPECT_ROOT_INC, + EXPECT_STATS(1, ROOT(1, 1, 1)), + }, /* r: 1 d: */ + { + 7, ACTION_GET, EXPECT_DELTA_SAME, EXPECT_ROOT_INC, + EXPECT_STATS(2, ROOT(1, 1, 1), ROOT(7, 1, 1)), + }, /* r: 1, 7 d: */ + { + 3, ACTION_GET, EXPECT_DELTA_INC, EXPECT_ROOT_SAME, + EXPECT_STATS(3, ROOT(1, 1, 2), ROOT(7, 1, 1), + DELTA(3, 1)), + }, /* r: 1, 7 d: 3^1 */ + { + 5, ACTION_GET, EXPECT_DELTA_INC, EXPECT_ROOT_SAME, + EXPECT_STATS(4, ROOT(1, 1, 3), ROOT(7, 1, 1), + DELTA(3, 1), DELTA(5, 1)), + }, /* r: 1, 7 d: 3^1, 5^1 */ + { + 3, ACTION_GET, EXPECT_DELTA_SAME, EXPECT_ROOT_SAME, + EXPECT_STATS(4, ROOT(1, 1, 4), ROOT(7, 1, 1), + DELTA(3, 2), DELTA(5, 1)), + }, /* r: 1, 7 d: 3^1, 3^1, 5^1 */ + { + 1, ACTION_GET, EXPECT_DELTA_SAME, EXPECT_ROOT_SAME, + EXPECT_STATS(4, ROOT(1, 2, 5), ROOT(7, 1, 1), + DELTA(3, 2), DELTA(5, 1)), + }, /* r: 1, 1, 7 d: 3^1, 3^1, 5^1 */ + { + 30, ACTION_GET, EXPECT_DELTA_SAME, EXPECT_ROOT_INC, + EXPECT_STATS(5, ROOT(1, 2, 5), ROOT(7, 1, 1), ROOT(30, 1, 1), + DELTA(3, 2), DELTA(5, 1)), + }, /* r: 1, 1, 7, 30 d: 3^1, 3^1, 5^1 */ + { + 8, ACTION_GET, EXPECT_DELTA_INC, EXPECT_ROOT_SAME, + EXPECT_STATS(6, ROOT(1, 2, 5), ROOT(7, 1, 2), ROOT(30, 1, 1), + DELTA(3, 2), DELTA(5, 1), DELTA(8, 1)), + }, /* r: 1, 1, 7, 30 d: 3^1, 3^1, 5^1, 8^7 */ + { + 8, ACTION_GET, EXPECT_DELTA_SAME, EXPECT_ROOT_SAME, + EXPECT_STATS(6, ROOT(1, 2, 5), ROOT(7, 1, 3), ROOT(30, 1, 1), + DELTA(3, 2), DELTA(8, 2), DELTA(5, 1)), + }, /* r: 1, 1, 7, 30 d: 3^1, 3^1, 5^1, 8^7, 8^7 */ + { + 3, ACTION_PUT, EXPECT_DELTA_SAME, EXPECT_ROOT_SAME, + EXPECT_STATS(6, ROOT(1, 2, 4), ROOT(7, 1, 3), ROOT(30, 1, 1), + DELTA(8, 2), DELTA(3, 1), DELTA(5, 1)), + }, /* r: 1, 1, 7, 30 d: 3^1, 5^1, 8^7, 8^7 */ + { + 3, ACTION_PUT, EXPECT_DELTA_DEC, EXPECT_ROOT_SAME, + EXPECT_STATS(5, ROOT(1, 2, 3), ROOT(7, 1, 3), ROOT(30, 1, 1), + DELTA(8, 2), DELTA(5, 1)), + }, /* r: 1, 1, 7, 30 d: 5^1, 8^7, 8^7 */ + { + 1, ACTION_PUT, EXPECT_DELTA_SAME, EXPECT_ROOT_SAME, + EXPECT_STATS(5, ROOT(7, 1, 3), ROOT(1, 1, 2), ROOT(30, 1, 1), + DELTA(8, 2), DELTA(5, 1)), + }, /* r: 1, 7, 30 d: 5^1, 8^7, 8^7 */ + { + 1, ACTION_PUT, EXPECT_DELTA_SAME, EXPECT_ROOT_SAME, + EXPECT_STATS(5, ROOT(7, 1, 3), ROOT(30, 1, 1), ROOT(1, 0, 1), + DELTA(8, 2), DELTA(5, 1)), + }, /* r: 7, 30 d: 5^1, 8^7, 8^7 */ + { + 5, ACTION_PUT, EXPECT_DELTA_DEC, EXPECT_ROOT_DEC, + EXPECT_STATS(3, ROOT(7, 1, 3), ROOT(30, 1, 1), + DELTA(8, 2)), + }, /* r: 7, 30 d: 8^7, 8^7 */ + { + 5, ACTION_GET, EXPECT_DELTA_SAME, EXPECT_ROOT_INC, + EXPECT_STATS(4, ROOT(7, 1, 3), ROOT(30, 1, 1), ROOT(5, 1, 1), + DELTA(8, 2)), + }, /* r: 7, 30, 5 d: 8^7, 8^7 */ + { + 6, ACTION_GET, EXPECT_DELTA_INC, EXPECT_ROOT_SAME, + EXPECT_STATS(5, ROOT(7, 1, 3), ROOT(5, 1, 2), ROOT(30, 1, 1), + DELTA(8, 2), DELTA(6, 1)), + }, /* r: 7, 30, 5 d: 8^7, 8^7, 6^5 */ + { + 8, ACTION_GET, EXPECT_DELTA_SAME, EXPECT_ROOT_SAME, + EXPECT_STATS(5, ROOT(7, 1, 4), ROOT(5, 1, 2), ROOT(30, 1, 1), + DELTA(8, 3), DELTA(6, 1)), + }, /* r: 7, 30, 5 d: 8^7, 8^7, 8^7, 6^5 */ + { + 8, ACTION_PUT, EXPECT_DELTA_SAME, EXPECT_ROOT_SAME, + EXPECT_STATS(5, ROOT(7, 1, 3), ROOT(5, 1, 2), ROOT(30, 1, 1), + DELTA(8, 2), DELTA(6, 1)), + }, /* r: 7, 30, 5 d: 8^7, 8^7, 6^5 */ + { + 8, ACTION_PUT, EXPECT_DELTA_SAME, EXPECT_ROOT_SAME, + EXPECT_STATS(5, ROOT(7, 1, 2), ROOT(5, 1, 2), ROOT(30, 1, 1), + DELTA(8, 1), DELTA(6, 1)), + }, /* r: 7, 30, 5 d: 8^7, 6^5 */ + { + 8, ACTION_PUT, EXPECT_DELTA_DEC, EXPECT_ROOT_SAME, + EXPECT_STATS(4, ROOT(5, 1, 2), ROOT(7, 1, 1), ROOT(30, 1, 1), + DELTA(6, 1)), + }, /* r: 7, 30, 5 d: 6^5 */ + { + 8, ACTION_GET, EXPECT_DELTA_INC, EXPECT_ROOT_SAME, + EXPECT_STATS(5, ROOT(5, 1, 3), ROOT(7, 1, 1), ROOT(30, 1, 1), + DELTA(6, 1), DELTA(8, 1)), + }, /* r: 7, 30, 5 d: 6^5, 8^5 */ + { + 7, ACTION_PUT, EXPECT_DELTA_SAME, EXPECT_ROOT_DEC, + EXPECT_STATS(4, ROOT(5, 1, 3), ROOT(30, 1, 1), + DELTA(6, 1), DELTA(8, 1)), + }, /* r: 30, 5 d: 6^5, 8^5 */ + { + 30, ACTION_PUT, EXPECT_DELTA_SAME, EXPECT_ROOT_DEC, + EXPECT_STATS(3, ROOT(5, 1, 3), + DELTA(6, 1), DELTA(8, 1)), + }, /* r: 5 d: 6^5, 8^5 */ + { + 5, ACTION_PUT, EXPECT_DELTA_SAME, EXPECT_ROOT_SAME, + EXPECT_STATS(3, ROOT(5, 0, 2), + DELTA(6, 1), DELTA(8, 1)), + }, /* r: d: 6^5, 8^5 */ + { + 6, ACTION_PUT, EXPECT_DELTA_DEC, EXPECT_ROOT_SAME, + EXPECT_STATS(2, ROOT(5, 0, 1), + DELTA(8, 1)), + }, /* r: d: 6^5 */ + { + 8, ACTION_PUT, EXPECT_DELTA_DEC, EXPECT_ROOT_DEC, + EXPECT_STATS(0, ), + }, /* r: d: */ +}; + +static int check_expect(struct world *world, + const struct action_item *action_item, + unsigned int orig_delta_count, + unsigned int orig_root_count) +{ + unsigned int key_id = action_item->key_id; + + switch (action_item->expect_delta) { + case EXPECT_DELTA_SAME: + if (orig_delta_count != world->delta_count) { + pr_err("Key %u: Delta count changed while expected to remain the same.\n", + key_id); + return -EINVAL; + } + break; + case EXPECT_DELTA_INC: + if (WARN_ON(action_item->action == ACTION_PUT)) + return -EINVAL; + if (orig_delta_count + 1 != world->delta_count) { + pr_err("Key %u: Delta count was not incremented.\n", + key_id); + return -EINVAL; + } + break; + case EXPECT_DELTA_DEC: + if (WARN_ON(action_item->action == ACTION_GET)) + return -EINVAL; + if (orig_delta_count - 1 != world->delta_count) { + pr_err("Key %u: Delta count was not decremented.\n", + key_id); + return -EINVAL; + } + break; + } + + switch (action_item->expect_root) { + case EXPECT_ROOT_SAME: + if (orig_root_count != world->root_count) { + pr_err("Key %u: Root count changed while expected to remain the same.\n", + key_id); + return -EINVAL; + } + break; + case EXPECT_ROOT_INC: + if (WARN_ON(action_item->action == ACTION_PUT)) + return -EINVAL; + if (orig_root_count + 1 != world->root_count) { + pr_err("Key %u: Root count was not incremented.\n", + key_id); + return -EINVAL; + } + break; + case EXPECT_ROOT_DEC: + if (WARN_ON(action_item->action == ACTION_GET)) + return -EINVAL; + if (orig_root_count - 1 != world->root_count) { + pr_err("Key %u: Root count was not decremented.\n", + key_id); + return -EINVAL; + } + } + + return 0; +} + +static unsigned int obj_to_key_id(struct objagg_obj *objagg_obj) +{ + const struct tokey *root_key; + const struct delta *delta; + unsigned int key_id; + + root_key = objagg_obj_root_priv(objagg_obj); + key_id = root_key->id; + delta = objagg_obj_delta_priv(objagg_obj); + if (delta) + key_id += delta->key_id_diff; + return key_id; +} + +static int +check_expect_stats_nums(const struct objagg_obj_stats_info *stats_info, + const struct expect_stats_info *expect_stats_info, + const char **errmsg) +{ + if (stats_info->is_root != expect_stats_info->is_root) { + if (errmsg) + *errmsg = "Incorrect root/delta indication"; + return -EINVAL; + } + if (stats_info->stats.user_count != + expect_stats_info->stats.user_count) { + if (errmsg) + *errmsg = "Incorrect user count"; + return -EINVAL; + } + if (stats_info->stats.delta_user_count != + expect_stats_info->stats.delta_user_count) { + if (errmsg) + *errmsg = "Incorrect delta user count"; + return -EINVAL; + } + return 0; +} + +static int +check_expect_stats_key_id(const struct objagg_obj_stats_info *stats_info, + const struct expect_stats_info *expect_stats_info, + const char **errmsg) +{ + if (obj_to_key_id(stats_info->objagg_obj) != + expect_stats_info->key_id) { + if (errmsg) + *errmsg = "incorrect key id"; + return -EINVAL; + } + return 0; +} + +static int check_expect_stats_neigh(const struct objagg_stats *stats, + const struct expect_stats *expect_stats, + int pos) +{ + int i; + int err; + + for (i = pos - 1; i >= 0; i--) { + err = check_expect_stats_nums(&stats->stats_info[i], + &expect_stats->info[pos], NULL); + if (err) + break; + err = check_expect_stats_key_id(&stats->stats_info[i], + &expect_stats->info[pos], NULL); + if (!err) + return 0; + } + for (i = pos + 1; i < stats->stats_info_count; i++) { + err = check_expect_stats_nums(&stats->stats_info[i], + &expect_stats->info[pos], NULL); + if (err) + break; + err = check_expect_stats_key_id(&stats->stats_info[i], + &expect_stats->info[pos], NULL); + if (!err) + return 0; + } + return -EINVAL; +} + +static int __check_expect_stats(const struct objagg_stats *stats, + const struct expect_stats *expect_stats, + const char **errmsg) +{ + int i; + int err; + + if (stats->stats_info_count != expect_stats->info_count) { + *errmsg = "Unexpected object count"; + return -EINVAL; + } + + for (i = 0; i < stats->stats_info_count; i++) { + err = check_expect_stats_nums(&stats->stats_info[i], + &expect_stats->info[i], errmsg); + if (err) + return err; + err = check_expect_stats_key_id(&stats->stats_info[i], + &expect_stats->info[i], errmsg); + if (err) { + /* It is possible that one of the neighbor stats with + * same numbers have the correct key id, so check it + */ + err = check_expect_stats_neigh(stats, expect_stats, i); + if (err) + return err; + } + } + return 0; +} + +static int check_expect_stats(struct objagg *objagg, + const struct expect_stats *expect_stats, + const char **errmsg) +{ + const struct objagg_stats *stats; + int err; + + stats = objagg_stats_get(objagg); + if (IS_ERR(stats)) + return PTR_ERR(stats); + err = __check_expect_stats(stats, expect_stats, errmsg); + objagg_stats_put(stats); + return err; +} + +static int test_delta_action_item(struct world *world, + struct objagg *objagg, + const struct action_item *action_item, + bool inverse) +{ + unsigned int orig_delta_count = world->delta_count; + unsigned int orig_root_count = world->root_count; + unsigned int key_id = action_item->key_id; + enum action action = action_item->action; + struct objagg_obj *objagg_obj; + const char *errmsg; + int err; + + if (inverse) + action = action == ACTION_GET ? ACTION_PUT : ACTION_GET; + + switch (action) { + case ACTION_GET: + objagg_obj = world_obj_get(world, objagg, key_id); + if (IS_ERR(objagg_obj)) + return PTR_ERR(objagg_obj); + break; + case ACTION_PUT: + world_obj_put(world, objagg, key_id); + break; + } + + if (inverse) + return 0; + err = check_expect(world, action_item, + orig_delta_count, orig_root_count); + if (err) + goto errout; + + err = check_expect_stats(objagg, &action_item->expect_stats, &errmsg); + if (err) { + pr_err("Key %u: Stats: %s\n", action_item->key_id, errmsg); + goto errout; + } + + return 0; + +errout: + /* This can only happen when action is not inversed. + * So in case of an error, cleanup by doing inverse action. + */ + test_delta_action_item(world, objagg, action_item, true); + return err; +} + +static int test_delta(void) +{ + struct world world = {}; + struct objagg *objagg; + int i; + int err; + + objagg = objagg_create(&delta_ops, &world); + if (IS_ERR(objagg)) + return PTR_ERR(objagg); + + for (i = 0; i < ARRAY_SIZE(action_items); i++) { + err = test_delta_action_item(&world, objagg, + &action_items[i], false); + if (err) + goto err_do_action_item; + } + + objagg_destroy(objagg); + return 0; + +err_do_action_item: + for (i--; i >= 0; i--) + test_delta_action_item(&world, objagg, &action_items[i], true); + + objagg_destroy(objagg); + return err; +} + +static int __init test_objagg_init(void) +{ + int err; + + err = test_nodelta(); + if (err) + return err; + return test_delta(); +} + +static void __exit test_objagg_exit(void) +{ +} + +module_init(test_objagg_init); +module_exit(test_objagg_exit); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Jiri Pirko "); +MODULE_DESCRIPTION("Test module for objagg"); -- cgit v1.2.3 From 110e2b4b94c623b36d0345a604970db34935f158 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Thu, 15 Nov 2018 14:57:09 -0800 Subject: test_objagg: Fix warning. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib/test_objagg.c: In function ‘test_delta_action_item’: ./include/linux/printk.h:308:2: warning: ‘errmsg’ may be used uninitialized in this function [-Wmaybe-uninitialized] Signed-off-by: David S. Miller --- lib/test_objagg.c | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/test_objagg.c b/lib/test_objagg.c index aac5d8e8800c..ab57144bb0cd 100644 --- a/lib/test_objagg.c +++ b/lib/test_objagg.c @@ -769,6 +769,7 @@ static int test_delta_action_item(struct world *world, if (err) goto errout; + errmsg = NULL; err = check_expect_stats(objagg, &action_item->expect_stats, &errmsg); if (err) { pr_err("Key %u: Stats: %s\n", action_item->key_id, errmsg); -- cgit v1.2.3 From 0c4b2d370514cb4f3454dd3b18f031d2651fab73 Mon Sep 17 00:00:00 2001 From: Michał Mirosław Date: Sat, 10 Nov 2018 19:58:36 +0100 Subject: net: remove VLAN_TAG_PRESENT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace VLAN_TAG_PRESENT with single bit flag and free up VLAN.CFI overload. Now VLAN.CFI is visible in networking stack and can be passed around intact. Signed-off-by: Michał Mirosław Signed-off-by: David S. Miller --- arch/mips/net/bpf_jit.c | 3 --- arch/powerpc/net/bpf_jit_comp.c | 3 --- arch/sparc/net/bpf_jit_comp_32.c | 4 ---- include/linux/if_vlan.h | 11 ++++++----- include/linux/skbuff.h | 16 +++++++++------- lib/test_bpf.c | 14 ++++++++------ net/core/filter.c | 6 ------ 7 files changed, 23 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/arch/mips/net/bpf_jit.c b/arch/mips/net/bpf_jit.c index de4c6372ad9a..3a0e34f4e615 100644 --- a/arch/mips/net/bpf_jit.c +++ b/arch/mips/net/bpf_jit.c @@ -1164,9 +1164,6 @@ jmp_cmp: vlan_tci) != 2); off = offsetof(struct sk_buff, vlan_tci); emit_half_load_unsigned(r_A, r_skb, off, ctx); -#ifdef VLAN_TAG_PRESENT - emit_andi(r_A, r_A, (u16)~VLAN_TAG_PRESENT, ctx); -#endif break; case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: ctx->flags |= SEEN_SKB | SEEN_A; diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index dc4a2f54e829..91d223cf512b 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -383,9 +383,6 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, vlan_tci)); -#ifdef VLAN_TAG_PRESENT - PPC_ANDI(r_A, r_A, ~VLAN_TAG_PRESENT); -#endif break; case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: PPC_LBZ_OFFS(r_A, r_skb, PKT_VLAN_PRESENT_OFFSET()); diff --git a/arch/sparc/net/bpf_jit_comp_32.c b/arch/sparc/net/bpf_jit_comp_32.c index 48f3c04dd179..84cc8f7f83e9 100644 --- a/arch/sparc/net/bpf_jit_comp_32.c +++ b/arch/sparc/net/bpf_jit_comp_32.c @@ -553,10 +553,6 @@ void bpf_jit_compile(struct bpf_prog *fp) break; case BPF_ANC | SKF_AD_VLAN_TAG: emit_skb_load16(vlan_tci, r_A); -#ifdef VLAN_TAG_PRESENT - emit_loadimm(~VLAN_TAG_PRESENT, r_TMP); - emit_and(r_A, r_TMP, r_A); -#endif break; case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: __emit_skb_load8(__pkt_vlan_present_offset, r_A); diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 1be5230921b5..7a541eadf78e 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -66,7 +66,6 @@ static inline struct vlan_ethhdr *vlan_eth_hdr(const struct sk_buff *skb) #define VLAN_PRIO_MASK 0xe000 /* Priority Code Point */ #define VLAN_PRIO_SHIFT 13 #define VLAN_CFI_MASK 0x1000 /* Canonical Format Indicator */ -#define VLAN_TAG_PRESENT VLAN_CFI_MASK #define VLAN_VID_MASK 0x0fff /* VLAN Identifier */ #define VLAN_N_VID 4096 @@ -78,8 +77,8 @@ static inline bool is_vlan_dev(const struct net_device *dev) return dev->priv_flags & IFF_802_1Q_VLAN; } -#define skb_vlan_tag_present(__skb) ((__skb)->vlan_tci & VLAN_TAG_PRESENT) -#define skb_vlan_tag_get(__skb) ((__skb)->vlan_tci & ~VLAN_TAG_PRESENT) +#define skb_vlan_tag_present(__skb) ((__skb)->vlan_present) +#define skb_vlan_tag_get(__skb) ((__skb)->vlan_tci) #define skb_vlan_tag_get_id(__skb) ((__skb)->vlan_tci & VLAN_VID_MASK) #define skb_vlan_tag_get_prio(__skb) (((__skb)->vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT) @@ -480,7 +479,7 @@ static inline struct sk_buff *vlan_insert_tag_set_proto(struct sk_buff *skb, */ static inline void __vlan_hwaccel_clear_tag(struct sk_buff *skb) { - skb->vlan_tci = 0; + skb->vlan_present = 0; } /** @@ -492,6 +491,7 @@ static inline void __vlan_hwaccel_clear_tag(struct sk_buff *skb) */ static inline void __vlan_hwaccel_copy_tag(struct sk_buff *dst, const struct sk_buff *src) { + dst->vlan_present = src->vlan_present; dst->vlan_proto = src->vlan_proto; dst->vlan_tci = src->vlan_tci; } @@ -526,7 +526,8 @@ static inline void __vlan_hwaccel_put_tag(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) { skb->vlan_proto = vlan_proto; - skb->vlan_tci = VLAN_TAG_PRESENT | vlan_tci; + skb->vlan_tci = vlan_tci; + skb->vlan_present = 1; } /** diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 99f38779332c..b9aa0d1b21cf 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -777,6 +777,14 @@ struct sk_buff { __u8 encap_hdr_csum:1; __u8 csum_valid:1; +#ifdef __BIG_ENDIAN_BITFIELD +#define PKT_VLAN_PRESENT_BIT 7 +#else +#define PKT_VLAN_PRESENT_BIT 0 +#endif +#define PKT_VLAN_PRESENT_OFFSET() offsetof(struct sk_buff, __pkt_vlan_present_offset) + __u8 __pkt_vlan_present_offset[0]; + __u8 vlan_present:1; __u8 csum_complete_sw:1; __u8 csum_level:2; __u8 csum_not_inet:1; @@ -784,8 +792,8 @@ struct sk_buff { #ifdef CONFIG_IPV6_NDISC_NODETYPE __u8 ndisc_nodetype:2; #endif - __u8 ipvs_property:1; + __u8 ipvs_property:1; __u8 inner_protocol_type:1; __u8 remcsum_offload:1; #ifdef CONFIG_NET_SWITCHDEV @@ -816,12 +824,6 @@ struct sk_buff { __u32 priority; int skb_iif; __u32 hash; -#define PKT_VLAN_PRESENT_BIT 4 // CFI (12-th bit) in TCI -#ifdef __BIG_ENDIAN -#define PKT_VLAN_PRESENT_OFFSET() offsetof(struct sk_buff, vlan_tci) -#else -#define PKT_VLAN_PRESENT_OFFSET() (offsetof(struct sk_buff, vlan_tci) + 1) -#endif __be16 vlan_proto; __u16 vlan_tci; #if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS) diff --git a/lib/test_bpf.c b/lib/test_bpf.c index aa22bcaec1dc..f3e570722a7e 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -39,6 +39,7 @@ #define SKB_HASH 0x1234aaab #define SKB_QUEUE_MAP 123 #define SKB_VLAN_TCI 0xffff +#define SKB_VLAN_PRESENT 1 #define SKB_DEV_IFINDEX 577 #define SKB_DEV_TYPE 588 @@ -725,8 +726,8 @@ static struct bpf_test tests[] = { CLASSIC, { }, { - { 1, SKB_VLAN_TCI & ~VLAN_TAG_PRESENT }, - { 10, SKB_VLAN_TCI & ~VLAN_TAG_PRESENT } + { 1, SKB_VLAN_TCI }, + { 10, SKB_VLAN_TCI } }, }, { @@ -739,8 +740,8 @@ static struct bpf_test tests[] = { CLASSIC, { }, { - { 1, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) }, - { 10, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) } + { 1, SKB_VLAN_PRESENT }, + { 10, SKB_VLAN_PRESENT } }, }, { @@ -5289,8 +5290,8 @@ static struct bpf_test tests[] = { #endif { }, { - { 1, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) }, - { 10, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) } + { 1, SKB_VLAN_PRESENT }, + { 10, SKB_VLAN_PRESENT } }, .fill_helper = bpf_fill_maxinsns6, .expected_errcode = -ENOTSUPP, @@ -6493,6 +6494,7 @@ static struct sk_buff *populate_skb(char *buf, int size) skb->hash = SKB_HASH; skb->queue_mapping = SKB_QUEUE_MAP; skb->vlan_tci = SKB_VLAN_TCI; + skb->vlan_present = SKB_VLAN_PRESENT; skb->vlan_proto = htons(ETH_P_IP); dev_net_set(&dev, &init_net); skb->dev = &dev; diff --git a/net/core/filter.c b/net/core/filter.c index c151b906df53..10acbc00ff6c 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -301,9 +301,6 @@ static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg, /* dst_reg = *(u16 *) (src_reg + offsetof(vlan_tci)) */ *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, offsetof(struct sk_buff, vlan_tci)); -#ifdef VLAN_TAG_PRESENT - *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, ~VLAN_TAG_PRESENT); -#endif break; case SKF_AD_VLAN_TAG_PRESENT: *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_VLAN_PRESENT_OFFSET()); @@ -6152,9 +6149,6 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type, *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg, bpf_target_off(struct sk_buff, vlan_tci, 2, target_size)); -#ifdef VLAN_TAG_PRESENT - *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, ~VLAN_TAG_PRESENT); -#endif break; case offsetof(struct __sk_buff, cb[0]) ... -- cgit v1.2.3 From dd333449d0fb667c5250c42488a7e90470e16c77 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 16 Nov 2018 17:26:18 -0800 Subject: crypto: chacha20-generic - add HChaCha20 library function Refactor the unkeyed permutation part of chacha20_block() into its own function, then add hchacha20_block() which is the ChaCha equivalent of HSalsa20 and is an intermediate step towards XChaCha20 (see https://cr.yp.to/snuffle/xsalsa-20081128.pdf). HChaCha20 skips the final addition of the initial state, and outputs only certain words of the state. It should not be used for streaming directly. Reviewed-by: Ard Biesheuvel Acked-by: Martin Willi Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- include/crypto/chacha20.h | 2 ++ lib/chacha20.c | 50 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/include/crypto/chacha20.h b/include/crypto/chacha20.h index 2d3129442a52..56073814eef0 100644 --- a/include/crypto/chacha20.h +++ b/include/crypto/chacha20.h @@ -20,6 +20,8 @@ struct chacha20_ctx { }; void chacha20_block(u32 *state, u8 *stream); +void hchacha20_block(const u32 *in, u32 *out); + void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv); int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keysize); diff --git a/lib/chacha20.c b/lib/chacha20.c index d907fec6a9ed..6a484e16171d 100644 --- a/lib/chacha20.c +++ b/lib/chacha20.c @@ -1,5 +1,5 @@ /* - * ChaCha20 256-bit cipher algorithm, RFC7539 + * The "hash function" used as the core of the ChaCha20 stream cipher (RFC7539) * * Copyright (C) 2015 Martin Willi * @@ -16,14 +16,10 @@ #include #include -void chacha20_block(u32 *state, u8 *stream) +static void chacha20_permute(u32 *x) { - u32 x[16]; int i; - for (i = 0; i < ARRAY_SIZE(x); i++) - x[i] = state[i]; - for (i = 0; i < 20; i += 2) { x[0] += x[4]; x[12] = rol32(x[12] ^ x[0], 16); x[1] += x[5]; x[13] = rol32(x[13] ^ x[1], 16); @@ -65,6 +61,25 @@ void chacha20_block(u32 *state, u8 *stream) x[8] += x[13]; x[7] = rol32(x[7] ^ x[8], 7); x[9] += x[14]; x[4] = rol32(x[4] ^ x[9], 7); } +} + +/** + * chacha20_block - generate one keystream block and increment block counter + * @state: input state matrix (16 32-bit words) + * @stream: output keystream block (64 bytes) + * + * This is the ChaCha20 core, a function from 64-byte strings to 64-byte + * strings. The caller has already converted the endianness of the input. This + * function also handles incrementing the block counter in the input matrix. + */ +void chacha20_block(u32 *state, u8 *stream) +{ + u32 x[16]; + int i; + + memcpy(x, state, 64); + + chacha20_permute(x); for (i = 0; i < ARRAY_SIZE(x); i++) put_unaligned_le32(x[i] + state[i], &stream[i * sizeof(u32)]); @@ -72,3 +87,26 @@ void chacha20_block(u32 *state, u8 *stream) state[12]++; } EXPORT_SYMBOL(chacha20_block); + +/** + * hchacha20_block - abbreviated ChaCha20 core, for XChaCha20 + * @in: input state matrix (16 32-bit words) + * @out: output (8 32-bit words) + * + * HChaCha20 is the ChaCha equivalent of HSalsa20 and is an intermediate step + * towards XChaCha20 (see https://cr.yp.to/snuffle/xsalsa-20081128.pdf). + * HChaCha20 skips the final addition of the initial state, and outputs only + * certain words of the state. It should not be used for streaming directly. + */ +void hchacha20_block(const u32 *in, u32 *out) +{ + u32 x[16]; + + memcpy(x, in, 64); + + chacha20_permute(x); + + memcpy(&out[0], &x[0], 16); + memcpy(&out[4], &x[12], 16); +} +EXPORT_SYMBOL(hchacha20_block); -- cgit v1.2.3 From 1ca1b917940c24ca3d1f490118c5474168622953 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 16 Nov 2018 17:26:21 -0800 Subject: crypto: chacha20-generic - refactor to allow varying number of rounds In preparation for adding XChaCha12 support, rename/refactor chacha20-generic to support different numbers of rounds. The justification for needing XChaCha12 support is explained in more detail in the patch "crypto: chacha - add XChaCha12 support". The only difference between ChaCha{8,12,20} are the number of rounds itself; all other parts of the algorithm are the same. Therefore, remove the "20" from all definitions, structures, functions, files, etc. that will be shared by all ChaCha versions. Also make ->setkey() store the round count in the chacha_ctx (previously chacha20_ctx). The generic code then passes the round count through to chacha_block(). There will be a ->setkey() function for each explicitly allowed round count; the encrypt/decrypt functions will be the same. I decided not to do it the opposite way (same ->setkey() function for all round counts, with different encrypt/decrypt functions) because that would have required more boilerplate code in architecture-specific implementations of ChaCha and XChaCha. Reviewed-by: Ard Biesheuvel Acked-by: Martin Willi Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/crypto/chacha20-neon-glue.c | 40 +++---- arch/arm64/crypto/chacha20-neon-glue.c | 40 +++---- arch/x86/crypto/chacha20_glue.c | 48 ++++---- crypto/Makefile | 2 +- crypto/chacha20_generic.c | 185 ------------------------------- crypto/chacha20poly1305.c | 10 +- crypto/chacha_generic.c | 193 +++++++++++++++++++++++++++++++++ drivers/char/random.c | 51 +++++---- drivers/crypto/caam/caamalg.c | 2 +- drivers/crypto/caam/caamalg_qi2.c | 8 +- drivers/crypto/caam/compat.h | 2 +- include/crypto/chacha.h | 47 ++++++++ include/crypto/chacha20.h | 42 ------- lib/Makefile | 2 +- lib/chacha.c | 117 ++++++++++++++++++++ lib/chacha20.c | 112 ------------------- 16 files changed, 459 insertions(+), 442 deletions(-) delete mode 100644 crypto/chacha20_generic.c create mode 100644 crypto/chacha_generic.c create mode 100644 include/crypto/chacha.h delete mode 100644 include/crypto/chacha20.h create mode 100644 lib/chacha.c delete mode 100644 lib/chacha20.c (limited to 'lib') diff --git a/arch/arm/crypto/chacha20-neon-glue.c b/arch/arm/crypto/chacha20-neon-glue.c index 59a7be08e80c..7386eb1c1889 100644 --- a/arch/arm/crypto/chacha20-neon-glue.c +++ b/arch/arm/crypto/chacha20-neon-glue.c @@ -19,7 +19,7 @@ */ #include -#include +#include #include #include #include @@ -34,20 +34,20 @@ asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src); static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src, unsigned int bytes) { - u8 buf[CHACHA20_BLOCK_SIZE]; + u8 buf[CHACHA_BLOCK_SIZE]; - while (bytes >= CHACHA20_BLOCK_SIZE * 4) { + while (bytes >= CHACHA_BLOCK_SIZE * 4) { chacha20_4block_xor_neon(state, dst, src); - bytes -= CHACHA20_BLOCK_SIZE * 4; - src += CHACHA20_BLOCK_SIZE * 4; - dst += CHACHA20_BLOCK_SIZE * 4; + bytes -= CHACHA_BLOCK_SIZE * 4; + src += CHACHA_BLOCK_SIZE * 4; + dst += CHACHA_BLOCK_SIZE * 4; state[12] += 4; } - while (bytes >= CHACHA20_BLOCK_SIZE) { + while (bytes >= CHACHA_BLOCK_SIZE) { chacha20_block_xor_neon(state, dst, src); - bytes -= CHACHA20_BLOCK_SIZE; - src += CHACHA20_BLOCK_SIZE; - dst += CHACHA20_BLOCK_SIZE; + bytes -= CHACHA_BLOCK_SIZE; + src += CHACHA_BLOCK_SIZE; + dst += CHACHA_BLOCK_SIZE; state[12]++; } if (bytes) { @@ -60,17 +60,17 @@ static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src, static int chacha20_neon(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm); + struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk walk; u32 state[16]; int err; - if (req->cryptlen <= CHACHA20_BLOCK_SIZE || !may_use_simd()) - return crypto_chacha20_crypt(req); + if (req->cryptlen <= CHACHA_BLOCK_SIZE || !may_use_simd()) + return crypto_chacha_crypt(req); err = skcipher_walk_virt(&walk, req, true); - crypto_chacha20_init(state, ctx, walk.iv); + crypto_chacha_init(state, ctx, walk.iv); kernel_neon_begin(); while (walk.nbytes > 0) { @@ -93,14 +93,14 @@ static struct skcipher_alg alg = { .base.cra_driver_name = "chacha20-neon", .base.cra_priority = 300, .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha20_ctx), + .base.cra_ctxsize = sizeof(struct chacha_ctx), .base.cra_module = THIS_MODULE, - .min_keysize = CHACHA20_KEY_SIZE, - .max_keysize = CHACHA20_KEY_SIZE, - .ivsize = CHACHA20_IV_SIZE, - .chunksize = CHACHA20_BLOCK_SIZE, - .walksize = 4 * CHACHA20_BLOCK_SIZE, + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .walksize = 4 * CHACHA_BLOCK_SIZE, .setkey = crypto_chacha20_setkey, .encrypt = chacha20_neon, .decrypt = chacha20_neon, diff --git a/arch/arm64/crypto/chacha20-neon-glue.c b/arch/arm64/crypto/chacha20-neon-glue.c index 727579c93ded..96e0cfb8c3f5 100644 --- a/arch/arm64/crypto/chacha20-neon-glue.c +++ b/arch/arm64/crypto/chacha20-neon-glue.c @@ -19,7 +19,7 @@ */ #include -#include +#include #include #include #include @@ -34,15 +34,15 @@ asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src); static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src, unsigned int bytes) { - u8 buf[CHACHA20_BLOCK_SIZE]; + u8 buf[CHACHA_BLOCK_SIZE]; - while (bytes >= CHACHA20_BLOCK_SIZE * 4) { + while (bytes >= CHACHA_BLOCK_SIZE * 4) { kernel_neon_begin(); chacha20_4block_xor_neon(state, dst, src); kernel_neon_end(); - bytes -= CHACHA20_BLOCK_SIZE * 4; - src += CHACHA20_BLOCK_SIZE * 4; - dst += CHACHA20_BLOCK_SIZE * 4; + bytes -= CHACHA_BLOCK_SIZE * 4; + src += CHACHA_BLOCK_SIZE * 4; + dst += CHACHA_BLOCK_SIZE * 4; state[12] += 4; } @@ -50,11 +50,11 @@ static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src, return; kernel_neon_begin(); - while (bytes >= CHACHA20_BLOCK_SIZE) { + while (bytes >= CHACHA_BLOCK_SIZE) { chacha20_block_xor_neon(state, dst, src); - bytes -= CHACHA20_BLOCK_SIZE; - src += CHACHA20_BLOCK_SIZE; - dst += CHACHA20_BLOCK_SIZE; + bytes -= CHACHA_BLOCK_SIZE; + src += CHACHA_BLOCK_SIZE; + dst += CHACHA_BLOCK_SIZE; state[12]++; } if (bytes) { @@ -68,17 +68,17 @@ static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src, static int chacha20_neon(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm); + struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk walk; u32 state[16]; int err; - if (!may_use_simd() || req->cryptlen <= CHACHA20_BLOCK_SIZE) - return crypto_chacha20_crypt(req); + if (!may_use_simd() || req->cryptlen <= CHACHA_BLOCK_SIZE) + return crypto_chacha_crypt(req); err = skcipher_walk_virt(&walk, req, false); - crypto_chacha20_init(state, ctx, walk.iv); + crypto_chacha_init(state, ctx, walk.iv); while (walk.nbytes > 0) { unsigned int nbytes = walk.nbytes; @@ -99,14 +99,14 @@ static struct skcipher_alg alg = { .base.cra_driver_name = "chacha20-neon", .base.cra_priority = 300, .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha20_ctx), + .base.cra_ctxsize = sizeof(struct chacha_ctx), .base.cra_module = THIS_MODULE, - .min_keysize = CHACHA20_KEY_SIZE, - .max_keysize = CHACHA20_KEY_SIZE, - .ivsize = CHACHA20_IV_SIZE, - .chunksize = CHACHA20_BLOCK_SIZE, - .walksize = 4 * CHACHA20_BLOCK_SIZE, + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .walksize = 4 * CHACHA_BLOCK_SIZE, .setkey = crypto_chacha20_setkey, .encrypt = chacha20_neon, .decrypt = chacha20_neon, diff --git a/arch/x86/crypto/chacha20_glue.c b/arch/x86/crypto/chacha20_glue.c index 9fd84fe6ec09..1e9e66509226 100644 --- a/arch/x86/crypto/chacha20_glue.c +++ b/arch/x86/crypto/chacha20_glue.c @@ -10,7 +10,7 @@ */ #include -#include +#include #include #include #include @@ -35,8 +35,8 @@ static bool chacha20_use_avx2; static unsigned int chacha20_advance(unsigned int len, unsigned int maxblocks) { - len = min(len, maxblocks * CHACHA20_BLOCK_SIZE); - return round_up(len, CHACHA20_BLOCK_SIZE) / CHACHA20_BLOCK_SIZE; + len = min(len, maxblocks * CHACHA_BLOCK_SIZE); + return round_up(len, CHACHA_BLOCK_SIZE) / CHACHA_BLOCK_SIZE; } static void chacha20_dosimd(u32 *state, u8 *dst, const u8 *src, @@ -44,38 +44,38 @@ static void chacha20_dosimd(u32 *state, u8 *dst, const u8 *src, { #ifdef CONFIG_AS_AVX2 if (chacha20_use_avx2) { - while (bytes >= CHACHA20_BLOCK_SIZE * 8) { + while (bytes >= CHACHA_BLOCK_SIZE * 8) { chacha20_8block_xor_avx2(state, dst, src, bytes); - bytes -= CHACHA20_BLOCK_SIZE * 8; - src += CHACHA20_BLOCK_SIZE * 8; - dst += CHACHA20_BLOCK_SIZE * 8; + bytes -= CHACHA_BLOCK_SIZE * 8; + src += CHACHA_BLOCK_SIZE * 8; + dst += CHACHA_BLOCK_SIZE * 8; state[12] += 8; } - if (bytes > CHACHA20_BLOCK_SIZE * 4) { + if (bytes > CHACHA_BLOCK_SIZE * 4) { chacha20_8block_xor_avx2(state, dst, src, bytes); state[12] += chacha20_advance(bytes, 8); return; } - if (bytes > CHACHA20_BLOCK_SIZE * 2) { + if (bytes > CHACHA_BLOCK_SIZE * 2) { chacha20_4block_xor_avx2(state, dst, src, bytes); state[12] += chacha20_advance(bytes, 4); return; } - if (bytes > CHACHA20_BLOCK_SIZE) { + if (bytes > CHACHA_BLOCK_SIZE) { chacha20_2block_xor_avx2(state, dst, src, bytes); state[12] += chacha20_advance(bytes, 2); return; } } #endif - while (bytes >= CHACHA20_BLOCK_SIZE * 4) { + while (bytes >= CHACHA_BLOCK_SIZE * 4) { chacha20_4block_xor_ssse3(state, dst, src, bytes); - bytes -= CHACHA20_BLOCK_SIZE * 4; - src += CHACHA20_BLOCK_SIZE * 4; - dst += CHACHA20_BLOCK_SIZE * 4; + bytes -= CHACHA_BLOCK_SIZE * 4; + src += CHACHA_BLOCK_SIZE * 4; + dst += CHACHA_BLOCK_SIZE * 4; state[12] += 4; } - if (bytes > CHACHA20_BLOCK_SIZE) { + if (bytes > CHACHA_BLOCK_SIZE) { chacha20_4block_xor_ssse3(state, dst, src, bytes); state[12] += chacha20_advance(bytes, 4); return; @@ -89,7 +89,7 @@ static void chacha20_dosimd(u32 *state, u8 *dst, const u8 *src, static int chacha20_simd(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm); + struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); u32 *state, state_buf[16 + 2] __aligned(8); struct skcipher_walk walk; int err; @@ -97,12 +97,12 @@ static int chacha20_simd(struct skcipher_request *req) BUILD_BUG_ON(CHACHA20_STATE_ALIGN != 16); state = PTR_ALIGN(state_buf + 0, CHACHA20_STATE_ALIGN); - if (req->cryptlen <= CHACHA20_BLOCK_SIZE || !may_use_simd()) - return crypto_chacha20_crypt(req); + if (req->cryptlen <= CHACHA_BLOCK_SIZE || !may_use_simd()) + return crypto_chacha_crypt(req); err = skcipher_walk_virt(&walk, req, true); - crypto_chacha20_init(state, ctx, walk.iv); + crypto_chacha_init(state, ctx, walk.iv); kernel_fpu_begin(); @@ -128,13 +128,13 @@ static struct skcipher_alg alg = { .base.cra_driver_name = "chacha20-simd", .base.cra_priority = 300, .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha20_ctx), + .base.cra_ctxsize = sizeof(struct chacha_ctx), .base.cra_module = THIS_MODULE, - .min_keysize = CHACHA20_KEY_SIZE, - .max_keysize = CHACHA20_KEY_SIZE, - .ivsize = CHACHA20_IV_SIZE, - .chunksize = CHACHA20_BLOCK_SIZE, + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, .setkey = crypto_chacha20_setkey, .encrypt = chacha20_simd, .decrypt = chacha20_simd, diff --git a/crypto/Makefile b/crypto/Makefile index abbd86fdbad2..102e8525814f 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -117,7 +117,7 @@ obj-$(CONFIG_CRYPTO_KHAZAD) += khazad.o obj-$(CONFIG_CRYPTO_ANUBIS) += anubis.o obj-$(CONFIG_CRYPTO_SEED) += seed.o obj-$(CONFIG_CRYPTO_SALSA20) += salsa20_generic.o -obj-$(CONFIG_CRYPTO_CHACHA20) += chacha20_generic.o +obj-$(CONFIG_CRYPTO_CHACHA20) += chacha_generic.o obj-$(CONFIG_CRYPTO_POLY1305) += poly1305_generic.o obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c deleted file mode 100644 index 4305b1b62b16..000000000000 --- a/crypto/chacha20_generic.c +++ /dev/null @@ -1,185 +0,0 @@ -/* - * ChaCha20 (RFC7539) and XChaCha20 stream cipher algorithms - * - * Copyright (C) 2015 Martin Willi - * Copyright (C) 2018 Google LLC - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ - -#include -#include -#include -#include -#include - -static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src, - unsigned int bytes) -{ - /* aligned to potentially speed up crypto_xor() */ - u8 stream[CHACHA20_BLOCK_SIZE] __aligned(sizeof(long)); - - if (dst != src) - memcpy(dst, src, bytes); - - while (bytes >= CHACHA20_BLOCK_SIZE) { - chacha20_block(state, stream); - crypto_xor(dst, stream, CHACHA20_BLOCK_SIZE); - bytes -= CHACHA20_BLOCK_SIZE; - dst += CHACHA20_BLOCK_SIZE; - } - if (bytes) { - chacha20_block(state, stream); - crypto_xor(dst, stream, bytes); - } -} - -static int chacha20_stream_xor(struct skcipher_request *req, - struct chacha20_ctx *ctx, u8 *iv) -{ - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - crypto_chacha20_init(state, ctx, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); - - chacha20_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr, - nbytes); - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv) -{ - state[0] = 0x61707865; /* "expa" */ - state[1] = 0x3320646e; /* "nd 3" */ - state[2] = 0x79622d32; /* "2-by" */ - state[3] = 0x6b206574; /* "te k" */ - state[4] = ctx->key[0]; - state[5] = ctx->key[1]; - state[6] = ctx->key[2]; - state[7] = ctx->key[3]; - state[8] = ctx->key[4]; - state[9] = ctx->key[5]; - state[10] = ctx->key[6]; - state[11] = ctx->key[7]; - state[12] = get_unaligned_le32(iv + 0); - state[13] = get_unaligned_le32(iv + 4); - state[14] = get_unaligned_le32(iv + 8); - state[15] = get_unaligned_le32(iv + 12); -} -EXPORT_SYMBOL_GPL(crypto_chacha20_init); - -int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize) -{ - struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm); - int i; - - if (keysize != CHACHA20_KEY_SIZE) - return -EINVAL; - - for (i = 0; i < ARRAY_SIZE(ctx->key); i++) - ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); - - return 0; -} -EXPORT_SYMBOL_GPL(crypto_chacha20_setkey); - -int crypto_chacha20_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha20_stream_xor(req, ctx, req->iv); -} -EXPORT_SYMBOL_GPL(crypto_chacha20_crypt); - -int crypto_xchacha20_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha20_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - /* Compute the subkey given the original key and first 128 nonce bits */ - crypto_chacha20_init(state, ctx, req->iv); - hchacha20_block(state, subctx.key); - - /* Build the real IV */ - memcpy(&real_iv[0], req->iv + 24, 8); /* stream position */ - memcpy(&real_iv[8], req->iv + 16, 8); /* remaining 64 nonce bits */ - - /* Generate the stream and XOR it with the data */ - return chacha20_stream_xor(req, &subctx, real_iv); -} -EXPORT_SYMBOL_GPL(crypto_xchacha20_crypt); - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-generic", - .base.cra_priority = 100, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha20_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA20_KEY_SIZE, - .max_keysize = CHACHA20_KEY_SIZE, - .ivsize = CHACHA20_IV_SIZE, - .chunksize = CHACHA20_BLOCK_SIZE, - .setkey = crypto_chacha20_setkey, - .encrypt = crypto_chacha20_crypt, - .decrypt = crypto_chacha20_crypt, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-generic", - .base.cra_priority = 100, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha20_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA20_KEY_SIZE, - .max_keysize = CHACHA20_KEY_SIZE, - .ivsize = XCHACHA20_IV_SIZE, - .chunksize = CHACHA20_BLOCK_SIZE, - .setkey = crypto_chacha20_setkey, - .encrypt = crypto_xchacha20_crypt, - .decrypt = crypto_xchacha20_crypt, - } -}; - -static int __init chacha20_generic_mod_init(void) -{ - return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); -} - -static void __exit chacha20_generic_mod_fini(void) -{ - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); -} - -module_init(chacha20_generic_mod_init); -module_exit(chacha20_generic_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Martin Willi "); -MODULE_DESCRIPTION("ChaCha20 and XChaCha20 stream ciphers (generic)"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-generic"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-generic"); diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index f9dd5453046a..fef11446ab1b 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -49,7 +49,7 @@ struct poly_req { }; struct chacha_req { - u8 iv[CHACHA20_IV_SIZE]; + u8 iv[CHACHA_IV_SIZE]; struct scatterlist src[1]; struct skcipher_request req; /* must be last member */ }; @@ -89,7 +89,7 @@ static void chacha_iv(u8 *iv, struct aead_request *req, u32 icb) memcpy(iv, &leicb, sizeof(leicb)); memcpy(iv + sizeof(leicb), ctx->salt, ctx->saltlen); memcpy(iv + sizeof(leicb) + ctx->saltlen, req->iv, - CHACHA20_IV_SIZE - sizeof(leicb) - ctx->saltlen); + CHACHA_IV_SIZE - sizeof(leicb) - ctx->saltlen); } static int poly_verify_tag(struct aead_request *req) @@ -492,7 +492,7 @@ static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, struct chachapoly_ctx *ctx = crypto_aead_ctx(aead); int err; - if (keylen != ctx->saltlen + CHACHA20_KEY_SIZE) + if (keylen != ctx->saltlen + CHACHA_KEY_SIZE) return -EINVAL; keylen -= ctx->saltlen; @@ -637,7 +637,7 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb, err = -EINVAL; /* Need 16-byte IV size, including Initial Block Counter value */ - if (crypto_skcipher_alg_ivsize(chacha) != CHACHA20_IV_SIZE) + if (crypto_skcipher_alg_ivsize(chacha) != CHACHA_IV_SIZE) goto out_drop_chacha; /* Not a stream cipher? */ if (chacha->base.cra_blocksize != 1) diff --git a/crypto/chacha_generic.c b/crypto/chacha_generic.c new file mode 100644 index 000000000000..438f15a14054 --- /dev/null +++ b/crypto/chacha_generic.c @@ -0,0 +1,193 @@ +/* + * ChaCha20 (RFC7539) and XChaCha20 stream cipher algorithms + * + * Copyright (C) 2015 Martin Willi + * Copyright (C) 2018 Google LLC + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#include + +static void chacha_docrypt(u32 *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds) +{ + /* aligned to potentially speed up crypto_xor() */ + u8 stream[CHACHA_BLOCK_SIZE] __aligned(sizeof(long)); + + if (dst != src) + memcpy(dst, src, bytes); + + while (bytes >= CHACHA_BLOCK_SIZE) { + chacha_block(state, stream, nrounds); + crypto_xor(dst, stream, CHACHA_BLOCK_SIZE); + bytes -= CHACHA_BLOCK_SIZE; + dst += CHACHA_BLOCK_SIZE; + } + if (bytes) { + chacha_block(state, stream, nrounds); + crypto_xor(dst, stream, bytes); + } +} + +static int chacha_stream_xor(struct skcipher_request *req, + struct chacha_ctx *ctx, u8 *iv) +{ + struct skcipher_walk walk; + u32 state[16]; + int err; + + err = skcipher_walk_virt(&walk, req, false); + + crypto_chacha_init(state, ctx, iv); + + while (walk.nbytes > 0) { + unsigned int nbytes = walk.nbytes; + + if (nbytes < walk.total) + nbytes = round_down(nbytes, walk.stride); + + chacha_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr, + nbytes, ctx->nrounds); + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); + } + + return err; +} + +void crypto_chacha_init(u32 *state, struct chacha_ctx *ctx, u8 *iv) +{ + state[0] = 0x61707865; /* "expa" */ + state[1] = 0x3320646e; /* "nd 3" */ + state[2] = 0x79622d32; /* "2-by" */ + state[3] = 0x6b206574; /* "te k" */ + state[4] = ctx->key[0]; + state[5] = ctx->key[1]; + state[6] = ctx->key[2]; + state[7] = ctx->key[3]; + state[8] = ctx->key[4]; + state[9] = ctx->key[5]; + state[10] = ctx->key[6]; + state[11] = ctx->key[7]; + state[12] = get_unaligned_le32(iv + 0); + state[13] = get_unaligned_le32(iv + 4); + state[14] = get_unaligned_le32(iv + 8); + state[15] = get_unaligned_le32(iv + 12); +} +EXPORT_SYMBOL_GPL(crypto_chacha_init); + +static int chacha_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keysize, int nrounds) +{ + struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); + int i; + + if (keysize != CHACHA_KEY_SIZE) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(ctx->key); i++) + ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); + + ctx->nrounds = nrounds; + return 0; +} + +int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keysize) +{ + return chacha_setkey(tfm, key, keysize, 20); +} +EXPORT_SYMBOL_GPL(crypto_chacha20_setkey); + +int crypto_chacha_crypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); + + return chacha_stream_xor(req, ctx, req->iv); +} +EXPORT_SYMBOL_GPL(crypto_chacha_crypt); + +int crypto_xchacha_crypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); + struct chacha_ctx subctx; + u32 state[16]; + u8 real_iv[16]; + + /* Compute the subkey given the original key and first 128 nonce bits */ + crypto_chacha_init(state, ctx, req->iv); + hchacha_block(state, subctx.key, ctx->nrounds); + subctx.nrounds = ctx->nrounds; + + /* Build the real IV */ + memcpy(&real_iv[0], req->iv + 24, 8); /* stream position */ + memcpy(&real_iv[8], req->iv + 16, 8); /* remaining 64 nonce bits */ + + /* Generate the stream and XOR it with the data */ + return chacha_stream_xor(req, &subctx, real_iv); +} +EXPORT_SYMBOL_GPL(crypto_xchacha_crypt); + +static struct skcipher_alg algs[] = { + { + .base.cra_name = "chacha20", + .base.cra_driver_name = "chacha20-generic", + .base.cra_priority = 100, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = crypto_chacha20_setkey, + .encrypt = crypto_chacha_crypt, + .decrypt = crypto_chacha_crypt, + }, { + .base.cra_name = "xchacha20", + .base.cra_driver_name = "xchacha20-generic", + .base.cra_priority = 100, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = XCHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = crypto_chacha20_setkey, + .encrypt = crypto_xchacha_crypt, + .decrypt = crypto_xchacha_crypt, + } +}; + +static int __init chacha_generic_mod_init(void) +{ + return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); +} + +static void __exit chacha_generic_mod_fini(void) +{ + crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); +} + +module_init(chacha_generic_mod_init); +module_exit(chacha_generic_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Martin Willi "); +MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (generic)"); +MODULE_ALIAS_CRYPTO("chacha20"); +MODULE_ALIAS_CRYPTO("chacha20-generic"); +MODULE_ALIAS_CRYPTO("xchacha20"); +MODULE_ALIAS_CRYPTO("xchacha20-generic"); diff --git a/drivers/char/random.c b/drivers/char/random.c index 2eb70e76ed35..38c6d1af6d1c 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -265,7 +265,7 @@ #include #include #include -#include +#include #include #include @@ -431,11 +431,10 @@ static int crng_init = 0; #define crng_ready() (likely(crng_init > 1)) static int crng_init_cnt = 0; static unsigned long crng_global_init_time = 0; -#define CRNG_INIT_CNT_THRESH (2*CHACHA20_KEY_SIZE) -static void _extract_crng(struct crng_state *crng, - __u8 out[CHACHA20_BLOCK_SIZE]); +#define CRNG_INIT_CNT_THRESH (2*CHACHA_KEY_SIZE) +static void _extract_crng(struct crng_state *crng, __u8 out[CHACHA_BLOCK_SIZE]); static void _crng_backtrack_protect(struct crng_state *crng, - __u8 tmp[CHACHA20_BLOCK_SIZE], int used); + __u8 tmp[CHACHA_BLOCK_SIZE], int used); static void process_random_ready_list(void); static void _get_random_bytes(void *buf, int nbytes); @@ -863,7 +862,7 @@ static int crng_fast_load(const char *cp, size_t len) } p = (unsigned char *) &primary_crng.state[4]; while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) { - p[crng_init_cnt % CHACHA20_KEY_SIZE] ^= *cp; + p[crng_init_cnt % CHACHA_KEY_SIZE] ^= *cp; cp++; crng_init_cnt++; len--; } spin_unlock_irqrestore(&primary_crng.lock, flags); @@ -895,7 +894,7 @@ static int crng_slow_load(const char *cp, size_t len) unsigned long flags; static unsigned char lfsr = 1; unsigned char tmp; - unsigned i, max = CHACHA20_KEY_SIZE; + unsigned i, max = CHACHA_KEY_SIZE; const char * src_buf = cp; char * dest_buf = (char *) &primary_crng.state[4]; @@ -913,8 +912,8 @@ static int crng_slow_load(const char *cp, size_t len) lfsr >>= 1; if (tmp & 1) lfsr ^= 0xE1; - tmp = dest_buf[i % CHACHA20_KEY_SIZE]; - dest_buf[i % CHACHA20_KEY_SIZE] ^= src_buf[i % len] ^ lfsr; + tmp = dest_buf[i % CHACHA_KEY_SIZE]; + dest_buf[i % CHACHA_KEY_SIZE] ^= src_buf[i % len] ^ lfsr; lfsr += (tmp << 3) | (tmp >> 5); } spin_unlock_irqrestore(&primary_crng.lock, flags); @@ -926,7 +925,7 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r) unsigned long flags; int i, num; union { - __u8 block[CHACHA20_BLOCK_SIZE]; + __u8 block[CHACHA_BLOCK_SIZE]; __u32 key[8]; } buf; @@ -937,7 +936,7 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r) } else { _extract_crng(&primary_crng, buf.block); _crng_backtrack_protect(&primary_crng, buf.block, - CHACHA20_KEY_SIZE); + CHACHA_KEY_SIZE); } spin_lock_irqsave(&crng->lock, flags); for (i = 0; i < 8; i++) { @@ -973,7 +972,7 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r) } static void _extract_crng(struct crng_state *crng, - __u8 out[CHACHA20_BLOCK_SIZE]) + __u8 out[CHACHA_BLOCK_SIZE]) { unsigned long v, flags; @@ -990,7 +989,7 @@ static void _extract_crng(struct crng_state *crng, spin_unlock_irqrestore(&crng->lock, flags); } -static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE]) +static void extract_crng(__u8 out[CHACHA_BLOCK_SIZE]) { struct crng_state *crng = NULL; @@ -1008,14 +1007,14 @@ static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE]) * enough) to mutate the CRNG key to provide backtracking protection. */ static void _crng_backtrack_protect(struct crng_state *crng, - __u8 tmp[CHACHA20_BLOCK_SIZE], int used) + __u8 tmp[CHACHA_BLOCK_SIZE], int used) { unsigned long flags; __u32 *s, *d; int i; used = round_up(used, sizeof(__u32)); - if (used + CHACHA20_KEY_SIZE > CHACHA20_BLOCK_SIZE) { + if (used + CHACHA_KEY_SIZE > CHACHA_BLOCK_SIZE) { extract_crng(tmp); used = 0; } @@ -1027,7 +1026,7 @@ static void _crng_backtrack_protect(struct crng_state *crng, spin_unlock_irqrestore(&crng->lock, flags); } -static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used) +static void crng_backtrack_protect(__u8 tmp[CHACHA_BLOCK_SIZE], int used) { struct crng_state *crng = NULL; @@ -1042,8 +1041,8 @@ static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used) static ssize_t extract_crng_user(void __user *buf, size_t nbytes) { - ssize_t ret = 0, i = CHACHA20_BLOCK_SIZE; - __u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4); + ssize_t ret = 0, i = CHACHA_BLOCK_SIZE; + __u8 tmp[CHACHA_BLOCK_SIZE] __aligned(4); int large_request = (nbytes > 256); while (nbytes) { @@ -1057,7 +1056,7 @@ static ssize_t extract_crng_user(void __user *buf, size_t nbytes) } extract_crng(tmp); - i = min_t(int, nbytes, CHACHA20_BLOCK_SIZE); + i = min_t(int, nbytes, CHACHA_BLOCK_SIZE); if (copy_to_user(buf, tmp, i)) { ret = -EFAULT; break; @@ -1622,14 +1621,14 @@ static void _warn_unseeded_randomness(const char *func_name, void *caller, */ static void _get_random_bytes(void *buf, int nbytes) { - __u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4); + __u8 tmp[CHACHA_BLOCK_SIZE] __aligned(4); trace_get_random_bytes(nbytes, _RET_IP_); - while (nbytes >= CHACHA20_BLOCK_SIZE) { + while (nbytes >= CHACHA_BLOCK_SIZE) { extract_crng(buf); - buf += CHACHA20_BLOCK_SIZE; - nbytes -= CHACHA20_BLOCK_SIZE; + buf += CHACHA_BLOCK_SIZE; + nbytes -= CHACHA_BLOCK_SIZE; } if (nbytes > 0) { @@ -1637,7 +1636,7 @@ static void _get_random_bytes(void *buf, int nbytes) memcpy(buf, tmp, nbytes); crng_backtrack_protect(tmp, nbytes); } else - crng_backtrack_protect(tmp, CHACHA20_BLOCK_SIZE); + crng_backtrack_protect(tmp, CHACHA_BLOCK_SIZE); memzero_explicit(tmp, sizeof(tmp)); } @@ -2208,8 +2207,8 @@ struct ctl_table random_table[] = { struct batched_entropy { union { - u64 entropy_u64[CHACHA20_BLOCK_SIZE / sizeof(u64)]; - u32 entropy_u32[CHACHA20_BLOCK_SIZE / sizeof(u32)]; + u64 entropy_u64[CHACHA_BLOCK_SIZE / sizeof(u64)]; + u32 entropy_u32[CHACHA_BLOCK_SIZE / sizeof(u32)]; }; unsigned int position; }; diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 523565ce0060..92e593e2069a 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -559,7 +559,7 @@ static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, unsigned int ivsize = crypto_aead_ivsize(aead); unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; - if (keylen != CHACHA20_KEY_SIZE + saltlen) { + if (keylen != CHACHA_KEY_SIZE + saltlen) { crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 2598640aa98b..425d5d974613 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -591,7 +591,7 @@ static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, unsigned int ivsize = crypto_aead_ivsize(aead); unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; - if (keylen != CHACHA20_KEY_SIZE + saltlen) { + if (keylen != CHACHA_KEY_SIZE + saltlen) { crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } @@ -1577,9 +1577,9 @@ static struct caam_skcipher_alg driver_algs[] = { .setkey = skcipher_setkey, .encrypt = skcipher_encrypt, .decrypt = skcipher_decrypt, - .min_keysize = CHACHA20_KEY_SIZE, - .max_keysize = CHACHA20_KEY_SIZE, - .ivsize = CHACHA20_IV_SIZE, + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = CHACHA_IV_SIZE, }, .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20, }, diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index 8bde903f9f4a..87d9efe4c7aa 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h new file mode 100644 index 000000000000..b722a23e54bb --- /dev/null +++ b/include/crypto/chacha.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common values and helper functions for the ChaCha and XChaCha stream ciphers. + * + * XChaCha extends ChaCha's nonce to 192 bits, while provably retaining ChaCha's + * security. Here they share the same key size, tfm context, and setkey + * function; only their IV size and encrypt/decrypt function differ. + */ + +#ifndef _CRYPTO_CHACHA_H +#define _CRYPTO_CHACHA_H + +#include +#include +#include + +/* 32-bit stream position, then 96-bit nonce (RFC7539 convention) */ +#define CHACHA_IV_SIZE 16 + +#define CHACHA_KEY_SIZE 32 +#define CHACHA_BLOCK_SIZE 64 +#define CHACHAPOLY_IV_SIZE 12 + +/* 192-bit nonce, then 64-bit stream position */ +#define XCHACHA_IV_SIZE 32 + +struct chacha_ctx { + u32 key[8]; + int nrounds; +}; + +void chacha_block(u32 *state, u8 *stream, int nrounds); +static inline void chacha20_block(u32 *state, u8 *stream) +{ + chacha_block(state, stream, 20); +} +void hchacha_block(const u32 *in, u32 *out, int nrounds); + +void crypto_chacha_init(u32 *state, struct chacha_ctx *ctx, u8 *iv); + +int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keysize); + +int crypto_chacha_crypt(struct skcipher_request *req); +int crypto_xchacha_crypt(struct skcipher_request *req); + +#endif /* _CRYPTO_CHACHA_H */ diff --git a/include/crypto/chacha20.h b/include/crypto/chacha20.h deleted file mode 100644 index c24b4ac03b85..000000000000 --- a/include/crypto/chacha20.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Common values and helper functions for the ChaCha20 and XChaCha20 algorithms. - * - * XChaCha20 extends ChaCha20's nonce to 192 bits, while provably retaining - * ChaCha20's security. Here they share the same key size, tfm context, and - * setkey function; only their IV size and encrypt/decrypt function differ. - */ - -#ifndef _CRYPTO_CHACHA20_H -#define _CRYPTO_CHACHA20_H - -#include -#include -#include - -/* 32-bit stream position, then 96-bit nonce (RFC7539 convention) */ -#define CHACHA20_IV_SIZE 16 - -#define CHACHA20_KEY_SIZE 32 -#define CHACHA20_BLOCK_SIZE 64 -#define CHACHAPOLY_IV_SIZE 12 - -/* 192-bit nonce, then 64-bit stream position */ -#define XCHACHA20_IV_SIZE 32 - -struct chacha20_ctx { - u32 key[8]; -}; - -void chacha20_block(u32 *state, u8 *stream); -void hchacha20_block(const u32 *in, u32 *out); - -void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv); - -int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize); - -int crypto_chacha20_crypt(struct skcipher_request *req); -int crypto_xchacha20_crypt(struct skcipher_request *req); - -#endif diff --git a/lib/Makefile b/lib/Makefile index db06d1237898..4c2b6fc5cde9 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -20,7 +20,7 @@ KCOV_INSTRUMENT_dynamic_debug.o := n lib-y := ctype.o string.o vsprintf.o cmdline.o \ rbtree.o radix-tree.o timerqueue.o xarray.o \ idr.o int_sqrt.o extable.o \ - sha1.o chacha20.o irq_regs.o argv_split.o \ + sha1.o chacha.o irq_regs.o argv_split.o \ flex_proportions.o ratelimit.o show_mem.o \ is_single_threaded.o plist.o decompress.o kobject_uevent.o \ earlycpio.o seq_buf.o siphash.o dec_and_lock.o \ diff --git a/lib/chacha.c b/lib/chacha.c new file mode 100644 index 000000000000..1bdc688c18df --- /dev/null +++ b/lib/chacha.c @@ -0,0 +1,117 @@ +/* + * The "hash function" used as the core of the ChaCha stream cipher (RFC7539) + * + * Copyright (C) 2015 Martin Willi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#include +#include + +static void chacha_permute(u32 *x, int nrounds) +{ + int i; + + /* whitelist the allowed round counts */ + WARN_ON_ONCE(nrounds != 20); + + for (i = 0; i < nrounds; i += 2) { + x[0] += x[4]; x[12] = rol32(x[12] ^ x[0], 16); + x[1] += x[5]; x[13] = rol32(x[13] ^ x[1], 16); + x[2] += x[6]; x[14] = rol32(x[14] ^ x[2], 16); + x[3] += x[7]; x[15] = rol32(x[15] ^ x[3], 16); + + x[8] += x[12]; x[4] = rol32(x[4] ^ x[8], 12); + x[9] += x[13]; x[5] = rol32(x[5] ^ x[9], 12); + x[10] += x[14]; x[6] = rol32(x[6] ^ x[10], 12); + x[11] += x[15]; x[7] = rol32(x[7] ^ x[11], 12); + + x[0] += x[4]; x[12] = rol32(x[12] ^ x[0], 8); + x[1] += x[5]; x[13] = rol32(x[13] ^ x[1], 8); + x[2] += x[6]; x[14] = rol32(x[14] ^ x[2], 8); + x[3] += x[7]; x[15] = rol32(x[15] ^ x[3], 8); + + x[8] += x[12]; x[4] = rol32(x[4] ^ x[8], 7); + x[9] += x[13]; x[5] = rol32(x[5] ^ x[9], 7); + x[10] += x[14]; x[6] = rol32(x[6] ^ x[10], 7); + x[11] += x[15]; x[7] = rol32(x[7] ^ x[11], 7); + + x[0] += x[5]; x[15] = rol32(x[15] ^ x[0], 16); + x[1] += x[6]; x[12] = rol32(x[12] ^ x[1], 16); + x[2] += x[7]; x[13] = rol32(x[13] ^ x[2], 16); + x[3] += x[4]; x[14] = rol32(x[14] ^ x[3], 16); + + x[10] += x[15]; x[5] = rol32(x[5] ^ x[10], 12); + x[11] += x[12]; x[6] = rol32(x[6] ^ x[11], 12); + x[8] += x[13]; x[7] = rol32(x[7] ^ x[8], 12); + x[9] += x[14]; x[4] = rol32(x[4] ^ x[9], 12); + + x[0] += x[5]; x[15] = rol32(x[15] ^ x[0], 8); + x[1] += x[6]; x[12] = rol32(x[12] ^ x[1], 8); + x[2] += x[7]; x[13] = rol32(x[13] ^ x[2], 8); + x[3] += x[4]; x[14] = rol32(x[14] ^ x[3], 8); + + x[10] += x[15]; x[5] = rol32(x[5] ^ x[10], 7); + x[11] += x[12]; x[6] = rol32(x[6] ^ x[11], 7); + x[8] += x[13]; x[7] = rol32(x[7] ^ x[8], 7); + x[9] += x[14]; x[4] = rol32(x[4] ^ x[9], 7); + } +} + +/** + * chacha_block - generate one keystream block and increment block counter + * @state: input state matrix (16 32-bit words) + * @stream: output keystream block (64 bytes) + * @nrounds: number of rounds (currently must be 20) + * + * This is the ChaCha core, a function from 64-byte strings to 64-byte strings. + * The caller has already converted the endianness of the input. This function + * also handles incrementing the block counter in the input matrix. + */ +void chacha_block(u32 *state, u8 *stream, int nrounds) +{ + u32 x[16]; + int i; + + memcpy(x, state, 64); + + chacha_permute(x, nrounds); + + for (i = 0; i < ARRAY_SIZE(x); i++) + put_unaligned_le32(x[i] + state[i], &stream[i * sizeof(u32)]); + + state[12]++; +} +EXPORT_SYMBOL(chacha_block); + +/** + * hchacha_block - abbreviated ChaCha core, for XChaCha + * @in: input state matrix (16 32-bit words) + * @out: output (8 32-bit words) + * @nrounds: number of rounds (currently must be 20) + * + * HChaCha is the ChaCha equivalent of HSalsa and is an intermediate step + * towards XChaCha (see https://cr.yp.to/snuffle/xsalsa-20081128.pdf). HChaCha + * skips the final addition of the initial state, and outputs only certain words + * of the state. It should not be used for streaming directly. + */ +void hchacha_block(const u32 *in, u32 *out, int nrounds) +{ + u32 x[16]; + + memcpy(x, in, 64); + + chacha_permute(x, nrounds); + + memcpy(&out[0], &x[0], 16); + memcpy(&out[4], &x[12], 16); +} +EXPORT_SYMBOL(hchacha_block); diff --git a/lib/chacha20.c b/lib/chacha20.c deleted file mode 100644 index 6a484e16171d..000000000000 --- a/lib/chacha20.c +++ /dev/null @@ -1,112 +0,0 @@ -/* - * The "hash function" used as the core of the ChaCha20 stream cipher (RFC7539) - * - * Copyright (C) 2015 Martin Willi - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ - -#include -#include -#include -#include -#include -#include - -static void chacha20_permute(u32 *x) -{ - int i; - - for (i = 0; i < 20; i += 2) { - x[0] += x[4]; x[12] = rol32(x[12] ^ x[0], 16); - x[1] += x[5]; x[13] = rol32(x[13] ^ x[1], 16); - x[2] += x[6]; x[14] = rol32(x[14] ^ x[2], 16); - x[3] += x[7]; x[15] = rol32(x[15] ^ x[3], 16); - - x[8] += x[12]; x[4] = rol32(x[4] ^ x[8], 12); - x[9] += x[13]; x[5] = rol32(x[5] ^ x[9], 12); - x[10] += x[14]; x[6] = rol32(x[6] ^ x[10], 12); - x[11] += x[15]; x[7] = rol32(x[7] ^ x[11], 12); - - x[0] += x[4]; x[12] = rol32(x[12] ^ x[0], 8); - x[1] += x[5]; x[13] = rol32(x[13] ^ x[1], 8); - x[2] += x[6]; x[14] = rol32(x[14] ^ x[2], 8); - x[3] += x[7]; x[15] = rol32(x[15] ^ x[3], 8); - - x[8] += x[12]; x[4] = rol32(x[4] ^ x[8], 7); - x[9] += x[13]; x[5] = rol32(x[5] ^ x[9], 7); - x[10] += x[14]; x[6] = rol32(x[6] ^ x[10], 7); - x[11] += x[15]; x[7] = rol32(x[7] ^ x[11], 7); - - x[0] += x[5]; x[15] = rol32(x[15] ^ x[0], 16); - x[1] += x[6]; x[12] = rol32(x[12] ^ x[1], 16); - x[2] += x[7]; x[13] = rol32(x[13] ^ x[2], 16); - x[3] += x[4]; x[14] = rol32(x[14] ^ x[3], 16); - - x[10] += x[15]; x[5] = rol32(x[5] ^ x[10], 12); - x[11] += x[12]; x[6] = rol32(x[6] ^ x[11], 12); - x[8] += x[13]; x[7] = rol32(x[7] ^ x[8], 12); - x[9] += x[14]; x[4] = rol32(x[4] ^ x[9], 12); - - x[0] += x[5]; x[15] = rol32(x[15] ^ x[0], 8); - x[1] += x[6]; x[12] = rol32(x[12] ^ x[1], 8); - x[2] += x[7]; x[13] = rol32(x[13] ^ x[2], 8); - x[3] += x[4]; x[14] = rol32(x[14] ^ x[3], 8); - - x[10] += x[15]; x[5] = rol32(x[5] ^ x[10], 7); - x[11] += x[12]; x[6] = rol32(x[6] ^ x[11], 7); - x[8] += x[13]; x[7] = rol32(x[7] ^ x[8], 7); - x[9] += x[14]; x[4] = rol32(x[4] ^ x[9], 7); - } -} - -/** - * chacha20_block - generate one keystream block and increment block counter - * @state: input state matrix (16 32-bit words) - * @stream: output keystream block (64 bytes) - * - * This is the ChaCha20 core, a function from 64-byte strings to 64-byte - * strings. The caller has already converted the endianness of the input. This - * function also handles incrementing the block counter in the input matrix. - */ -void chacha20_block(u32 *state, u8 *stream) -{ - u32 x[16]; - int i; - - memcpy(x, state, 64); - - chacha20_permute(x); - - for (i = 0; i < ARRAY_SIZE(x); i++) - put_unaligned_le32(x[i] + state[i], &stream[i * sizeof(u32)]); - - state[12]++; -} -EXPORT_SYMBOL(chacha20_block); - -/** - * hchacha20_block - abbreviated ChaCha20 core, for XChaCha20 - * @in: input state matrix (16 32-bit words) - * @out: output (8 32-bit words) - * - * HChaCha20 is the ChaCha equivalent of HSalsa20 and is an intermediate step - * towards XChaCha20 (see https://cr.yp.to/snuffle/xsalsa-20081128.pdf). - * HChaCha20 skips the final addition of the initial state, and outputs only - * certain words of the state. It should not be used for streaming directly. - */ -void hchacha20_block(const u32 *in, u32 *out) -{ - u32 x[16]; - - memcpy(x, in, 64); - - chacha20_permute(x); - - memcpy(&out[0], &x[0], 16); - memcpy(&out[4], &x[12], 16); -} -EXPORT_SYMBOL(hchacha20_block); -- cgit v1.2.3 From aa7624093cb7fbf4fea95e612580d8d29a819f67 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 16 Nov 2018 17:26:22 -0800 Subject: crypto: chacha - add XChaCha12 support Now that the generic implementation of ChaCha20 has been refactored to allow varying the number of rounds, add support for XChaCha12, which is the XSalsa construction applied to ChaCha12. ChaCha12 is one of the three ciphers specified by the original ChaCha paper (https://cr.yp.to/chacha/chacha-20080128.pdf: "ChaCha, a variant of Salsa20"), alongside ChaCha8 and ChaCha20. ChaCha12 is faster than ChaCha20 but has a lower, but still large, security margin. We need XChaCha12 support so that it can be used in the Adiantum encryption mode, which enables disk/file encryption on low-end mobile devices where AES-XTS is too slow as the CPUs lack AES instructions. We'd prefer XChaCha20 (the more popular variant), but it's too slow on some of our target devices, so at least in some cases we do need the XChaCha12-based version. In more detail, the problem is that Adiantum is still much slower than we're happy with, and encryption still has a quite noticeable effect on the feel of low-end devices. Users and vendors push back hard against encryption that degrades the user experience, which always risks encryption being disabled entirely. So we need to choose the fastest option that gives us a solid margin of security, and here that's XChaCha12. The best known attack on ChaCha breaks only 7 rounds and has 2^235 time complexity, so ChaCha12's security margin is still better than AES-256's. Much has been learned about cryptanalysis of ARX ciphers since Salsa20 was originally designed in 2005, and it now seems we can be comfortable with a smaller number of rounds. The eSTREAM project also suggests the 12-round version of Salsa20 as providing the best balance among the different variants: combining very good performance with a "comfortable margin of security". Note that it would be trivial to add vanilla ChaCha12 in addition to XChaCha12. However, it's unneeded for now and therefore is omitted. As discussed in the patch that introduced XChaCha20 support, I considered splitting the code into separate chacha-common, chacha20, xchacha20, and xchacha12 modules, so that these algorithms could be enabled/disabled independently. However, since nearly all the code is shared anyway, I ultimately decided there would have been little benefit to the added complexity. Reviewed-by: Ard Biesheuvel Acked-by: Martin Willi Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 8 +- crypto/chacha_generic.c | 26 ++- crypto/testmgr.c | 6 + crypto/testmgr.h | 578 ++++++++++++++++++++++++++++++++++++++++++++++++ include/crypto/chacha.h | 7 + lib/chacha.c | 6 +- 6 files changed, 625 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/crypto/Kconfig b/crypto/Kconfig index 75ebd1a2746c..4431c0db56b7 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1403,10 +1403,10 @@ config CRYPTO_SALSA20 Bernstein . See config CRYPTO_CHACHA20 - tristate "ChaCha20 stream cipher algorithms" + tristate "ChaCha stream cipher algorithms" select CRYPTO_BLKCIPHER help - The ChaCha20 and XChaCha20 stream cipher algorithms. + The ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms. ChaCha20 is a 256-bit high-speed stream cipher designed by Daniel J. Bernstein and further specified in RFC7539 for use in IETF protocols. @@ -1419,6 +1419,10 @@ config CRYPTO_CHACHA20 while provably retaining ChaCha20's security. See also: + XChaCha12 is XChaCha20 reduced to 12 rounds, with correspondingly + reduced security margin but increased performance. It can be needed + in some performance-sensitive scenarios. + config CRYPTO_CHACHA20_X86_64 tristate "ChaCha20 cipher algorithm (x86_64/SSSE3/AVX2)" depends on X86 && 64BIT diff --git a/crypto/chacha_generic.c b/crypto/chacha_generic.c index 438f15a14054..35b583101f4f 100644 --- a/crypto/chacha_generic.c +++ b/crypto/chacha_generic.c @@ -1,5 +1,5 @@ /* - * ChaCha20 (RFC7539) and XChaCha20 stream cipher algorithms + * ChaCha and XChaCha stream ciphers, including ChaCha20 (RFC7539) * * Copyright (C) 2015 Martin Willi * Copyright (C) 2018 Google LLC @@ -106,6 +106,13 @@ int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, } EXPORT_SYMBOL_GPL(crypto_chacha20_setkey); +int crypto_chacha12_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keysize) +{ + return chacha_setkey(tfm, key, keysize, 12); +} +EXPORT_SYMBOL_GPL(crypto_chacha12_setkey); + int crypto_chacha_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -168,6 +175,21 @@ static struct skcipher_alg algs[] = { .setkey = crypto_chacha20_setkey, .encrypt = crypto_xchacha_crypt, .decrypt = crypto_xchacha_crypt, + }, { + .base.cra_name = "xchacha12", + .base.cra_driver_name = "xchacha12-generic", + .base.cra_priority = 100, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct chacha_ctx), + .base.cra_module = THIS_MODULE, + + .min_keysize = CHACHA_KEY_SIZE, + .max_keysize = CHACHA_KEY_SIZE, + .ivsize = XCHACHA_IV_SIZE, + .chunksize = CHACHA_BLOCK_SIZE, + .setkey = crypto_chacha12_setkey, + .encrypt = crypto_xchacha_crypt, + .decrypt = crypto_xchacha_crypt, } }; @@ -191,3 +213,5 @@ MODULE_ALIAS_CRYPTO("chacha20"); MODULE_ALIAS_CRYPTO("chacha20-generic"); MODULE_ALIAS_CRYPTO("xchacha20"); MODULE_ALIAS_CRYPTO("xchacha20-generic"); +MODULE_ALIAS_CRYPTO("xchacha12"); +MODULE_ALIAS_CRYPTO("xchacha12-generic"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 11f5c8b0f4dc..6ff60c3745f1 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -3576,6 +3576,12 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .hash = __VECS(aes_xcbc128_tv_template) } + }, { + .alg = "xchacha12", + .test = alg_test_skcipher, + .suite = { + .cipher = __VECS(xchacha12_tv_template) + }, }, { .alg = "xchacha20", .test = alg_test_skcipher, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index df0dc44a9b7b..a23dca2b11d0 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -31571,6 +31571,584 @@ static const struct cipher_testvec xchacha20_tv_template[] = { }, }; +/* + * Same as XChaCha20 test vectors above, but recomputed the ciphertext with + * XChaCha12, using a modified libsodium. + */ +static const struct cipher_testvec xchacha12_tv_template[] = { + { + .key = "\x79\xc9\x97\x98\xac\x67\x30\x0b" + "\xbb\x27\x04\xc9\x5c\x34\x1e\x32" + "\x45\xf3\xdc\xb2\x17\x61\xb9\x8e" + "\x52\xff\x45\xb2\x4f\x30\x4f\xc4", + .klen = 32, + .iv = "\xb3\x3f\xfd\x30\x96\x47\x9b\xcf" + "\xbc\x9a\xee\x49\x41\x76\x88\xa0" + "\xa2\x55\x4f\x8d\x95\x38\x94\x19" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00", + .ctext = "\x1b\x78\x7f\xd7\xa1\x41\x68\xab" + "\x3d\x3f\xd1\x7b\x69\x56\xb2\xd5" + "\x43\xce\xeb\xaf\x36\xf0\x29\x9d" + "\x3a\xfb\x18\xae\x1b", + .len = 29, + }, { + .key = "\x9d\x23\xbd\x41\x49\xcb\x97\x9c" + "\xcf\x3c\x5c\x94\xdd\x21\x7e\x98" + "\x08\xcb\x0e\x50\xcd\x0f\x67\x81" + "\x22\x35\xea\xaf\x60\x1d\x62\x32", + .klen = 32, + .iv = "\xc0\x47\x54\x82\x66\xb7\xc3\x70" + "\xd3\x35\x66\xa2\x42\x5c\xbf\x30" + "\xd8\x2d\x1e\xaf\x52\x94\x10\x9e" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00", + .ctext = "\xfb\x32\x09\x1d\x83\x05\xae\x4c" + "\x13\x1f\x12\x71\xf2\xca\xb2\xeb" + "\x5b\x83\x14\x7d\x83\xf6\x57\x77" + "\x2e\x40\x1f\x92\x2c\xf9\xec\x35" + "\x34\x1f\x93\xdf\xfb\x30\xd7\x35" + "\x03\x05\x78\xc1\x20\x3b\x7a\xe3" + "\x62\xa3\x89\xdc\x11\x11\x45\xa8" + "\x82\x89\xa0\xf1\x4e\xc7\x0f\x11" + "\x69\xdd\x0c\x84\x2b\x89\x5c\xdc" + "\xf0\xde\x01\xef\xc5\x65\x79\x23" + "\x87\x67\xd6\x50\xd9\x8d\xd9\x92" + "\x54\x5b\x0e", + .len = 91, + }, { + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .klen = 32, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x67\xc6\x69\x73" + "\x51\xff\x4a\xec\x29\xcd\xba\xab" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .ctext = "\xdf\x2d\xc6\x21\x2a\x9d\xa1\xbb" + "\xc2\x77\x66\x0c\x5c\x46\xef\xa7" + "\x79\x1b\xb9\xdf\x55\xe2\xf9\x61" + "\x4c\x7b\xa4\x52\x24\xaf\xa2\xda" + "\xd1\x8f\x8f\xa2\x9e\x53\x4d\xc4" + "\xb8\x55\x98\x08\x7c\x08\xd4\x18" + "\x67\x8f\xef\x50\xb1\x5f\xa5\x77" + "\x4c\x25\xe7\x86\x26\x42\xca\x44", + .len = 64, + }, { + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x01", + .klen = 32, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x02\xf2\xfb\xe3\x46" + "\x7c\xc2\x54\xf8\x1b\xe8\xe7\x8d" + "\x01\x00\x00\x00\x00\x00\x00\x00", + .ptext = "\x41\x6e\x79\x20\x73\x75\x62\x6d" + "\x69\x73\x73\x69\x6f\x6e\x20\x74" + "\x6f\x20\x74\x68\x65\x20\x49\x45" + "\x54\x46\x20\x69\x6e\x74\x65\x6e" + "\x64\x65\x64\x20\x62\x79\x20\x74" + "\x68\x65\x20\x43\x6f\x6e\x74\x72" + "\x69\x62\x75\x74\x6f\x72\x20\x66" + "\x6f\x72\x20\x70\x75\x62\x6c\x69" + "\x63\x61\x74\x69\x6f\x6e\x20\x61" + "\x73\x20\x61\x6c\x6c\x20\x6f\x72" + "\x20\x70\x61\x72\x74\x20\x6f\x66" + "\x20\x61\x6e\x20\x49\x45\x54\x46" + "\x20\x49\x6e\x74\x65\x72\x6e\x65" + "\x74\x2d\x44\x72\x61\x66\x74\x20" + "\x6f\x72\x20\x52\x46\x43\x20\x61" + "\x6e\x64\x20\x61\x6e\x79\x20\x73" + "\x74\x61\x74\x65\x6d\x65\x6e\x74" + "\x20\x6d\x61\x64\x65\x20\x77\x69" + "\x74\x68\x69\x6e\x20\x74\x68\x65" + "\x20\x63\x6f\x6e\x74\x65\x78\x74" + "\x20\x6f\x66\x20\x61\x6e\x20\x49" + "\x45\x54\x46\x20\x61\x63\x74\x69" + "\x76\x69\x74\x79\x20\x69\x73\x20" + "\x63\x6f\x6e\x73\x69\x64\x65\x72" + "\x65\x64\x20\x61\x6e\x20\x22\x49" + "\x45\x54\x46\x20\x43\x6f\x6e\x74" + "\x72\x69\x62\x75\x74\x69\x6f\x6e" + "\x22\x2e\x20\x53\x75\x63\x68\x20" + "\x73\x74\x61\x74\x65\x6d\x65\x6e" + "\x74\x73\x20\x69\x6e\x63\x6c\x75" + "\x64\x65\x20\x6f\x72\x61\x6c\x20" + "\x73\x74\x61\x74\x65\x6d\x65\x6e" + "\x74\x73\x20\x69\x6e\x20\x49\x45" + "\x54\x46\x20\x73\x65\x73\x73\x69" + "\x6f\x6e\x73\x2c\x20\x61\x73\x20" + "\x77\x65\x6c\x6c\x20\x61\x73\x20" + "\x77\x72\x69\x74\x74\x65\x6e\x20" + "\x61\x6e\x64\x20\x65\x6c\x65\x63" + "\x74\x72\x6f\x6e\x69\x63\x20\x63" + "\x6f\x6d\x6d\x75\x6e\x69\x63\x61" + "\x74\x69\x6f\x6e\x73\x20\x6d\x61" + "\x64\x65\x20\x61\x74\x20\x61\x6e" + "\x79\x20\x74\x69\x6d\x65\x20\x6f" + "\x72\x20\x70\x6c\x61\x63\x65\x2c" + "\x20\x77\x68\x69\x63\x68\x20\x61" + "\x72\x65\x20\x61\x64\x64\x72\x65" + "\x73\x73\x65\x64\x20\x74\x6f", + .ctext = "\xe4\xa6\xc8\x30\xc4\x23\x13\xd6" + "\x08\x4d\xc9\xb7\xa5\x64\x7c\xb9" + "\x71\xe2\xab\x3e\xa8\x30\x8a\x1c" + "\x4a\x94\x6d\x9b\xe0\xb3\x6f\xf1" + "\xdc\xe3\x1b\xb3\xa9\x6d\x0d\xd6" + "\xd0\xca\x12\xef\xe7\x5f\xd8\x61" + "\x3c\x82\xd3\x99\x86\x3c\x6f\x66" + "\x02\x06\xdc\x55\xf9\xed\xdf\x38" + "\xb4\xa6\x17\x00\x7f\xef\xbf\x4f" + "\xf8\x36\xf1\x60\x7e\x47\xaf\xdb" + "\x55\x9b\x12\xcb\x56\x44\xa7\x1f" + "\xd3\x1a\x07\x3b\x00\xec\xe6\x4c" + "\xa2\x43\x27\xdf\x86\x19\x4f\x16" + "\xed\xf9\x4a\xf3\x63\x6f\xfa\x7f" + "\x78\x11\xf6\x7d\x97\x6f\xec\x6f" + "\x85\x0f\x5c\x36\x13\x8d\x87\xe0" + "\x80\xb1\x69\x0b\x98\x89\x9c\x4e" + "\xf8\xdd\xee\x5c\x0a\x85\xce\xd4" + "\xea\x1b\x48\xbe\x08\xf8\xe2\xa8" + "\xa5\xb0\x3c\x79\xb1\x15\xb4\xb9" + "\x75\x10\x95\x35\x81\x7e\x26\xe6" + "\x78\xa4\x88\xcf\xdb\x91\x34\x18" + "\xad\xd7\x8e\x07\x7d\xab\x39\xf9" + "\xa3\x9e\xa5\x1d\xbb\xed\x61\xfd" + "\xdc\xb7\x5a\x27\xfc\xb5\xc9\x10" + "\xa8\xcc\x52\x7f\x14\x76\x90\xe7" + "\x1b\x29\x60\x74\xc0\x98\x77\xbb" + "\xe0\x54\xbb\x27\x49\x59\x1e\x62" + "\x3d\xaf\x74\x06\xa4\x42\x6f\xc6" + "\x52\x97\xc4\x1d\xc4\x9f\xe2\xe5" + "\x38\x57\x91\xd1\xa2\x28\xcc\x40" + "\xcc\x70\x59\x37\xfc\x9f\x4b\xda" + "\xa0\xeb\x97\x9a\x7d\xed\x14\x5c" + "\x9c\xb7\x93\x26\x41\xa8\x66\xdd" + "\x87\x6a\xc0\xd3\xc2\xa9\x3e\xae" + "\xe9\x72\xfe\xd1\xb3\xac\x38\xea" + "\x4d\x15\xa9\xd5\x36\x61\xe9\x96" + "\x6c\x23\xf8\x43\xe4\x92\x29\xd9" + "\x8b\x78\xf7\x0a\x52\xe0\x19\x5b" + "\x59\x69\x5b\x5d\xa1\x53\xc4\x68" + "\xe1\xbb\xac\x89\x14\xe2\xe2\x85" + "\x41\x18\xf5\xb3\xd1\xfa\x68\x19" + "\x44\x78\xdc\xcf\xe7\x88\x2d\x52" + "\x5f\x40\xb5\x7e\xf8\x88\xa2\xae" + "\x4a\xb2\x07\x35\x9d\x9b\x07\x88" + "\xb7\x00\xd0\x0c\xb6\xa0\x47\x59" + "\xda\x4e\xc9\xab\x9b\x8a\x7b", + + .len = 375, + .also_non_np = 1, + .np = 3, + .tap = { 375 - 20, 4, 16 }, + + }, { + .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" + "\xf3\x33\x88\x86\x04\xf6\xb5\xf0" + "\x47\x39\x17\xc1\x40\x2b\x80\x09" + "\x9d\xca\x5c\xbc\x20\x70\x75\xc0", + .klen = 32, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x02\x76\x5a\x2e\x63" + "\x33\x9f\xc9\x9a\x66\x32\x0d\xb7" + "\x2a\x00\x00\x00\x00\x00\x00\x00", + .ptext = "\x27\x54\x77\x61\x73\x20\x62\x72" + "\x69\x6c\x6c\x69\x67\x2c\x20\x61" + "\x6e\x64\x20\x74\x68\x65\x20\x73" + "\x6c\x69\x74\x68\x79\x20\x74\x6f" + "\x76\x65\x73\x0a\x44\x69\x64\x20" + "\x67\x79\x72\x65\x20\x61\x6e\x64" + "\x20\x67\x69\x6d\x62\x6c\x65\x20" + "\x69\x6e\x20\x74\x68\x65\x20\x77" + "\x61\x62\x65\x3a\x0a\x41\x6c\x6c" + "\x20\x6d\x69\x6d\x73\x79\x20\x77" + "\x65\x72\x65\x20\x74\x68\x65\x20" + "\x62\x6f\x72\x6f\x67\x6f\x76\x65" + "\x73\x2c\x0a\x41\x6e\x64\x20\x74" + "\x68\x65\x20\x6d\x6f\x6d\x65\x20" + "\x72\x61\x74\x68\x73\x20\x6f\x75" + "\x74\x67\x72\x61\x62\x65\x2e", + .ctext = "\xb9\x68\xbc\x6a\x24\xbc\xcc\xd8" + "\x9b\x2a\x8d\x5b\x96\xaf\x56\xe3" + "\x11\x61\xe7\xa7\x9b\xce\x4e\x7d" + "\x60\x02\x48\xac\xeb\xd5\x3a\x26" + "\x9d\x77\x3b\xb5\x32\x13\x86\x8e" + "\x20\x82\x26\x72\xae\x64\x1b\x7e" + "\x2e\x01\x68\xb4\x87\x45\xa1\x24" + "\xe4\x48\x40\xf0\xaa\xac\xee\xa9" + "\xfc\x31\xad\x9d\x89\xa3\xbb\xd2" + "\xe4\x25\x13\xad\x0f\x5e\xdf\x3c" + "\x27\xab\xb8\x62\x46\x22\x30\x48" + "\x55\x2c\x4e\x84\x78\x1d\x0d\x34" + "\x8d\x3c\x91\x0a\x7f\x5b\x19\x9f" + "\x97\x05\x4c\xa7\x62\x47\x8b\xc5" + "\x44\x2e\x20\x33\xdd\xa0\x82\xa9" + "\x25\x76\x37\xe6\x3c\x67\x5b", + .len = 127, + }, { + .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" + "\xf3\x33\x88\x86\x04\xf6\xb5\xf0" + "\x47\x39\x17\xc1\x40\x2b\x80\x09" + "\x9d\xca\x5c\xbc\x20\x70\x75\xc0", + .klen = 32, + .iv = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x01\x31\x58\xa3\x5a" + "\x25\x5d\x05\x17\x58\xe9\x5e\xd4" + "\x1c\x00\x00\x00\x00\x00\x00\x00", + .ptext = "\x49\xee\xe0\xdc\x24\x90\x40\xcd" + "\xc5\x40\x8f\x47\x05\xbc\xdd\x81" + "\x47\xc6\x8d\xe6\xb1\x8f\xd7\xcb" + "\x09\x0e\x6e\x22\x48\x1f\xbf\xb8" + "\x5c\xf7\x1e\x8a\xc1\x23\xf2\xd4" + "\x19\x4b\x01\x0f\x4e\xa4\x43\xce" + "\x01\xc6\x67\xda\x03\x91\x18\x90" + "\xa5\xa4\x8e\x45\x03\xb3\x2d\xac" + "\x74\x92\xd3\x53\x47\xc8\xdd\x25" + "\x53\x6c\x02\x03\x87\x0d\x11\x0c" + "\x58\xe3\x12\x18\xfd\x2a\x5b\x40" + "\x0c\x30\xf0\xb8\x3f\x43\xce\xae" + "\x65\x3a\x7d\x7c\xf4\x54\xaa\xcc" + "\x33\x97\xc3\x77\xba\xc5\x70\xde" + "\xd7\xd5\x13\xa5\x65\xc4\x5f\x0f" + "\x46\x1a\x0d\x97\xb5\xf3\xbb\x3c" + "\x84\x0f\x2b\xc5\xaa\xea\xf2\x6c" + "\xc9\xb5\x0c\xee\x15\xf3\x7d\xbe" + "\x9f\x7b\x5a\xa6\xae\x4f\x83\xb6" + "\x79\x49\x41\xf4\x58\x18\xcb\x86" + "\x7f\x30\x0e\xf8\x7d\x44\x36\xea" + "\x75\xeb\x88\x84\x40\x3c\xad\x4f" + "\x6f\x31\x6b\xaa\x5d\xe5\xa5\xc5" + "\x21\x66\xe9\xa7\xe3\xb2\x15\x88" + "\x78\xf6\x79\xa1\x59\x47\x12\x4e" + "\x9f\x9f\x64\x1a\xa0\x22\x5b\x08" + "\xbe\x7c\x36\xc2\x2b\x66\x33\x1b" + "\xdd\x60\x71\xf7\x47\x8c\x61\xc3" + "\xda\x8a\x78\x1e\x16\xfa\x1e\x86" + "\x81\xa6\x17\x2a\xa7\xb5\xc2\xe7" + "\xa4\xc7\x42\xf1\xcf\x6a\xca\xb4" + "\x45\xcf\xf3\x93\xf0\xe7\xea\xf6" + "\xf4\xe6\x33\x43\x84\x93\xa5\x67" + "\x9b\x16\x58\x58\x80\x0f\x2b\x5c" + "\x24\x74\x75\x7f\x95\x81\xb7\x30" + "\x7a\x33\xa7\xf7\x94\x87\x32\x27" + "\x10\x5d\x14\x4c\x43\x29\xdd\x26" + "\xbd\x3e\x3c\x0e\xfe\x0e\xa5\x10" + "\xea\x6b\x64\xfd\x73\xc6\xed\xec" + "\xa8\xc9\xbf\xb3\xba\x0b\x4d\x07" + "\x70\xfc\x16\xfd\x79\x1e\xd7\xc5" + "\x49\x4e\x1c\x8b\x8d\x79\x1b\xb1" + "\xec\xca\x60\x09\x4c\x6a\xd5\x09" + "\x49\x46\x00\x88\x22\x8d\xce\xea" + "\xb1\x17\x11\xde\x42\xd2\x23\xc1" + "\x72\x11\xf5\x50\x73\x04\x40\x47" + "\xf9\x5d\xe7\xa7\x26\xb1\x7e\xb0" + "\x3f\x58\xc1\x52\xab\x12\x67\x9d" + "\x3f\x43\x4b\x68\xd4\x9c\x68\x38" + "\x07\x8a\x2d\x3e\xf3\xaf\x6a\x4b" + "\xf9\xe5\x31\x69\x22\xf9\xa6\x69" + "\xc6\x9c\x96\x9a\x12\x35\x95\x1d" + "\x95\xd5\xdd\xbe\xbf\x93\x53\x24" + "\xfd\xeb\xc2\x0a\x64\xb0\x77\x00" + "\x6f\x88\xc4\x37\x18\x69\x7c\xd7" + "\x41\x92\x55\x4c\x03\xa1\x9a\x4b" + "\x15\xe5\xdf\x7f\x37\x33\x72\xc1" + "\x8b\x10\x67\xa3\x01\x57\x94\x25" + "\x7b\x38\x71\x7e\xdd\x1e\xcc\x73" + "\x55\xd2\x8e\xeb\x07\xdd\xf1\xda" + "\x58\xb1\x47\x90\xfe\x42\x21\x72" + "\xa3\x54\x7a\xa0\x40\xec\x9f\xdd" + "\xc6\x84\x6e\xca\xae\xe3\x68\xb4" + "\x9d\xe4\x78\xff\x57\xf2\xf8\x1b" + "\x03\xa1\x31\xd9\xde\x8d\xf5\x22" + "\x9c\xdd\x20\xa4\x1e\x27\xb1\x76" + "\x4f\x44\x55\xe2\x9b\xa1\x9c\xfe" + "\x54\xf7\x27\x1b\xf4\xde\x02\xf5" + "\x1b\x55\x48\x5c\xdc\x21\x4b\x9e" + "\x4b\x6e\xed\x46\x23\xdc\x65\xb2" + "\xcf\x79\x5f\x28\xe0\x9e\x8b\xe7" + "\x4c\x9d\x8a\xff\xc1\xa6\x28\xb8" + "\x65\x69\x8a\x45\x29\xef\x74\x85" + "\xde\x79\xc7\x08\xae\x30\xb0\xf4" + "\xa3\x1d\x51\x41\xab\xce\xcb\xf6" + "\xb5\xd8\x6d\xe0\x85\xe1\x98\xb3" + "\x43\xbb\x86\x83\x0a\xa0\xf5\xb7" + "\x04\x0b\xfa\x71\x1f\xb0\xf6\xd9" + "\x13\x00\x15\xf0\xc7\xeb\x0d\x5a" + "\x9f\xd7\xb9\x6c\x65\x14\x22\x45" + "\x6e\x45\x32\x3e\x7e\x60\x1a\x12" + "\x97\x82\x14\xfb\xaa\x04\x22\xfa" + "\xa0\xe5\x7e\x8c\x78\x02\x48\x5d" + "\x78\x33\x5a\x7c\xad\xdb\x29\xce" + "\xbb\x8b\x61\xa4\xb7\x42\xe2\xac" + "\x8b\x1a\xd9\x2f\x0b\x8b\x62\x21" + "\x83\x35\x7e\xad\x73\xc2\xb5\x6c" + "\x10\x26\x38\x07\xe5\xc7\x36\x80" + "\xe2\x23\x12\x61\xf5\x48\x4b\x2b" + "\xc5\xdf\x15\xd9\x87\x01\xaa\xac" + "\x1e\x7c\xad\x73\x78\x18\x63\xe0" + "\x8b\x9f\x81\xd8\x12\x6a\x28\x10" + "\xbe\x04\x68\x8a\x09\x7c\x1b\x1c" + "\x83\x66\x80\x47\x80\xe8\xfd\x35" + "\x1c\x97\x6f\xae\x49\x10\x66\xcc" + "\xc6\xd8\xcc\x3a\x84\x91\x20\x77" + "\x72\xe4\x24\xd2\x37\x9f\xc5\xc9" + "\x25\x94\x10\x5f\x40\x00\x64\x99" + "\xdc\xae\xd7\x21\x09\x78\x50\x15" + "\xac\x5f\xc6\x2c\xa2\x0b\xa9\x39" + "\x87\x6e\x6d\xab\xde\x08\x51\x16" + "\xc7\x13\xe9\xea\xed\x06\x8e\x2c" + "\xf8\x37\x8c\xf0\xa6\x96\x8d\x43" + "\xb6\x98\x37\xb2\x43\xed\xde\xdf" + "\x89\x1a\xe7\xeb\x9d\xa1\x7b\x0b" + "\x77\xb0\xe2\x75\xc0\xf1\x98\xd9" + "\x80\x55\xc9\x34\x91\xd1\x59\xe8" + "\x4b\x0f\xc1\xa9\x4b\x7a\x84\x06" + "\x20\xa8\x5d\xfa\xd1\xde\x70\x56" + "\x2f\x9e\x91\x9c\x20\xb3\x24\xd8" + "\x84\x3d\xe1\x8c\x7e\x62\x52\xe5" + "\x44\x4b\x9f\xc2\x93\x03\xea\x2b" + "\x59\xc5\xfa\x3f\x91\x2b\xbb\x23" + "\xf5\xb2\x7b\xf5\x38\xaf\xb3\xee" + "\x63\xdc\x7b\xd1\xff\xaa\x8b\xab" + "\x82\x6b\x37\x04\xeb\x74\xbe\x79" + "\xb9\x83\x90\xef\x20\x59\x46\xff" + "\xe9\x97\x3e\x2f\xee\xb6\x64\x18" + "\x38\x4c\x7a\x4a\xf9\x61\xe8\x9a" + "\xa1\xb5\x01\xa6\x47\xd3\x11\xd4" + "\xce\xd3\x91\x49\x88\xc7\xb8\x4d" + "\xb1\xb9\x07\x6d\x16\x72\xae\x46" + "\x5e\x03\xa1\x4b\xb6\x02\x30\xa8" + "\x3d\xa9\x07\x2a\x7c\x19\xe7\x62" + "\x87\xe3\x82\x2f\x6f\xe1\x09\xd9" + "\x94\x97\xea\xdd\x58\x9e\xae\x76" + "\x7e\x35\xe5\xb4\xda\x7e\xf4\xde" + "\xf7\x32\x87\xcd\x93\xbf\x11\x56" + "\x11\xbe\x08\x74\xe1\x69\xad\xe2" + "\xd7\xf8\x86\x75\x8a\x3c\xa4\xbe" + "\x70\xa7\x1b\xfc\x0b\x44\x2a\x76" + "\x35\xea\x5d\x85\x81\xaf\x85\xeb" + "\xa0\x1c\x61\xc2\xf7\x4f\xa5\xdc" + "\x02\x7f\xf6\x95\x40\x6e\x8a\x9a" + "\xf3\x5d\x25\x6e\x14\x3a\x22\xc9" + "\x37\x1c\xeb\x46\x54\x3f\xa5\x91" + "\xc2\xb5\x8c\xfe\x53\x08\x97\x32" + "\x1b\xb2\x30\x27\xfe\x25\x5d\xdc" + "\x08\x87\xd0\xe5\x94\x1a\xd4\xf1" + "\xfe\xd6\xb4\xa3\xe6\x74\x81\x3c" + "\x1b\xb7\x31\xa7\x22\xfd\xd4\xdd" + "\x20\x4e\x7c\x51\xb0\x60\x73\xb8" + "\x9c\xac\x91\x90\x7e\x01\xb0\xe1" + "\x8a\x2f\x75\x1c\x53\x2a\x98\x2a" + "\x06\x52\x95\x52\xb2\xe9\x25\x2e" + "\x4c\xe2\x5a\x00\xb2\x13\x81\x03" + "\x77\x66\x0d\xa5\x99\xda\x4e\x8c" + "\xac\xf3\x13\x53\x27\x45\xaf\x64" + "\x46\xdc\xea\x23\xda\x97\xd1\xab" + "\x7d\x6c\x30\x96\x1f\xbc\x06\x34" + "\x18\x0b\x5e\x21\x35\x11\x8d\x4c" + "\xe0\x2d\xe9\x50\x16\x74\x81\xa8" + "\xb4\x34\xb9\x72\x42\xa6\xcc\xbc" + "\xca\x34\x83\x27\x10\x5b\x68\x45" + "\x8f\x52\x22\x0c\x55\x3d\x29\x7c" + "\xe3\xc0\x66\x05\x42\x91\x5f\x58" + "\xfe\x4a\x62\xd9\x8c\xa9\x04\x19" + "\x04\xa9\x08\x4b\x57\xfc\x67\x53" + "\x08\x7c\xbc\x66\x8a\xb0\xb6\x9f" + "\x92\xd6\x41\x7c\x5b\x2a\x00\x79" + "\x72", + .ctext = "\xe1\xb6\x8b\x5c\x80\xb8\xcc\x08" + "\x1b\x84\xb2\xd1\xad\xa4\x70\xac" + "\x67\xa9\x39\x27\xac\xb4\x5b\xb7" + "\x4c\x26\x77\x23\x1d\xce\x0a\xbe" + "\x18\x9e\x42\x8b\xbd\x7f\xd6\xf1" + "\xf1\x6b\xe2\x6d\x7f\x92\x0e\xcb" + "\xb8\x79\xba\xb4\xac\x7e\x2d\xc0" + "\x9e\x83\x81\x91\xd5\xea\xc3\x12" + "\x8d\xa4\x26\x70\xa4\xf9\x71\x0b" + "\xbd\x2e\xe1\xb3\x80\x42\x25\xb3" + "\x0b\x31\x99\xe1\x0d\xde\xa6\x90" + "\xf2\xa3\x10\xf7\xe5\xf3\x83\x1e" + "\x2c\xfb\x4d\xf0\x45\x3d\x28\x3c" + "\xb8\xf1\xcb\xbf\x67\xd8\x43\x5a" + "\x9d\x7b\x73\x29\x88\x0f\x13\x06" + "\x37\x50\x0d\x7c\xe6\x9b\x07\xdd" + "\x7e\x01\x1f\x81\x90\x10\x69\xdb" + "\xa4\xad\x8a\x5e\xac\x30\x72\xf2" + "\x36\xcd\xe3\x23\x49\x02\x93\xfa" + "\x3d\xbb\xe2\x98\x83\xeb\xe9\x8d" + "\xb3\x8f\x11\xaa\x53\xdb\xaf\x2e" + "\x95\x13\x99\x3d\x71\xbd\x32\x92" + "\xdd\xfc\x9d\x5e\x6f\x63\x2c\xee" + "\x91\x1f\x4c\x64\x3d\x87\x55\x0f" + "\xcc\x3d\x89\x61\x53\x02\x57\x8f" + "\xe4\x77\x29\x32\xaf\xa6\x2f\x0a" + "\xae\x3c\x3f\x3f\xf4\xfb\x65\x52" + "\xc5\xc1\x78\x78\x53\x28\xad\xed" + "\xd1\x67\x37\xc7\x59\x70\xcd\x0a" + "\xb8\x0f\x80\x51\x9f\xc0\x12\x5e" + "\x06\x0a\x7e\xec\x24\x5f\x73\x00" + "\xb1\x0b\x31\x47\x4f\x73\x8d\xb4" + "\xce\xf3\x55\x45\x6c\x84\x27\xba" + "\xb9\x6f\x03\x4a\xeb\x98\x88\x6e" + "\x53\xed\x25\x19\x0d\x8f\xfe\xca" + "\x60\xe5\x00\x93\x6e\x3c\xff\x19" + "\xae\x08\x3b\x8a\xa6\x84\x05\xfe" + "\x9b\x59\xa0\x8c\xc8\x05\x45\xf5" + "\x05\x37\xdc\x45\x6f\x8b\x95\x8c" + "\x4e\x11\x45\x7a\xce\x21\xa5\xf7" + "\x71\x67\xb9\xce\xd7\xf9\xe9\x5e" + "\x60\xf5\x53\x7a\xa8\x85\x14\x03" + "\xa0\x92\xec\xf3\x51\x80\x84\xc4" + "\xdc\x11\x9e\x57\xce\x4b\x45\xcf" + "\x90\x95\x85\x0b\x96\xe9\xee\x35" + "\x10\xb8\x9b\xf2\x59\x4a\xc6\x7e" + "\x85\xe5\x6f\x38\x51\x93\x40\x0c" + "\x99\xd7\x7f\x32\xa8\x06\x27\xd1" + "\x2b\xd5\xb5\x3a\x1a\xe1\x5e\xda" + "\xcd\x5a\x50\x30\x3c\xc7\xe7\x65" + "\xa6\x07\x0b\x98\x91\xc6\x20\x27" + "\x2a\x03\x63\x1b\x1e\x3d\xaf\xc8" + "\x71\x48\x46\x6a\x64\x28\xf9\x3d" + "\xd1\x1d\xab\xc8\x40\x76\xc2\x39" + "\x4e\x00\x75\xd2\x0e\x82\x58\x8c" + "\xd3\x73\x5a\xea\x46\x89\xbe\xfd" + "\x4e\x2c\x0d\x94\xaa\x9b\x68\xac" + "\x86\x87\x30\x7e\xa9\x16\xcd\x59" + "\xd2\xa6\xbe\x0a\xd8\xf5\xfd\x2d" + "\x49\x69\xd2\x1a\x90\xd2\x1b\xed" + "\xff\x71\x04\x87\x87\x21\xc4\xb8" + "\x1f\x5b\x51\x33\xd0\xd6\x59\x9a" + "\x03\x0e\xd3\x8b\xfb\x57\x73\xfd" + "\x5a\x52\x63\x82\xc8\x85\x2f\xcb" + "\x74\x6d\x4e\xd9\x68\x37\x85\x6a" + "\xd4\xfb\x94\xed\x8d\xd1\x1a\xaf" + "\x76\xa7\xb7\x88\xd0\x2b\x4e\xda" + "\xec\x99\x94\x27\x6f\x87\x8c\xdf" + "\x4b\x5e\xa6\x66\xdd\xcb\x33\x7b" + "\x64\x94\x31\xa8\x37\xa6\x1d\xdb" + "\x0d\x5c\x93\xa4\x40\xf9\x30\x53" + "\x4b\x74\x8d\xdd\xf6\xde\x3c\xac" + "\x5c\x80\x01\x3a\xef\xb1\x9a\x02" + "\x0c\x22\x8e\xe7\x44\x09\x74\x4c" + "\xf2\x9a\x27\x69\x7f\x12\x32\x36" + "\xde\x92\xdf\xde\x8f\x5b\x31\xab" + "\x4a\x01\x26\xe0\xb1\xda\xe8\x37" + "\x21\x64\xe8\xff\x69\xfc\x9e\x41" + "\xd2\x96\x2d\x18\x64\x98\x33\x78" + "\x24\x61\x73\x9b\x47\x29\xf1\xa7" + "\xcb\x27\x0f\xf0\x85\x6d\x8c\x9d" + "\x2c\x95\x9e\xe5\xb2\x8e\x30\x29" + "\x78\x8a\x9d\x65\xb4\x8e\xde\x7b" + "\xd9\x00\x50\xf5\x7f\x81\xc3\x1b" + "\x25\x85\xeb\xc2\x8c\x33\x22\x1e" + "\x68\x38\x22\x30\xd8\x2e\x00\x98" + "\x85\x16\x06\x56\xb4\x81\x74\x20" + "\x95\xdb\x1c\x05\x19\xe8\x23\x4d" + "\x65\x5d\xcc\xd8\x7f\xc4\x2d\x0f" + "\x57\x26\x71\x07\xad\xaa\x71\x9f" + "\x19\x76\x2f\x25\x51\x88\xe4\xc0" + "\x82\x6e\x08\x05\x37\x04\xee\x25" + "\x23\x90\xe9\x4e\xce\x9b\x16\xc1" + "\x31\xe7\x6e\x2c\x1b\xe1\x85\x9a" + "\x0c\x8c\xbb\x12\x1e\x68\x7b\x93" + "\xa9\x3c\x39\x56\x23\x3e\x6e\xc7" + "\x77\x84\xd3\xe0\x86\x59\xaa\xb9" + "\xd5\x53\x58\xc9\x0a\x83\x5f\x85" + "\xd8\x47\x14\x67\x8a\x3c\x17\xe0" + "\xab\x02\x51\xea\xf1\xf0\x4f\x30" + "\x7d\xe0\x92\xc2\x5f\xfb\x19\x5a" + "\x3f\xbd\xf4\x39\xa4\x31\x0c\x39" + "\xd1\xae\x4e\xf7\x65\x7f\x1f\xce" + "\xc2\x39\xd1\x84\xd4\xe5\x02\xe0" + "\x58\xaa\xf1\x5e\x81\xaf\x7f\x72" + "\x0f\x08\x99\x43\xb9\xd8\xac\x41" + "\x35\x55\xf2\xb2\xd4\x98\xb8\x3b" + "\x2b\x3c\x3e\x16\x06\x31\xfc\x79" + "\x47\x38\x63\x51\xc5\xd0\x26\xd7" + "\x43\xb4\x2b\xd9\xc5\x05\xf2\x9d" + "\x18\xc9\x26\x82\x56\xd2\x11\x05" + "\xb6\x89\xb4\x43\x9c\xb5\x9d\x11" + "\x6c\x83\x37\x71\x27\x1c\xae\xbf" + "\xcd\x57\xd2\xee\x0d\x5a\x15\x26" + "\x67\x88\x80\x80\x1b\xdc\xc1\x62" + "\xdd\x4c\xff\x92\x5c\x6c\xe1\xa0" + "\xe3\x79\xa9\x65\x8c\x8c\x14\x42" + "\xe5\x11\xd2\x1a\xad\xa9\x56\x6f" + "\x98\xfc\x8a\x7b\x56\x1f\xc6\xc1" + "\x52\x12\x92\x9b\x41\x0f\x4b\xae" + "\x1b\x4a\xbc\xfe\x23\xb6\x94\x70" + "\x04\x30\x9e\x69\x47\xbe\xb8\x8f" + "\xca\x45\xd7\x8a\xf4\x78\x3e\xaa" + "\x71\x17\xd8\x1e\xb8\x11\x8f\xbc" + "\xc8\x1a\x65\x7b\x41\x89\x72\xc7" + "\x5f\xbe\xc5\x2a\xdb\x5c\x54\xf9" + "\x25\xa3\x7a\x80\x56\x9c\x8c\xab" + "\x26\x19\x10\x36\xa6\xf3\x14\x79" + "\x40\x98\x70\x68\xb7\x35\xd9\xb9" + "\x27\xd4\xe7\x74\x5b\x3d\x97\xb4" + "\xd9\xaa\xd9\xf2\xb5\x14\x84\x1f" + "\xa9\xde\x12\x44\x5b\x00\xc0\xbc" + "\xc8\x11\x25\x1b\x67\x7a\x15\x72" + "\xa6\x31\x6f\xf4\x68\x7a\x86\x9d" + "\x43\x1c\x5f\x16\xd3\xad\x2e\x52" + "\xf3\xb4\xc3\xfa\x27\x2e\x68\x6c" + "\x06\xe7\x4c\x4f\xa2\xe0\xe4\x21" + "\x5d\x9e\x33\x58\x8d\xbf\xd5\x70" + "\xf8\x80\xa5\xdd\xe7\x18\x79\xfa" + "\x7b\xfd\x09\x69\x2c\x37\x32\xa8" + "\x65\xfa\x8d\x8b\x5c\xcc\xe8\xf3" + "\x37\xf6\xa6\xc6\x5c\xa2\x66\x79" + "\xfa\x8a\xa7\xd1\x0b\x2e\x1b\x5e" + "\x95\x35\x00\x76\xae\x42\xf7\x50" + "\x51\x78\xfb\xb4\x28\x24\xde\x1a" + "\x70\x8b\xed\xca\x3c\x5e\xe4\xbd" + "\x28\xb5\xf3\x76\x4f\x67\x5d\x81" + "\xb2\x60\x87\xd9\x7b\x19\x1a\xa7" + "\x79\xa2\xfa\x3f\x9e\xa9\xd7\x25" + "\x61\xe1\x74\x31\xa2\x77\xa0\x1b" + "\xf6\xf7\xcb\xc5\xaa\x9e\xce\xf9" + "\x9b\x96\xef\x51\xc3\x1a\x44\x96" + "\xae\x17\x50\xab\x29\x08\xda\xcc" + "\x1a\xb3\x12\xd0\x24\xe4\xe2\xe0" + "\xc6\xe3\xcc\x82\xd0\xba\x47\x4c" + "\x3f\x49\xd7\xe8\xb6\x61\xaa\x65" + "\x25\x18\x40\x2d\x62\x25\x02\x71" + "\x61\xa2\xc1\xb2\x13\xd2\x71\x3f" + "\x43\x1a\xc9\x09\x92\xff\xd5\x57" + "\xf0\xfc\x5e\x1c\xf1\xf5\xf9\xf3" + "\x5b", + .len = 1281, + .also_non_np = 1, + .np = 3, + .tap = { 1200, 1, 80 }, + }, +}; + /* * CTS (Cipher Text Stealing) mode tests */ diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index b722a23e54bb..1fc70a69d550 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -5,6 +5,11 @@ * XChaCha extends ChaCha's nonce to 192 bits, while provably retaining ChaCha's * security. Here they share the same key size, tfm context, and setkey * function; only their IV size and encrypt/decrypt function differ. + * + * The ChaCha paper specifies 20, 12, and 8-round variants. In general, it is + * recommended to use the 20-round variant ChaCha20. However, the other + * variants can be needed in some performance-sensitive scenarios. The generic + * ChaCha code currently allows only the 20 and 12-round variants. */ #ifndef _CRYPTO_CHACHA_H @@ -40,6 +45,8 @@ void crypto_chacha_init(u32 *state, struct chacha_ctx *ctx, u8 *iv); int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keysize); +int crypto_chacha12_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keysize); int crypto_chacha_crypt(struct skcipher_request *req); int crypto_xchacha_crypt(struct skcipher_request *req); diff --git a/lib/chacha.c b/lib/chacha.c index 1bdc688c18df..a46d2832dbab 100644 --- a/lib/chacha.c +++ b/lib/chacha.c @@ -21,7 +21,7 @@ static void chacha_permute(u32 *x, int nrounds) int i; /* whitelist the allowed round counts */ - WARN_ON_ONCE(nrounds != 20); + WARN_ON_ONCE(nrounds != 20 && nrounds != 12); for (i = 0; i < nrounds; i += 2) { x[0] += x[4]; x[12] = rol32(x[12] ^ x[0], 16); @@ -70,7 +70,7 @@ static void chacha_permute(u32 *x, int nrounds) * chacha_block - generate one keystream block and increment block counter * @state: input state matrix (16 32-bit words) * @stream: output keystream block (64 bytes) - * @nrounds: number of rounds (currently must be 20) + * @nrounds: number of rounds (20 or 12; 20 is recommended) * * This is the ChaCha core, a function from 64-byte strings to 64-byte strings. * The caller has already converted the endianness of the input. This function @@ -96,7 +96,7 @@ EXPORT_SYMBOL(chacha_block); * hchacha_block - abbreviated ChaCha core, for XChaCha * @in: input state matrix (16 32-bit words) * @out: output (8 32-bit words) - * @nrounds: number of rounds (currently must be 20) + * @nrounds: number of rounds (20 or 12; 20 is recommended) * * HChaCha is the ChaCha equivalent of HSalsa and is an intermediate step * towards XChaCha (see https://cr.yp.to/snuffle/xsalsa-20081128.pdf). HChaCha -- cgit v1.2.3 From 36bd1a8e91c66e9def12958547548aa549de9cbf Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 6 Nov 2018 19:22:23 -0800 Subject: percpu-refcount: Replace call_rcu_sched() with call_rcu() Now that call_rcu()'s callback is not invoked until after all preempt-disable regions of code have completed (in addition to explicitly marked RCU read-side critical sections), call_rcu() can be used in place of call_rcu_sched(). This commit therefore makes that change. Signed-off-by: Paul E. McKenney Cc: Ming Lei Cc: Bart Van Assche Cc: Jens Axboe Acked-by: Tejun Heo --- lib/percpu-refcount.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index de10b8c0bff6..9877682e49c7 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -181,7 +181,7 @@ static void __percpu_ref_switch_to_atomic(struct percpu_ref *ref, ref->confirm_switch = confirm_switch ?: percpu_ref_noop_confirm_switch; percpu_ref_get(ref); /* put after confirmation */ - call_rcu_sched(&ref->rcu, percpu_ref_switch_to_atomic_rcu); + call_rcu(&ref->rcu, percpu_ref_switch_to_atomic_rcu); } static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref) -- cgit v1.2.3 From f91528955d0094ff2200632661d62ee64019c985 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Nov 2018 22:32:59 -0500 Subject: iov_iter: reduce code duplication The same combination of csum_partial_copy_nocheck() with csum_add_block() is used in a bunch of places. Add a helper doing just that and use it. Signed-off-by: Al Viro --- lib/iov_iter.c | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 54c248526b55..621984743268 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -560,13 +560,20 @@ static size_t copy_pipe_to_iter(const void *addr, size_t bytes, return bytes; } +static __wsum csum_and_memcpy(void *to, const void *from, size_t len, + __wsum sum, size_t off) +{ + __wsum next = csum_partial_copy_nocheck(from, to, len, 0); + return csum_block_add(sum, next, off); +} + static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes, __wsum *csum, struct iov_iter *i) { struct pipe_inode_info *pipe = i->pipe; size_t n, r; size_t off = 0; - __wsum sum = *csum, next; + __wsum sum = *csum; int idx; if (!sanity(i)) @@ -578,8 +585,7 @@ static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes, for ( ; n; idx = next_idx(idx, pipe), r = 0) { size_t chunk = min_t(size_t, n, PAGE_SIZE - r); char *p = kmap_atomic(pipe->bufs[idx].page); - next = csum_partial_copy_nocheck(addr, p + r, chunk, 0); - sum = csum_block_add(sum, next, off); + sum = csum_and_memcpy(p + r, addr, chunk, sum, off); kunmap_atomic(p); i->idx = idx; i->iov_offset = r + chunk; @@ -1400,17 +1406,15 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, err ? v.iov_len : 0; }), ({ char *p = kmap_atomic(v.bv_page); - next = csum_partial_copy_nocheck(p + v.bv_offset, - (to += v.bv_len) - v.bv_len, - v.bv_len, 0); + sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, + p + v.bv_offset, v.bv_len, + sum, off); kunmap_atomic(p); - sum = csum_block_add(sum, next, off); off += v.bv_len; }),({ - next = csum_partial_copy_nocheck(v.iov_base, - (to += v.iov_len) - v.iov_len, - v.iov_len, 0); - sum = csum_block_add(sum, next, off); + sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, + v.iov_base, v.iov_len, + sum, off); off += v.iov_len; }) ) @@ -1444,17 +1448,15 @@ bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, 0; }), ({ char *p = kmap_atomic(v.bv_page); - next = csum_partial_copy_nocheck(p + v.bv_offset, - (to += v.bv_len) - v.bv_len, - v.bv_len, 0); + sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, + p + v.bv_offset, v.bv_len, + sum, off); kunmap_atomic(p); - sum = csum_block_add(sum, next, off); off += v.bv_len; }),({ - next = csum_partial_copy_nocheck(v.iov_base, - (to += v.iov_len) - v.iov_len, - v.iov_len, 0); - sum = csum_block_add(sum, next, off); + sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, + v.iov_base, v.iov_len, + sum, off); off += v.iov_len; }) ) @@ -1491,17 +1493,15 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum, err ? v.iov_len : 0; }), ({ char *p = kmap_atomic(v.bv_page); - next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len, - p + v.bv_offset, - v.bv_len, 0); + sum = csum_and_memcpy(p + v.bv_offset, + (from += v.bv_len) - v.bv_len, + v.bv_len, sum, off); kunmap_atomic(p); - sum = csum_block_add(sum, next, off); off += v.bv_len; }),({ - next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len, - v.iov_base, - v.iov_len, 0); - sum = csum_block_add(sum, next, off); + sum = csum_and_memcpy(v.iov_base, + (from += v.iov_len) - v.iov_len, + v.iov_len, sum, off); off += v.iov_len; }) ) -- cgit v1.2.3 From 58d81d64e06ffaea6bddc85ae2b7295c371bcc55 Mon Sep 17 00:00:00 2001 From: Priit Laes Date: Mon, 19 Nov 2018 20:01:22 +0200 Subject: lib: cordic: Move cordic macros and defines to header file Now that these macros are in header file, we can eventually clean up the duplicate macros present in the drivers that utilize the same cordic algorithm implementation. Also add CORDIC_ prefix to nonprefixed macros. Reviewed-by: Arend van Spriel Signed-off-by: Priit Laes Acked-by: Larry Finger Signed-off-by: Kalle Valo --- include/linux/cordic.h | 9 +++++++++ lib/cordic.c | 23 +++++++---------------- 2 files changed, 16 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/include/linux/cordic.h b/include/linux/cordic.h index cf68ca4a508c..3d656f54d64f 100644 --- a/include/linux/cordic.h +++ b/include/linux/cordic.h @@ -18,6 +18,15 @@ #include +#define CORDIC_ANGLE_GEN 39797 +#define CORDIC_PRECISION_SHIFT 16 +#define CORDIC_NUM_ITER (CORDIC_PRECISION_SHIFT + 2) + +#define CORDIC_FIXED(X) ((s32)((X) << CORDIC_PRECISION_SHIFT)) +#define CORDIC_FLOAT(X) (((X) >= 0) \ + ? ((((X) >> (CORDIC_PRECISION_SHIFT - 1)) + 1) >> 1) \ + : -((((-(X)) >> (CORDIC_PRECISION_SHIFT - 1)) + 1) >> 1)) + /** * struct cordic_iq - i/q coordinate. * diff --git a/lib/cordic.c b/lib/cordic.c index 6cf477839ebd..8ef27c12956f 100644 --- a/lib/cordic.c +++ b/lib/cordic.c @@ -16,15 +16,6 @@ #include #include -#define CORDIC_ANGLE_GEN 39797 -#define CORDIC_PRECISION_SHIFT 16 -#define CORDIC_NUM_ITER (CORDIC_PRECISION_SHIFT + 2) - -#define FIXED(X) ((s32)((X) << CORDIC_PRECISION_SHIFT)) -#define FLOAT(X) (((X) >= 0) \ - ? ((((X) >> (CORDIC_PRECISION_SHIFT - 1)) + 1) >> 1) \ - : -((((-(X)) >> (CORDIC_PRECISION_SHIFT - 1)) + 1) >> 1)) - static const s32 arctan_table[] = { 2949120, 1740967, @@ -64,16 +55,16 @@ struct cordic_iq cordic_calc_iq(s32 theta) coord.q = 0; angle = 0; - theta = FIXED(theta); + theta = CORDIC_FIXED(theta); signtheta = (theta < 0) ? -1 : 1; - theta = ((theta + FIXED(180) * signtheta) % FIXED(360)) - - FIXED(180) * signtheta; + theta = ((theta + CORDIC_FIXED(180) * signtheta) % CORDIC_FIXED(360)) - + CORDIC_FIXED(180) * signtheta; - if (FLOAT(theta) > 90) { - theta -= FIXED(180); + if (CORDIC_FLOAT(theta) > 90) { + theta -= CORDIC_FIXED(180); signx = -1; - } else if (FLOAT(theta) < -90) { - theta += FIXED(180); + } else if (CORDIC_FLOAT(theta) < -90) { + theta += CORDIC_FIXED(180); signx = -1; } -- cgit v1.2.3 From 27fae429acee1e9418059e7fa545438075af5256 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 29 Nov 2018 12:35:16 -0700 Subject: sbitmap: don't loop for find_next_zero_bit() for !round_robin If we aren't forced to do round robin tag allocation, just use the allocation hint to find the index for the tag word, don't use it for the offset inside the word. This avoids a potential extra round trip in the bit looping, and since we're fetching this cacheline, we may as well check the whole word from the start. Reviewed-by: Omar Sandoval Signed-off-by: Jens Axboe --- lib/sbitmap.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/sbitmap.c b/lib/sbitmap.c index fdd1b8aa8ac6..45cab6bbc1c7 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -118,10 +118,19 @@ int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint, bool round_robin) index = SB_NR_TO_INDEX(sb, alloc_hint); + /* + * Unless we're doing round robin tag allocation, just use the + * alloc_hint to find the right word index. No point in looping + * twice in find_next_zero_bit() for that case. + */ + if (round_robin) + alloc_hint = SB_NR_TO_BIT(sb, alloc_hint); + else + alloc_hint = 0; + for (i = 0; i < sb->map_nr; i++) { nr = __sbitmap_get_word(&sb->map[index].word, - sb->map[index].depth, - SB_NR_TO_BIT(sb, alloc_hint), + sb->map[index].depth, alloc_hint, !round_robin); if (nr != -1) { nr += index << sb->shift; @@ -129,13 +138,9 @@ int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint, bool round_robin) } /* Jump to next index. */ - index++; - alloc_hint = index << sb->shift; - - if (index >= sb->map_nr) { + alloc_hint = 0; + if (++index >= sb->map_nr) index = 0; - alloc_hint = 0; - } } return nr; -- cgit v1.2.3 From 5b39fc049ce1232a56628408058ae04f353ac705 Mon Sep 17 00:00:00 2001 From: Sergey Senozhatsky Date: Thu, 25 Oct 2018 16:05:43 +0900 Subject: s390: use common bust_spinlocks() s390 is the only architecture that is using own bust_spinlocks() variant, while other arch-s seem to be OK with the common implementation. Heiko Carstens [1] said he would prefer s390 to use the common bust_spinlocks() as well: I did some code archaeology and this function is unchanged since ~17 years. When it was introduced it was close to identical to the x86 variant. All other architectures use the common code variant in the meantime. So if we change this I'd prefer that we switch s390 to the common code variant as well. Right now I can't see a reason for not doing that This patch removes s390 bust_spinlocks() and drops the weak attribute from the common bust_spinlocks() version. [1] lkml.kernel.org/r/20181025062800.GB4037@osiris Signed-off-by: Sergey Senozhatsky Signed-off-by: Heiko Carstens Signed-off-by: Martin Schwidefsky --- arch/s390/mm/fault.c | 24 ------------------------ lib/bust_spinlocks.c | 6 +++--- 2 files changed, 3 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 2b8f32f56e0c..11613362c4e7 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -81,30 +81,6 @@ static inline int notify_page_fault(struct pt_regs *regs) return ret; } - -/* - * Unlock any spinlocks which will prevent us from getting the - * message out. - */ -void bust_spinlocks(int yes) -{ - if (yes) { - oops_in_progress = 1; - } else { - int loglevel_save = console_loglevel; - console_unblank(); - oops_in_progress = 0; - /* - * OK, the message is on the console. Now we call printk() - * without oops_in_progress set so that printk will give klogd - * a poke. Hold onto your hats... - */ - console_loglevel = 15; - printk(" "); - console_loglevel = loglevel_save; - } -} - /* * Find out which address space caused the exception. * Access register mode is impossible, ignore space == 3. diff --git a/lib/bust_spinlocks.c b/lib/bust_spinlocks.c index ab719495e2cb..8be59f84eaea 100644 --- a/lib/bust_spinlocks.c +++ b/lib/bust_spinlocks.c @@ -2,7 +2,8 @@ /* * lib/bust_spinlocks.c * - * Provides a minimal bust_spinlocks for architectures which don't have one of their own. + * Provides a minimal bust_spinlocks for architectures which don't + * have one of their own. * * bust_spinlocks() clears any spinlocks which would prevent oops, die(), BUG() * and panic() information from reaching the user. @@ -16,8 +17,7 @@ #include #include - -void __attribute__((weak)) bust_spinlocks(int yes) +void bust_spinlocks(int yes) { if (yes) { ++oops_in_progress; -- cgit v1.2.3 From ea86ea2cdced20057da4d2c32965c1219c238197 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 30 Nov 2018 13:18:06 -0700 Subject: sbitmap: ammortize cost of clearing bits sbitmap maintains a set of words that we use to set and clear bits, with each bit representing a tag for blk-mq. Even though we spread the bits out and maintain a hint cache, one particular bit allocated will end up being cleared in the exact same spot. This introduces batched clearing of bits. Instead of clearing a given bit, the same bit is set in a cleared/free mask instead. If we fail allocating a bit from a given word, then we check the free mask, and batch move those cleared bits at that time. This trades 64 atomic bitops for 2 cmpxchg(). In a threaded poll test case, half the overhead of getting and clearing tags is removed with this change. On another poll test case with a single thread, performance is unchanged. Reviewed-by: Omar Sandoval Signed-off-by: Jens Axboe --- include/linux/sbitmap.h | 33 ++++++++++++++++---- lib/sbitmap.c | 81 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 100 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index 804a50983ec5..81359d45751e 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -30,14 +30,24 @@ struct seq_file; */ struct sbitmap_word { /** - * @word: The bitmap word itself. + * @depth: Number of bits being used in @word/@cleared */ - unsigned long word; + unsigned long depth; /** - * @depth: Number of bits being used in @word. + * @word: word holding free bits */ - unsigned long depth; + unsigned long word ____cacheline_aligned_in_smp; + + /** + * @cleared: word holding cleared bits + */ + unsigned long cleared ____cacheline_aligned_in_smp; + + /** + * @swap_lock: Held while swapping word <-> cleared + */ + spinlock_t swap_lock; } ____cacheline_aligned_in_smp; /** @@ -310,6 +320,19 @@ static inline void sbitmap_clear_bit(struct sbitmap *sb, unsigned int bitnr) clear_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr)); } +/* + * This one is special, since it doesn't actually clear the bit, rather it + * sets the corresponding bit in the ->cleared mask instead. Paired with + * the caller doing sbitmap_batch_clear() if a given index is full, which + * will clear the previously freed entries in the corresponding ->word. + */ +static inline void sbitmap_deferred_clear_bit(struct sbitmap *sb, unsigned int bitnr) +{ + unsigned long *addr = &sb->map[SB_NR_TO_INDEX(sb, bitnr)].cleared; + + set_bit(SB_NR_TO_BIT(sb, bitnr), addr); +} + static inline void sbitmap_clear_bit_unlock(struct sbitmap *sb, unsigned int bitnr) { @@ -321,8 +344,6 @@ static inline int sbitmap_test_bit(struct sbitmap *sb, unsigned int bitnr) return test_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr)); } -unsigned int sbitmap_weight(const struct sbitmap *sb); - /** * sbitmap_show() - Dump &struct sbitmap information to a &struct seq_file. * @sb: Bitmap to show. diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 45cab6bbc1c7..f99382e59314 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -59,6 +59,7 @@ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, for (i = 0; i < sb->map_nr; i++) { sb->map[i].depth = min(depth, bits_per_word); depth -= sb->map[i].depth; + spin_lock_init(&sb->map[i].swap_lock); } return 0; } @@ -111,6 +112,57 @@ static int __sbitmap_get_word(unsigned long *word, unsigned long depth, return nr; } +/* + * See if we have deferred clears that we can batch move + */ +static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index) +{ + unsigned long mask, val; + bool ret = false; + + spin_lock(&sb->map[index].swap_lock); + + if (!sb->map[index].cleared) + goto out_unlock; + + /* + * First get a stable cleared mask, setting the old mask to 0. + */ + do { + mask = sb->map[index].cleared; + } while (cmpxchg(&sb->map[index].cleared, mask, 0) != mask); + + /* + * Now clear the masked bits in our free word + */ + do { + val = sb->map[index].word; + } while (cmpxchg(&sb->map[index].word, val, val & ~mask) != val); + + ret = true; +out_unlock: + spin_unlock(&sb->map[index].swap_lock); + return ret; +} + +static int sbitmap_find_bit_in_index(struct sbitmap *sb, int index, + unsigned int alloc_hint, bool round_robin) +{ + int nr; + + do { + nr = __sbitmap_get_word(&sb->map[index].word, + sb->map[index].depth, alloc_hint, + !round_robin); + if (nr != -1) + break; + if (!sbitmap_deferred_clear(sb, index)) + break; + } while (1); + + return nr; +} + int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint, bool round_robin) { unsigned int i, index; @@ -129,9 +181,8 @@ int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint, bool round_robin) alloc_hint = 0; for (i = 0; i < sb->map_nr; i++) { - nr = __sbitmap_get_word(&sb->map[index].word, - sb->map[index].depth, alloc_hint, - !round_robin); + nr = sbitmap_find_bit_in_index(sb, index, alloc_hint, + round_robin); if (nr != -1) { nr += index << sb->shift; break; @@ -206,23 +257,36 @@ bool sbitmap_any_bit_clear(const struct sbitmap *sb) } EXPORT_SYMBOL_GPL(sbitmap_any_bit_clear); -unsigned int sbitmap_weight(const struct sbitmap *sb) +static unsigned int __sbitmap_weight(const struct sbitmap *sb, bool set) { unsigned int i, weight = 0; for (i = 0; i < sb->map_nr; i++) { const struct sbitmap_word *word = &sb->map[i]; - weight += bitmap_weight(&word->word, word->depth); + if (set) + weight += bitmap_weight(&word->word, word->depth); + else + weight += bitmap_weight(&word->cleared, word->depth); } return weight; } -EXPORT_SYMBOL_GPL(sbitmap_weight); + +static unsigned int sbitmap_weight(const struct sbitmap *sb) +{ + return __sbitmap_weight(sb, true); +} + +static unsigned int sbitmap_cleared(const struct sbitmap *sb) +{ + return __sbitmap_weight(sb, false); +} void sbitmap_show(struct sbitmap *sb, struct seq_file *m) { seq_printf(m, "depth=%u\n", sb->depth); - seq_printf(m, "busy=%u\n", sbitmap_weight(sb)); + seq_printf(m, "busy=%u\n", sbitmap_weight(sb) - sbitmap_cleared(sb)); + seq_printf(m, "cleared=%u\n", sbitmap_cleared(sb)); seq_printf(m, "bits_per_word=%u\n", 1U << sb->shift); seq_printf(m, "map_nr=%u\n", sb->map_nr); } @@ -514,7 +578,8 @@ EXPORT_SYMBOL_GPL(sbitmap_queue_wake_up); void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, unsigned int cpu) { - sbitmap_clear_bit_unlock(&sbq->sb, nr); + sbitmap_deferred_clear_bit(&sbq->sb, nr); + /* * Pairs with the memory barrier in set_current_state() to ensure the * proper ordering of clear_bit_unlock()/waitqueue_active() in the waker -- cgit v1.2.3 From 5d2ee7122c73be6a3b6bfe90d237e8aed737cfaa Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 29 Nov 2018 17:36:41 -0700 Subject: sbitmap: optimize wakeup check Even if we have no waiters on any of the sbitmap_queue wait states, we still have to loop every entry to check. We do this for every IO, so the cost adds up. Shift a bit of the cost to the slow path, when we actually have waiters. Wrap prepare_to_wait_exclusive() and finish_wait(), so we can maintain an internal count of how many are currently active. Then we can simply check this count in sbq_wake_ptr() and not have to loop if we don't have any sleepers. Convert the two users of sbitmap with waiting, blk-mq-tag and iSCSI. Reviewed-by: Omar Sandoval Signed-off-by: Jens Axboe --- block/blk-mq-tag.c | 11 +++++------ drivers/target/iscsi/iscsi_target_util.c | 12 ++++++----- include/linux/sbitmap.h | 34 ++++++++++++++++++++++++++++++++ lib/sbitmap.c | 28 ++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 87bc5df72d48..2089c6c62f44 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -110,7 +110,7 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) struct blk_mq_tags *tags = blk_mq_tags_from_data(data); struct sbitmap_queue *bt; struct sbq_wait_state *ws; - DEFINE_WAIT(wait); + DEFINE_SBQ_WAIT(wait); unsigned int tag_offset; bool drop_ctx; int tag; @@ -154,8 +154,7 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) if (tag != -1) break; - prepare_to_wait_exclusive(&ws->wait, &wait, - TASK_UNINTERRUPTIBLE); + sbitmap_prepare_to_wait(bt, ws, &wait, TASK_UNINTERRUPTIBLE); tag = __blk_mq_get_tag(data, bt); if (tag != -1) @@ -167,6 +166,8 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) bt_prev = bt; io_schedule(); + sbitmap_finish_wait(bt, ws, &wait); + data->ctx = blk_mq_get_ctx(data->q); data->hctx = blk_mq_map_queue(data->q, data->cmd_flags, data->ctx->cpu); @@ -176,8 +177,6 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) else bt = &tags->bitmap_tags; - finish_wait(&ws->wait, &wait); - /* * If destination hw queue is changed, fake wake up on * previous queue for compensating the wake up miss, so @@ -192,7 +191,7 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) if (drop_ctx && data->ctx) blk_mq_put_ctx(data->ctx); - finish_wait(&ws->wait, &wait); + sbitmap_finish_wait(bt, ws, &wait); found_tag: return tag + tag_offset; diff --git a/drivers/target/iscsi/iscsi_target_util.c b/drivers/target/iscsi/iscsi_target_util.c index 36b742932c72..86987da86dd6 100644 --- a/drivers/target/iscsi/iscsi_target_util.c +++ b/drivers/target/iscsi/iscsi_target_util.c @@ -150,24 +150,26 @@ void iscsit_free_r2ts_from_list(struct iscsi_cmd *cmd) static int iscsit_wait_for_tag(struct se_session *se_sess, int state, int *cpup) { int tag = -1; - DEFINE_WAIT(wait); + DEFINE_SBQ_WAIT(wait); struct sbq_wait_state *ws; + struct sbitmap_queue *sbq; if (state == TASK_RUNNING) return tag; - ws = &se_sess->sess_tag_pool.ws[0]; + sbq = &se_sess->sess_tag_pool; + ws = &sbq->ws[0]; for (;;) { - prepare_to_wait_exclusive(&ws->wait, &wait, state); + sbitmap_prepare_to_wait(sbq, ws, &wait, state); if (signal_pending_state(state, current)) break; - tag = sbitmap_queue_get(&se_sess->sess_tag_pool, cpup); + tag = sbitmap_queue_get(sbq, cpup); if (tag >= 0) break; schedule(); } - finish_wait(&ws->wait, &wait); + sbitmap_finish_wait(sbq, ws, &wait); return tag; } diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index 81359d45751e..92806a2dbab7 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -135,6 +135,11 @@ struct sbitmap_queue { */ struct sbq_wait_state *ws; + /* + * @ws_active: count of currently active ws waitqueues + */ + atomic_t ws_active; + /** * @round_robin: Allocate bits in strict round-robin order. */ @@ -552,4 +557,33 @@ void sbitmap_queue_wake_up(struct sbitmap_queue *sbq); */ void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m); +struct sbq_wait { + int accounted; + struct wait_queue_entry wait; +}; + +#define DEFINE_SBQ_WAIT(name) \ + struct sbq_wait name = { \ + .accounted = 0, \ + .wait = { \ + .private = current, \ + .func = autoremove_wake_function, \ + .entry = LIST_HEAD_INIT((name).wait.entry), \ + } \ + } + +/* + * Wrapper around prepare_to_wait_exclusive(), which maintains some extra + * internal state. + */ +void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq, + struct sbq_wait_state *ws, + struct sbq_wait *sbq_wait, int state); + +/* + * Must be paired with sbitmap_prepare_to_wait(). + */ +void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, + struct sbq_wait *sbq_wait); + #endif /* __LINUX_SCALE_BITMAP_H */ diff --git a/lib/sbitmap.c b/lib/sbitmap.c index f99382e59314..a89fbe7cf6ca 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -394,6 +394,7 @@ int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth, sbq->min_shallow_depth = UINT_MAX; sbq->wake_batch = sbq_calc_wake_batch(sbq, depth); atomic_set(&sbq->wake_index, 0); + atomic_set(&sbq->ws_active, 0); sbq->ws = kzalloc_node(SBQ_WAIT_QUEUES * sizeof(*sbq->ws), flags, node); if (!sbq->ws) { @@ -509,6 +510,9 @@ static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq) { int i, wake_index; + if (!atomic_read(&sbq->ws_active)) + return NULL; + wake_index = atomic_read(&sbq->wake_index); for (i = 0; i < SBQ_WAIT_QUEUES; i++) { struct sbq_wait_state *ws = &sbq->ws[wake_index]; @@ -634,6 +638,7 @@ void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m) seq_printf(m, "wake_batch=%u\n", sbq->wake_batch); seq_printf(m, "wake_index=%d\n", atomic_read(&sbq->wake_index)); + seq_printf(m, "ws_active=%d\n", atomic_read(&sbq->ws_active)); seq_puts(m, "ws={\n"); for (i = 0; i < SBQ_WAIT_QUEUES; i++) { @@ -649,3 +654,26 @@ void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m) seq_printf(m, "min_shallow_depth=%u\n", sbq->min_shallow_depth); } EXPORT_SYMBOL_GPL(sbitmap_queue_show); + +void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq, + struct sbq_wait_state *ws, + struct sbq_wait *sbq_wait, int state) +{ + if (!sbq_wait->accounted) { + atomic_inc(&sbq->ws_active); + sbq_wait->accounted = 1; + } + prepare_to_wait_exclusive(&ws->wait, &sbq_wait->wait, state); +} +EXPORT_SYMBOL_GPL(sbitmap_prepare_to_wait); + +void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, + struct sbq_wait *sbq_wait) +{ + finish_wait(&ws->wait, &sbq_wait->wait); + if (sbq_wait->accounted) { + atomic_dec(&sbq->ws_active); + sbq_wait->accounted = 0; + } +} +EXPORT_SYMBOL_GPL(sbitmap_finish_wait); -- cgit v1.2.3 From 82208d0d54ab85d8fedbb1c9a1960bd401a4ca1a Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 30 Nov 2018 10:26:50 +1100 Subject: rhashtable: detect when object movement between tables might have invalidated a lookup Some users of rhashtables might need to move an object from one table to another - this appears to be the reason for the incomplete usage of NULLS markers. To support these, we store a unique NULLS_MARKER at the end of each chain, and when a search fails to find a match, we check if the NULLS marker found was the expected one. If not, the search may not have examined all objects in the target bucket, so it is repeated. The unique NULLS_MARKER is derived from the address of the head of the chain. As this cannot be derived at load-time the static rhnull in rht_bucket_nested() needs to be initialised at run time. Any caller of a lookup function must still be prepared for the possibility that the object returned is in a different table - it might have been there for some time. Note that this does NOT provide support for other uses of NULLS_MARKERs such as allocating with SLAB_TYPESAFE_BY_RCU or changing the key of an object and re-inserting it in the same table. These could only be done safely if new objects were inserted at the *start* of a hash chain, and that is not currently the case. Signed-off-by: NeilBrown Acked-by: Herbert Xu Signed-off-by: David S. Miller --- include/linux/rhashtable.h | 34 ++++++++++++++++++++++++++-------- lib/rhashtable.c | 8 +++++--- 2 files changed, 31 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h index eb7111039247..20f9c6af7473 100644 --- a/include/linux/rhashtable.h +++ b/include/linux/rhashtable.h @@ -75,8 +75,19 @@ struct bucket_table { struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp; }; +/* + * NULLS_MARKER() expects a hash value with the low + * bits mostly likely to be significant, and it discards + * the msb. + * We git it an address, in which the bottom 2 bits are + * always 0, and the msb might be significant. + * So we shift the address down one bit to align with + * expectations and avoid losing a significant bit. + */ +#define RHT_NULLS_MARKER(ptr) \ + ((void *)NULLS_MARKER(((unsigned long) (ptr)) >> 1)) #define INIT_RHT_NULLS_HEAD(ptr) \ - ((ptr) = (typeof(ptr)) NULLS_MARKER(0)) + ((ptr) = RHT_NULLS_MARKER(&(ptr))) static inline bool rht_is_a_nulls(const struct rhash_head *ptr) { @@ -471,6 +482,7 @@ static inline struct rhash_head *__rhashtable_lookup( .ht = ht, .key = key, }; + struct rhash_head __rcu * const *head; struct bucket_table *tbl; struct rhash_head *he; unsigned int hash; @@ -478,13 +490,19 @@ static inline struct rhash_head *__rhashtable_lookup( tbl = rht_dereference_rcu(ht->tbl, ht); restart: hash = rht_key_hashfn(ht, tbl, key, params); - rht_for_each_rcu(he, tbl, hash) { - if (params.obj_cmpfn ? - params.obj_cmpfn(&arg, rht_obj(ht, he)) : - rhashtable_compare(&arg, rht_obj(ht, he))) - continue; - return he; - } + head = rht_bucket(tbl, hash); + do { + rht_for_each_rcu_continue(he, *head, tbl, hash) { + if (params.obj_cmpfn ? + params.obj_cmpfn(&arg, rht_obj(ht, he)) : + rhashtable_compare(&arg, rht_obj(ht, he))) + continue; + return he; + } + /* An object might have been moved to a different hash chain, + * while we walk along it - better check and retry. + */ + } while (he != RHT_NULLS_MARKER(head)); /* Ensure we see any new tables. */ smp_rmb(); diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 30526afa8343..852ffa5160f1 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -1179,8 +1179,7 @@ struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, unsigned int hash) { const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); - static struct rhash_head __rcu *rhnull = - (struct rhash_head __rcu *)NULLS_MARKER(0); + static struct rhash_head __rcu *rhnull; unsigned int index = hash & ((1 << tbl->nest) - 1); unsigned int size = tbl->size >> tbl->nest; unsigned int subhash = hash; @@ -1198,8 +1197,11 @@ struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, subhash >>= shift; } - if (!ntbl) + if (!ntbl) { + if (!rhnull) + INIT_RHT_NULLS_HEAD(rhnull); return &rhnull; + } return &ntbl[subhash].bucket; -- cgit v1.2.3 From 186bddb28ff9f61250d1b33554321d0bf5d085f6 Mon Sep 17 00:00:00 2001 From: Ezequiel Garcia Date: Mon, 3 Dec 2018 13:44:35 -0300 Subject: kref/kobject: Improve documentation The current kref and kobject documentation may be insufficient to understand these common pitfalls regarding object lifetime and object releasing. Add a bit more documentation and improve the warnings seen by the user, pointing to the right piece of documentation. Also, it's important to understand that making fun of people publicly is not at all helpful, doesn't provide any value, and it's not a healthy way of encouraging developers to do better. "Mocking mercilessly" will, if anything, make developers feel bad and go away. This kind of behavior should not be encouraged or justified. Signed-off-by: Ezequiel Garcia Signed-off-by: Enric Balletbo i Serra Signed-off-by: Gustavo Padovan Signed-off-by: Matthias Brugger Acked-by: Daniel Vetter Acked-by: Guenter Roeck Signed-off-by: Greg Kroah-Hartman --- Documentation/kobject.txt | 10 +++++++--- drivers/base/core.c | 3 +-- include/linux/kref.h | 5 +---- lib/kobject.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/Documentation/kobject.txt b/Documentation/kobject.txt index fc9485d79061..ff4c25098119 100644 --- a/Documentation/kobject.txt +++ b/Documentation/kobject.txt @@ -279,10 +279,14 @@ such a method has a form like:: One important point cannot be overstated: every kobject must have a release() method, and the kobject must persist (in a consistent state) until that method is called. If these constraints are not met, the code is -flawed. Note that the kernel will warn you if you forget to provide a +flawed. Note that the kernel will warn you if you forget to provide a release() method. Do not try to get rid of this warning by providing an -"empty" release function; you will be mocked mercilessly by the kobject -maintainer if you attempt this. +"empty" release function. + +If all your cleanup function needs to do is call kfree(), then you must +create a wrapper function which uses container_of() to upcast to the correct +type (as shown in the example above) and then calls kfree() on the overall +structure. Note, the name of the kobject is available in the release function, but it must NOT be changed within this callback. Otherwise there will be a memory diff --git a/drivers/base/core.c b/drivers/base/core.c index ed145fbfeddf..e2285059161d 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -897,8 +897,7 @@ static void device_release(struct kobject *kobj) else if (dev->class && dev->class->dev_release) dev->class->dev_release(dev); else - WARN(1, KERN_ERR "Device '%s' does not have a release() " - "function, it is broken and must be fixed.\n", + WARN(1, KERN_ERR "Device '%s' does not have a release() function, it is broken and must be fixed. See Documentation/kobject.txt.\n", dev_name(dev)); kfree(p); } diff --git a/include/linux/kref.h b/include/linux/kref.h index 29220724bf1c..cb00a0268061 100644 --- a/include/linux/kref.h +++ b/include/linux/kref.h @@ -53,10 +53,7 @@ static inline void kref_get(struct kref *kref) * @release: pointer to the function that will clean up the object when the * last reference to the object is released. * This pointer is required, and it is not acceptable to pass kfree - * in as this function. If the caller does pass kfree to this - * function, you will be publicly mocked mercilessly by the kref - * maintainer, and anyone else who happens to notice it. You have - * been warned. + * in as this function. * * Decrement the refcount, and if 0, call release(). * Return 1 if the object was removed, otherwise return 0. Beware, if this diff --git a/lib/kobject.c b/lib/kobject.c index 97d86dc17c42..b72e00fd7d09 100644 --- a/lib/kobject.c +++ b/lib/kobject.c @@ -639,7 +639,7 @@ static void kobject_cleanup(struct kobject *kobj) kobject_name(kobj), kobj, __func__, kobj->parent); if (t && !t->release) - pr_debug("kobject: '%s' (%p): does not have a release() function, it is broken and must be fixed.\n", + pr_debug("kobject: '%s' (%p): does not have a release() function, it is broken and must be fixed. See Documentation/kobject.txt.\n", kobject_name(kobj), kobj); /* send "remove" if the caller did not do it but sent "add" */ -- cgit v1.2.3 From 7c703e54cc71df5baa962e24a5663d88173bba5c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 9 Nov 2018 09:51:00 +0100 Subject: arch: switch the default on ARCH_HAS_SG_CHAIN These days architectures are mostly out of the business of dealing with struct scatterlist at all, unless they have architecture specific iommu drivers. Replace the ARCH_HAS_SG_CHAIN symbol with a ARCH_NO_SG_CHAIN one only enabled for architectures with horrible legacy iommu drivers like alpha and parisc, and conditionally for arm which wants to keep it disable for legacy platforms. Signed-off-by: Christoph Hellwig Reviewed-by: Palmer Dabbelt --- .../features/io/sg-chain/arch-support.txt | 33 ---------------------- arch/alpha/Kconfig | 1 + arch/arc/Kconfig | 1 - arch/arm/Kconfig | 2 +- arch/arm64/Kconfig | 1 - arch/ia64/Kconfig | 1 - arch/parisc/Kconfig | 1 + arch/powerpc/Kconfig | 1 - arch/s390/Kconfig | 1 - arch/sparc/Kconfig | 1 - arch/x86/Kconfig | 1 - arch/xtensa/Kconfig | 1 - include/linux/scatterlist.h | 6 ++-- lib/Kconfig | 2 +- lib/scatterlist.c | 2 +- 15 files changed, 8 insertions(+), 47 deletions(-) delete mode 100644 Documentation/features/io/sg-chain/arch-support.txt (limited to 'lib') diff --git a/Documentation/features/io/sg-chain/arch-support.txt b/Documentation/features/io/sg-chain/arch-support.txt deleted file mode 100644 index 6554f0372c3f..000000000000 --- a/Documentation/features/io/sg-chain/arch-support.txt +++ /dev/null @@ -1,33 +0,0 @@ -# -# Feature name: sg-chain -# Kconfig: ARCH_HAS_SG_CHAIN -# description: arch supports chained scatter-gather lists -# - ----------------------- - | arch |status| - ----------------------- - | alpha: | TODO | - | arc: | ok | - | arm: | ok | - | arm64: | ok | - | c6x: | TODO | - | h8300: | TODO | - | hexagon: | TODO | - | ia64: | ok | - | m68k: | TODO | - | microblaze: | TODO | - | mips: | TODO | - | nds32: | TODO | - | nios2: | TODO | - | openrisc: | TODO | - | parisc: | TODO | - | powerpc: | ok | - | riscv: | TODO | - | s390: | ok | - | sh: | TODO | - | sparc: | ok | - | um: | TODO | - | unicore32: | TODO | - | x86: | ok | - | xtensa: | TODO | - ----------------------- diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index 5b4f88363453..a7e748a46c18 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -5,6 +5,7 @@ config ALPHA select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO select ARCH_NO_PREEMPT + select ARCH_NO_SG_CHAIN select ARCH_USE_CMPXCHG_LOCKREF select HAVE_AOUT select HAVE_IDE diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index c9e2a1323536..fd48d698da29 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -13,7 +13,6 @@ config ARC select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE - select ARCH_HAS_SG_CHAIN select ARCH_SUPPORTS_ATOMIC_RMW if ARC_HAS_LLSC select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 3b2852df6eb3..a858ee791ef0 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -19,6 +19,7 @@ config ARM select ARCH_HAVE_CUSTOM_GPIO_H select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_MIGHT_HAVE_PC_PARPORT + select ARCH_NO_SG_CHAIN if !ARM_HAS_SG_CHAIN select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT if CPU_V7 select ARCH_SUPPORTS_ATOMIC_RMW @@ -119,7 +120,6 @@ config ARM . config ARM_HAS_SG_CHAIN - select ARCH_HAS_SG_CHAIN bool config ARM_DMA_USE_IOMMU diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 2e645ea693ea..06cf0ef24367 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -23,7 +23,6 @@ config ARM64 select ARCH_HAS_MEMBARRIER_SYNC_CORE select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SET_MEMORY - select ARCH_HAS_SG_CHAIN select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_STRICT_MODULE_RWX select ARCH_HAS_SYNC_DMA_FOR_DEVICE diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 36773def6920..d6f203658994 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -29,7 +29,6 @@ config IA64 select HAVE_MEMBLOCK_NODE_MAP select HAVE_VIRT_CPU_ACCOUNTING select ARCH_HAS_DMA_MARK_CLEAN - select ARCH_HAS_SG_CHAIN select VIRT_TO_BUS select ARCH_DISCARD_MEMBLOCK select GENERIC_IRQ_PROBE diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 92a339ee28b3..428ee50fc3db 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -11,6 +11,7 @@ config PARISC select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_UBSAN_SANITIZE_ALL + select ARCH_NO_SG_CHAIN select ARCH_SUPPORTS_MEMORY_FAILURE select RTC_CLASS select RTC_DRV_GENERIC diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 8be31261aec8..4bc8edd83cee 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -138,7 +138,6 @@ config PPC select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_MEMBARRIER_CALLBACKS select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE && PPC64 - select ARCH_HAS_SG_CHAIN select ARCH_HAS_STRICT_KERNEL_RWX if ((PPC_BOOK3S_64 || PPC32) && !RELOCATABLE && !HIBERNATION) select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UACCESS_FLUSHCACHE if PPC64 diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 5173366af8f3..5624e8607054 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -73,7 +73,6 @@ config S390 select ARCH_HAS_KCOV select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SET_MEMORY - select ARCH_HAS_SG_CHAIN select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_STRICT_MODULE_RWX select ARCH_HAS_UBSAN_SANITIZE_ALL diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 490b2c95c212..8853b6ceae17 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -40,7 +40,6 @@ config SPARC select MODULES_USE_ELF_RELA select ODD_RT_SIGACTION select OLD_SIGSUSPEND - select ARCH_HAS_SG_CHAIN select CPU_NO_EFFICIENT_FFS select LOCKDEP_SMALL if LOCKDEP select NEED_DMA_MAP_STATE diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 9d734f3c8234..adc845b66f01 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -66,7 +66,6 @@ config X86 select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 select ARCH_HAS_UACCESS_MCSAFE if X86_64 && X86_MCE select ARCH_HAS_SET_MEMORY - select ARCH_HAS_SG_CHAIN select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_STRICT_MODULE_RWX select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index 239bfb16c58b..75488b606edc 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 config XTENSA def_bool y - select ARCH_HAS_SG_CHAIN select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_NO_COHERENT_DMA_MMAP if !MMU diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h index 093aa57120b0..b96f0d0b5b8f 100644 --- a/include/linux/scatterlist.h +++ b/include/linux/scatterlist.h @@ -324,10 +324,10 @@ size_t sg_zero_buffer(struct scatterlist *sgl, unsigned int nents, * Like SG_CHUNK_SIZE, but for archs that have sg chaining. This limit * is totally arbitrary, a setting of 2048 will get you at least 8mb ios. */ -#ifdef CONFIG_ARCH_HAS_SG_CHAIN -#define SG_MAX_SEGMENTS 2048 -#else +#ifdef CONFIG_ARCH_NO_SG_CHAIN #define SG_MAX_SEGMENTS SG_CHUNK_SIZE +#else +#define SG_MAX_SEGMENTS 2048 #endif #ifdef CONFIG_SG_POOL diff --git a/lib/Kconfig b/lib/Kconfig index a9965f4af4dd..d5a5e2ebf286 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -577,7 +577,7 @@ config SG_POOL # sg chaining option # -config ARCH_HAS_SG_CHAIN +config ARCH_NO_SG_CHAIN def_bool n config ARCH_HAS_PMEM_API diff --git a/lib/scatterlist.c b/lib/scatterlist.c index 7c6096a71704..9ba349e775ef 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -271,7 +271,7 @@ int __sg_alloc_table(struct sg_table *table, unsigned int nents, if (nents == 0) return -EINVAL; -#ifndef CONFIG_ARCH_HAS_SG_CHAIN +#ifdef CONFIG_ARCH_NO_SG_CHAIN if (WARN_ON_ONCE(nents > max_ents)) return -EINVAL; #endif -- cgit v1.2.3 From 58ab5e32e6fd83e33943614e7257f2ac5823824a Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 9 Dec 2018 17:43:20 -0700 Subject: sbitmap: silence bogus lockdep IRQ warning Ming reports that lockdep spews the following trace. What this essentially says is that the sbitmap swap_lock was used inconsistently in IRQ enabled and disabled context, and that is usually indicative of a bug that will cause a deadlock. For this case, it's a false positive. The swap_lock is used from process context only, when we swap the bits in the word and cleared mask. We also end up doing that when we are getting a driver tag, from the blk_mq_mark_tag_wait(), and from there we hold the waitqueue lock with IRQs disabled. However, this isn't from an actual IRQ, it's still process context. In lieu of a better way to fix this, simply always disable interrupts when grabbing the swap_lock if lockdep is enabled. [ 100.967642] ================start test sanity/001================ [ 101.238280] null: module loaded [ 106.093735] [ 106.094012] ===================================================== [ 106.094854] WARNING: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected [ 106.095759] 4.20.0-rc3_5d2ee7122c73_for-next+ #1 Not tainted [ 106.096551] ----------------------------------------------------- [ 106.097386] fio/1043 [HC0[0]:SC0[0]:HE0:SE1] is trying to acquire: [ 106.098231] 000000004c43fa71 (&(&sb->map[i].swap_lock)->rlock){+.+.}, at: sbitmap_get+0xd5/0x22c [ 106.099431] [ 106.099431] and this task is already holding: [ 106.100229] 000000007eec8b2f (&(&hctx->dispatch_wait_lock)->rlock){....}, at: blk_mq_dispatch_rq_list+0x4c1/0xd7c [ 106.101630] which would create a new lock dependency: [ 106.102326] (&(&hctx->dispatch_wait_lock)->rlock){....} -> (&(&sb->map[i].swap_lock)->rlock){+.+.} [ 106.103553] [ 106.103553] but this new dependency connects a SOFTIRQ-irq-safe lock: [ 106.104580] (&sbq->ws[i].wait){..-.} [ 106.104582] [ 106.104582] ... which became SOFTIRQ-irq-safe at: [ 106.105751] _raw_spin_lock_irqsave+0x4b/0x82 [ 106.106284] __wake_up_common_lock+0x119/0x1b9 [ 106.106825] sbitmap_queue_wake_up+0x33f/0x383 [ 106.107456] sbitmap_queue_clear+0x4c/0x9a [ 106.108046] __blk_mq_free_request+0x188/0x1d3 [ 106.108581] blk_mq_free_request+0x23b/0x26b [ 106.109102] scsi_end_request+0x345/0x5d7 [ 106.109587] scsi_io_completion+0x4b5/0x8f0 [ 106.110099] scsi_finish_command+0x412/0x456 [ 106.110615] scsi_softirq_done+0x23f/0x29b [ 106.111115] blk_done_softirq+0x2a7/0x2e6 [ 106.111608] __do_softirq+0x360/0x6ad [ 106.112062] run_ksoftirqd+0x2f/0x5b [ 106.112499] smpboot_thread_fn+0x3a5/0x3db [ 106.113000] kthread+0x1d4/0x1e4 [ 106.113457] ret_from_fork+0x3a/0x50 [ 106.113969] [ 106.113969] to a SOFTIRQ-irq-unsafe lock: [ 106.114672] (&(&sb->map[i].swap_lock)->rlock){+.+.} [ 106.114674] [ 106.114674] ... which became SOFTIRQ-irq-unsafe at: [ 106.116000] ... [ 106.116003] _raw_spin_lock+0x33/0x64 [ 106.116676] sbitmap_get+0xd5/0x22c [ 106.117134] __sbitmap_queue_get+0xe8/0x177 [ 106.117731] __blk_mq_get_tag+0x1e6/0x22d [ 106.118286] blk_mq_get_tag+0x1db/0x6e4 [ 106.118756] blk_mq_get_driver_tag+0x161/0x258 [ 106.119383] blk_mq_dispatch_rq_list+0x28e/0xd7c [ 106.120043] blk_mq_do_dispatch_sched+0x23a/0x287 [ 106.120607] blk_mq_sched_dispatch_requests+0x379/0x3fc [ 106.121234] __blk_mq_run_hw_queue+0x137/0x17e [ 106.121781] __blk_mq_delay_run_hw_queue+0x80/0x25f [ 106.122366] blk_mq_run_hw_queue+0x151/0x187 [ 106.122887] blk_mq_sched_insert_requests+0x13f/0x175 [ 106.123492] blk_mq_flush_plug_list+0x7d6/0x81b [ 106.124042] blk_flush_plug_list+0x392/0x3d7 [ 106.124557] blk_finish_plug+0x37/0x4f [ 106.125019] read_pages+0x3ef/0x430 [ 106.125446] __do_page_cache_readahead+0x18e/0x2fc [ 106.126027] force_page_cache_readahead+0x121/0x133 [ 106.126621] page_cache_sync_readahead+0x35f/0x3bb [ 106.127229] generic_file_buffered_read+0x410/0x1860 [ 106.127932] __vfs_read+0x319/0x38f [ 106.128415] vfs_read+0xd2/0x19a [ 106.128817] ksys_read+0xb9/0x135 [ 106.129225] do_syscall_64+0x140/0x385 [ 106.129684] entry_SYSCALL_64_after_hwframe+0x49/0xbe [ 106.130292] [ 106.130292] other info that might help us debug this: [ 106.130292] [ 106.131226] Chain exists of: [ 106.131226] &sbq->ws[i].wait --> &(&hctx->dispatch_wait_lock)->rlock --> &(&sb->map[i].swap_lock)->rlock [ 106.131226] [ 106.132865] Possible interrupt unsafe locking scenario: [ 106.132865] [ 106.133659] CPU0 CPU1 [ 106.134194] ---- ---- [ 106.134733] lock(&(&sb->map[i].swap_lock)->rlock); [ 106.135318] local_irq_disable(); [ 106.136014] lock(&sbq->ws[i].wait); [ 106.136747] lock(&(&hctx->dispatch_wait_lock)->rlock); [ 106.137742] [ 106.138110] lock(&sbq->ws[i].wait); [ 106.138625] [ 106.138625] *** DEADLOCK *** [ 106.138625] [ 106.139430] 3 locks held by fio/1043: [ 106.139947] #0: 0000000076ff0fd9 (rcu_read_lock){....}, at: hctx_lock+0x29/0xe8 [ 106.140813] #1: 000000002feb1016 (&sbq->ws[i].wait){..-.}, at: blk_mq_dispatch_rq_list+0x4ad/0xd7c [ 106.141877] #2: 000000007eec8b2f (&(&hctx->dispatch_wait_lock)->rlock){....}, at: blk_mq_dispatch_rq_list+0x4c1/0xd7c [ 106.143267] [ 106.143267] the dependencies between SOFTIRQ-irq-safe lock and the holding lock: [ 106.144351] -> (&sbq->ws[i].wait){..-.} ops: 82 { [ 106.144926] IN-SOFTIRQ-W at: [ 106.145314] _raw_spin_lock_irqsave+0x4b/0x82 [ 106.146042] __wake_up_common_lock+0x119/0x1b9 [ 106.146785] sbitmap_queue_wake_up+0x33f/0x383 [ 106.147567] sbitmap_queue_clear+0x4c/0x9a [ 106.148379] __blk_mq_free_request+0x188/0x1d3 [ 106.149148] blk_mq_free_request+0x23b/0x26b [ 106.149864] scsi_end_request+0x345/0x5d7 [ 106.150546] scsi_io_completion+0x4b5/0x8f0 [ 106.151367] scsi_finish_command+0x412/0x456 [ 106.152157] scsi_softirq_done+0x23f/0x29b [ 106.152855] blk_done_softirq+0x2a7/0x2e6 [ 106.153537] __do_softirq+0x360/0x6ad [ 106.154280] run_ksoftirqd+0x2f/0x5b [ 106.155020] smpboot_thread_fn+0x3a5/0x3db [ 106.155828] kthread+0x1d4/0x1e4 [ 106.156526] ret_from_fork+0x3a/0x50 [ 106.157267] INITIAL USE at: [ 106.157713] _raw_spin_lock_irqsave+0x4b/0x82 [ 106.158542] prepare_to_wait_exclusive+0xa8/0x215 [ 106.159421] blk_mq_get_tag+0x34f/0x6e4 [ 106.160186] blk_mq_get_request+0x48e/0xaef [ 106.160997] blk_mq_make_request+0x27e/0xbd2 [ 106.161828] generic_make_request+0x4d1/0x873 [ 106.162661] submit_bio+0x20c/0x253 [ 106.163379] mpage_bio_submit+0x44/0x4b [ 106.164142] mpage_readpages+0x3c2/0x407 [ 106.164919] read_pages+0x13a/0x430 [ 106.165633] __do_page_cache_readahead+0x18e/0x2fc [ 106.166530] force_page_cache_readahead+0x121/0x133 [ 106.167439] page_cache_sync_readahead+0x35f/0x3bb [ 106.168337] generic_file_buffered_read+0x410/0x1860 [ 106.169255] __vfs_read+0x319/0x38f [ 106.169977] vfs_read+0xd2/0x19a [ 106.170662] ksys_read+0xb9/0x135 [ 106.171356] do_syscall_64+0x140/0x385 [ 106.172120] entry_SYSCALL_64_after_hwframe+0x49/0xbe [ 106.173051] } [ 106.173308] ... key at: [] __key.26481+0x0/0x40 [ 106.174219] ... acquired at: [ 106.174646] _raw_spin_lock+0x33/0x64 [ 106.175183] blk_mq_dispatch_rq_list+0x4c1/0xd7c [ 106.175843] blk_mq_do_dispatch_sched+0x23a/0x287 [ 106.176518] blk_mq_sched_dispatch_requests+0x379/0x3fc [ 106.177262] __blk_mq_run_hw_queue+0x137/0x17e [ 106.177900] __blk_mq_delay_run_hw_queue+0x80/0x25f [ 106.178591] blk_mq_run_hw_queue+0x151/0x187 [ 106.179207] blk_mq_sched_insert_requests+0x13f/0x175 [ 106.179926] blk_mq_flush_plug_list+0x7d6/0x81b [ 106.180571] blk_flush_plug_list+0x392/0x3d7 [ 106.181187] blk_finish_plug+0x37/0x4f [ 106.181737] __se_sys_io_submit+0x171/0x304 [ 106.182346] do_syscall_64+0x140/0x385 [ 106.182895] entry_SYSCALL_64_after_hwframe+0x49/0xbe [ 106.183607] [ 106.183830] -> (&(&hctx->dispatch_wait_lock)->rlock){....} ops: 1 { [ 106.184691] INITIAL USE at: [ 106.185119] _raw_spin_lock+0x33/0x64 [ 106.185838] blk_mq_dispatch_rq_list+0x4c1/0xd7c [ 106.186697] blk_mq_do_dispatch_sched+0x23a/0x287 [ 106.187551] blk_mq_sched_dispatch_requests+0x379/0x3fc [ 106.188481] __blk_mq_run_hw_queue+0x137/0x17e [ 106.189307] __blk_mq_delay_run_hw_queue+0x80/0x25f [ 106.190189] blk_mq_run_hw_queue+0x151/0x187 [ 106.190989] blk_mq_sched_insert_requests+0x13f/0x175 [ 106.191902] blk_mq_flush_plug_list+0x7d6/0x81b [ 106.192739] blk_flush_plug_list+0x392/0x3d7 [ 106.193535] blk_finish_plug+0x37/0x4f [ 106.194269] __se_sys_io_submit+0x171/0x304 [ 106.195059] do_syscall_64+0x140/0x385 [ 106.195794] entry_SYSCALL_64_after_hwframe+0x49/0xbe [ 106.196705] } [ 106.196950] ... key at: [] __key.51231+0x0/0x40 [ 106.197853] ... acquired at: [ 106.198270] lock_acquire+0x280/0x2f3 [ 106.198806] _raw_spin_lock+0x33/0x64 [ 106.199337] sbitmap_get+0xd5/0x22c [ 106.199850] __sbitmap_queue_get+0xe8/0x177 [ 106.200450] __blk_mq_get_tag+0x1e6/0x22d [ 106.201035] blk_mq_get_tag+0x1db/0x6e4 [ 106.201589] blk_mq_get_driver_tag+0x161/0x258 [ 106.202237] blk_mq_dispatch_rq_list+0x5b9/0xd7c [ 106.202902] blk_mq_do_dispatch_sched+0x23a/0x287 [ 106.203572] blk_mq_sched_dispatch_requests+0x379/0x3fc [ 106.204316] __blk_mq_run_hw_queue+0x137/0x17e [ 106.204956] __blk_mq_delay_run_hw_queue+0x80/0x25f [ 106.205649] blk_mq_run_hw_queue+0x151/0x187 [ 106.206269] blk_mq_sched_insert_requests+0x13f/0x175 [ 106.206997] blk_mq_flush_plug_list+0x7d6/0x81b [ 106.207644] blk_flush_plug_list+0x392/0x3d7 [ 106.208264] blk_finish_plug+0x37/0x4f [ 106.208814] __se_sys_io_submit+0x171/0x304 [ 106.209415] do_syscall_64+0x140/0x385 [ 106.209965] entry_SYSCALL_64_after_hwframe+0x49/0xbe [ 106.210684] [ 106.210904] [ 106.210904] the dependencies between the lock to be acquired [ 106.210905] and SOFTIRQ-irq-unsafe lock: [ 106.212541] -> (&(&sb->map[i].swap_lock)->rlock){+.+.} ops: 1969 { [ 106.213393] HARDIRQ-ON-W at: [ 106.213840] _raw_spin_lock+0x33/0x64 [ 106.214570] sbitmap_get+0xd5/0x22c [ 106.215282] __sbitmap_queue_get+0xe8/0x177 [ 106.216086] __blk_mq_get_tag+0x1e6/0x22d [ 106.216876] blk_mq_get_tag+0x1db/0x6e4 [ 106.217627] blk_mq_get_driver_tag+0x161/0x258 [ 106.218465] blk_mq_dispatch_rq_list+0x28e/0xd7c [ 106.219326] blk_mq_do_dispatch_sched+0x23a/0x287 [ 106.220198] blk_mq_sched_dispatch_requests+0x379/0x3fc [ 106.221138] __blk_mq_run_hw_queue+0x137/0x17e [ 106.221975] __blk_mq_delay_run_hw_queue+0x80/0x25f [ 106.222874] blk_mq_run_hw_queue+0x151/0x187 [ 106.223686] blk_mq_sched_insert_requests+0x13f/0x175 [ 106.224597] blk_mq_flush_plug_list+0x7d6/0x81b [ 106.225444] blk_flush_plug_list+0x392/0x3d7 [ 106.226255] blk_finish_plug+0x37/0x4f [ 106.227006] read_pages+0x3ef/0x430 [ 106.227717] __do_page_cache_readahead+0x18e/0x2fc [ 106.228595] force_page_cache_readahead+0x121/0x133 [ 106.229491] page_cache_sync_readahead+0x35f/0x3bb [ 106.230373] generic_file_buffered_read+0x410/0x1860 [ 106.231277] __vfs_read+0x319/0x38f [ 106.231986] vfs_read+0xd2/0x19a [ 106.232666] ksys_read+0xb9/0x135 [ 106.233350] do_syscall_64+0x140/0x385 [ 106.234097] entry_SYSCALL_64_after_hwframe+0x49/0xbe [ 106.235012] SOFTIRQ-ON-W at: [ 106.235460] _raw_spin_lock+0x33/0x64 [ 106.236195] sbitmap_get+0xd5/0x22c [ 106.236913] __sbitmap_queue_get+0xe8/0x177 [ 106.237715] __blk_mq_get_tag+0x1e6/0x22d [ 106.238488] blk_mq_get_tag+0x1db/0x6e4 [ 106.239244] blk_mq_get_driver_tag+0x161/0x258 [ 106.240079] blk_mq_dispatch_rq_list+0x28e/0xd7c [ 106.240937] blk_mq_do_dispatch_sched+0x23a/0x287 [ 106.241806] blk_mq_sched_dispatch_requests+0x379/0x3fc [ 106.242751] __blk_mq_run_hw_queue+0x137/0x17e [ 106.243579] __blk_mq_delay_run_hw_queue+0x80/0x25f [ 106.244469] blk_mq_run_hw_queue+0x151/0x187 [ 106.245277] blk_mq_sched_insert_requests+0x13f/0x175 [ 106.246191] blk_mq_flush_plug_list+0x7d6/0x81b [ 106.247044] blk_flush_plug_list+0x392/0x3d7 [ 106.247859] blk_finish_plug+0x37/0x4f [ 106.248749] read_pages+0x3ef/0x430 [ 106.249463] __do_page_cache_readahead+0x18e/0x2fc [ 106.250357] force_page_cache_readahead+0x121/0x133 [ 106.251263] page_cache_sync_readahead+0x35f/0x3bb [ 106.252157] generic_file_buffered_read+0x410/0x1860 [ 106.253084] __vfs_read+0x319/0x38f [ 106.253808] vfs_read+0xd2/0x19a [ 106.254488] ksys_read+0xb9/0x135 [ 106.255186] do_syscall_64+0x140/0x385 [ 106.255943] entry_SYSCALL_64_after_hwframe+0x49/0xbe [ 106.256867] INITIAL USE at: [ 106.257300] _raw_spin_lock+0x33/0x64 [ 106.258033] sbitmap_get+0xd5/0x22c [ 106.258747] __sbitmap_queue_get+0xe8/0x177 [ 106.259542] __blk_mq_get_tag+0x1e6/0x22d [ 106.260320] blk_mq_get_tag+0x1db/0x6e4 [ 106.261072] blk_mq_get_driver_tag+0x161/0x258 [ 106.261902] blk_mq_dispatch_rq_list+0x28e/0xd7c [ 106.262762] blk_mq_do_dispatch_sched+0x23a/0x287 [ 106.263626] blk_mq_sched_dispatch_requests+0x379/0x3fc [ 106.264571] __blk_mq_run_hw_queue+0x137/0x17e [ 106.265409] __blk_mq_delay_run_hw_queue+0x80/0x25f [ 106.266302] blk_mq_run_hw_queue+0x151/0x187 [ 106.267111] blk_mq_sched_insert_requests+0x13f/0x175 [ 106.268028] blk_mq_flush_plug_list+0x7d6/0x81b [ 106.268878] blk_flush_plug_list+0x392/0x3d7 [ 106.269694] blk_finish_plug+0x37/0x4f [ 106.270432] read_pages+0x3ef/0x430 [ 106.271139] __do_page_cache_readahead+0x18e/0x2fc [ 106.272040] force_page_cache_readahead+0x121/0x133 [ 106.272932] page_cache_sync_readahead+0x35f/0x3bb [ 106.273811] generic_file_buffered_read+0x410/0x1860 [ 106.274709] __vfs_read+0x319/0x38f [ 106.275407] vfs_read+0xd2/0x19a [ 106.276074] ksys_read+0xb9/0x135 [ 106.276764] do_syscall_64+0x140/0x385 [ 106.277500] entry_SYSCALL_64_after_hwframe+0x49/0xbe [ 106.278417] } [ 106.278676] ... key at: [] __key.26212+0x0/0x40 [ 106.279586] ... acquired at: [ 106.280026] lock_acquire+0x280/0x2f3 [ 106.280559] _raw_spin_lock+0x33/0x64 [ 106.281101] sbitmap_get+0xd5/0x22c [ 106.281610] __sbitmap_queue_get+0xe8/0x177 [ 106.282221] __blk_mq_get_tag+0x1e6/0x22d [ 106.282809] blk_mq_get_tag+0x1db/0x6e4 [ 106.283368] blk_mq_get_driver_tag+0x161/0x258 [ 106.284018] blk_mq_dispatch_rq_list+0x5b9/0xd7c [ 106.284685] blk_mq_do_dispatch_sched+0x23a/0x287 [ 106.285371] blk_mq_sched_dispatch_requests+0x379/0x3fc [ 106.286135] __blk_mq_run_hw_queue+0x137/0x17e [ 106.286806] __blk_mq_delay_run_hw_queue+0x80/0x25f [ 106.287515] blk_mq_run_hw_queue+0x151/0x187 [ 106.288149] blk_mq_sched_insert_requests+0x13f/0x175 [ 106.289041] blk_mq_flush_plug_list+0x7d6/0x81b [ 106.289912] blk_flush_plug_list+0x392/0x3d7 [ 106.290590] blk_finish_plug+0x37/0x4f [ 106.291238] __se_sys_io_submit+0x171/0x304 [ 106.291864] do_syscall_64+0x140/0x385 [ 106.292534] entry_SYSCALL_64_after_hwframe+0x49/0xbe Reported-by: Ming Lei Tested-by: Guenter Roeck Signed-off-by: Jens Axboe --- lib/sbitmap.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/sbitmap.c b/lib/sbitmap.c index a89fbe7cf6ca..2261136ae067 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -118,8 +118,13 @@ static int __sbitmap_get_word(unsigned long *word, unsigned long depth, static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index) { unsigned long mask, val; + unsigned long __maybe_unused flags; bool ret = false; + /* Silence bogus lockdep warning */ +#if defined(CONFIG_LOCKDEP) + local_irq_save(flags); +#endif spin_lock(&sb->map[index].swap_lock); if (!sb->map[index].cleared) @@ -142,6 +147,9 @@ static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index) ret = true; out_unlock: spin_unlock(&sb->map[index].swap_lock); +#if defined(CONFIG_LOCKDEP) + local_irq_restore(flags); +#endif return ret; } -- cgit v1.2.3 From 4d42c44727a062e233e446c6c86da1c84d762d79 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 4 Dec 2018 23:23:11 +0200 Subject: lib/vsprintf: Print time and date in human readable format via %pt There are users which print time and date represented by content of struct rtc_time in human readable format. Instead of open coding that each time introduce %ptR[dt][r] specifier. Cc: Arnd Bergmann Cc: Bartlomiej Zolnierkiewicz Cc: Dmitry Torokhov Cc: Geert Uytterhoeven Cc: Guan Xuetao Cc: Ingo Molnar Cc: Jason Wessel Cc: Jonathan Corbet Cc: Jonathan Hunter Cc: Krzysztof Kozlowski Cc: "Rafael J. Wysocki" Cc: Thierry Reding Cc: Petr Mladek Signed-off-by: Andy Shevchenko Reviewed-by: Petr Mladek Signed-off-by: Alexandre Belloni --- Documentation/core-api/printk-formats.rst | 18 ++++++ lib/test_printf.c | 61 +++++++++++++++++- lib/vsprintf.c | 100 ++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/Documentation/core-api/printk-formats.rst b/Documentation/core-api/printk-formats.rst index ff48b55040ef..a7fae4538946 100644 --- a/Documentation/core-api/printk-formats.rst +++ b/Documentation/core-api/printk-formats.rst @@ -412,6 +412,24 @@ Examples:: Passed by reference. +Time and date (struct rtc_time) +------------------------------- + +:: + + %ptR YYYY-mm-ddTHH:MM:SS + %ptRd YYYY-mm-dd + %ptRt HH:MM:SS + %ptR[dt][r] + +For printing date and time as represented by struct rtc_time structure in +human readable format. + +By default year will be incremented by 1900 and month by 1. Use %ptRr (raw) +to suppress this behaviour. + +Passed by reference. + struct clk ---------- diff --git a/lib/test_printf.c b/lib/test_printf.c index 53527ea822b5..659b6cc0d483 100644 --- a/lib/test_printf.c +++ b/lib/test_printf.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -249,12 +250,11 @@ plain_format(void) #endif /* BITS_PER_LONG == 64 */ static int __init -plain_hash(void) +plain_hash_to_buffer(const void *p, char *buf, size_t len) { - char buf[PLAIN_BUF_SIZE]; int nchars; - nchars = snprintf(buf, PLAIN_BUF_SIZE, "%p", PTR); + nchars = snprintf(buf, len, "%p", p); if (nchars != PTR_WIDTH) return -1; @@ -265,6 +265,20 @@ plain_hash(void) return 0; } + return 0; +} + + +static int __init +plain_hash(void) +{ + char buf[PLAIN_BUF_SIZE]; + int ret; + + ret = plain_hash_to_buffer(PTR, buf, PLAIN_BUF_SIZE); + if (ret) + return ret; + if (strncmp(buf, PTR_STR, PTR_WIDTH) == 0) return -1; @@ -294,6 +308,23 @@ plain(void) } } +static void __init +test_hashed(const char *fmt, const void *p) +{ + char buf[PLAIN_BUF_SIZE]; + int ret; + + /* + * No need to increase failed test counter since this is assumed + * to be called after plain(). + */ + ret = plain_hash_to_buffer(p, buf, PLAIN_BUF_SIZE); + if (ret) + return; + + test(buf, fmt, p); +} + static void __init symbol_ptr(void) { @@ -418,6 +449,29 @@ struct_va_format(void) { } +static void __init +struct_rtc_time(void) +{ + /* 1543210543 */ + const struct rtc_time tm = { + .tm_sec = 43, + .tm_min = 35, + .tm_hour = 5, + .tm_mday = 26, + .tm_mon = 10, + .tm_year = 118, + }; + + test_hashed("%pt", &tm); + + test("2018-11-26T05:35:43", "%ptR", &tm); + test("0118-10-26T05:35:43", "%ptRr", &tm); + test("05:35:43|2018-11-26", "%ptRt|%ptRd", &tm, &tm); + test("05:35:43|0118-10-26", "%ptRtr|%ptRdr", &tm, &tm); + test("05:35:43|2018-11-26", "%ptRttr|%ptRdtr", &tm, &tm); + test("05:35:43 tr|2018-11-26 tr", "%ptRt tr|%ptRd tr", &tm, &tm); +} + static void __init struct_clk(void) { @@ -529,6 +583,7 @@ test_pointer(void) uuid(); dentry(); struct_va_format(); + struct_rtc_time(); struct_clk(); bitmap(); netdev_features(); diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 37a54a6dd594..3add92329bae 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -822,6 +823,20 @@ static const struct printf_spec default_dec_spec = { .precision = -1, }; +static const struct printf_spec default_dec02_spec = { + .base = 10, + .field_width = 2, + .precision = -1, + .flags = ZEROPAD, +}; + +static const struct printf_spec default_dec04_spec = { + .base = 10, + .field_width = 4, + .precision = -1, + .flags = ZEROPAD, +}; + static noinline_for_stack char *resource_string(char *buf, char *end, struct resource *res, struct printf_spec spec, const char *fmt) @@ -1549,6 +1564,87 @@ char *address_val(char *buf, char *end, const void *addr, const char *fmt) return special_hex_number(buf, end, num, size); } +static noinline_for_stack +char *date_str(char *buf, char *end, const struct rtc_time *tm, bool r) +{ + int year = tm->tm_year + (r ? 0 : 1900); + int mon = tm->tm_mon + (r ? 0 : 1); + + buf = number(buf, end, year, default_dec04_spec); + if (buf < end) + *buf = '-'; + buf++; + + buf = number(buf, end, mon, default_dec02_spec); + if (buf < end) + *buf = '-'; + buf++; + + return number(buf, end, tm->tm_mday, default_dec02_spec); +} + +static noinline_for_stack +char *time_str(char *buf, char *end, const struct rtc_time *tm, bool r) +{ + buf = number(buf, end, tm->tm_hour, default_dec02_spec); + if (buf < end) + *buf = ':'; + buf++; + + buf = number(buf, end, tm->tm_min, default_dec02_spec); + if (buf < end) + *buf = ':'; + buf++; + + return number(buf, end, tm->tm_sec, default_dec02_spec); +} + +static noinline_for_stack +char *rtc_str(char *buf, char *end, const struct rtc_time *tm, const char *fmt) +{ + bool have_t = true, have_d = true; + bool raw = false; + int count = 2; + + switch (fmt[count]) { + case 'd': + have_t = false; + count++; + break; + case 't': + have_d = false; + count++; + break; + } + + raw = fmt[count] == 'r'; + + if (have_d) + buf = date_str(buf, end, tm, raw); + if (have_d && have_t) { + /* Respect ISO 8601 */ + if (buf < end) + *buf = 'T'; + buf++; + } + if (have_t) + buf = time_str(buf, end, tm, raw); + + return buf; +} + +static noinline_for_stack +char *time_and_date(char *buf, char *end, void *ptr, struct printf_spec spec, + const char *fmt) +{ + switch (fmt[1]) { + case 'R': + return rtc_str(buf, end, (const struct rtc_time *)ptr, fmt); + default: + return ptr_to_id(buf, end, ptr, spec); + } +} + static noinline_for_stack char *clock(char *buf, char *end, struct clk *clk, struct printf_spec spec, const char *fmt) @@ -1828,6 +1924,8 @@ char *device_node_string(char *buf, char *end, struct device_node *dn, * - 'd[234]' For a dentry name (optionally 2-4 last components) * - 'D[234]' Same as 'd' but for a struct file * - 'g' For block_device name (gendisk + partition number) + * - 't[R][dt][r]' For time and date as represented: + * R struct rtc_time * - 'C' For a clock, it prints the name (Common Clock Framework) or address * (legacy clock framework) of the clock * - 'Cn' For a clock, it prints the name (Common Clock Framework) or address @@ -1952,6 +2050,8 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, return address_val(buf, end, ptr, fmt); case 'd': return dentry_name(buf, end, ptr, spec, fmt); + case 't': + return time_and_date(buf, end, ptr, spec, fmt); case 'C': return clock(buf, end, ptr, spec, fmt); case 'D': -- cgit v1.2.3 From b2dbff1bb893d5dfdf501231ff5505ca10cdede3 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 11 Dec 2018 18:39:41 -0700 Subject: sbitmap: flush deferred clears for resize and shallow gets We're missing a deferred clear off the shallow get, which can cause a hang. Additionally, when we resize the sbitmap, we should also flush deferred clears for good measure. Ensure we have full coverage on batch clears, even for paths where we would not be doing deferred clear. This makes it less error prone for future additions. Reported-by: Bart Van Assche Tested-by: Ming Lei Signed-off-by: Jens Axboe --- lib/sbitmap.c | 94 ++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 43 deletions(-) (limited to 'lib') diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 2261136ae067..5b3e56d68dab 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -20,6 +20,47 @@ #include #include +/* + * See if we have deferred clears that we can batch move + */ +static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index) +{ + unsigned long mask, val; + unsigned long __maybe_unused flags; + bool ret = false; + + /* Silence bogus lockdep warning */ +#if defined(CONFIG_LOCKDEP) + local_irq_save(flags); +#endif + spin_lock(&sb->map[index].swap_lock); + + if (!sb->map[index].cleared) + goto out_unlock; + + /* + * First get a stable cleared mask, setting the old mask to 0. + */ + do { + mask = sb->map[index].cleared; + } while (cmpxchg(&sb->map[index].cleared, mask, 0) != mask); + + /* + * Now clear the masked bits in our free word + */ + do { + val = sb->map[index].word; + } while (cmpxchg(&sb->map[index].word, val, val & ~mask) != val); + + ret = true; +out_unlock: + spin_unlock(&sb->map[index].swap_lock); +#if defined(CONFIG_LOCKDEP) + local_irq_restore(flags); +#endif + return ret; +} + int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, gfp_t flags, int node) { @@ -70,6 +111,9 @@ void sbitmap_resize(struct sbitmap *sb, unsigned int depth) unsigned int bits_per_word = 1U << sb->shift; unsigned int i; + for (i = 0; i < sb->map_nr; i++) + sbitmap_deferred_clear(sb, i); + sb->depth = depth; sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word); @@ -112,47 +156,6 @@ static int __sbitmap_get_word(unsigned long *word, unsigned long depth, return nr; } -/* - * See if we have deferred clears that we can batch move - */ -static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index) -{ - unsigned long mask, val; - unsigned long __maybe_unused flags; - bool ret = false; - - /* Silence bogus lockdep warning */ -#if defined(CONFIG_LOCKDEP) - local_irq_save(flags); -#endif - spin_lock(&sb->map[index].swap_lock); - - if (!sb->map[index].cleared) - goto out_unlock; - - /* - * First get a stable cleared mask, setting the old mask to 0. - */ - do { - mask = sb->map[index].cleared; - } while (cmpxchg(&sb->map[index].cleared, mask, 0) != mask); - - /* - * Now clear the masked bits in our free word - */ - do { - val = sb->map[index].word; - } while (cmpxchg(&sb->map[index].word, val, val & ~mask) != val); - - ret = true; -out_unlock: - spin_unlock(&sb->map[index].swap_lock); -#if defined(CONFIG_LOCKDEP) - local_irq_restore(flags); -#endif - return ret; -} - static int sbitmap_find_bit_in_index(struct sbitmap *sb, int index, unsigned int alloc_hint, bool round_robin) { @@ -215,6 +218,7 @@ int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint, index = SB_NR_TO_INDEX(sb, alloc_hint); for (i = 0; i < sb->map_nr; i++) { +again: nr = __sbitmap_get_word(&sb->map[index].word, min(sb->map[index].depth, shallow_depth), SB_NR_TO_BIT(sb, alloc_hint), true); @@ -223,6 +227,9 @@ int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint, break; } + if (sbitmap_deferred_clear(sb, index)) + goto again; + /* Jump to next index. */ index++; alloc_hint = index << sb->shift; @@ -242,7 +249,7 @@ bool sbitmap_any_bit_set(const struct sbitmap *sb) unsigned int i; for (i = 0; i < sb->map_nr; i++) { - if (sb->map[i].word) + if (sb->map[i].word & ~sb->map[i].cleared) return true; } return false; @@ -255,9 +262,10 @@ bool sbitmap_any_bit_clear(const struct sbitmap *sb) for (i = 0; i < sb->map_nr; i++) { const struct sbitmap_word *word = &sb->map[i]; + unsigned long mask = word->word & ~word->cleared; unsigned long ret; - ret = find_first_zero_bit(&word->word, word->depth); + ret = find_first_zero_bit(&mask, word->depth); if (ret < word->depth) return true; } -- cgit v1.2.3 From cb002d074dabfaa2248507fd9478d16a542e4f1e Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Mon, 3 Dec 2018 17:52:07 -0800 Subject: iov_iter: pass void csum pointer to csum_and_copy_to_iter The single caller to csum_and_copy_to_iter is skb_copy_and_csum_datagram and we are trying to unite its logic with skb_copy_datagram_iter by passing a callback to the copy function that we want to apply. Thus, we need to make the checksum pointer private to the function. Acked-by: David S. Miller Signed-off-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- include/linux/uio.h | 2 +- lib/iov_iter.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/include/linux/uio.h b/include/linux/uio.h index 55ce99ddb912..41d1f8d3313d 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -266,7 +266,7 @@ static inline void iov_iter_reexpand(struct iov_iter *i, size_t count) { i->count = count; } -size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum, struct iov_iter *i); +size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump, struct iov_iter *i); size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i); bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i); diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 54c248526b55..63a8999a234d 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1464,10 +1464,11 @@ bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, } EXPORT_SYMBOL(csum_and_copy_from_iter_full); -size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum, +size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump, struct iov_iter *i) { const char *from = addr; + __wsum *csum = csump; __wsum sum, next; size_t off = 0; -- cgit v1.2.3 From d05f443554b3c7dc6d46e3ba9c3c4de468875d4f Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Mon, 3 Dec 2018 17:52:09 -0800 Subject: iov_iter: introduce hash_and_copy_to_iter helper Allow consumers that want to use iov iterator helpers and also update a predefined hash calculation online when copying data. This is useful when copying incoming network buffers to a local iterator and calculate a digest on the incoming stream. nvme-tcp host driver that will be introduced in following patches is the first consumer via skb_copy_and_hash_datagram_iter. Acked-by: David S. Miller Signed-off-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- include/linux/uio.h | 3 +++ lib/iov_iter.c | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) (limited to 'lib') diff --git a/include/linux/uio.h b/include/linux/uio.h index 41d1f8d3313d..ecf584f6b82d 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -11,6 +11,7 @@ #include #include +#include #include struct page; @@ -269,6 +270,8 @@ static inline void iov_iter_reexpand(struct iov_iter *i, size_t count) size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump, struct iov_iter *i); size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i); bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i); +size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, + struct iov_iter *i); int import_iovec(int type, const struct iovec __user * uvector, unsigned nr_segs, unsigned fast_segs, diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 63a8999a234d..1928009f506e 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -6,6 +6,7 @@ #include #include #include +#include #define PIPE_PARANOIA /* for now */ @@ -1511,6 +1512,21 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump, } EXPORT_SYMBOL(csum_and_copy_to_iter); +size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, + struct iov_iter *i) +{ + struct ahash_request *hash = hashp; + struct scatterlist sg; + size_t copied; + + copied = copy_to_iter(addr, bytes, i); + sg_init_one(&sg, addr, copied); + ahash_request_set_crypt(hash, &sg, NULL, copied); + crypto_ahash_update(hash); + return copied; +} +EXPORT_SYMBOL(hash_and_copy_to_iter); + int iov_iter_npages(const struct iov_iter *i, int maxpages) { size_t size = i->count; -- cgit v1.2.3 From 10fdf838e5f540beca466e9d1325999c072e5d3f Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 10 Dec 2018 08:08:28 +0000 Subject: lib: fix build failure in CONFIG_DEBUG_VIRTUAL test On several arches, virt_to_phys() is in io.h Build fails without it: CC lib/test_debug_virtual.o lib/test_debug_virtual.c: In function 'test_debug_virtual_init': lib/test_debug_virtual.c:26:7: error: implicit declaration of function 'virt_to_phys' [-Werror=implicit-function-declaration] pa = virt_to_phys(va); ^ Fixes: e4dace361552 ("lib: add test module for CONFIG_DEBUG_VIRTUAL") CC: stable@vger.kernel.org Signed-off-by: Christophe Leroy Reviewed-by: Kees Cook Signed-off-by: Michael Ellerman --- lib/test_debug_virtual.c | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/test_debug_virtual.c b/lib/test_debug_virtual.c index d5a06addeb27..bf864c73e462 100644 --- a/lib/test_debug_virtual.c +++ b/lib/test_debug_virtual.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #ifdef CONFIG_MIPS -- cgit v1.2.3 From 809c67059162e7ba85c61a83ad7547b4ffbb1e6e Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Sun, 16 Dec 2018 20:48:21 +0100 Subject: test_rhashtable: remove semaphore usage This is one of only two files that initialize a semaphore to a negative value. We don't really need the two semaphores here at all, but can do the same thing in more conventional and more effient way, by using a single waitqueue and an atomic thread counter. This gets us a little bit closer to eliminating classic semaphores from the kernel. It also fixes a corner case where we fail to continue after one of the threads fails to start up. An alternative would be to use a split kthread_create()+wake_up_process() and completely eliminate the separate synchronization. Acked-by: Phil Sutter Signed-off-by: Arnd Bergmann Acked-by: Herbert Xu Signed-off-by: David S. Miller --- lib/test_rhashtable.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/test_rhashtable.c b/lib/test_rhashtable.c index 82ac39ce5310..6a8ac7626797 100644 --- a/lib/test_rhashtable.c +++ b/lib/test_rhashtable.c @@ -20,11 +20,11 @@ #include #include #include -#include #include #include #include #include +#include #define MAX_ENTRIES 1000000 #define TEST_INSERT_FAIL INT_MAX @@ -112,8 +112,8 @@ static struct rhashtable_params test_rht_params_dup = { .automatic_shrinking = false, }; -static struct semaphore prestart_sem; -static struct semaphore startup_sem = __SEMAPHORE_INITIALIZER(startup_sem, 0); +static atomic_t startup_count; +static DECLARE_WAIT_QUEUE_HEAD(startup_wait); static int insert_retry(struct rhashtable *ht, struct test_obj *obj, const struct rhashtable_params params) @@ -634,9 +634,12 @@ static int threadfunc(void *data) int i, step, err = 0, insert_retries = 0; struct thread_data *tdata = data; - up(&prestart_sem); - if (down_interruptible(&startup_sem)) - pr_err(" thread[%d]: down_interruptible failed\n", tdata->id); + if (atomic_dec_and_test(&startup_count)) + wake_up(&startup_wait); + if (wait_event_interruptible(startup_wait, atomic_read(&startup_count) == -1)) { + pr_err(" thread[%d]: interrupted\n", tdata->id); + goto out; + } for (i = 0; i < tdata->entries; i++) { tdata->objs[i].value.id = i; @@ -755,7 +758,7 @@ static int __init test_rht_init(void) pr_info("Testing concurrent rhashtable access from %d threads\n", tcount); - sema_init(&prestart_sem, 1 - tcount); + atomic_set(&startup_count, tcount); tdata = vzalloc(array_size(tcount, sizeof(struct thread_data))); if (!tdata) return -ENOMEM; @@ -781,15 +784,18 @@ static int __init test_rht_init(void) tdata[i].objs = objs + i * entries; tdata[i].task = kthread_run(threadfunc, &tdata[i], "rhashtable_thrad[%d]", i); - if (IS_ERR(tdata[i].task)) + if (IS_ERR(tdata[i].task)) { pr_err(" kthread_run failed for thread %d\n", i); - else + atomic_dec(&startup_count); + } else { started_threads++; + } } - if (down_interruptible(&prestart_sem)) - pr_err(" down interruptible failed\n"); - for (i = 0; i < tcount; i++) - up(&startup_sem); + if (wait_event_interruptible(startup_wait, atomic_read(&startup_count) == 0)) + pr_err(" wait_event interruptible failed\n"); + /* count is 0 now, set it to -1 and wake up all threads together */ + atomic_dec(&startup_count); + wake_up_all(&startup_wait); for (i = 0; i < tcount; i++) { if (IS_ERR(tdata[i].task)) continue; -- cgit v1.2.3 From ac8b6f148fc97e9e10b48bd337ef571b1d1136aa Mon Sep 17 00:00:00 2001 From: Amanoel Dawod Date: Wed, 5 Dec 2018 18:56:37 -0500 Subject: Fonts: New Terminus large console font This patch adds an option to compile-in a high resolution and large Terminus (ter16x32) bitmap console font for use with HiDPI and Retina screens. The font was convereted from standard Terminus ter-i32b.psf (size 16x32) with the help of psftools and minor hand editing deleting useless characters. This patch is non-intrusive, no options are enabled by default so most users won't notice a thing. I am placing my changes under the GPL 2.0 just as source Terminus font. Signed-off-by: Amanoel Dawod Signed-off-by: Greg Kroah-Hartman --- include/linux/font.h | 4 +- lib/fonts/Kconfig | 10 + lib/fonts/Makefile | 1 + lib/fonts/font_ter16x32.c | 2072 +++++++++++++++++++++++++++++++++++++++++++++ lib/fonts/fonts.c | 4 + 5 files changed, 2090 insertions(+), 1 deletion(-) create mode 100644 lib/fonts/font_ter16x32.c (limited to 'lib') diff --git a/include/linux/font.h b/include/linux/font.h index d6821769dd1e..51b91c8b69d5 100644 --- a/include/linux/font.h +++ b/include/linux/font.h @@ -32,6 +32,7 @@ struct font_desc { #define ACORN8x8_IDX 8 #define MINI4x6_IDX 9 #define FONT6x10_IDX 10 +#define TER16x32_IDX 11 extern const struct font_desc font_vga_8x8, font_vga_8x16, @@ -43,7 +44,8 @@ extern const struct font_desc font_vga_8x8, font_sun_12x22, font_acorn_8x8, font_mini_4x6, - font_6x10; + font_6x10, + font_ter_16x32; /* Find a font with a specific name */ diff --git a/lib/fonts/Kconfig b/lib/fonts/Kconfig index 8fa0791e8a1e..3ecdd5204ec5 100644 --- a/lib/fonts/Kconfig +++ b/lib/fonts/Kconfig @@ -109,6 +109,15 @@ config FONT_SUN12x22 big letters (like the letters used in the SPARC PROM). If the standard font is unreadable for you, say Y, otherwise say N. +config FONT_TER16x32 + bool "Terminus 16x32 font (not supported by all drivers)" + depends on FRAMEBUFFER_CONSOLE && (!SPARC && FONTS || SPARC) + help + Terminus Font is a clean, fixed width bitmap font, designed + for long (8 and more hours per day) work with computers. + This is the high resolution, large version for use with HiDPI screens. + If the standard font is unreadable for you, say Y, otherwise say N. + config FONT_AUTOSELECT def_bool y depends on !FONT_8x8 @@ -121,6 +130,7 @@ config FONT_AUTOSELECT depends on !FONT_SUN8x16 depends on !FONT_SUN12x22 depends on !FONT_10x18 + depends on !FONT_TER16x32 select FONT_8x16 endif # FONT_SUPPORT diff --git a/lib/fonts/Makefile b/lib/fonts/Makefile index d56f02dea83a..ed95070860de 100644 --- a/lib/fonts/Makefile +++ b/lib/fonts/Makefile @@ -14,6 +14,7 @@ font-objs-$(CONFIG_FONT_PEARL_8x8) += font_pearl_8x8.o font-objs-$(CONFIG_FONT_ACORN_8x8) += font_acorn_8x8.o font-objs-$(CONFIG_FONT_MINI_4x6) += font_mini_4x6.o font-objs-$(CONFIG_FONT_6x10) += font_6x10.o +font-objs-$(CONFIG_FONT_TER16x32) += font_ter16x32.o font-objs += $(font-objs-y) diff --git a/lib/fonts/font_ter16x32.c b/lib/fonts/font_ter16x32.c new file mode 100644 index 000000000000..3f0cf1ccdf3a --- /dev/null +++ b/lib/fonts/font_ter16x32.c @@ -0,0 +1,2072 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#define FONTDATAMAX 16384 + +static const unsigned char fontdata_ter16x32[FONTDATAMAX] = { + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3f, 0xf8, 0x7f, 0xfc, + 0xf0, 0x1e, 0xe0, 0x0e, 0xe0, 0x0e, 0xe0, 0x0e, + 0xee, 0xee, 0xee, 0xee, 0xe0, 0x0e, 0xe0, 0x0e, + 0xe0, 0x0e, 0xe0, 0x0e, 0xef, 0xee, 0xe7, 0xce, + 0xe0, 0x0e, 0xe0, 0x0e, 0xe0, 0x0e, 0xf0, 0x1e, + 0x7f, 0xfc, 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 1 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3f, 0xf8, 0x7f, 0xfc, + 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, + 0xe3, 0x8e, 0xe3, 0x8e, 0xff, 0xfe, 0xff, 0xfe, + 0xff, 0xfe, 0xff, 0xfe, 0xe0, 0x0e, 0xf0, 0x1e, + 0xf8, 0x3e, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, + 0x7f, 0xfc, 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 2 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x78, 0x3c, 0xfc, 0x7e, 0xfe, 0xfe, 0xff, 0xfe, + 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, + 0x7f, 0xfc, 0x7f, 0xfc, 0x3f, 0xf8, 0x1f, 0xf0, + 0x0f, 0xe0, 0x07, 0xc0, 0x03, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x03, 0x80, 0x07, 0xc0, 0x0f, 0xe0, + 0x1f, 0xf0, 0x3f, 0xf8, 0x7f, 0xfc, 0xff, 0xfe, + 0xff, 0xfe, 0x7f, 0xfc, 0x3f, 0xf8, 0x1f, 0xf0, + 0x0f, 0xe0, 0x07, 0xc0, 0x03, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 4 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x07, 0xc0, 0x0f, 0xe0, + 0x0f, 0xe0, 0x0f, 0xe0, 0x0f, 0xe0, 0x0f, 0xe0, + 0x07, 0xc0, 0x03, 0x80, 0x3b, 0xb8, 0x7f, 0xfc, + 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, + 0x7f, 0xfc, 0x3b, 0xb8, 0x03, 0x80, 0x03, 0x80, + 0x0f, 0xe0, 0x0f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 5 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x07, 0xc0, 0x0f, 0xe0, 0x1f, 0xf0, 0x3f, 0xf8, + 0x7f, 0xfc, 0x7f, 0xfc, 0xff, 0xfe, 0xff, 0xfe, + 0xff, 0xfe, 0xff, 0xfe, 0xff, 0xfe, 0x7b, 0xbc, + 0x3b, 0xb8, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x0f, 0xe0, 0x0f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 6 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0xc0, 0x07, 0xe0, 0x0f, 0xf0, 0x0f, 0xf0, + 0x0f, 0xf0, 0x0f, 0xf0, 0x07, 0xe0, 0x03, 0xc0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfc, 0x3f, 0xf8, 0x1f, 0xf0, 0x0f, 0xf0, 0x0f, + 0xf0, 0x0f, 0xf0, 0x0f, 0xf8, 0x1f, 0xfc, 0x3f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 8 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0xc0, 0x07, 0xe0, 0x0e, 0x70, 0x0c, 0x30, + 0x0c, 0x30, 0x0e, 0x70, 0x07, 0xe0, 0x03, 0xc0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 9 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfc, 0x3f, 0xf8, 0x1f, 0xf1, 0x8f, 0xf3, 0xcf, + 0xf3, 0xcf, 0xf1, 0x8f, 0xf8, 0x1f, 0xfc, 0x3f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 10 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0xfe, 0x03, 0xfe, + 0x00, 0x1e, 0x00, 0x3e, 0x00, 0x76, 0x00, 0xe6, + 0x01, 0xc6, 0x03, 0x86, 0x3f, 0xe0, 0x7f, 0xf0, + 0xf0, 0x78, 0xe0, 0x38, 0xe0, 0x38, 0xe0, 0x38, + 0xe0, 0x38, 0xe0, 0x38, 0xe0, 0x38, 0xf0, 0x78, + 0x7f, 0xf0, 0x3f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 11 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, 0x3f, 0xf8, + 0x1f, 0xf0, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x7f, 0xfc, 0x7f, 0xfc, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 12 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3f, 0xfc, 0x3f, 0xfc, + 0x38, 0x1c, 0x38, 0x1c, 0x38, 0x1c, 0x38, 0x1c, + 0x3f, 0xfc, 0x3f, 0xfc, 0x38, 0x00, 0x38, 0x00, + 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, + 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, + 0xf8, 0x00, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfe, 0x7f, 0xfe, + 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, + 0x7f, 0xfe, 0x7f, 0xfe, 0x70, 0x0e, 0x70, 0x0e, + 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, + 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x3e, + 0xf0, 0x3c, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 14 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x73, 0x9c, 0x73, 0x9c, + 0x3b, 0xb8, 0x1f, 0xf0, 0x0f, 0xe0, 0x7c, 0x7c, + 0x7c, 0x7c, 0x0f, 0xe0, 0x1f, 0xf0, 0x3b, 0xb8, + 0x73, 0x9c, 0x73, 0x9c, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 15 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc0, 0x00, 0xf0, 0x00, 0xfc, 0x00, 0xff, 0x00, + 0xff, 0xc0, 0xff, 0xf0, 0xff, 0xfc, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xfc, 0xff, 0xf0, 0xff, 0xc0, + 0xff, 0x00, 0xfc, 0x00, 0xf0, 0x00, 0xc0, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 16 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x00, 0x0f, 0x00, 0x3f, 0x00, 0xff, + 0x03, 0xff, 0x0f, 0xff, 0x3f, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x3f, 0xff, 0x0f, 0xff, 0x03, 0xff, + 0x00, 0xff, 0x00, 0x3f, 0x00, 0x0f, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x07, 0xc0, + 0x0f, 0xe0, 0x1f, 0xf0, 0x3b, 0xb8, 0x73, 0x9c, + 0x63, 0x8c, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x63, 0x8c, + 0x73, 0x9c, 0x3b, 0xb8, 0x1f, 0xf0, 0x0f, 0xe0, + 0x07, 0xc0, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 19 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xfe, 0x3f, 0xfe, + 0x79, 0xce, 0x71, 0xce, 0x71, 0xce, 0x71, 0xce, + 0x71, 0xce, 0x71, 0xce, 0x79, 0xce, 0x3f, 0xce, + 0x1f, 0xce, 0x01, 0xce, 0x01, 0xce, 0x01, 0xce, + 0x01, 0xce, 0x01, 0xce, 0x01, 0xce, 0x01, 0xce, + 0x01, 0xce, 0x01, 0xce, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 20 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0xe0, 0x0f, 0xf0, 0x1e, 0x78, 0x1c, 0x38, + 0x1c, 0x00, 0x1e, 0x00, 0x0f, 0xc0, 0x0f, 0xe0, + 0x1c, 0xf0, 0x1c, 0x78, 0x1c, 0x38, 0x1c, 0x38, + 0x1c, 0x38, 0x1e, 0x38, 0x0f, 0x38, 0x07, 0xf0, + 0x03, 0xf0, 0x00, 0x78, 0x00, 0x38, 0x1c, 0x38, + 0x1e, 0x78, 0x0f, 0xf0, 0x07, 0xe0, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 21 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfe, 0x7f, 0xfe, + 0x7f, 0xfe, 0x7f, 0xfe, 0x7f, 0xfe, 0x7f, 0xfe, + 0x7f, 0xfe, 0x7f, 0xfe, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 22 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x07, 0xc0, + 0x0f, 0xe0, 0x1f, 0xf0, 0x3b, 0xb8, 0x73, 0x9c, + 0x63, 0x8c, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x63, 0x8c, 0x73, 0x9c, 0x3b, 0xb8, + 0x1f, 0xf0, 0x0f, 0xe0, 0x07, 0xc0, 0x03, 0x80, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x07, 0xc0, + 0x0f, 0xe0, 0x1f, 0xf0, 0x3b, 0xb8, 0x73, 0x9c, + 0x63, 0x8c, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 24 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x63, 0x8c, + 0x73, 0x9c, 0x3b, 0xb8, 0x1f, 0xf0, 0x0f, 0xe0, + 0x07, 0xc0, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 25 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc0, 0x00, 0xe0, 0x00, 0x70, + 0x00, 0x38, 0x00, 0x1c, 0x7f, 0xfe, 0x7f, 0xfe, + 0x7f, 0xfe, 0x00, 0x1c, 0x00, 0x38, 0x00, 0x70, + 0x00, 0xe0, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 26 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x03, 0x00, 0x07, 0x00, 0x0e, 0x00, + 0x1c, 0x00, 0x38, 0x00, 0x7f, 0xfe, 0x7f, 0xfe, + 0x7f, 0xfe, 0x38, 0x00, 0x1c, 0x00, 0x0e, 0x00, + 0x07, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x60, 0x0e, 0x70, 0x1c, 0x38, + 0x38, 0x1c, 0x70, 0x0e, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x70, 0x0e, 0x38, 0x1c, 0x1c, 0x38, + 0x0e, 0x70, 0x06, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 29 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x80, 0x01, 0x80, 0x03, 0xc0, 0x03, 0xc0, + 0x07, 0xe0, 0x07, 0xe0, 0x0f, 0xf0, 0x0f, 0xf0, + 0x1f, 0xf8, 0x1f, 0xf8, 0x3f, 0xfc, 0x3f, 0xfc, + 0x7f, 0xfe, 0x7f, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 30 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x7f, 0xfe, 0x7f, 0xfe, + 0x3f, 0xfc, 0x3f, 0xfc, 0x1f, 0xf8, 0x1f, 0xf8, + 0x0f, 0xf0, 0x0f, 0xf0, 0x07, 0xe0, 0x07, 0xe0, + 0x03, 0xc0, 0x03, 0xc0, 0x01, 0x80, 0x01, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 31 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 32 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 34 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x7f, 0xfc, + 0x7f, 0xfc, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x7f, 0xfc, + 0x7f, 0xfc, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 35 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x1f, 0xf0, + 0x3f, 0xf8, 0x7b, 0xbc, 0x73, 0x9c, 0x73, 0x80, + 0x73, 0x80, 0x73, 0x80, 0x7b, 0x80, 0x3f, 0xf0, + 0x1f, 0xf8, 0x03, 0xbc, 0x03, 0x9c, 0x03, 0x9c, + 0x03, 0x9c, 0x73, 0x9c, 0x7b, 0xbc, 0x3f, 0xf8, + 0x1f, 0xf0, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 36 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0x1c, 0x3f, 0x9c, + 0x3b, 0xb8, 0x3b, 0xb8, 0x3f, 0xf0, 0x1f, 0x70, + 0x00, 0xe0, 0x00, 0xe0, 0x01, 0xc0, 0x01, 0xc0, + 0x03, 0x80, 0x03, 0x80, 0x07, 0x00, 0x07, 0x00, + 0x0e, 0xf8, 0x0f, 0xfc, 0x1d, 0xdc, 0x1d, 0xdc, + 0x39, 0xfc, 0x38, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0f, 0xc0, 0x1f, 0xe0, + 0x38, 0x70, 0x38, 0x70, 0x38, 0x70, 0x38, 0x70, + 0x38, 0x70, 0x1c, 0xe0, 0x0f, 0xc0, 0x0f, 0x80, + 0x1f, 0xce, 0x38, 0xee, 0x70, 0x7c, 0x70, 0x38, + 0x70, 0x38, 0x70, 0x38, 0x70, 0x38, 0x78, 0x7c, + 0x3f, 0xee, 0x1f, 0xce, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 39 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x01, 0xc0, + 0x03, 0x80, 0x07, 0x00, 0x07, 0x00, 0x0e, 0x00, + 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, + 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, + 0x0e, 0x00, 0x07, 0x00, 0x07, 0x00, 0x03, 0x80, + 0x01, 0xc0, 0x00, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 40 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x07, 0x00, + 0x03, 0x80, 0x01, 0xc0, 0x01, 0xc0, 0x00, 0xe0, + 0x00, 0xe0, 0x00, 0xe0, 0x00, 0xe0, 0x00, 0xe0, + 0x00, 0xe0, 0x00, 0xe0, 0x00, 0xe0, 0x00, 0xe0, + 0x00, 0xe0, 0x01, 0xc0, 0x01, 0xc0, 0x03, 0x80, + 0x07, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 41 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x38, 0x38, 0x1c, 0x70, + 0x0e, 0xe0, 0x07, 0xc0, 0x03, 0x80, 0x7f, 0xfc, + 0x7f, 0xfc, 0x03, 0x80, 0x07, 0xc0, 0x0e, 0xe0, + 0x1c, 0x70, 0x38, 0x38, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 42 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x7f, 0xfc, + 0x7f, 0xfc, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x07, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 44 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, + 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 45 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 46 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x38, 0x00, 0x38, 0x00, 0x70, 0x00, 0x70, + 0x00, 0xe0, 0x00, 0xe0, 0x01, 0xc0, 0x01, 0xc0, + 0x03, 0x80, 0x03, 0x80, 0x07, 0x00, 0x07, 0x00, + 0x0e, 0x00, 0x0e, 0x00, 0x1c, 0x00, 0x1c, 0x00, + 0x38, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x3c, + 0x70, 0x7c, 0x70, 0xfc, 0x71, 0xdc, 0x73, 0x9c, + 0x77, 0x1c, 0x7e, 0x1c, 0x7c, 0x1c, 0x78, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 48 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x07, 0x80, + 0x0f, 0x80, 0x1f, 0x80, 0x1f, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x1f, 0xf0, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 49 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x00, 0x1c, 0x00, 0x38, 0x00, 0x70, + 0x00, 0xe0, 0x01, 0xc0, 0x03, 0x80, 0x07, 0x00, + 0x0e, 0x00, 0x1c, 0x00, 0x38, 0x00, 0x70, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 50 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x1c, + 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x3c, 0x0f, 0xf8, + 0x0f, 0xf8, 0x00, 0x3c, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 51 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x3c, + 0x00, 0x7c, 0x00, 0xfc, 0x01, 0xdc, 0x03, 0x9c, + 0x07, 0x1c, 0x0e, 0x1c, 0x1c, 0x1c, 0x38, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x7f, 0xfc, + 0x7f, 0xfc, 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 52 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x7f, 0xf0, 0x7f, 0xf8, + 0x00, 0x3c, 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x1c, 0x00, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf8, 0x3f, 0xf8, + 0x78, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x7f, 0xf0, 0x7f, 0xf8, + 0x70, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 54 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x38, + 0x00, 0x38, 0x00, 0x70, 0x00, 0x70, 0x00, 0xe0, + 0x00, 0xe0, 0x01, 0xc0, 0x01, 0xc0, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 55 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, 0x3f, 0xf8, + 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 56 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x3c, + 0x3f, 0xf8, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 58 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x07, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 59 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x38, + 0x00, 0x70, 0x00, 0xe0, 0x01, 0xc0, 0x03, 0x80, + 0x07, 0x00, 0x0e, 0x00, 0x1c, 0x00, 0x38, 0x00, + 0x38, 0x00, 0x1c, 0x00, 0x0e, 0x00, 0x07, 0x00, + 0x03, 0x80, 0x01, 0xc0, 0x00, 0xe0, 0x00, 0x70, + 0x00, 0x38, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 60 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 61 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, 0x1c, 0x00, + 0x0e, 0x00, 0x07, 0x00, 0x03, 0x80, 0x01, 0xc0, + 0x00, 0xe0, 0x00, 0x70, 0x00, 0x38, 0x00, 0x1c, + 0x00, 0x1c, 0x00, 0x38, 0x00, 0x70, 0x00, 0xe0, + 0x01, 0xc0, 0x03, 0x80, 0x07, 0x00, 0x0e, 0x00, + 0x1c, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 62 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x00, 0x38, 0x00, 0x70, 0x00, 0xe0, + 0x01, 0xc0, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf8, 0x3f, 0xfc, + 0x78, 0x0e, 0x70, 0x06, 0x71, 0xfe, 0x73, 0xfe, + 0x77, 0x8e, 0x77, 0x0e, 0x77, 0x0e, 0x77, 0x0e, + 0x77, 0x0e, 0x77, 0x0e, 0x77, 0x0e, 0x77, 0x9e, + 0x73, 0xfe, 0x71, 0xf6, 0x70, 0x00, 0x78, 0x00, + 0x3f, 0xfe, 0x1f, 0xfe, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 64 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x7f, 0xfc, 0x7f, 0xfc, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 65 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xf0, 0x7f, 0xf8, + 0x70, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x38, 0x7f, 0xf0, 0x7f, 0xf0, + 0x70, 0x38, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x3c, + 0x7f, 0xf8, 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 66 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xc0, 0x7f, 0xf0, + 0x70, 0x78, 0x70, 0x38, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x38, 0x70, 0x78, + 0x7f, 0xf0, 0x7f, 0xc0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 68 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x7f, 0xe0, + 0x7f, 0xe0, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 69 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x7f, 0xe0, + 0x7f, 0xe0, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 70 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x71, 0xfc, + 0x71, 0xfc, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 71 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x7f, 0xfc, + 0x7f, 0xfc, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 72 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0f, 0xe0, 0x0f, 0xe0, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x0f, 0xe0, 0x0f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x00, 0xfe, + 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, + 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, + 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, + 0x70, 0x38, 0x70, 0x38, 0x70, 0x38, 0x78, 0x78, + 0x3f, 0xf0, 0x1f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 74 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x0c, 0x70, 0x1c, + 0x70, 0x38, 0x70, 0x70, 0x70, 0xe0, 0x71, 0xc0, + 0x73, 0x80, 0x77, 0x00, 0x7e, 0x00, 0x7c, 0x00, + 0x7c, 0x00, 0x7e, 0x00, 0x77, 0x00, 0x73, 0x80, + 0x71, 0xc0, 0x70, 0xe0, 0x70, 0x70, 0x70, 0x38, + 0x70, 0x1c, 0x70, 0x0c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 75 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 76 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x0e, 0x70, 0x0e, + 0x78, 0x1e, 0x7c, 0x3e, 0x7e, 0x7e, 0x7e, 0x7e, + 0x77, 0xee, 0x73, 0xce, 0x73, 0xce, 0x71, 0x8e, + 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, + 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, + 0x70, 0x0e, 0x70, 0x0e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x7c, 0x1c, 0x7e, 0x1c, 0x77, 0x1c, 0x73, 0x9c, + 0x71, 0xdc, 0x70, 0xfc, 0x70, 0x7c, 0x70, 0x3c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 79 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xf0, 0x7f, 0xf8, + 0x70, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x3c, + 0x7f, 0xf8, 0x7f, 0xf0, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 80 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x73, 0x9c, 0x79, 0xfc, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x38, 0x00, 0x1c, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 81 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xf0, 0x7f, 0xf8, + 0x70, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x3c, + 0x7f, 0xf8, 0x7f, 0xf0, 0x7e, 0x00, 0x77, 0x00, + 0x73, 0x80, 0x71, 0xc0, 0x70, 0xe0, 0x70, 0x70, + 0x70, 0x38, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 82 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x78, 0x00, 0x3f, 0xf0, + 0x1f, 0xf8, 0x00, 0x3c, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 84 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 85 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, + 0x0e, 0xe0, 0x0e, 0xe0, 0x0e, 0xe0, 0x07, 0xc0, + 0x07, 0xc0, 0x07, 0xc0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 86 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x0e, 0x70, 0x0e, + 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, + 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, + 0x71, 0x8e, 0x73, 0xce, 0x73, 0xce, 0x77, 0xee, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7c, 0x3e, 0x78, 0x1e, + 0x70, 0x0e, 0x70, 0x0e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x1c, 0x70, 0x1c, + 0x38, 0x38, 0x38, 0x38, 0x1c, 0x70, 0x1c, 0x70, + 0x0e, 0xe0, 0x0e, 0xe0, 0x07, 0xc0, 0x07, 0xc0, + 0x07, 0xc0, 0x07, 0xc0, 0x0e, 0xe0, 0x0e, 0xe0, + 0x1c, 0x70, 0x1c, 0x70, 0x38, 0x38, 0x38, 0x38, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 88 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x38, 0x38, 0x38, 0x38, 0x1c, 0x70, + 0x1c, 0x70, 0x0e, 0xe0, 0x0e, 0xe0, 0x07, 0xc0, + 0x07, 0xc0, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 89 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, + 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x38, + 0x00, 0x70, 0x00, 0xe0, 0x01, 0xc0, 0x03, 0x80, + 0x07, 0x00, 0x0e, 0x00, 0x1c, 0x00, 0x38, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 90 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0f, 0xf0, 0x0f, 0xf0, + 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, + 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, + 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, + 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, 0x0e, 0x00, + 0x0f, 0xf0, 0x0f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 91 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, 0x38, 0x00, + 0x1c, 0x00, 0x1c, 0x00, 0x0e, 0x00, 0x0e, 0x00, + 0x07, 0x00, 0x07, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x01, 0xc0, 0x01, 0xc0, 0x00, 0xe0, 0x00, 0xe0, + 0x00, 0x70, 0x00, 0x70, 0x00, 0x38, 0x00, 0x38, + 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 92 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0f, 0xf0, 0x0f, 0xf0, + 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, + 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, + 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, + 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, + 0x0f, 0xf0, 0x0f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x80, 0x07, 0xc0, 0x0e, 0xe0, 0x1c, 0x70, + 0x38, 0x38, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 94 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, + 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 95 */ + 0x00, 0x00, 0x1c, 0x00, 0x0e, 0x00, 0x07, 0x00, + 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 96 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x3f, 0xf0, 0x3f, 0xf8, 0x00, 0x3c, 0x00, 0x1c, + 0x00, 0x1c, 0x1f, 0xfc, 0x3f, 0xfc, 0x78, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x7f, 0xf0, 0x7f, 0xf8, 0x70, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x3c, + 0x7f, 0xf8, 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 98 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 99 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x1c, + 0x1f, 0xfc, 0x3f, 0xfc, 0x78, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 100 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 101 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x01, 0xfe, + 0x03, 0xc0, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x3f, 0xf8, 0x3f, 0xf8, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 102 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xfc, 0x3f, 0xfc, 0x78, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x3c, 0x3f, 0xf8, 0x3f, 0xf0, 0x00, 0x00, /* 103 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x7f, 0xf0, 0x7f, 0xf8, 0x70, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 104 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0x80, 0x0f, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x0f, 0xe0, 0x0f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 105 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, 0x38, + 0x00, 0x38, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xf8, 0x00, 0xf8, 0x00, 0x38, 0x00, 0x38, + 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, + 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, + 0x00, 0x38, 0x00, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x3c, 0x78, 0x1f, 0xf0, 0x0f, 0xe0, 0x00, 0x00, /* 106 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, 0x38, 0x00, + 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, + 0x38, 0x1c, 0x38, 0x38, 0x38, 0x70, 0x38, 0xe0, + 0x39, 0xc0, 0x3b, 0x80, 0x3f, 0x00, 0x3f, 0x00, + 0x3b, 0x80, 0x39, 0xc0, 0x38, 0xe0, 0x38, 0x70, + 0x38, 0x38, 0x38, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 107 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0f, 0x80, 0x0f, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x0f, 0xe0, 0x0f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 108 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xf0, 0x7f, 0xf8, 0x73, 0xbc, 0x73, 0x9c, + 0x73, 0x9c, 0x73, 0x9c, 0x73, 0x9c, 0x73, 0x9c, + 0x73, 0x9c, 0x73, 0x9c, 0x73, 0x9c, 0x73, 0x9c, + 0x73, 0x9c, 0x73, 0x9c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 109 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xf0, 0x7f, 0xf8, 0x70, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 110 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 111 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xf0, 0x7f, 0xf8, 0x70, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x3c, + 0x7f, 0xf8, 0x7f, 0xf0, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x00, 0x00, /* 112 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xfc, 0x3f, 0xfc, 0x78, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x00, /* 113 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x73, 0xfc, 0x77, 0xfc, 0x7e, 0x00, 0x7c, 0x00, + 0x78, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 114 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x00, + 0x70, 0x00, 0x78, 0x00, 0x3f, 0xf0, 0x1f, 0xf8, + 0x00, 0x3c, 0x00, 0x1c, 0x00, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 115 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x07, 0x00, + 0x07, 0x00, 0x07, 0x00, 0x07, 0x00, 0x07, 0x00, + 0x7f, 0xf0, 0x7f, 0xf0, 0x07, 0x00, 0x07, 0x00, + 0x07, 0x00, 0x07, 0x00, 0x07, 0x00, 0x07, 0x00, + 0x07, 0x00, 0x07, 0x00, 0x07, 0x00, 0x07, 0x80, + 0x03, 0xfc, 0x01, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 116 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 117 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x0e, 0xe0, 0x0e, 0xe0, 0x07, 0xc0, + 0x07, 0xc0, 0x07, 0xc0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 118 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x73, 0x9c, 0x73, 0x9c, 0x73, 0x9c, 0x73, 0x9c, + 0x73, 0x9c, 0x73, 0x9c, 0x73, 0x9c, 0x7b, 0xbc, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 119 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x38, 0x38, + 0x1c, 0x70, 0x0e, 0xe0, 0x07, 0xc0, 0x07, 0xc0, + 0x0e, 0xe0, 0x1c, 0x70, 0x38, 0x38, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 120 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x3c, 0x3f, 0xf8, 0x3f, 0xf0, 0x00, 0x00, /* 121 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x38, 0x00, 0x70, + 0x00, 0xe0, 0x01, 0xc0, 0x03, 0x80, 0x07, 0x00, + 0x0e, 0x00, 0x1c, 0x00, 0x38, 0x00, 0x70, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 122 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0xf0, 0x03, 0xf0, + 0x07, 0x80, 0x07, 0x00, 0x07, 0x00, 0x07, 0x00, + 0x07, 0x00, 0x07, 0x00, 0x07, 0x00, 0x3e, 0x00, + 0x3e, 0x00, 0x07, 0x00, 0x07, 0x00, 0x07, 0x00, + 0x07, 0x00, 0x07, 0x00, 0x07, 0x00, 0x07, 0x80, + 0x03, 0xf0, 0x01, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 123 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 124 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x3f, 0x00, + 0x07, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x01, 0xf0, + 0x01, 0xf0, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x07, 0x80, + 0x3f, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 125 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1e, 0x1c, 0x3f, 0x1c, 0x77, 0x9c, 0x73, 0xdc, + 0x71, 0xf8, 0x70, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 126 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x07, 0xc0, + 0x0f, 0xe0, 0x1e, 0xf0, 0x3c, 0x78, 0x78, 0x3c, + 0xf0, 0x1e, 0xe0, 0x0e, 0xe0, 0x0e, 0xe0, 0x0e, + 0xe0, 0x0e, 0xe0, 0x0e, 0xe0, 0x0e, 0xe0, 0x0e, + 0xff, 0xfe, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 127 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x07, 0x00, 0x0e, 0x00, 0x00, 0x00, /* 128 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 129 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0xe0, + 0x01, 0xc0, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 130 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x07, 0xc0, + 0x0e, 0xe0, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x3f, 0xf0, 0x3f, 0xf8, 0x00, 0x3c, 0x00, 0x1c, + 0x00, 0x1c, 0x1f, 0xfc, 0x3f, 0xfc, 0x78, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 131 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x3f, 0xf0, 0x3f, 0xf8, 0x00, 0x3c, 0x00, 0x1c, + 0x00, 0x1c, 0x1f, 0xfc, 0x3f, 0xfc, 0x78, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 132 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x0e, 0x00, + 0x07, 0x00, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x3f, 0xf0, 0x3f, 0xf8, 0x00, 0x3c, 0x00, 0x1c, + 0x00, 0x1c, 0x1f, 0xfc, 0x3f, 0xfc, 0x78, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 133 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x07, 0xc0, 0x0e, 0xe0, + 0x0e, 0xe0, 0x0e, 0xe0, 0x07, 0xc0, 0x00, 0x00, + 0x3f, 0xf0, 0x3f, 0xf8, 0x00, 0x3c, 0x00, 0x1c, + 0x00, 0x1c, 0x1f, 0xfc, 0x3f, 0xfc, 0x78, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 134 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x07, 0x00, 0x0e, 0x00, 0x00, 0x00, /* 135 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x07, 0xc0, + 0x0e, 0xe0, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 136 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 137 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x0e, 0x00, + 0x07, 0x00, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 138 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0x80, 0x0f, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x0f, 0xe0, 0x0f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 139 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x07, 0xc0, + 0x0e, 0xe0, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0x80, 0x0f, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x0f, 0xe0, 0x0f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 140 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x0e, 0x00, + 0x07, 0x00, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0x80, 0x0f, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x0f, 0xe0, 0x0f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 141 */ + 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x7f, 0xfc, 0x7f, 0xfc, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 142 */ + 0x00, 0x00, 0x07, 0xc0, 0x0e, 0xe0, 0x0e, 0xe0, + 0x0e, 0xe0, 0x07, 0xc0, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x7f, 0xfc, 0x7f, 0xfc, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 143 */ + 0x00, 0x00, 0x00, 0x70, 0x00, 0xe0, 0x01, 0xc0, + 0x03, 0x80, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x7f, 0xe0, + 0x7f, 0xe0, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 144 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xf8, 0x7f, 0xfc, 0x03, 0x9e, 0x03, 0x8e, + 0x03, 0x8e, 0x3f, 0x8e, 0x7f, 0xfe, 0xf3, 0xfe, + 0xe3, 0x80, 0xe3, 0x80, 0xe3, 0x80, 0xf3, 0xce, + 0x7f, 0xfe, 0x3e, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 145 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3f, 0xfe, 0x7f, 0xfe, + 0xf1, 0xc0, 0xe1, 0xc0, 0xe1, 0xc0, 0xe1, 0xc0, + 0xe1, 0xc0, 0xe1, 0xc0, 0xe1, 0xc0, 0xff, 0xfe, + 0xff, 0xfe, 0xe1, 0xc0, 0xe1, 0xc0, 0xe1, 0xc0, + 0xe1, 0xc0, 0xe1, 0xc0, 0xe1, 0xc0, 0xe1, 0xc0, + 0xe1, 0xfe, 0xe1, 0xfe, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 146 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x07, 0xc0, + 0x0e, 0xe0, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 147 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 148 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x0e, 0x00, + 0x07, 0x00, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 149 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x07, 0xc0, + 0x0e, 0xe0, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 150 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x0e, 0x00, + 0x07, 0x00, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 151 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x3c, 0x3f, 0xf8, 0x3f, 0xf0, 0x00, 0x00, /* 152 */ + 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 153 */ + 0x00, 0x00, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x00, 0x00, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 154 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x1f, 0xf0, 0x3f, 0xf8, 0x7b, 0xbc, 0x73, 0x9c, + 0x73, 0x80, 0x73, 0x80, 0x73, 0x80, 0x73, 0x80, + 0x73, 0x80, 0x73, 0x80, 0x73, 0x9c, 0x7b, 0xbc, + 0x3f, 0xf8, 0x1f, 0xf0, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 155 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x07, 0xe0, 0x0f, 0xf0, + 0x1e, 0x78, 0x1c, 0x38, 0x1c, 0x00, 0x1c, 0x00, + 0x1c, 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x7f, 0xe0, + 0x7f, 0xe0, 0x1c, 0x00, 0x1c, 0x00, 0x1c, 0x00, + 0x1c, 0x00, 0x1c, 0x00, 0x1c, 0x1c, 0x1c, 0x1c, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 156 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x1c, 0x70, 0x1c, + 0x38, 0x38, 0x38, 0x38, 0x1c, 0x70, 0x1c, 0x70, + 0x0e, 0xe0, 0x0e, 0xe0, 0x07, 0xc0, 0x07, 0xc0, + 0x03, 0x80, 0x03, 0x80, 0x3f, 0xf8, 0x3f, 0xf8, + 0x03, 0x80, 0x03, 0x80, 0x3f, 0xf8, 0x3f, 0xf8, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 157 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0xff, 0x80, + 0xe3, 0xc0, 0xe1, 0xc0, 0xe1, 0xc0, 0xe1, 0xc0, + 0xe1, 0xc0, 0xe1, 0xc0, 0xe3, 0xc0, 0xff, 0xf0, + 0xff, 0x70, 0xe0, 0x70, 0xe3, 0xfe, 0xe3, 0xfe, + 0xe0, 0x70, 0xe0, 0x70, 0xe0, 0x70, 0xe0, 0x70, + 0xe0, 0x7e, 0xe0, 0x3e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 158 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0xf8, 0x03, 0xfc, + 0x03, 0x9c, 0x03, 0x9c, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x1f, 0xf0, 0x1f, 0xf0, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x73, 0x80, 0x73, 0x80, + 0x7f, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, /* 159 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0xe0, + 0x01, 0xc0, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x3f, 0xf0, 0x3f, 0xf8, 0x00, 0x3c, 0x00, 0x1c, + 0x00, 0x1c, 0x1f, 0xfc, 0x3f, 0xfc, 0x78, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 160 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0xe0, + 0x01, 0xc0, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0x80, 0x0f, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x0f, 0xe0, 0x0f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 161 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0xe0, + 0x01, 0xc0, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xf0, 0x3f, 0xf8, 0x78, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 162 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0xe0, + 0x01, 0xc0, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x3f, 0xfc, 0x1f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 163 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0x38, 0x3b, 0xb8, + 0x3b, 0xb8, 0x39, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xf0, 0x7f, 0xf8, 0x70, 0x3c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 164 */ + 0x00, 0x00, 0x1f, 0x38, 0x3b, 0xb8, 0x3b, 0xb8, + 0x39, 0xf0, 0x00, 0x00, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x1c, + 0x7c, 0x1c, 0x7e, 0x1c, 0x77, 0x1c, 0x73, 0x9c, + 0x71, 0xdc, 0x70, 0xfc, 0x70, 0x7c, 0x70, 0x3c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 165 */ + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xe0, 0x1f, 0xf0, + 0x00, 0x38, 0x00, 0x38, 0x0f, 0xf8, 0x1f, 0xf8, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x1f, 0xf8, + 0x0f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xf8, + 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 166 */ + 0x00, 0x00, 0x00, 0x00, 0x0f, 0xe0, 0x1f, 0xf0, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, + 0x38, 0x38, 0x38, 0x38, 0x38, 0x38, 0x1f, 0xf0, + 0x0f, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xf8, + 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 167 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x07, 0x00, + 0x0e, 0x00, 0x1c, 0x00, 0x38, 0x00, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 168 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 169 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 170 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x3c, 0x00, + 0x7c, 0x06, 0x1c, 0x0e, 0x1c, 0x1c, 0x1c, 0x38, + 0x1c, 0x70, 0x1c, 0xe0, 0x1d, 0xc0, 0x03, 0x80, + 0x07, 0x00, 0x0e, 0xfc, 0x1d, 0xfe, 0x39, 0xce, + 0x71, 0xce, 0x60, 0x1c, 0x00, 0x38, 0x00, 0x70, + 0x00, 0xfe, 0x01, 0xfe, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 171 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x1e, 0x00, + 0x3e, 0x00, 0x0e, 0x00, 0x0e, 0x06, 0x0e, 0x0e, + 0x0e, 0x1c, 0x0e, 0x38, 0x0e, 0x70, 0x00, 0xe0, + 0x01, 0xce, 0x03, 0x9e, 0x07, 0x3e, 0x0e, 0x7e, + 0x1c, 0xee, 0x39, 0xce, 0x73, 0xfe, 0x63, 0xfe, + 0x00, 0x0e, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 172 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 173 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0xce, 0x03, 0x9c, 0x07, 0x38, 0x0e, 0x70, + 0x1c, 0xe0, 0x39, 0xc0, 0x73, 0x80, 0x73, 0x80, + 0x39, 0xc0, 0x1c, 0xe0, 0x0e, 0x70, 0x07, 0x38, + 0x03, 0x9c, 0x01, 0xce, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 174 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x73, 0x80, 0x39, 0xc0, 0x1c, 0xe0, 0x0e, 0x70, + 0x07, 0x38, 0x03, 0x9c, 0x01, 0xce, 0x01, 0xce, + 0x03, 0x9c, 0x07, 0x38, 0x0e, 0x70, 0x1c, 0xe0, + 0x39, 0xc0, 0x73, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 175 */ + 0xaa, 0xaa, 0x00, 0x00, 0xaa, 0xaa, 0x00, 0x00, + 0xaa, 0xaa, 0x00, 0x00, 0xaa, 0xaa, 0x00, 0x00, + 0xaa, 0xaa, 0x00, 0x00, 0xaa, 0xaa, 0x00, 0x00, + 0xaa, 0xaa, 0x00, 0x00, 0xaa, 0xaa, 0x00, 0x00, + 0xaa, 0xaa, 0x00, 0x00, 0xaa, 0xaa, 0x00, 0x00, + 0xaa, 0xaa, 0x00, 0x00, 0xaa, 0xaa, 0x00, 0x00, + 0xaa, 0xaa, 0x00, 0x00, 0xaa, 0xaa, 0x00, 0x00, + 0xaa, 0xaa, 0x00, 0x00, 0xaa, 0xaa, 0x00, 0x00, /* 176 */ + 0xaa, 0xaa, 0x55, 0x55, 0xaa, 0xaa, 0x55, 0x55, + 0xaa, 0xaa, 0x55, 0x55, 0xaa, 0xaa, 0x55, 0x55, + 0xaa, 0xaa, 0x55, 0x55, 0xaa, 0xaa, 0x55, 0x55, + 0xaa, 0xaa, 0x55, 0x55, 0xaa, 0xaa, 0x55, 0x55, + 0xaa, 0xaa, 0x55, 0x55, 0xaa, 0xaa, 0x55, 0x55, + 0xaa, 0xaa, 0x55, 0x55, 0xaa, 0xaa, 0x55, 0x55, + 0xaa, 0xaa, 0x55, 0x55, 0xaa, 0xaa, 0x55, 0x55, + 0xaa, 0xaa, 0x55, 0x55, 0xaa, 0xaa, 0x55, 0x55, /* 177 */ + 0xff, 0xff, 0xaa, 0xaa, 0xff, 0xff, 0xaa, 0xaa, + 0xff, 0xff, 0xaa, 0xaa, 0xff, 0xff, 0xaa, 0xaa, + 0xff, 0xff, 0xaa, 0xaa, 0xff, 0xff, 0xaa, 0xaa, + 0xff, 0xff, 0xaa, 0xaa, 0xff, 0xff, 0xaa, 0xaa, + 0xff, 0xff, 0xaa, 0xaa, 0xff, 0xff, 0xaa, 0xaa, + 0xff, 0xff, 0xaa, 0xaa, 0xff, 0xff, 0xaa, 0xaa, + 0xff, 0xff, 0xaa, 0xaa, 0xff, 0xff, 0xaa, 0xaa, + 0xff, 0xff, 0xaa, 0xaa, 0xff, 0xff, 0xaa, 0xaa, /* 178 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 179 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0xff, 0x80, 0xff, 0x80, + 0xff, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 180 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0xff, 0x80, 0xff, 0x80, 0xff, 0x80, 0x03, 0x80, + 0x03, 0x80, 0xff, 0x80, 0xff, 0x80, 0xff, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 181 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0xfe, 0x70, 0xfe, 0x70, + 0xfe, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 182 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xf0, 0xff, 0xf0, + 0xff, 0xf0, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 183 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0x80, 0xff, 0x80, 0xff, 0x80, 0x03, 0x80, + 0x03, 0x80, 0xff, 0x80, 0xff, 0x80, 0xff, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 184 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0xfe, 0x70, 0xfe, 0x70, 0xfe, 0x70, 0x00, 0x70, + 0x00, 0x70, 0xfe, 0x70, 0xfe, 0x70, 0xfe, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 185 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 186 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xf0, 0xff, 0xf0, 0xff, 0xf0, 0x00, 0x70, + 0x00, 0x70, 0xfe, 0x70, 0xfe, 0x70, 0xfe, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 187 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0xfe, 0x70, 0xfe, 0x70, 0xfe, 0x70, 0x00, 0x70, + 0x00, 0x70, 0xff, 0xf0, 0xff, 0xf0, 0xff, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 188 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0xff, 0xf0, 0xff, 0xf0, + 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 189 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0xff, 0x80, 0xff, 0x80, 0xff, 0x80, 0x03, 0x80, + 0x03, 0x80, 0xff, 0x80, 0xff, 0x80, 0xff, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 190 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0x80, 0xff, 0x80, + 0xff, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 191 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0xff, 0x03, 0xff, + 0x03, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 192 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 193 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 194 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0xff, 0x03, 0xff, + 0x03, 0xff, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 195 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 196 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 197 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0xff, 0x03, 0xff, 0x03, 0xff, 0x03, 0x80, + 0x03, 0x80, 0x03, 0xff, 0x03, 0xff, 0x03, 0xff, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 198 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x7f, 0x0e, 0x7f, + 0x0e, 0x7f, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 199 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x7f, 0x0e, 0x7f, 0x0e, 0x7f, 0x0e, 0x00, + 0x0e, 0x00, 0x0f, 0xff, 0x0f, 0xff, 0x0f, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 200 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0xff, 0x0f, 0xff, 0x0f, 0xff, 0x0e, 0x00, + 0x0e, 0x00, 0x0e, 0x7f, 0x0e, 0x7f, 0x0e, 0x7f, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 201 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0xfe, 0x7f, 0xfe, 0x7f, 0xfe, 0x7f, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 202 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0xfe, 0x7f, 0xfe, 0x7f, 0xfe, 0x7f, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 203 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x7f, 0x0e, 0x7f, 0x0e, 0x7f, 0x0e, 0x00, + 0x0e, 0x00, 0x0e, 0x7f, 0x0e, 0x7f, 0x0e, 0x7f, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 204 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 205 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0xfe, 0x7f, 0xfe, 0x7f, 0xfe, 0x7f, 0x00, 0x00, + 0x00, 0x00, 0xfe, 0x7f, 0xfe, 0x7f, 0xfe, 0x7f, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 206 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 207 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 208 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 209 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 210 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0f, 0xff, 0x0f, 0xff, + 0x0f, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 211 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0xff, 0x03, 0xff, 0x03, 0xff, 0x03, 0x80, + 0x03, 0x80, 0x03, 0xff, 0x03, 0xff, 0x03, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 212 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0xff, 0x03, 0xff, 0x03, 0xff, 0x03, 0x80, + 0x03, 0x80, 0x03, 0xff, 0x03, 0xff, 0x03, 0xff, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 213 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0x0f, 0xff, + 0x0f, 0xff, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 214 */ + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, + 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, 0x0e, 0x70, /* 215 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03, 0x80, + 0x03, 0x80, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 216 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0xff, 0x80, 0xff, 0x80, + 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 217 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0x03, 0xff, + 0x03, 0xff, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 218 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 219 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 220 */ + 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, + 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, + 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, + 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, + 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, + 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, + 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, + 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, /* 221 */ + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, + 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff, /* 222 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 223 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xee, 0x3f, 0xfe, 0x78, 0x3c, 0x70, 0x38, + 0x70, 0x38, 0x70, 0x38, 0x70, 0x38, 0x70, 0x38, + 0x70, 0x38, 0x70, 0x38, 0x70, 0x38, 0x78, 0x3c, + 0x3f, 0xfe, 0x1f, 0xee, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 224 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3f, 0xe0, 0x7f, 0xf0, + 0x70, 0x78, 0x70, 0x38, 0x70, 0x38, 0x70, 0x38, + 0x70, 0x38, 0x70, 0x70, 0x7f, 0xf0, 0x7f, 0xf0, + 0x70, 0x38, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x3c, + 0x7f, 0xf8, 0x7f, 0xf0, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x00, 0x00, /* 225 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 226 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 227 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, + 0x70, 0x00, 0x38, 0x00, 0x1c, 0x00, 0x0e, 0x00, + 0x07, 0x00, 0x03, 0x80, 0x01, 0xc0, 0x00, 0xe0, + 0x00, 0xe0, 0x01, 0xc0, 0x03, 0x80, 0x07, 0x00, + 0x0e, 0x00, 0x1c, 0x00, 0x38, 0x00, 0x70, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 228 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xfe, 0x3f, 0xfe, 0x78, 0xf0, 0x70, 0x78, + 0x70, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 229 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x3c, 0x70, 0x7c, 0x70, 0xfc, + 0x7f, 0xdc, 0x7f, 0x9c, 0x70, 0x00, 0x70, 0x00, + 0x70, 0x00, 0x70, 0x00, 0x70, 0x00, 0x00, 0x00, /* 230 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0xc0, + 0x01, 0xf8, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 231 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x1f, 0xf0, 0x3f, 0xf8, 0x7b, 0xbc, 0x73, 0x9c, + 0x73, 0x9c, 0x73, 0x9c, 0x73, 0x9c, 0x73, 0x9c, + 0x73, 0x9c, 0x73, 0x9c, 0x73, 0x9c, 0x73, 0x9c, + 0x73, 0x9c, 0x7b, 0xbc, 0x3f, 0xf8, 0x1f, 0xf0, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 232 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x77, 0xdc, + 0x77, 0xdc, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 233 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x3f, 0xf8, + 0x78, 0x3c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x38, 0x38, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, + 0x7c, 0x7c, 0x7c, 0x7c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 234 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf0, 0x1f, 0xf0, + 0x0e, 0x00, 0x07, 0x00, 0x03, 0x80, 0x01, 0xc0, + 0x0f, 0xe0, 0x1f, 0xf0, 0x38, 0x38, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x78, 0x3c, + 0x3f, 0xf8, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 235 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0xf8, + 0x7f, 0xfc, 0xe7, 0xce, 0xe3, 0x8e, 0xe3, 0x8e, + 0xe3, 0x8e, 0xe3, 0x8e, 0xe7, 0xce, 0x7f, 0xfc, + 0x3e, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 236 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x1c, + 0x00, 0x38, 0x00, 0x38, 0x0f, 0xf0, 0x1f, 0xf8, + 0x38, 0xfc, 0x38, 0xfc, 0x39, 0xdc, 0x39, 0xdc, + 0x3b, 0x9c, 0x3b, 0x9c, 0x3f, 0x1c, 0x3f, 0x1c, + 0x1f, 0xf8, 0x0f, 0xf0, 0x1c, 0x00, 0x1c, 0x00, + 0x38, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 237 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x07, 0xfc, 0x1f, 0xfc, 0x3c, 0x00, + 0x38, 0x00, 0x70, 0x00, 0x70, 0x00, 0x7f, 0xfc, + 0x7f, 0xfc, 0x70, 0x00, 0x70, 0x00, 0x38, 0x00, + 0x3c, 0x00, 0x1f, 0xfc, 0x07, 0xfc, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 238 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x07, 0xc0, 0x1f, 0xf0, + 0x3c, 0x78, 0x38, 0x38, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, + 0x70, 0x1c, 0x70, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 239 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, + 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 240 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x7f, 0xfc, + 0x7f, 0xfc, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0xfc, 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 241 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x0e, 0x00, + 0x07, 0x00, 0x03, 0x80, 0x01, 0xc0, 0x00, 0xe0, + 0x00, 0x70, 0x00, 0x38, 0x00, 0x38, 0x00, 0x70, + 0x00, 0xe0, 0x01, 0xc0, 0x03, 0x80, 0x07, 0x00, + 0x0e, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x3f, 0xfc, 0x3f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 242 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, 0x70, + 0x00, 0xe0, 0x01, 0xc0, 0x03, 0x80, 0x07, 0x00, + 0x0e, 0x00, 0x1c, 0x00, 0x1c, 0x00, 0x0e, 0x00, + 0x07, 0x00, 0x03, 0x80, 0x01, 0xc0, 0x00, 0xe0, + 0x00, 0x70, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, + 0x3f, 0xfc, 0x3f, 0xfc, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 243 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0xf8, 0x03, 0xfc, + 0x03, 0x9c, 0x03, 0x9c, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, /* 244 */ + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x73, 0x80, 0x73, 0x80, + 0x7f, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 245 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, + 0x7f, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 246 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x1c, + 0x7f, 0xbc, 0x7b, 0xfc, 0x70, 0xf8, 0x00, 0x00, + 0x00, 0x00, 0x3e, 0x1c, 0x7f, 0xbc, 0x7b, 0xfc, + 0x70, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 247 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0xe0, 0x1f, 0xf0, 0x1c, 0x70, 0x1c, 0x70, + 0x1c, 0x70, 0x1c, 0x70, 0x1f, 0xf0, 0x0f, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 248 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x03, 0xc0, 0x07, 0xe0, 0x07, 0xe0, + 0x07, 0xe0, 0x07, 0xe0, 0x03, 0xc0, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 249 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x80, 0x03, 0x80, + 0x03, 0x80, 0x03, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 250 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, + 0x00, 0x3e, 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, + 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, + 0x00, 0x38, 0x00, 0x38, 0x00, 0x38, 0x70, 0x38, + 0x70, 0x38, 0x70, 0x38, 0x78, 0x38, 0x3c, 0x38, + 0x1e, 0x38, 0x0f, 0x38, 0x07, 0xb8, 0x03, 0xf8, + 0x01, 0xf8, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 251 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0xe0, 0x1f, 0xf0, 0x1c, 0x38, 0x1c, 0x38, + 0x1c, 0x38, 0x1c, 0x38, 0x1c, 0x38, 0x1c, 0x38, + 0x1c, 0x38, 0x1c, 0x38, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 252 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xe0, + 0x1f, 0xf0, 0x1c, 0x70, 0x1c, 0x70, 0x00, 0xe0, + 0x01, 0xc0, 0x03, 0x80, 0x07, 0x00, 0x0e, 0x00, + 0x1f, 0xf0, 0x1f, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 253 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xf8, 0x1f, 0xf8, + 0x1f, 0xf8, 0x1f, 0xf8, 0x1f, 0xf8, 0x1f, 0xf8, + 0x1f, 0xf8, 0x1f, 0xf8, 0x1f, 0xf8, 0x1f, 0xf8, + 0x1f, 0xf8, 0x1f, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 254 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 255 */ + +}; + + +const struct font_desc font_ter_16x32 = { + .idx = TER16x32_IDX, + .name = "TER16x32", + .width = 16, + .height = 32, + .data = fontdata_ter16x32, +#ifdef __sparc__ + .pref = 5, +#else + .pref = -1, +#endif +}; diff --git a/lib/fonts/fonts.c b/lib/fonts/fonts.c index 823376ca0a8b..9969358a7af5 100644 --- a/lib/fonts/fonts.c +++ b/lib/fonts/fonts.c @@ -67,6 +67,10 @@ static const struct font_desc *fonts[] = { #undef NO_FONTS &font_6x10, #endif +#ifdef CONFIG_FONT_TER16x32 +#undef NO_FONTS + &font_ter_16x32, +#endif }; #define num_fonts ARRAY_SIZE(fonts) -- cgit v1.2.3 From e213574a449f7a57d4202c1869bbc7680b6b5521 Mon Sep 17 00:00:00 2001 From: Joel Stanley Date: Fri, 2 Nov 2018 11:14:55 +1030 Subject: raid6/ppc: Fix build for clang We cannot build these files with clang as it does not allow altivec instructions in assembly when -msoft-float is passed. Jinsong Ji wrote: > We currently disable Altivec/VSX support when enabling soft-float. So > any usage of vector builtins will break. > > Enable Altivec/VSX with soft-float may need quite some clean up work, so > I guess this is currently a limitation. > > Removing -msoft-float will make it work (and we are lucky that no > floating point instructions will be generated as well). This is a workaround until the issue is resolved in clang. Link: https://bugs.llvm.org/show_bug.cgi?id=31177 Link: https://github.com/ClangBuiltLinux/linux/issues/239 Signed-off-by: Joel Stanley Reviewed-by: Nick Desaulniers Signed-off-by: Michael Ellerman --- lib/raid6/Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'lib') diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile index 2f8b61dfd9b0..7ed43eaa02ef 100644 --- a/lib/raid6/Makefile +++ b/lib/raid6/Makefile @@ -18,6 +18,21 @@ quiet_cmd_unroll = UNROLL $@ ifeq ($(CONFIG_ALTIVEC),y) altivec_flags := -maltivec $(call cc-option,-mabi=altivec) + +ifdef CONFIG_CC_IS_CLANG +# clang ppc port does not yet support -maltivec when -msoft-float is +# enabled. A future release of clang will resolve this +# https://bugs.llvm.org/show_bug.cgi?id=31177 +CFLAGS_REMOVE_altivec1.o += -msoft-float +CFLAGS_REMOVE_altivec2.o += -msoft-float +CFLAGS_REMOVE_altivec4.o += -msoft-float +CFLAGS_REMOVE_altivec8.o += -msoft-float +CFLAGS_REMOVE_altivec8.o += -msoft-float +CFLAGS_REMOVE_vpermxor1.o += -msoft-float +CFLAGS_REMOVE_vpermxor2.o += -msoft-float +CFLAGS_REMOVE_vpermxor4.o += -msoft-float +CFLAGS_REMOVE_vpermxor8.o += -msoft-float +endif endif # The GCC option -ffreestanding is required in order to compile code containing -- cgit v1.2.3 From 86919f9dd2dbb88bbbe4e381dc8e6b9e9a1b9eb5 Mon Sep 17 00:00:00 2001 From: Daniel Verkamp Date: Mon, 12 Nov 2018 15:22:18 -0800 Subject: lib/raid6: check for assembler SSSE3 support Allow the x86 SSSE3 recovery function to be tested in raid6test. Signed-off-by: Daniel Verkamp Signed-off-by: Shaohua Li --- lib/raid6/test/Makefile | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile index 79777645cac9..3ab8720aa2f8 100644 --- a/lib/raid6/test/Makefile +++ b/lib/raid6/test/Makefile @@ -34,6 +34,9 @@ endif ifeq ($(IS_X86),yes) OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o + CFLAGS += $(shell echo "pshufb %xmm0, %xmm0" | \ + gcc -c -x assembler - >&/dev/null && \ + rm ./-.o && echo -DCONFIG_AS_SSSE3=1) CFLAGS += $(shell echo "vpbroadcastb %xmm0, %ymm1" | \ gcc -c -x assembler - >&/dev/null && \ rm ./-.o && echo -DCONFIG_AS_AVX2=1) -- cgit v1.2.3 From 0437de4fa09fe59b57d12b785e4afb73b0f34c05 Mon Sep 17 00:00:00 2001 From: Daniel Verkamp Date: Mon, 12 Nov 2018 15:26:51 -0800 Subject: lib/raid6: sort algos in rough performance order Sort the list of RAID6 algorithms in roughly decreasing order of expected performance: newer instruction sets first (within each architecture) and wider unrollings first. This doesn't make any difference right now, since all functions are benchmarked; a follow-up change will make use of this by optionally choosing the first valid function rather than testing all of them. The Itanium raid6_intx{16,32} entries are also moved down to be near the other raid6_intx entries for clarity. Signed-off-by: Daniel Verkamp Signed-off-by: Shaohua Li --- lib/raid6/algos.c | 76 +++++++++++++++++++++++++++---------------------------- 1 file changed, 38 insertions(+), 38 deletions(-) (limited to 'lib') diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c index 5065b1e7e327..a753ff56670f 100644 --- a/lib/raid6/algos.c +++ b/lib/raid6/algos.c @@ -34,64 +34,64 @@ struct raid6_calls raid6_call; EXPORT_SYMBOL_GPL(raid6_call); const struct raid6_calls * const raid6_algos[] = { -#if defined(__ia64__) - &raid6_intx16, - &raid6_intx32, -#endif #if defined(__i386__) && !defined(__arch_um__) - &raid6_mmxx1, - &raid6_mmxx2, - &raid6_sse1x1, - &raid6_sse1x2, - &raid6_sse2x1, - &raid6_sse2x2, -#ifdef CONFIG_AS_AVX2 - &raid6_avx2x1, - &raid6_avx2x2, -#endif #ifdef CONFIG_AS_AVX512 - &raid6_avx512x1, &raid6_avx512x2, + &raid6_avx512x1, #endif -#endif -#if defined(__x86_64__) && !defined(__arch_um__) - &raid6_sse2x1, - &raid6_sse2x2, - &raid6_sse2x4, #ifdef CONFIG_AS_AVX2 - &raid6_avx2x1, &raid6_avx2x2, - &raid6_avx2x4, + &raid6_avx2x1, #endif + &raid6_sse2x2, + &raid6_sse2x1, + &raid6_sse1x2, + &raid6_sse1x1, + &raid6_mmxx2, + &raid6_mmxx1, +#endif +#if defined(__x86_64__) && !defined(__arch_um__) #ifdef CONFIG_AS_AVX512 - &raid6_avx512x1, - &raid6_avx512x2, &raid6_avx512x4, + &raid6_avx512x2, + &raid6_avx512x1, +#endif +#ifdef CONFIG_AS_AVX2 + &raid6_avx2x4, + &raid6_avx2x2, + &raid6_avx2x1, #endif + &raid6_sse2x4, + &raid6_sse2x2, + &raid6_sse2x1, #endif #ifdef CONFIG_ALTIVEC - &raid6_altivec1, - &raid6_altivec2, - &raid6_altivec4, - &raid6_altivec8, - &raid6_vpermxor1, - &raid6_vpermxor2, - &raid6_vpermxor4, &raid6_vpermxor8, + &raid6_vpermxor4, + &raid6_vpermxor2, + &raid6_vpermxor1, + &raid6_altivec8, + &raid6_altivec4, + &raid6_altivec2, + &raid6_altivec1, #endif #if defined(CONFIG_S390) &raid6_s390vx8, #endif - &raid6_intx1, - &raid6_intx2, - &raid6_intx4, - &raid6_intx8, #ifdef CONFIG_KERNEL_MODE_NEON - &raid6_neonx1, - &raid6_neonx2, - &raid6_neonx4, &raid6_neonx8, + &raid6_neonx4, + &raid6_neonx2, + &raid6_neonx1, +#endif +#if defined(__ia64__) + &raid6_intx32, + &raid6_intx16, #endif + &raid6_intx8, + &raid6_intx4, + &raid6_intx2, + &raid6_intx1, NULL }; -- cgit v1.2.3 From be85f93ae2df32dea0b20908316f1d894c3e0f64 Mon Sep 17 00:00:00 2001 From: Daniel Verkamp Date: Mon, 12 Nov 2018 15:26:52 -0800 Subject: lib/raid6: add option to skip algo benchmarking This is helpful for systems where fast startup time is important. It is especially nice to avoid benchmarking RAID functions that are never used (for example, BTRFS selects RAID6_PQ even if the parity RAID mode is not in use). This saves 250+ milliseconds of boot time on modern x86 and ARM systems with a dozen or more available implementations. The new option is defaulted to 'y' to match the previous behavior of always benchmarking on init. Signed-off-by: Daniel Verkamp Signed-off-by: Shaohua Li --- include/linux/raid/pq.h | 3 +++ lib/Kconfig | 8 ++++++++ lib/raid6/algos.c | 5 +++++ 3 files changed, 16 insertions(+) (limited to 'lib') diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h index d7c99161bba2..605cf46c17bd 100644 --- a/include/linux/raid/pq.h +++ b/include/linux/raid/pq.h @@ -70,6 +70,9 @@ extern const char raid6_empty_zero_page[PAGE_SIZE]; #define MODULE_DESCRIPTION(desc) #define subsys_initcall(x) #define module_exit(x) + +#define IS_ENABLED(x) (x) +#define CONFIG_RAID6_PQ_BENCHMARK 1 #endif /* __KERNEL__ */ /* Routine choices */ diff --git a/lib/Kconfig b/lib/Kconfig index a9965f4af4dd..fcb05305a5a2 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -10,6 +10,14 @@ menu "Library routines" config RAID6_PQ tristate +config RAID6_PQ_BENCHMARK + bool "Automatically choose fastest RAID6 PQ functions" + depends on RAID6_PQ + default y + help + Benchmark all available RAID6 PQ functions on init and choose the + fastest one. + config BITREVERSE tristate diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c index a753ff56670f..7e4f7a8ffa8e 100644 --- a/lib/raid6/algos.c +++ b/lib/raid6/algos.c @@ -163,6 +163,11 @@ static inline const struct raid6_calls *raid6_choose_gen( if ((*algo)->valid && !(*algo)->valid()) continue; + if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { + best = *algo; + break; + } + perf = 0; preempt_disable(); -- cgit v1.2.3 From 9f6b7ef6c3ebe35be77b0ae3cf12e4d25ae80420 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 20 Dec 2018 08:49:00 -0700 Subject: sbitmap: add helpers for add/del wait queue handling After commit 5d2ee7122c73, users of sbitmap that need wait queue handling must use the provided helpers. But we only added prepare_to_wait()/finish_wait() style helpers, add the equivalent add_wait_queue/list_del wrappers as we.. This is needed to ensure kyber plays by the sbitmap waitqueue rules. Tested-by: Ming Lei Reviewed-by: Omar Sandoval Signed-off-by: Jens Axboe --- include/linux/sbitmap.h | 16 ++++++++++++++-- lib/sbitmap.c | 30 ++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index 03f50fcedc79..14d558146aea 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -560,13 +560,13 @@ void sbitmap_queue_wake_up(struct sbitmap_queue *sbq); void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m); struct sbq_wait { - int accounted; + struct sbitmap_queue *sbq; /* if set, sbq_wait is accounted */ struct wait_queue_entry wait; }; #define DEFINE_SBQ_WAIT(name) \ struct sbq_wait name = { \ - .accounted = 0, \ + .sbq = NULL, \ .wait = { \ .private = current, \ .func = autoremove_wake_function, \ @@ -588,4 +588,16 @@ void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq, void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, struct sbq_wait *sbq_wait); +/* + * Wrapper around add_wait_queue(), which maintains some extra internal state + */ +void sbitmap_add_wait_queue(struct sbitmap_queue *sbq, + struct sbq_wait_state *ws, + struct sbq_wait *sbq_wait); + +/* + * Must be paired with sbitmap_add_wait_queue() + */ +void sbitmap_del_wait_queue(struct sbq_wait *sbq_wait); + #endif /* __LINUX_SCALE_BITMAP_H */ diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 5b3e56d68dab..65c2d06250a6 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -671,13 +671,35 @@ void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m) } EXPORT_SYMBOL_GPL(sbitmap_queue_show); +void sbitmap_add_wait_queue(struct sbitmap_queue *sbq, + struct sbq_wait_state *ws, + struct sbq_wait *sbq_wait) +{ + if (!sbq_wait->sbq) { + sbq_wait->sbq = sbq; + atomic_inc(&sbq->ws_active); + } + add_wait_queue(&ws->wait, &sbq_wait->wait); +} +EXPORT_SYMBOL_GPL(sbitmap_add_wait_queue); + +void sbitmap_del_wait_queue(struct sbq_wait *sbq_wait) +{ + list_del_init(&sbq_wait->wait.entry); + if (sbq_wait->sbq) { + atomic_dec(&sbq_wait->sbq->ws_active); + sbq_wait->sbq = NULL; + } +} +EXPORT_SYMBOL_GPL(sbitmap_del_wait_queue); + void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, struct sbq_wait *sbq_wait, int state) { - if (!sbq_wait->accounted) { + if (!sbq_wait->sbq) { atomic_inc(&sbq->ws_active); - sbq_wait->accounted = 1; + sbq_wait->sbq = sbq; } prepare_to_wait_exclusive(&ws->wait, &sbq_wait->wait, state); } @@ -687,9 +709,9 @@ void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, struct sbq_wait *sbq_wait) { finish_wait(&ws->wait, &sbq_wait->wait); - if (sbq_wait->accounted) { + if (sbq_wait->sbq) { atomic_dec(&sbq->ws_active); - sbq_wait->accounted = 0; + sbq_wait->sbq = NULL; } } EXPORT_SYMBOL_GPL(sbitmap_finish_wait); -- cgit v1.2.3 From 8636a1f9677db4f883f29a072f401303acfc2edd Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 11 Dec 2018 20:01:04 +0900 Subject: treewide: surround Kconfig file paths with double quotes The Kconfig lexer supports special characters such as '.' and '/' in the parameter context. In my understanding, the reason is just to support bare file paths in the source statement. I do not see a good reason to complicate Kconfig for the room of ambiguity. The majority of code already surrounds file paths with double quotes, and it makes sense since file paths are constant string literals. Make it treewide consistent now. Signed-off-by: Masahiro Yamada Acked-by: Wolfram Sang Acked-by: Geert Uytterhoeven Acked-by: Ingo Molnar --- arch/arm/Kconfig | 2 +- arch/arm/kvm/Kconfig | 2 +- arch/arm64/Kconfig | 2 +- arch/arm64/kvm/Kconfig | 2 +- arch/ia64/Kconfig | 2 +- arch/m68k/Kconfig | 6 +++--- arch/mips/kvm/Kconfig | 2 +- arch/openrisc/Kconfig | 2 +- arch/powerpc/Kconfig | 4 ++-- arch/powerpc/kvm/Kconfig | 2 +- arch/riscv/Kconfig | 2 +- arch/s390/Kconfig | 2 +- arch/s390/kvm/Kconfig | 2 +- arch/sh/Kconfig | 2 +- arch/sparc/Kconfig | 2 +- arch/x86/Kconfig | 2 +- arch/x86/kvm/Kconfig | 2 +- block/Kconfig | 2 +- crypto/Kconfig | 4 ++-- drivers/crypto/Kconfig | 2 +- drivers/gpu/drm/i915/Kconfig | 2 +- drivers/hwmon/Kconfig | 2 +- drivers/i2c/Kconfig | 6 +++--- drivers/pps/Kconfig | 4 ++-- drivers/ras/Kconfig | 2 +- drivers/thermal/Kconfig | 2 +- drivers/w1/Kconfig | 4 ++-- lib/Kconfig.debug | 4 ++-- security/Kconfig | 16 ++++++++-------- security/integrity/Kconfig | 4 ++-- 30 files changed, 47 insertions(+), 47 deletions(-) (limited to 'lib') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 91be74d8df65..0a7faf82250e 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -910,7 +910,7 @@ config PLAT_VERSATILE source "arch/arm/firmware/Kconfig" -source arch/arm/mm/Kconfig +source "arch/arm/mm/Kconfig" config IWMMXT bool "Enable iWMMXt support" diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig index e2bd35b6780c..3f5320f46de2 100644 --- a/arch/arm/kvm/Kconfig +++ b/arch/arm/kvm/Kconfig @@ -55,6 +55,6 @@ config KVM_ARM_HOST ---help--- Provides host support for ARM processors. -source drivers/vhost/Kconfig +source "drivers/vhost/Kconfig" endif # VIRTUALIZATION diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 787d7850e064..04e525fc5a0f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -782,7 +782,7 @@ config NEED_PER_CPU_EMBED_FIRST_CHUNK config HOLES_IN_ZONE def_bool y -source kernel/Kconfig.hz +source "kernel/Kconfig.hz" config ARCH_SUPPORTS_DEBUG_PAGEALLOC def_bool y diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 47b23bf617c7..a3f85624313e 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -61,6 +61,6 @@ config KVM_ARM_PMU config KVM_INDIRECT_VECTORS def_bool KVM && (HARDEN_BRANCH_PREDICTOR || HARDEN_EL2_VECTORS) -source drivers/vhost/Kconfig +source "drivers/vhost/Kconfig" endif # VIRTUALIZATION diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 36773def6920..0ef105ac40f6 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -261,7 +261,7 @@ config HZ endif if !IA64_HP_SIM -source kernel/Kconfig.hz +source "kernel/Kconfig.hz" endif config IA64_BRL_EMU diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 1bc9f1ba759a..6f18c45f7703 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -123,11 +123,11 @@ config BOOTINFO_PROC menu "Platform setup" -source arch/m68k/Kconfig.cpu +source "arch/m68k/Kconfig.cpu" -source arch/m68k/Kconfig.machine +source "arch/m68k/Kconfig.machine" -source arch/m68k/Kconfig.bus +source "arch/m68k/Kconfig.bus" endmenu diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig index 76b93a9c8c9b..c36930226b7b 100644 --- a/arch/mips/kvm/Kconfig +++ b/arch/mips/kvm/Kconfig @@ -72,6 +72,6 @@ config KVM_MIPS_DEBUG_COP0_COUNTERS If unsure, say N. -source drivers/vhost/Kconfig +source "drivers/vhost/Kconfig" endif # VIRTUALIZATION diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig index 285f7d05c8ed..d765b4a5e05f 100644 --- a/arch/openrisc/Kconfig +++ b/arch/openrisc/Kconfig @@ -139,7 +139,7 @@ config SMP If you don't know what to do here, say N. -source kernel/Kconfig.hz +source "kernel/Kconfig.hz" config OPENRISC_NO_SPR_SR_DSX bool "use SPR_SR_DSX software emulation" if OR1K_1200 diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 8be31261aec8..e1307d66c76b 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -393,7 +393,7 @@ config HIGHMEM bool "High memory support" depends on PPC32 -source kernel/Kconfig.hz +source "kernel/Kconfig.hz" config HUGETLB_PAGE_SIZE_VARIABLE bool @@ -816,7 +816,7 @@ config ARCH_WANTS_FREEZER_CONTROL def_bool y depends on ADB_PMU -source kernel/power/Kconfig +source "kernel/power/Kconfig" config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index 68a0e9d5b440..bfdde04e4905 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -204,6 +204,6 @@ config KVM_XIVE default y depends on KVM_XICS && PPC_XIVE_NATIVE && KVM_BOOK3S_HV_POSSIBLE -source drivers/vhost/Kconfig +source "drivers/vhost/Kconfig" endif # VIRTUALIZATION diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 55da93f4e818..4f428ab4429f 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -287,6 +287,6 @@ endmenu menu "Power management options" -source kernel/power/Kconfig +source "kernel/power/Kconfig" endmenu diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 5173366af8f3..48de9d32b833 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -520,7 +520,7 @@ config SCHED_TOPOLOGY making when dealing with machines that have multi-threading, multiple cores or multiple books. -source kernel/Kconfig.hz +source "kernel/Kconfig.hz" config KEXEC def_bool y diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index a3dbd459cce9..767453faacfc 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -57,6 +57,6 @@ config KVM_S390_UCONTROL # OK, it's a little counter-intuitive to do this, but it puts it neatly under # the virtualization menu. -source drivers/vhost/Kconfig +source "drivers/vhost/Kconfig" endif # VIRTUALIZATION diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index f82a4da7adf3..b2581b14e464 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -597,7 +597,7 @@ endmenu menu "Kernel features" -source kernel/Kconfig.hz +source "kernel/Kconfig.hz" config KEXEC bool "kexec system call (EXPERIMENTAL)" diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 490b2c95c212..29b97f1dd9c5 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -187,7 +187,7 @@ config NR_CPUS default 32 if SPARC32 default 4096 if SPARC64 -source kernel/Kconfig.hz +source "kernel/Kconfig.hz" config RWSEM_GENERIC_SPINLOCK bool diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 9d734f3c8234..fd5ac1d89d74 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1975,7 +1975,7 @@ config SECCOMP If unsure, say Y. Only embedded should say N here. -source kernel/Kconfig.hz +source "kernel/Kconfig.hz" config KEXEC bool "kexec system call" diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 1bbec387d289..72fa955f4a15 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -98,6 +98,6 @@ config KVM_MMU_AUDIT # OK, it's a little counter-intuitive to do this, but it puts it neatly under # the virtualization menu. -source drivers/vhost/Kconfig +source "drivers/vhost/Kconfig" endif # VIRTUALIZATION diff --git a/block/Kconfig b/block/Kconfig index f7045aa47edb..1b9149810999 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -224,4 +224,4 @@ config BLK_MQ_RDMA config BLK_PM def_bool BLOCK && PM -source block/Kconfig.iosched +source "block/Kconfig.iosched" diff --git a/crypto/Kconfig b/crypto/Kconfig index f7a235db56aa..2c63b52a5fcc 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1826,7 +1826,7 @@ config CRYPTO_HASH_INFO bool source "drivers/crypto/Kconfig" -source crypto/asymmetric_keys/Kconfig -source certs/Kconfig +source "crypto/asymmetric_keys/Kconfig" +source "certs/Kconfig" endif # if CRYPTO diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index caa98a7fe392..4db49de39b3e 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -258,7 +258,7 @@ config CRYPTO_DEV_HIFN_795X_RNG Select this option if you want to enable the random number generator on the HIFN 795x crypto adapters. -source drivers/crypto/caam/Kconfig +source "drivers/crypto/caam/Kconfig" config CRYPTO_DEV_TALITOS tristate "Talitos Freescale Security Engine (SEC)" diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 33a458b7f1fc..148be8e1a090 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -131,5 +131,5 @@ config DRM_I915_GVT_KVMGT menu "drm/i915 Debugging" depends on DRM_I915 depends on EXPERT -source drivers/gpu/drm/i915/Kconfig.debug +source "drivers/gpu/drm/i915/Kconfig.debug" endmenu diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 81da17a42dc9..9ccbbd39b3c6 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -1306,7 +1306,7 @@ config SENSORS_PCF8591 These devices are hard to detect and rarely found on mainstream hardware. If unsure, say N. -source drivers/hwmon/pmbus/Kconfig +source "drivers/hwmon/pmbus/Kconfig" config SENSORS_PWM_FAN tristate "PWM fan" diff --git a/drivers/i2c/Kconfig b/drivers/i2c/Kconfig index efc3354d60ae..c6b7fc7b67d6 100644 --- a/drivers/i2c/Kconfig +++ b/drivers/i2c/Kconfig @@ -68,7 +68,7 @@ config I2C_MUX This support is also available as a module. If so, the module will be called i2c-mux. -source drivers/i2c/muxes/Kconfig +source "drivers/i2c/muxes/Kconfig" config I2C_HELPER_AUTO bool "Autoselect pertinent helper modules" @@ -94,8 +94,8 @@ config I2C_SMBUS This support is also available as a module. If so, the module will be called i2c-smbus. -source drivers/i2c/algos/Kconfig -source drivers/i2c/busses/Kconfig +source "drivers/i2c/algos/Kconfig" +source "drivers/i2c/busses/Kconfig" config I2C_STUB tristate "I2C/SMBus Test Stub" diff --git a/drivers/pps/Kconfig b/drivers/pps/Kconfig index c6008f296605..965aa086a1e0 100644 --- a/drivers/pps/Kconfig +++ b/drivers/pps/Kconfig @@ -37,8 +37,8 @@ config NTP_PPS It doesn't work on tickless systems at the moment. -source drivers/pps/clients/Kconfig +source "drivers/pps/clients/Kconfig" -source drivers/pps/generators/Kconfig +source "drivers/pps/generators/Kconfig" endif # PPS diff --git a/drivers/ras/Kconfig b/drivers/ras/Kconfig index 4c3c67d13254..b834ff555188 100644 --- a/drivers/ras/Kconfig +++ b/drivers/ras/Kconfig @@ -30,6 +30,6 @@ menuconfig RAS if RAS -source arch/x86/ras/Kconfig +source "arch/x86/ras/Kconfig" endif diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 5422523c03f8..5fbfabbf627b 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -383,7 +383,7 @@ config INTEL_QUARK_DTS_THERMAL underlying BIOS/Firmware. menu "ACPI INT340X thermal drivers" -source drivers/thermal/int340x_thermal/Kconfig +source "drivers/thermal/int340x_thermal/Kconfig" endmenu config INTEL_BXT_PMIC_THERMAL diff --git a/drivers/w1/Kconfig b/drivers/w1/Kconfig index 6743bde038cc..dbb41f45af8a 100644 --- a/drivers/w1/Kconfig +++ b/drivers/w1/Kconfig @@ -25,7 +25,7 @@ config W1_CON 2. Userspace commands. Includes read/write and search/alarm search commands. 3. Replies to userspace commands. -source drivers/w1/masters/Kconfig -source drivers/w1/slaves/Kconfig +source "drivers/w1/masters/Kconfig" +source "drivers/w1/slaves/Kconfig" endif # W1 diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 1af29b8224fd..08a95da1adcc 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -439,7 +439,7 @@ config DEBUG_KERNEL menu "Memory Debugging" -source mm/Kconfig.debug +source "mm/Kconfig.debug" config DEBUG_OBJECTS bool "Debug object operations" @@ -1609,7 +1609,7 @@ config LATENCYTOP Enable this option if you want to use the LatencyTOP tool to find out which userspace is blocking on what kernel operations. -source kernel/trace/Kconfig +source "kernel/trace/Kconfig" config PROVIDE_OHCI1394_DMA_INIT bool "Remote debugging over FireWire early on boot" diff --git a/security/Kconfig b/security/Kconfig index d9aa521b5206..e4fe2f3c2c65 100644 --- a/security/Kconfig +++ b/security/Kconfig @@ -4,7 +4,7 @@ menu "Security options" -source security/keys/Kconfig +source "security/keys/Kconfig" config SECURITY_DMESG_RESTRICT bool "Restrict unprivileged access to the kernel syslog" @@ -230,14 +230,14 @@ config STATIC_USERMODEHELPER_PATH If you wish for all usermode helper programs to be disabled, specify an empty string here (i.e. ""). -source security/selinux/Kconfig -source security/smack/Kconfig -source security/tomoyo/Kconfig -source security/apparmor/Kconfig -source security/loadpin/Kconfig -source security/yama/Kconfig +source "security/selinux/Kconfig" +source "security/smack/Kconfig" +source "security/tomoyo/Kconfig" +source "security/apparmor/Kconfig" +source "security/loadpin/Kconfig" +source "security/yama/Kconfig" -source security/integrity/Kconfig +source "security/integrity/Kconfig" choice prompt "Default security module" diff --git a/security/integrity/Kconfig b/security/integrity/Kconfig index da9565891738..877af1f2d904 100644 --- a/security/integrity/Kconfig +++ b/security/integrity/Kconfig @@ -66,7 +66,7 @@ config INTEGRITY_AUDIT be enabled by specifying 'integrity_audit=1' on the kernel command line. -source security/integrity/ima/Kconfig -source security/integrity/evm/Kconfig +source "security/integrity/ima/Kconfig" +source "security/integrity/evm/Kconfig" endif # if INTEGRITY -- cgit v1.2.3 From 0464ed24380905d640030d368cd84a4e4d1e15e2 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Fri, 19 Oct 2018 15:21:08 +1100 Subject: seq_buf: Make seq_buf_puts() null-terminate the buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently seq_buf_puts() will happily create a non null-terminated string for you in the buffer. This is particularly dangerous if the buffer is on the stack. For example: char buf[8]; char secret = "secret"; struct seq_buf s; seq_buf_init(&s, buf, sizeof(buf)); seq_buf_puts(&s, "foo"); printk("Message is %s\n", buf); Can result in: Message is fooªªªªªsecret We could require all users to memset() their buffer to zero before use. But that seems likely to be forgotten and lead to bugs. Instead we can change seq_buf_puts() to always leave the buffer in a null-terminated state. The only downside is that this makes the buffer 1 character smaller for seq_buf_puts(), but that seems like a good trade off. Link: http://lkml.kernel.org/r/20181019042109.8064-1-mpe@ellerman.id.au Acked-by: Kees Cook Signed-off-by: Michael Ellerman Signed-off-by: Steven Rostedt (VMware) --- lib/seq_buf.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 11f2ae0f9099..6aabb609dd87 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -144,9 +144,13 @@ int seq_buf_puts(struct seq_buf *s, const char *str) WARN_ON(s->size == 0); + /* Add 1 to len for the trailing null byte which must be there */ + len += 1; + if (seq_buf_can_fit(s, len)) { memcpy(s->buffer + s->len, str, len); - s->len += len; + /* Don't count the trailing null byte against the capacity */ + s->len += len - 1; return 0; } seq_buf_set_overflow(s); -- cgit v1.2.3 From 29924e5030969c55dbe68074215be5a1f14f1ff1 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Fri, 19 Oct 2018 15:21:09 +1100 Subject: seq_buf: Use size_t for len in seq_buf_puts() Jann Horn points out that we're using unsigned int for len in seq_buf_puts(), which could potentially overflow if we're passed a UINT_MAX sized string. The rest of the code already uses size_t, so we should also use that in seq_buf_puts() to avoid any issues. Link: http://lkml.kernel.org/r/20181019042109.8064-2-mpe@ellerman.id.au Suggested-by: Jann Horn Signed-off-by: Michael Ellerman Signed-off-by: Steven Rostedt (VMware) --- lib/seq_buf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 6aabb609dd87..bd807f545a9d 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -140,7 +140,7 @@ int seq_buf_bprintf(struct seq_buf *s, const char *fmt, const u32 *binary) */ int seq_buf_puts(struct seq_buf *s, const char *str) { - unsigned int len = strlen(str); + size_t len = strlen(str); WARN_ON(s->size == 0); -- cgit v1.2.3 From 2bd926b439b4cb6b9ed240a9781cd01958b53d85 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Fri, 28 Dec 2018 00:29:53 -0800 Subject: kasan: add CONFIG_KASAN_GENERIC and CONFIG_KASAN_SW_TAGS This commit splits the current CONFIG_KASAN config option into two: 1. CONFIG_KASAN_GENERIC, that enables the generic KASAN mode (the one that exists now); 2. CONFIG_KASAN_SW_TAGS, that enables the software tag-based KASAN mode. The name CONFIG_KASAN_SW_TAGS is chosen as in the future we will have another hardware tag-based KASAN mode, that will rely on hardware memory tagging support in arm64. With CONFIG_KASAN_SW_TAGS enabled, compiler options are changed to instrument kernel files with -fsantize=kernel-hwaddress (except the ones for which KASAN_SANITIZE := n is set). Both CONFIG_KASAN_GENERIC and CONFIG_KASAN_SW_TAGS support both CONFIG_KASAN_INLINE and CONFIG_KASAN_OUTLINE instrumentation modes. This commit also adds empty placeholder (for now) implementation of tag-based KASAN specific hooks inserted by the compiler and adjusts common hooks implementation. While this commit adds the CONFIG_KASAN_SW_TAGS config option, this option is not selectable, as it depends on HAVE_ARCH_KASAN_SW_TAGS, which we will enable once all the infrastracture code has been added. Link: http://lkml.kernel.org/r/b2550106eb8a68b10fefbabce820910b115aa853.1544099024.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Andrey Ryabinin Reviewed-by: Dmitry Vyukov Cc: Christoph Lameter Cc: Mark Rutland Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/compiler-clang.h | 6 ++- include/linux/compiler-gcc.h | 6 +++ include/linux/compiler_attributes.h | 13 ----- include/linux/kasan.h | 16 ++++-- lib/Kconfig.kasan | 98 ++++++++++++++++++++++++++++--------- mm/kasan/Makefile | 6 ++- mm/kasan/generic.c | 2 +- mm/kasan/kasan.h | 3 +- mm/kasan/tags.c | 75 ++++++++++++++++++++++++++++ mm/slub.c | 2 +- scripts/Makefile.kasan | 53 +++++++++++--------- 11 files changed, 214 insertions(+), 66 deletions(-) create mode 100644 mm/kasan/tags.c (limited to 'lib') diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index 3e7dafb3ea80..39f668d5066b 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -16,9 +16,13 @@ /* all clang versions usable with the kernel support KASAN ABI version 5 */ #define KASAN_ABI_VERSION 5 +#if __has_feature(address_sanitizer) || __has_feature(hwaddress_sanitizer) /* emulate gcc's __SANITIZE_ADDRESS__ flag */ -#if __has_feature(address_sanitizer) #define __SANITIZE_ADDRESS__ +#define __no_sanitize_address \ + __attribute__((no_sanitize("address", "hwaddress"))) +#else +#define __no_sanitize_address #endif /* diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h index 2010493e1040..5776da43da97 100644 --- a/include/linux/compiler-gcc.h +++ b/include/linux/compiler-gcc.h @@ -143,6 +143,12 @@ #define KASAN_ABI_VERSION 3 #endif +#if __has_attribute(__no_sanitize_address__) +#define __no_sanitize_address __attribute__((no_sanitize_address)) +#else +#define __no_sanitize_address +#endif + #if GCC_VERSION >= 50100 #define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1 #endif diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index fe07b680dd4a..19f32b0c29af 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -199,19 +199,6 @@ */ #define __noreturn __attribute__((__noreturn__)) -/* - * Optional: only supported since gcc >= 4.8 - * Optional: not supported by icc - * - * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-no_005fsanitize_005faddress-function-attribute - * clang: https://clang.llvm.org/docs/AttributeReference.html#no-sanitize-address-no-address-safety-analysis - */ -#if __has_attribute(__no_sanitize_address__) -# define __no_sanitize_address __attribute__((__no_sanitize_address__)) -#else -# define __no_sanitize_address -#endif - /* * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Type-Attributes.html#index-packed-type-attribute * clang: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-packed-variable-attribute diff --git a/include/linux/kasan.h b/include/linux/kasan.h index 52c86a568a4e..b66fdf5ea7ab 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -45,8 +45,6 @@ void kasan_free_pages(struct page *page, unsigned int order); void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, slab_flags_t *flags); -void kasan_cache_shrink(struct kmem_cache *cache); -void kasan_cache_shutdown(struct kmem_cache *cache); void kasan_poison_slab(struct page *page); void kasan_unpoison_object_data(struct kmem_cache *cache, void *object); @@ -97,8 +95,6 @@ static inline void kasan_free_pages(struct page *page, unsigned int order) {} static inline void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, slab_flags_t *flags) {} -static inline void kasan_cache_shrink(struct kmem_cache *cache) {} -static inline void kasan_cache_shutdown(struct kmem_cache *cache) {} static inline void kasan_poison_slab(struct page *page) {} static inline void kasan_unpoison_object_data(struct kmem_cache *cache, @@ -155,4 +151,16 @@ static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; } #endif /* CONFIG_KASAN */ +#ifdef CONFIG_KASAN_GENERIC + +void kasan_cache_shrink(struct kmem_cache *cache); +void kasan_cache_shutdown(struct kmem_cache *cache); + +#else /* CONFIG_KASAN_GENERIC */ + +static inline void kasan_cache_shrink(struct kmem_cache *cache) {} +static inline void kasan_cache_shutdown(struct kmem_cache *cache) {} + +#endif /* CONFIG_KASAN_GENERIC */ + #endif /* LINUX_KASAN_H */ diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index d0bad1bd9a2b..d8c474b6691e 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -1,36 +1,92 @@ +# This config refers to the generic KASAN mode. config HAVE_ARCH_KASAN bool -if HAVE_ARCH_KASAN +config HAVE_ARCH_KASAN_SW_TAGS + bool + +config CC_HAS_KASAN_GENERIC + def_bool $(cc-option, -fsanitize=kernel-address) + +config CC_HAS_KASAN_SW_TAGS + def_bool $(cc-option, -fsanitize=kernel-hwaddress) config KASAN - bool "KASan: runtime memory debugger" + bool "KASAN: runtime memory debugger" + depends on (HAVE_ARCH_KASAN && CC_HAS_KASAN_GENERIC) || \ + (HAVE_ARCH_KASAN_SW_TAGS && CC_HAS_KASAN_SW_TAGS) + depends on (SLUB && SYSFS) || (SLAB && !DEBUG_SLAB) + help + Enables KASAN (KernelAddressSANitizer) - runtime memory debugger, + designed to find out-of-bounds accesses and use-after-free bugs. + See Documentation/dev-tools/kasan.rst for details. + +choice + prompt "KASAN mode" + depends on KASAN + default KASAN_GENERIC + help + KASAN has two modes: generic KASAN (similar to userspace ASan, + x86_64/arm64/xtensa, enabled with CONFIG_KASAN_GENERIC) and + software tag-based KASAN (a version based on software memory + tagging, arm64 only, similar to userspace HWASan, enabled with + CONFIG_KASAN_SW_TAGS). + Both generic and tag-based KASAN are strictly debugging features. + +config KASAN_GENERIC + bool "Generic mode" + depends on HAVE_ARCH_KASAN && CC_HAS_KASAN_GENERIC depends on (SLUB && SYSFS) || (SLAB && !DEBUG_SLAB) select SLUB_DEBUG if SLUB select CONSTRUCTORS select STACKDEPOT help - Enables kernel address sanitizer - runtime memory debugger, - designed to find out-of-bounds accesses and use-after-free bugs. - This is strictly a debugging feature and it requires a gcc version - of 4.9.2 or later. Detection of out of bounds accesses to stack or - global variables requires gcc 5.0 or later. - This feature consumes about 1/8 of available memory and brings about - ~x3 performance slowdown. + Enables generic KASAN mode. + Supported in both GCC and Clang. With GCC it requires version 4.9.2 + or later for basic support and version 5.0 or later for detection of + out-of-bounds accesses for stack and global variables and for inline + instrumentation mode (CONFIG_KASAN_INLINE). With Clang it requires + version 3.7.0 or later and it doesn't support detection of + out-of-bounds accesses for global variables yet. + This mode consumes about 1/8th of available memory at kernel start + and introduces an overhead of ~x1.5 for the rest of the allocations. + The performance slowdown is ~x3. For better error detection enable CONFIG_STACKTRACE. - Currently CONFIG_KASAN doesn't work with CONFIG_DEBUG_SLAB + Currently CONFIG_KASAN_GENERIC doesn't work with CONFIG_DEBUG_SLAB (the resulting kernel does not boot). +config KASAN_SW_TAGS + bool "Software tag-based mode" + depends on HAVE_ARCH_KASAN_SW_TAGS && CC_HAS_KASAN_SW_TAGS + depends on (SLUB && SYSFS) || (SLAB && !DEBUG_SLAB) + select SLUB_DEBUG if SLUB + select CONSTRUCTORS + select STACKDEPOT + help + Enables software tag-based KASAN mode. + This mode requires Top Byte Ignore support by the CPU and therefore + is only supported for arm64. + This mode requires Clang version 7.0.0 or later. + This mode consumes about 1/16th of available memory at kernel start + and introduces an overhead of ~20% for the rest of the allocations. + This mode may potentially introduce problems relating to pointer + casting and comparison, as it embeds tags into the top byte of each + pointer. + For better error detection enable CONFIG_STACKTRACE. + Currently CONFIG_KASAN_SW_TAGS doesn't work with CONFIG_DEBUG_SLAB + (the resulting kernel does not boot). + +endchoice + config KASAN_EXTRA - bool "KAsan: extra checks" - depends on KASAN && DEBUG_KERNEL && !COMPILE_TEST + bool "KASAN: extra checks" + depends on KASAN_GENERIC && DEBUG_KERNEL && !COMPILE_TEST help - This enables further checks in the kernel address sanitizer, for now - it only includes the address-use-after-scope check that can lead - to excessive kernel stack usage, frame size warnings and longer + This enables further checks in generic KASAN, for now it only + includes the address-use-after-scope check that can lead to + excessive kernel stack usage, frame size warnings and longer compile time. - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 has more - + See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 choice prompt "Instrumentation type" @@ -53,7 +109,7 @@ config KASAN_INLINE memory accesses. This is faster than outline (in some workloads it gives about x2 boost over outline instrumentation), but make kernel's .text size much bigger. - This requires a gcc version of 5.0 or later. + For CONFIG_KASAN_GENERIC this requires GCC 5.0 or later. endchoice @@ -67,11 +123,9 @@ config KASAN_S390_4_LEVEL_PAGING 4-level paging instead. config TEST_KASAN - tristate "Module for testing kasan for bug detection" + tristate "Module for testing KASAN for bug detection" depends on m && KASAN help This is a test module doing various nasty things like out of bounds accesses, use after free. It is useful for testing - kernel debugging features like kernel address sanitizer. - -endif + kernel debugging features like KASAN. diff --git a/mm/kasan/Makefile b/mm/kasan/Makefile index d643530b24aa..68ba1822f003 100644 --- a/mm/kasan/Makefile +++ b/mm/kasan/Makefile @@ -2,6 +2,7 @@ KASAN_SANITIZE := n UBSAN_SANITIZE_common.o := n UBSAN_SANITIZE_generic.o := n +UBSAN_SANITIZE_tags.o := n KCOV_INSTRUMENT := n CFLAGS_REMOVE_generic.o = -pg @@ -10,5 +11,8 @@ CFLAGS_REMOVE_generic.o = -pg CFLAGS_common.o := $(call cc-option, -fno-conserve-stack -fno-stack-protector) CFLAGS_generic.o := $(call cc-option, -fno-conserve-stack -fno-stack-protector) +CFLAGS_tags.o := $(call cc-option, -fno-conserve-stack -fno-stack-protector) -obj-y := common.o generic.o report.o init.o quarantine.o +obj-$(CONFIG_KASAN) := common.o init.o report.o +obj-$(CONFIG_KASAN_GENERIC) += generic.o quarantine.o +obj-$(CONFIG_KASAN_SW_TAGS) += tags.o diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index 44ec228de0a2..b8de6d33c55c 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -1,5 +1,5 @@ /* - * This file contains core KASAN code. + * This file contains core generic KASAN code. * * Copyright (c) 2014 Samsung Electronics Co., Ltd. * Author: Andrey Ryabinin diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index 659463800f10..19b950eaccff 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -114,7 +114,8 @@ void kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip); void kasan_report_invalid_free(void *object, unsigned long ip); -#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB) +#if defined(CONFIG_KASAN_GENERIC) && \ + (defined(CONFIG_SLAB) || defined(CONFIG_SLUB)) void quarantine_put(struct kasan_free_meta *info, struct kmem_cache *cache); void quarantine_reduce(void); void quarantine_remove_cache(struct kmem_cache *cache); diff --git a/mm/kasan/tags.c b/mm/kasan/tags.c new file mode 100644 index 000000000000..04194923c543 --- /dev/null +++ b/mm/kasan/tags.c @@ -0,0 +1,75 @@ +/* + * This file contains core tag-based KASAN code. + * + * Copyright (c) 2018 Google, Inc. + * Author: Andrey Konovalov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define DISABLE_BRANCH_PROFILING + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kasan.h" +#include "../slab.h" + +void check_memory_region(unsigned long addr, size_t size, bool write, + unsigned long ret_ip) +{ +} + +#define DEFINE_HWASAN_LOAD_STORE(size) \ + void __hwasan_load##size##_noabort(unsigned long addr) \ + { \ + } \ + EXPORT_SYMBOL(__hwasan_load##size##_noabort); \ + void __hwasan_store##size##_noabort(unsigned long addr) \ + { \ + } \ + EXPORT_SYMBOL(__hwasan_store##size##_noabort) + +DEFINE_HWASAN_LOAD_STORE(1); +DEFINE_HWASAN_LOAD_STORE(2); +DEFINE_HWASAN_LOAD_STORE(4); +DEFINE_HWASAN_LOAD_STORE(8); +DEFINE_HWASAN_LOAD_STORE(16); + +void __hwasan_loadN_noabort(unsigned long addr, unsigned long size) +{ +} +EXPORT_SYMBOL(__hwasan_loadN_noabort); + +void __hwasan_storeN_noabort(unsigned long addr, unsigned long size) +{ +} +EXPORT_SYMBOL(__hwasan_storeN_noabort); + +void __hwasan_tag_memory(unsigned long addr, u8 tag, unsigned long size) +{ +} +EXPORT_SYMBOL(__hwasan_tag_memory); diff --git a/mm/slub.c b/mm/slub.c index 8561a32910dd..e739d46600b9 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2992,7 +2992,7 @@ static __always_inline void slab_free(struct kmem_cache *s, struct page *page, do_slab_free(s, page, head, tail, cnt, addr); } -#ifdef CONFIG_KASAN +#ifdef CONFIG_KASAN_GENERIC void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr) { do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr); diff --git a/scripts/Makefile.kasan b/scripts/Makefile.kasan index 69552a39951d..25c259df8ffa 100644 --- a/scripts/Makefile.kasan +++ b/scripts/Makefile.kasan @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -ifdef CONFIG_KASAN +ifdef CONFIG_KASAN_GENERIC + ifdef CONFIG_KASAN_INLINE call_threshold := 10000 else @@ -12,36 +13,44 @@ CFLAGS_KASAN_MINIMAL := -fsanitize=kernel-address cc-param = $(call cc-option, -mllvm -$(1), $(call cc-option, --param $(1))) -ifeq ($(call cc-option, $(CFLAGS_KASAN_MINIMAL) -Werror),) - ifneq ($(CONFIG_COMPILE_TEST),y) - $(warning Cannot use CONFIG_KASAN: \ - -fsanitize=kernel-address is not supported by compiler) - endif -else - # -fasan-shadow-offset fails without -fsanitize - CFLAGS_KASAN_SHADOW := $(call cc-option, -fsanitize=kernel-address \ +# -fasan-shadow-offset fails without -fsanitize +CFLAGS_KASAN_SHADOW := $(call cc-option, -fsanitize=kernel-address \ -fasan-shadow-offset=$(KASAN_SHADOW_OFFSET), \ $(call cc-option, -fsanitize=kernel-address \ -mllvm -asan-mapping-offset=$(KASAN_SHADOW_OFFSET))) - ifeq ($(strip $(CFLAGS_KASAN_SHADOW)),) - CFLAGS_KASAN := $(CFLAGS_KASAN_MINIMAL) - else - # Now add all the compiler specific options that are valid standalone - CFLAGS_KASAN := $(CFLAGS_KASAN_SHADOW) \ - $(call cc-param,asan-globals=1) \ - $(call cc-param,asan-instrumentation-with-call-threshold=$(call_threshold)) \ - $(call cc-param,asan-stack=1) \ - $(call cc-param,asan-use-after-scope=1) \ - $(call cc-param,asan-instrument-allocas=1) - endif - +ifeq ($(strip $(CFLAGS_KASAN_SHADOW)),) + CFLAGS_KASAN := $(CFLAGS_KASAN_MINIMAL) +else + # Now add all the compiler specific options that are valid standalone + CFLAGS_KASAN := $(CFLAGS_KASAN_SHADOW) \ + $(call cc-param,asan-globals=1) \ + $(call cc-param,asan-instrumentation-with-call-threshold=$(call_threshold)) \ + $(call cc-param,asan-stack=1) \ + $(call cc-param,asan-use-after-scope=1) \ + $(call cc-param,asan-instrument-allocas=1) endif ifdef CONFIG_KASAN_EXTRA CFLAGS_KASAN += $(call cc-option, -fsanitize-address-use-after-scope) endif -CFLAGS_KASAN_NOSANITIZE := -fno-builtin +endif # CONFIG_KASAN_GENERIC +ifdef CONFIG_KASAN_SW_TAGS + +ifdef CONFIG_KASAN_INLINE + instrumentation_flags := -mllvm -hwasan-mapping-offset=$(KASAN_SHADOW_OFFSET) +else + instrumentation_flags := -mllvm -hwasan-instrument-with-calls=1 +endif + +CFLAGS_KASAN := -fsanitize=kernel-hwaddress \ + -mllvm -hwasan-instrument-stack=0 \ + $(instrumentation_flags) + +endif # CONFIG_KASAN_SW_TAGS + +ifdef CONFIG_KASAN +CFLAGS_KASAN_NOSANITIZE := -fno-builtin endif -- cgit v1.2.3 From a9ee3a63dbfff5237bc682b88c02d91a3c798e35 Mon Sep 17 00:00:00 2001 From: Qian Cai Date: Fri, 28 Dec 2018 00:32:32 -0800 Subject: debugobjects: call debug_objects_mem_init eariler The current value of the early boot static pool size, 1024 is not big enough for systems with large number of CPUs with timer or/and workqueue objects selected. As the results, systems have 60+ CPUs with both timer and workqueue objects enabled could trigger "ODEBUG: Out of memory. ODEBUG disabled". Some debug objects are allocated during the early boot. Enabling some options like timers or workqueue objects may increase the size required significantly with large number of CPUs. For example, CONFIG_DEBUG_OBJECTS_TIMERS: No. CPUs x 2 (worker pool) objects: start_kernel workqueue_init_early init_worker_pool init_timer_key debug_object_init plus No. CPUs objects (CONFIG_HIGH_RES_TIMERS): sched_init hrtick_rq_init hrtimer_init CONFIG_DEBUG_OBJECTS_WORK: No. CPUs objects: vmalloc_init __init_work plus No. CPUs x 6 (workqueue) objects: workqueue_init_early alloc_workqueue __alloc_workqueue_key alloc_and_link_pwqs init_pwq Also, plus No. CPUs objects: perf_event_init __init_srcu_struct init_srcu_struct_fields init_srcu_struct_nodes __init_work However, none of the things are actually used or required before debug_objects_mem_init() is invoked, so just move the call right before vmalloc_init(). According to tglx, "the reason why the call is at this place in start_kernel() is historical. It's because back in the days when debugobjects were added the memory allocator was enabled way later than today." Link: http://lkml.kernel.org/r/20181126102407.1836-1-cai@gmx.us Signed-off-by: Qian Cai Suggested-by: Thomas Gleixner Cc: Waiman Long Cc: Yang Shi Cc: Arnd Bergmann Cc: Catalin Marinas Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- init/main.c | 2 +- lib/debugobjects.c | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/init/main.c b/init/main.c index 954d9b6c62c6..f6e901ec6b78 100644 --- a/init/main.c +++ b/init/main.c @@ -521,6 +521,7 @@ static void __init mm_init(void) mem_init(); kmem_cache_init(); pgtable_init(); + debug_objects_mem_init(); vmalloc_init(); ioremap_huge_init(); /* Should be run before the first non-init thread is created */ @@ -697,7 +698,6 @@ asmlinkage __visible void __init start_kernel(void) #endif page_ext_init(); kmemleak_init(); - debug_objects_mem_init(); setup_per_cpu_pageset(); numa_policy_init(); acpi_early_init(); diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 14afeeb7d6ef..55437fd5128b 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -1131,11 +1131,10 @@ static int __init debug_objects_replace_static_objects(void) } /* - * When debug_objects_mem_init() is called we know that only - * one CPU is up, so disabling interrupts is enough - * protection. This avoids the lockdep hell of lock ordering. + * debug_objects_mem_init() is now called early that only one CPU is up + * and interrupts have been disabled, so it is safe to replace the + * active object references. */ - local_irq_disable(); /* Remove the statically allocated objects from the pool */ hlist_for_each_entry_safe(obj, tmp, &obj_pool, node) @@ -1156,7 +1155,6 @@ static int __init debug_objects_replace_static_objects(void) cnt++; } } - local_irq_enable(); pr_debug("%d of %d active objects replaced\n", cnt, obj_pool_used); -- cgit v1.2.3 From 9705bea5f833f4fc21d5bef5fce7348427f76ea4 Mon Sep 17 00:00:00 2001 From: Arun KS Date: Fri, 28 Dec 2018 00:34:24 -0800 Subject: mm: convert zone->managed_pages to atomic variable totalram_pages, zone->managed_pages and totalhigh_pages updates are protected by managed_page_count_lock, but readers never care about it. Convert these variables to atomic to avoid readers potentially seeing a store tear. This patch converts zone->managed_pages. Subsequent patches will convert totalram_panges, totalhigh_pages and eventually managed_page_count_lock will be removed. Main motivation was that managed_page_count_lock handling was complicating things. It was discussed in length here, https://lore.kernel.org/patchwork/patch/995739/#1181785 So it seemes better to remove the lock and convert variables to atomic, with preventing poteintial store-to-read tearing as a bonus. Link: http://lkml.kernel.org/r/1542090790-21750-3-git-send-email-arunks@codeaurora.org Signed-off-by: Arun KS Suggested-by: Michal Hocko Suggested-by: Vlastimil Babka Reviewed-by: Konstantin Khlebnikov Reviewed-by: David Hildenbrand Acked-by: Michal Hocko Acked-by: Vlastimil Babka Reviewed-by: Pavel Tatashin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 2 +- include/linux/mmzone.h | 9 +++++-- lib/show_mem.c | 2 +- mm/memblock.c | 2 +- mm/page_alloc.c | 44 +++++++++++++++++------------------ mm/vmstat.c | 4 ++-- 6 files changed, 34 insertions(+), 29 deletions(-) (limited to 'lib') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index c02adbbeef2a..b7bc7d7d048f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -853,7 +853,7 @@ static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size, */ pgdat = NODE_DATA(numa_node_id); for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) - mem_in_bytes += pgdat->node_zones[zone_type].managed_pages; + mem_in_bytes += zone_managed_pages(&pgdat->node_zones[zone_type]); mem_in_bytes <<= PAGE_SHIFT; sub_type_hdr->length_low = lower_32_bits(mem_in_bytes); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 077d797d1f60..a23e34e21178 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -435,7 +435,7 @@ struct zone { * adjust_managed_page_count() should be used instead of directly * touching zone->managed_pages and totalram_pages. */ - unsigned long managed_pages; + atomic_long_t managed_pages; unsigned long spanned_pages; unsigned long present_pages; @@ -524,6 +524,11 @@ enum pgdat_flags { PGDAT_RECLAIM_LOCKED, /* prevents concurrent reclaim */ }; +static inline unsigned long zone_managed_pages(struct zone *zone) +{ + return (unsigned long)atomic_long_read(&zone->managed_pages); +} + static inline unsigned long zone_end_pfn(const struct zone *zone) { return zone->zone_start_pfn + zone->spanned_pages; @@ -820,7 +825,7 @@ static inline bool is_dev_zone(const struct zone *zone) */ static inline bool managed_zone(struct zone *zone) { - return zone->managed_pages; + return zone_managed_pages(zone); } /* Returns true if a zone has memory */ diff --git a/lib/show_mem.c b/lib/show_mem.c index 0beaa1d899aa..eefe67d50e84 100644 --- a/lib/show_mem.c +++ b/lib/show_mem.c @@ -28,7 +28,7 @@ void show_mem(unsigned int filter, nodemask_t *nodemask) continue; total += zone->present_pages; - reserved += zone->present_pages - zone->managed_pages; + reserved += zone->present_pages - zone_managed_pages(zone); if (is_highmem_idx(zoneid)) highmem += zone->present_pages; diff --git a/mm/memblock.c b/mm/memblock.c index 81ae63ca78d0..0068f87af1e8 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1950,7 +1950,7 @@ void reset_node_managed_pages(pg_data_t *pgdat) struct zone *z; for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) - z->managed_pages = 0; + atomic_long_set(&z->managed_pages, 0); } void __init reset_all_zones_managed_pages(void) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index b79e79caea99..4b5c4ff68f18 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1280,7 +1280,7 @@ static void __init __free_pages_boot_core(struct page *page, unsigned int order) __ClearPageReserved(p); set_page_count(p, 0); - page_zone(page)->managed_pages += nr_pages; + atomic_long_add(nr_pages, &page_zone(page)->managed_pages); set_page_refcounted(page); __free_pages(page, order); } @@ -2259,7 +2259,7 @@ static void reserve_highatomic_pageblock(struct page *page, struct zone *zone, * Limit the number reserved to 1 pageblock or roughly 1% of a zone. * Check is race-prone but harmless. */ - max_managed = (zone->managed_pages / 100) + pageblock_nr_pages; + max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages; if (zone->nr_reserved_highatomic >= max_managed) return; @@ -4661,7 +4661,7 @@ static unsigned long nr_free_zone_pages(int offset) struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); for_each_zone_zonelist(zone, z, zonelist, offset) { - unsigned long size = zone->managed_pages; + unsigned long size = zone_managed_pages(zone); unsigned long high = high_wmark_pages(zone); if (size > high) sum += size - high; @@ -4768,7 +4768,7 @@ void si_meminfo_node(struct sysinfo *val, int nid) pg_data_t *pgdat = NODE_DATA(nid); for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) - managed_pages += pgdat->node_zones[zone_type].managed_pages; + managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]); val->totalram = managed_pages; val->sharedram = node_page_state(pgdat, NR_SHMEM); val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES); @@ -4777,7 +4777,7 @@ void si_meminfo_node(struct sysinfo *val, int nid) struct zone *zone = &pgdat->node_zones[zone_type]; if (is_highmem(zone)) { - managed_highpages += zone->managed_pages; + managed_highpages += zone_managed_pages(zone); free_highpages += zone_page_state(zone, NR_FREE_PAGES); } } @@ -4984,7 +4984,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)), K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)), K(zone->present_pages), - K(zone->managed_pages), + K(zone_managed_pages(zone)), K(zone_page_state(zone, NR_MLOCK)), zone_page_state(zone, NR_KERNEL_STACK_KB), K(zone_page_state(zone, NR_PAGETABLE)), @@ -5656,7 +5656,7 @@ static int zone_batchsize(struct zone *zone) * The per-cpu-pages pools are set to around 1000th of the * size of the zone. */ - batch = zone->managed_pages / 1024; + batch = zone_managed_pages(zone) / 1024; /* But no more than a meg. */ if (batch * PAGE_SIZE > 1024 * 1024) batch = (1024 * 1024) / PAGE_SIZE; @@ -5766,7 +5766,7 @@ static void pageset_set_high_and_batch(struct zone *zone, { if (percpu_pagelist_fraction) pageset_set_high(pcp, - (zone->managed_pages / + (zone_managed_pages(zone) / percpu_pagelist_fraction)); else pageset_set_batch(pcp, zone_batchsize(zone)); @@ -6323,7 +6323,7 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat) static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, unsigned long remaining_pages) { - zone->managed_pages = remaining_pages; + atomic_long_set(&zone->managed_pages, remaining_pages); zone_set_nid(zone, nid); zone->name = zone_names[idx]; zone->zone_pgdat = NODE_DATA(nid); @@ -7076,7 +7076,7 @@ early_param("movablecore", cmdline_parse_movablecore); void adjust_managed_page_count(struct page *page, long count) { spin_lock(&managed_page_count_lock); - page_zone(page)->managed_pages += count; + atomic_long_add(count, &page_zone(page)->managed_pages); totalram_pages += count; #ifdef CONFIG_HIGHMEM if (PageHighMem(page)) @@ -7124,7 +7124,7 @@ void free_highmem_page(struct page *page) { __free_reserved_page(page); totalram_pages++; - page_zone(page)->managed_pages++; + atomic_long_inc(&page_zone(page)->managed_pages); totalhigh_pages++; } #endif @@ -7257,7 +7257,7 @@ static void calculate_totalreserve_pages(void) for (i = 0; i < MAX_NR_ZONES; i++) { struct zone *zone = pgdat->node_zones + i; long max = 0; - unsigned long managed_pages = zone->managed_pages; + unsigned long managed_pages = zone_managed_pages(zone); /* Find valid and maximum lowmem_reserve in the zone */ for (j = i; j < MAX_NR_ZONES; j++) { @@ -7293,7 +7293,7 @@ static void setup_per_zone_lowmem_reserve(void) for_each_online_pgdat(pgdat) { for (j = 0; j < MAX_NR_ZONES; j++) { struct zone *zone = pgdat->node_zones + j; - unsigned long managed_pages = zone->managed_pages; + unsigned long managed_pages = zone_managed_pages(zone); zone->lowmem_reserve[j] = 0; @@ -7311,7 +7311,7 @@ static void setup_per_zone_lowmem_reserve(void) lower_zone->lowmem_reserve[j] = managed_pages / sysctl_lowmem_reserve_ratio[idx]; } - managed_pages += lower_zone->managed_pages; + managed_pages += zone_managed_pages(lower_zone); } } } @@ -7330,14 +7330,14 @@ static void __setup_per_zone_wmarks(void) /* Calculate total number of !ZONE_HIGHMEM pages */ for_each_zone(zone) { if (!is_highmem(zone)) - lowmem_pages += zone->managed_pages; + lowmem_pages += zone_managed_pages(zone); } for_each_zone(zone) { u64 tmp; spin_lock_irqsave(&zone->lock, flags); - tmp = (u64)pages_min * zone->managed_pages; + tmp = (u64)pages_min * zone_managed_pages(zone); do_div(tmp, lowmem_pages); if (is_highmem(zone)) { /* @@ -7351,7 +7351,7 @@ static void __setup_per_zone_wmarks(void) */ unsigned long min_pages; - min_pages = zone->managed_pages / 1024; + min_pages = zone_managed_pages(zone) / 1024; min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); zone->watermark[WMARK_MIN] = min_pages; } else { @@ -7368,7 +7368,7 @@ static void __setup_per_zone_wmarks(void) * ensure a minimum size on small systems. */ tmp = max_t(u64, tmp >> 2, - mult_frac(zone->managed_pages, + mult_frac(zone_managed_pages(zone), watermark_scale_factor, 10000)); zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; @@ -7498,8 +7498,8 @@ static void setup_min_unmapped_ratio(void) pgdat->min_unmapped_pages = 0; for_each_zone(zone) - zone->zone_pgdat->min_unmapped_pages += (zone->managed_pages * - sysctl_min_unmapped_ratio) / 100; + zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * + sysctl_min_unmapped_ratio) / 100; } @@ -7526,8 +7526,8 @@ static void setup_min_slab_ratio(void) pgdat->min_slab_pages = 0; for_each_zone(zone) - zone->zone_pgdat->min_slab_pages += (zone->managed_pages * - sysctl_min_slab_ratio) / 100; + zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * + sysctl_min_slab_ratio) / 100; } int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, diff --git a/mm/vmstat.c b/mm/vmstat.c index 9c624595e904..83b30edc2f7f 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -227,7 +227,7 @@ int calculate_normal_threshold(struct zone *zone) * 125 1024 10 16-32 GB 9 */ - mem = zone->managed_pages >> (27 - PAGE_SHIFT); + mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT); threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); @@ -1569,7 +1569,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, high_wmark_pages(zone), zone->spanned_pages, zone->present_pages, - zone->managed_pages); + zone_managed_pages(zone)); seq_printf(m, "\n protection: (%ld", -- cgit v1.2.3 From c3a5c77afefa697bf87f15272c7257e1574cad56 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Fri, 28 Dec 2018 00:37:16 -0800 Subject: lib/show_mem.c: drop pgdat_resize_lock in show_mem() Function show_mem() is used to print system memory status when user requires or fail to allocate memory. Generally, this is a best effort information so any races with memory hotplug (or very theoretically an early initialization) should be tolerable and the worst that could happen is to print an imprecise node state. Drop the resize lock because this is the only place which might hold the lock from the interrupt context and so all other callers might use a simple spinlock. Even though this doesn't solve any real issue it makes the code easier to follow and tiny more effective. Link: http://lkml.kernel.org/r/20181129235532.9328-1-richard.weiyang@gmail.com Signed-off-by: Wei Yang Acked-by: Michal Hocko Reviewed-by: Oscar Salvador Cc: Johannes Weiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/show_mem.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib') diff --git a/lib/show_mem.c b/lib/show_mem.c index eefe67d50e84..6a042f53e7bb 100644 --- a/lib/show_mem.c +++ b/lib/show_mem.c @@ -18,10 +18,8 @@ void show_mem(unsigned int filter, nodemask_t *nodemask) show_free_areas(filter, nodemask); for_each_online_pgdat(pgdat) { - unsigned long flags; int zoneid; - pgdat_resize_lock(pgdat, &flags); for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { struct zone *zone = &pgdat->node_zones[zoneid]; if (!populated_zone(zone)) @@ -33,7 +31,6 @@ void show_mem(unsigned int filter, nodemask_t *nodemask) if (is_highmem_idx(zoneid)) highmem += zone->present_pages; } - pgdat_resize_unlock(pgdat, &flags); } printk("%lu pages RAM\n", total); -- cgit v1.2.3 From d239865ac804c91a621294ca7bece4241b006fae Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 28 Dec 2018 00:37:38 -0800 Subject: ioremap: rework pXd_free_pYd_page() API The recently merged API for ensuring break-before-make on page-table entries when installing huge mappings in the vmalloc/ioremap region is fairly counter-intuitive, resulting in the arch freeing functions (e.g. pmd_free_pte_page()) being called even on entries that aren't present. This resulted in a minor bug in the arm64 implementation, giving rise to spurious VM_WARN messages. This patch moves the pXd_present() checks out into the core code, refactoring the callsites at the same time so that we avoid the complex conjunctions when determining whether or not we can put down a huge mapping. Link: http://lkml.kernel.org/r/1544120495-17438-2-git-send-email-will.deacon@arm.com Signed-off-by: Will Deacon Reviewed-by: Toshi Kani Suggested-by: Linus Torvalds Cc: Chintan Pandya Cc: Toshi Kani Cc: Thomas Gleixner Cc: Michal Hocko Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Sean Christopherson Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/ioremap.c | 56 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/ioremap.c b/lib/ioremap.c index 517f5853ffed..6c72764af19c 100644 --- a/lib/ioremap.c +++ b/lib/ioremap.c @@ -76,6 +76,25 @@ static int ioremap_pte_range(pmd_t *pmd, unsigned long addr, return 0; } +static int ioremap_try_huge_pmd(pmd_t *pmd, unsigned long addr, + unsigned long end, phys_addr_t phys_addr, + pgprot_t prot) +{ + if (!ioremap_pmd_enabled()) + return 0; + + if ((end - addr) != PMD_SIZE) + return 0; + + if (!IS_ALIGNED(phys_addr, PMD_SIZE)) + return 0; + + if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) + return 0; + + return pmd_set_huge(pmd, phys_addr, prot); +} + static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot) { @@ -89,13 +108,8 @@ static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr, do { next = pmd_addr_end(addr, end); - if (ioremap_pmd_enabled() && - ((next - addr) == PMD_SIZE) && - IS_ALIGNED(phys_addr + addr, PMD_SIZE) && - pmd_free_pte_page(pmd, addr)) { - if (pmd_set_huge(pmd, phys_addr + addr, prot)) - continue; - } + if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr + addr, prot)) + continue; if (ioremap_pte_range(pmd, addr, next, phys_addr + addr, prot)) return -ENOMEM; @@ -103,6 +117,25 @@ static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr, return 0; } +static int ioremap_try_huge_pud(pud_t *pud, unsigned long addr, + unsigned long end, phys_addr_t phys_addr, + pgprot_t prot) +{ + if (!ioremap_pud_enabled()) + return 0; + + if ((end - addr) != PUD_SIZE) + return 0; + + if (!IS_ALIGNED(phys_addr, PUD_SIZE)) + return 0; + + if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) + return 0; + + return pud_set_huge(pud, phys_addr, prot); +} + static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot) { @@ -116,13 +149,8 @@ static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, do { next = pud_addr_end(addr, end); - if (ioremap_pud_enabled() && - ((next - addr) == PUD_SIZE) && - IS_ALIGNED(phys_addr + addr, PUD_SIZE) && - pud_free_pmd_page(pud, addr)) { - if (pud_set_huge(pud, phys_addr + addr, prot)) - continue; - } + if (ioremap_try_huge_pud(pud, addr, next, phys_addr + addr, prot)) + continue; if (ioremap_pmd_range(pud, addr, next, phys_addr + addr, prot)) return -ENOMEM; -- cgit v1.2.3 From 36ddc5a78c878e9b10c323d2fe88b0dae2f487eb Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 28 Dec 2018 00:37:49 -0800 Subject: lib/ioremap: ensure phys_addr actually corresponds to a physical address The current ioremap() code uses a phys_addr variable at each level of page table, which is confusingly offset by subtracting the base virtual address being mapped so that adding the current virtual address back on when iterating through the page table entries gives back the corresponding physical address. This is fairly confusing and results in all users of phys_addr having to add the current virtual address back on. Instead, this patch just updates phys_addr when iterating over the page table entries, ensuring that it's always up-to-date and doesn't require explicit offsetting. Link: http://lkml.kernel.org/r/1544120495-17438-5-git-send-email-will.deacon@arm.com Signed-off-by: Will Deacon Tested-by: Sean Christopherson Reviewed-by: Sean Christopherson Cc: Chintan Pandya Cc: Toshi Kani Cc: Thomas Gleixner Cc: Michal Hocko Cc: Sean Christopherson Cc: "H. Peter Anvin" Cc: Ingo Molnar Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/ioremap.c | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/ioremap.c b/lib/ioremap.c index 6c72764af19c..10d7c5485c39 100644 --- a/lib/ioremap.c +++ b/lib/ioremap.c @@ -101,19 +101,18 @@ static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr, pmd_t *pmd; unsigned long next; - phys_addr -= addr; pmd = pmd_alloc(&init_mm, pud, addr); if (!pmd) return -ENOMEM; do { next = pmd_addr_end(addr, end); - if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr + addr, prot)) + if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr, prot)) continue; - if (ioremap_pte_range(pmd, addr, next, phys_addr + addr, prot)) + if (ioremap_pte_range(pmd, addr, next, phys_addr, prot)) return -ENOMEM; - } while (pmd++, addr = next, addr != end); + } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); return 0; } @@ -142,19 +141,18 @@ static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, pud_t *pud; unsigned long next; - phys_addr -= addr; pud = pud_alloc(&init_mm, p4d, addr); if (!pud) return -ENOMEM; do { next = pud_addr_end(addr, end); - if (ioremap_try_huge_pud(pud, addr, next, phys_addr + addr, prot)) + if (ioremap_try_huge_pud(pud, addr, next, phys_addr, prot)) continue; - if (ioremap_pmd_range(pud, addr, next, phys_addr + addr, prot)) + if (ioremap_pmd_range(pud, addr, next, phys_addr, prot)) return -ENOMEM; - } while (pud++, addr = next, addr != end); + } while (pud++, phys_addr += (next - addr), addr = next, addr != end); return 0; } @@ -164,7 +162,6 @@ static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr, p4d_t *p4d; unsigned long next; - phys_addr -= addr; p4d = p4d_alloc(&init_mm, pgd, addr); if (!p4d) return -ENOMEM; @@ -173,14 +170,14 @@ static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr, if (ioremap_p4d_enabled() && ((next - addr) == P4D_SIZE) && - IS_ALIGNED(phys_addr + addr, P4D_SIZE)) { - if (p4d_set_huge(p4d, phys_addr + addr, prot)) + IS_ALIGNED(phys_addr, P4D_SIZE)) { + if (p4d_set_huge(p4d, phys_addr, prot)) continue; } - if (ioremap_pud_range(p4d, addr, next, phys_addr + addr, prot)) + if (ioremap_pud_range(p4d, addr, next, phys_addr, prot)) return -ENOMEM; - } while (p4d++, addr = next, addr != end); + } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); return 0; } @@ -196,14 +193,13 @@ int ioremap_page_range(unsigned long addr, BUG_ON(addr >= end); start = addr; - phys_addr -= addr; pgd = pgd_offset_k(addr); do { next = pgd_addr_end(addr, end); - err = ioremap_p4d_range(pgd, addr, next, phys_addr+addr, prot); + err = ioremap_p4d_range(pgd, addr, next, phys_addr, prot); if (err) break; - } while (pgd++, addr = next, addr != end); + } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); flush_cache_vmap(start, end); -- cgit v1.2.3 From 8e2d43405b22e98cf5f3730c1829ec1fdbe17ae7 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 28 Dec 2018 00:37:53 -0800 Subject: lib/ioremap: ensure break-before-make is used for huge p4d mappings Whilst no architectures actually enable support for huge p4d mappings in the vmap area, the code that is implemented should be using break-before-make, as we do for pud and pmd huge entries. Link: http://lkml.kernel.org/r/1544120495-17438-6-git-send-email-will.deacon@arm.com Signed-off-by: Will Deacon Reviewed-by: Toshi Kani Cc: Chintan Pandya Cc: Toshi Kani Cc: Thomas Gleixner Cc: Michal Hocko Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Sean Christopherson Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/arm64/mm/mmu.c | 5 +++++ arch/x86/mm/pgtable.c | 8 ++++++++ include/asm-generic/pgtable.h | 5 +++++ lib/ioremap.c | 27 +++++++++++++++++++++------ 4 files changed, 39 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 13b80361d9f5..b6f5aa52ac67 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1043,6 +1043,11 @@ int pud_free_pmd_page(pud_t *pudp, unsigned long addr) return 1; } +int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) +{ + return 0; /* Don't attempt a block mapping */ +} + #ifdef CONFIG_MEMORY_HOTPLUG int arch_add_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap, bool want_memblock) diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index e95a7d6ac8f8..b0284eab14dc 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -794,6 +794,14 @@ int pmd_clear_huge(pmd_t *pmd) return 0; } +/* + * Until we support 512GB pages, skip them in the vmap area. + */ +int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) +{ + return 0; +} + #ifdef CONFIG_X86_64 /** * pud_free_pmd_page - Clear pud entry and free pmd page. diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h index a9cac82e9a7a..05e61e6c843f 100644 --- a/include/asm-generic/pgtable.h +++ b/include/asm-generic/pgtable.h @@ -1057,6 +1057,7 @@ int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot); int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot); int pud_clear_huge(pud_t *pud); int pmd_clear_huge(pmd_t *pmd); +int p4d_free_pud_page(p4d_t *p4d, unsigned long addr); int pud_free_pmd_page(pud_t *pud, unsigned long addr); int pmd_free_pte_page(pmd_t *pmd, unsigned long addr); #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ @@ -1084,6 +1085,10 @@ static inline int pmd_clear_huge(pmd_t *pmd) { return 0; } +static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) +{ + return 0; +} static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr) { return 0; diff --git a/lib/ioremap.c b/lib/ioremap.c index 10d7c5485c39..063213685563 100644 --- a/lib/ioremap.c +++ b/lib/ioremap.c @@ -156,6 +156,25 @@ static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, return 0; } +static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr, + unsigned long end, phys_addr_t phys_addr, + pgprot_t prot) +{ + if (!ioremap_p4d_enabled()) + return 0; + + if ((end - addr) != P4D_SIZE) + return 0; + + if (!IS_ALIGNED(phys_addr, P4D_SIZE)) + return 0; + + if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) + return 0; + + return p4d_set_huge(p4d, phys_addr, prot); +} + static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot) { @@ -168,12 +187,8 @@ static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr, do { next = p4d_addr_end(addr, end); - if (ioremap_p4d_enabled() && - ((next - addr) == P4D_SIZE) && - IS_ALIGNED(phys_addr, P4D_SIZE)) { - if (p4d_set_huge(p4d, phys_addr, prot)) - continue; - } + if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot)) + continue; if (ioremap_pud_range(p4d, addr, next, phys_addr, prot)) return -ENOMEM; -- cgit v1.2.3 From d53ce042277a94eadf9a8a31fc41fac54c67dec5 Mon Sep 17 00:00:00 2001 From: Sri Krishna chowdary Date: Fri, 28 Dec 2018 00:38:54 -0800 Subject: kmemleak: add config to select auto scan Kmemleak scan can be cpu intensive and can stall user tasks at times. To prevent this, add config DEBUG_KMEMLEAK_AUTO_SCAN to enable/disable auto scan on boot up. Also protect first_run with DEBUG_KMEMLEAK_AUTO_SCAN as this is meant for only first automatic scan. Link: http://lkml.kernel.org/r/1540231723-7087-1-git-send-email-prpatel@nvidia.com Signed-off-by: Sri Krishna chowdary Signed-off-by: Sachin Nikam Signed-off-by: Prateek Reviewed-by: Catalin Marinas Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/Kconfig.debug | 15 +++++++++++++++ mm/kmemleak.c | 10 ++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index b3c91b9e32f8..2b5a4256e88b 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -593,6 +593,21 @@ config DEBUG_KMEMLEAK_DEFAULT_OFF Say Y here to disable kmemleak by default. It can then be enabled on the command line via kmemleak=on. +config DEBUG_KMEMLEAK_AUTO_SCAN + bool "Enable kmemleak auto scan thread on boot up" + default y + depends on DEBUG_KMEMLEAK + help + Depending on the cpu, kmemleak scan may be cpu intensive and can + stall user tasks at times. This option enables/disables automatic + kmemleak scan at boot up. + + Say N here to disable kmemleak auto scan thread to stop automatic + scanning. Disabling this option disables automatic reporting of + memory leaks. + + If unsure, say Y. + config DEBUG_STACK_USAGE bool "Stack utilization instrumentation" depends on DEBUG_KERNEL && !IA64 diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 9c3d2dea0861..f9d9dc250428 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1650,7 +1650,7 @@ static void kmemleak_scan(void) */ static int kmemleak_scan_thread(void *arg) { - static int first_run = 1; + static int first_run = IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN); pr_info("Automatic memory scanning thread started\n"); set_user_nice(current, 10); @@ -2144,9 +2144,11 @@ static int __init kmemleak_late_init(void) return -ENOMEM; } - mutex_lock(&scan_mutex); - start_scan_thread(); - mutex_unlock(&scan_mutex); + if (IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN)) { + mutex_lock(&scan_mutex); + start_scan_thread(); + mutex_unlock(&scan_mutex); + } pr_info("Kernel memory leak detector initialized\n"); -- cgit v1.2.3 From d8372ba8ce288acdfce67cb873b2a741785c2e88 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Sat, 29 Dec 2018 13:58:09 +1100 Subject: lib: don't depend on linux headers being installed. gen_crc64table requires linux include files to be installed in /usr/include/linux. This is a new requrement so hosts that could previously build the kernel, now cannot. gen_crc64table makes this requirement by including , but nothing from that header is actaully used. So remove the #include, so that the linux headers no longer need to be installed. Fixes: feba04fd2cf8 ("lib: add crc64 calculation routines") Signed-off-by: NeilBrown Acked-by: Coly Li Signed-off-by: Linus Torvalds --- lib/gen_crc64table.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/gen_crc64table.c b/lib/gen_crc64table.c index 9011926e4162..094b43aef8db 100644 --- a/lib/gen_crc64table.c +++ b/lib/gen_crc64table.c @@ -16,8 +16,6 @@ #include #include -#include - #define CRC64_ECMA182_POLY 0x42F0E1EBA9EA3693ULL static uint64_t crc64_table[256] = {0}; -- cgit v1.2.3 From 96d4f267e40f9509e8a66e2b39e8b95655617693 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Thu, 3 Jan 2019 18:57:57 -0800 Subject: Remove 'type' argument from access_ok() function Nobody has actually used the type (VERIFY_READ vs VERIFY_WRITE) argument of the user address range verification function since we got rid of the old racy i386-only code to walk page tables by hand. It existed because the original 80386 would not honor the write protect bit when in kernel mode, so you had to do COW by hand before doing any user access. But we haven't supported that in a long time, and these days the 'type' argument is a purely historical artifact. A discussion about extending 'user_access_begin()' to do the range checking resulted this patch, because there is no way we're going to move the old VERIFY_xyz interface to that model. And it's best done at the end of the merge window when I've done most of my merges, so let's just get this done once and for all. This patch was mostly done with a sed-script, with manual fix-ups for the cases that weren't of the trivial 'access_ok(VERIFY_xyz' form. There were a couple of notable cases: - csky still had the old "verify_area()" name as an alias. - the iter_iov code had magical hardcoded knowledge of the actual values of VERIFY_{READ,WRITE} (not that they mattered, since nothing really used it) - microblaze used the type argument for a debug printout but other than those oddities this should be a total no-op patch. I tried to fix up all architectures, did fairly extensive grepping for access_ok() uses, and the changes are trivial, but I may have missed something. Any missed conversion should be trivially fixable, though. Signed-off-by: Linus Torvalds --- arch/alpha/include/asm/futex.h | 2 +- arch/alpha/include/asm/uaccess.h | 2 +- arch/alpha/kernel/signal.c | 12 +-- arch/alpha/lib/csum_partial_copy.c | 2 +- arch/arc/include/asm/futex.h | 2 +- arch/arc/kernel/process.c | 2 +- arch/arc/kernel/signal.c | 4 +- arch/arm/include/asm/futex.h | 4 +- arch/arm/include/asm/uaccess.h | 4 +- arch/arm/kernel/perf_callchain.c | 2 +- arch/arm/kernel/signal.c | 6 +- arch/arm/kernel/swp_emulate.c | 2 +- arch/arm/kernel/sys_oabi-compat.c | 4 +- arch/arm/kernel/traps.c | 2 +- arch/arm/oprofile/common.c | 2 +- arch/arm64/include/asm/futex.h | 2 +- arch/arm64/include/asm/uaccess.h | 8 +- arch/arm64/kernel/armv8_deprecated.c | 2 +- arch/arm64/kernel/perf_callchain.c | 4 +- arch/arm64/kernel/signal.c | 6 +- arch/arm64/kernel/signal32.c | 6 +- arch/arm64/kernel/sys_compat.c | 2 +- arch/c6x/kernel/signal.c | 4 +- arch/csky/abiv1/alignment.c | 4 +- arch/csky/include/asm/uaccess.h | 16 +--- arch/csky/kernel/signal.c | 2 +- arch/csky/lib/usercopy.c | 8 +- arch/h8300/kernel/signal.c | 4 +- arch/hexagon/include/asm/futex.h | 2 +- arch/hexagon/include/asm/uaccess.h | 3 - arch/hexagon/kernel/signal.c | 4 +- arch/hexagon/mm/uaccess.c | 2 +- arch/ia64/include/asm/futex.h | 2 +- arch/ia64/include/asm/uaccess.h | 2 +- arch/ia64/kernel/ptrace.c | 4 +- arch/ia64/kernel/signal.c | 4 +- arch/m68k/include/asm/uaccess_mm.h | 2 +- arch/m68k/include/asm/uaccess_no.h | 2 +- arch/m68k/kernel/signal.c | 4 +- arch/microblaze/include/asm/futex.h | 2 +- arch/microblaze/include/asm/uaccess.h | 23 +++--- arch/microblaze/kernel/signal.c | 4 +- arch/mips/include/asm/checksum.h | 4 +- arch/mips/include/asm/futex.h | 2 +- arch/mips/include/asm/termios.h | 4 +- arch/mips/include/asm/uaccess.h | 12 +-- arch/mips/kernel/mips-r2-to-r6-emul.c | 24 +++--- arch/mips/kernel/ptrace.c | 12 +-- arch/mips/kernel/signal.c | 12 +-- arch/mips/kernel/signal32.c | 4 +- arch/mips/kernel/signal_n32.c | 4 +- arch/mips/kernel/signal_o32.c | 8 +- arch/mips/kernel/syscall.c | 2 +- arch/mips/kernel/unaligned.c | 98 ++++++++++++------------- arch/mips/math-emu/cp1emu.c | 16 ++-- arch/mips/mm/cache.c | 2 +- arch/mips/mm/gup.c | 3 +- arch/mips/oprofile/backtrace.c | 2 +- arch/mips/sibyte/common/sb_tbprof.c | 2 +- arch/nds32/include/asm/futex.h | 2 +- arch/nds32/include/asm/uaccess.h | 11 +-- arch/nds32/kernel/perf_event_cpu.c | 11 ++- arch/nds32/kernel/signal.c | 4 +- arch/nds32/mm/alignment.c | 8 +- arch/nios2/include/asm/uaccess.h | 8 +- arch/nios2/kernel/signal.c | 2 +- arch/openrisc/include/asm/futex.h | 2 +- arch/openrisc/include/asm/uaccess.h | 8 +- arch/openrisc/kernel/signal.c | 6 +- arch/parisc/include/asm/futex.h | 2 +- arch/parisc/include/asm/uaccess.h | 2 +- arch/powerpc/include/asm/futex.h | 2 +- arch/powerpc/include/asm/uaccess.h | 8 +- arch/powerpc/kernel/align.c | 3 +- arch/powerpc/kernel/rtas_flash.c | 2 +- arch/powerpc/kernel/rtasd.c | 2 +- arch/powerpc/kernel/signal.c | 2 +- arch/powerpc/kernel/signal_32.c | 12 +-- arch/powerpc/kernel/signal_64.c | 13 ++-- arch/powerpc/kernel/syscalls.c | 2 +- arch/powerpc/kernel/traps.c | 2 +- arch/powerpc/kvm/book3s_64_mmu_hv.c | 4 +- arch/powerpc/lib/checksum_wrappers.c | 4 +- arch/powerpc/mm/fault.c | 2 +- arch/powerpc/mm/subpage-prot.c | 2 +- arch/powerpc/oprofile/backtrace.c | 4 +- arch/powerpc/platforms/cell/spufs/file.c | 16 ++-- arch/powerpc/platforms/powernv/opal-lpc.c | 4 +- arch/powerpc/platforms/pseries/scanlog.c | 2 +- arch/riscv/include/asm/futex.h | 2 +- arch/riscv/include/asm/uaccess.h | 14 +--- arch/riscv/kernel/signal.c | 4 +- arch/s390/include/asm/uaccess.h | 2 +- arch/sh/include/asm/checksum_32.h | 2 +- arch/sh/include/asm/futex.h | 2 +- arch/sh/include/asm/uaccess.h | 9 +-- arch/sh/kernel/signal_32.c | 8 +- arch/sh/kernel/signal_64.c | 8 +- arch/sh/kernel/traps_64.c | 12 +-- arch/sh/mm/gup.c | 3 +- arch/sh/oprofile/backtrace.c | 2 +- arch/sparc/include/asm/checksum_32.h | 2 +- arch/sparc/include/asm/uaccess_32.h | 2 +- arch/sparc/include/asm/uaccess_64.h | 2 +- arch/sparc/kernel/sigutil_32.c | 2 +- arch/sparc/kernel/unaligned_32.c | 7 +- arch/um/kernel/ptrace.c | 4 +- arch/unicore32/kernel/signal.c | 4 +- arch/x86/entry/vsyscall/vsyscall_64.c | 2 +- arch/x86/ia32/ia32_aout.c | 4 +- arch/x86/ia32/ia32_signal.c | 8 +- arch/x86/ia32/sys_ia32.c | 2 +- arch/x86/include/asm/checksum_32.h | 2 +- arch/x86/include/asm/pgtable_32.h | 2 +- arch/x86/include/asm/uaccess.h | 7 +- arch/x86/kernel/fpu/signal.c | 4 +- arch/x86/kernel/signal.c | 14 ++-- arch/x86/kernel/stacktrace.c | 2 +- arch/x86/kernel/vm86_32.c | 4 +- arch/x86/lib/csum-wrappers_64.c | 4 +- arch/x86/lib/usercopy_32.c | 2 +- arch/x86/lib/usercopy_64.c | 2 +- arch/x86/math-emu/fpu_system.h | 4 +- arch/x86/math-emu/load_store.c | 6 +- arch/x86/math-emu/reg_ld_str.c | 48 ++++++------ arch/x86/mm/mpx.c | 2 +- arch/x86/um/asm/checksum_32.h | 2 +- arch/x86/um/signal.c | 6 +- arch/xtensa/include/asm/checksum.h | 2 +- arch/xtensa/include/asm/futex.h | 2 +- arch/xtensa/include/asm/uaccess.h | 10 +-- arch/xtensa/kernel/signal.c | 4 +- arch/xtensa/kernel/stacktrace.c | 2 +- drivers/acpi/acpi_dbg.c | 4 +- drivers/char/generic_nvram.c | 4 +- drivers/char/mem.c | 4 +- drivers/char/nwflash.c | 2 +- drivers/char/pcmcia/cm4000_cs.c | 4 +- drivers/crypto/ccp/psp-dev.c | 6 +- drivers/firewire/core-cdev.c | 2 +- drivers/firmware/efi/test/efi_test.c | 8 +- drivers/fpga/dfl-afu-dma-region.c | 2 +- drivers/fpga/dfl-fme-pr.c | 3 +- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 18 ++--- drivers/gpu/drm/armada/armada_gem.c | 2 +- drivers/gpu/drm/drm_file.c | 2 +- drivers/gpu/drm/etnaviv/etnaviv_drv.c | 8 +- drivers/gpu/drm/i915/i915_gem.c | 7 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 6 +- drivers/gpu/drm/i915/i915_gem_userptr.c | 3 +- drivers/gpu/drm/i915/i915_ioc32.c | 2 +- drivers/gpu/drm/i915/i915_perf.c | 2 +- drivers/gpu/drm/i915/i915_query.c | 2 +- drivers/gpu/drm/msm/msm_gem_submit.c | 2 +- drivers/gpu/drm/qxl/qxl_ioctl.c | 3 +- drivers/infiniband/core/uverbs_main.c | 3 +- drivers/infiniband/hw/hfi1/user_exp_rcv.c | 2 +- drivers/infiniband/hw/qib/qib_file_ops.c | 2 +- drivers/macintosh/ans-lcd.c | 2 +- drivers/macintosh/via-pmu.c | 2 +- drivers/media/pci/ivtv/ivtvfb.c | 2 +- drivers/media/v4l2-core/v4l2-compat-ioctl32.c | 46 ++++++------ drivers/misc/vmw_vmci/vmci_host.c | 2 +- drivers/pci/proc.c | 4 +- drivers/platform/goldfish/goldfish_pipe.c | 3 +- drivers/pnp/isapnp/proc.c | 2 +- drivers/scsi/pmcraid.c | 4 +- drivers/scsi/scsi_ioctl.c | 2 +- drivers/scsi/sg.c | 16 ++-- drivers/staging/comedi/comedi_compat32.c | 24 +++--- drivers/tty/n_hdlc.c | 2 +- drivers/usb/core/devices.c | 2 +- drivers/usb/core/devio.c | 7 +- drivers/usb/gadget/function/f_hid.c | 4 +- drivers/usb/gadget/udc/atmel_usba_udc.c | 2 +- drivers/vhost/vhost.c | 16 ++-- drivers/video/fbdev/amifb.c | 4 +- drivers/video/fbdev/omap2/omapfb/omapfb-ioctl.c | 2 +- drivers/xen/privcmd.c | 6 +- fs/binfmt_aout.c | 4 +- fs/btrfs/send.c | 2 +- fs/eventpoll.c | 2 +- fs/fat/dir.c | 4 +- fs/ioctl.c | 2 +- fs/namespace.c | 2 +- fs/ocfs2/dlmfs/dlmfs.c | 4 +- fs/pstore/pmsg.c | 2 +- fs/pstore/ram_core.c | 2 +- fs/read_write.c | 13 ++-- fs/readdir.c | 10 +-- fs/select.c | 11 +-- include/asm-generic/uaccess.h | 12 +-- include/linux/regset.h | 4 +- include/linux/uaccess.h | 9 +-- include/net/checksum.h | 4 +- kernel/bpf/syscall.c | 2 +- kernel/compat.c | 16 ++-- kernel/events/core.c | 2 +- kernel/exit.c | 4 +- kernel/futex.c | 35 +++++---- kernel/printk/printk.c | 4 +- kernel/ptrace.c | 4 +- kernel/rseq.c | 6 +- kernel/sched/core.c | 4 +- kernel/signal.c | 8 +- kernel/sys.c | 2 +- kernel/trace/bpf_trace.c | 2 +- lib/bitmap.c | 4 +- lib/iov_iter.c | 8 +- lib/usercopy.c | 4 +- mm/gup.c | 6 +- mm/mincore.c | 4 +- net/batman-adv/icmp_socket.c | 2 +- net/batman-adv/log.c | 2 +- net/compat.c | 30 ++++---- net/sunrpc/sysctl.c | 2 +- security/tomoyo/common.c | 2 +- sound/core/seq/seq_clientmgr.c | 2 +- sound/isa/sb/emu8000_patch.c | 4 +- tools/perf/util/include/asm/uaccess.h | 2 +- virt/kvm/kvm_main.c | 3 +- 221 files changed, 610 insertions(+), 679 deletions(-) (limited to 'lib') diff --git a/arch/alpha/include/asm/futex.h b/arch/alpha/include/asm/futex.h index ca3322536f72..bfd3c01038f8 100644 --- a/arch/alpha/include/asm/futex.h +++ b/arch/alpha/include/asm/futex.h @@ -68,7 +68,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, int ret = 0, cmp; u32 prev; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; __asm__ __volatile__ ( diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index 87d8c4f0307d..e69c4e13c328 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -36,7 +36,7 @@ #define __access_ok(addr, size) \ ((get_fs().seg & (addr | size | (addr+size))) == 0) -#define access_ok(type, addr, size) \ +#define access_ok(addr, size) \ ({ \ __chk_user_ptr(addr); \ __access_ok(((unsigned long)(addr)), (size)); \ diff --git a/arch/alpha/kernel/signal.c b/arch/alpha/kernel/signal.c index 8c0c4ee0be6e..33e904a05881 100644 --- a/arch/alpha/kernel/signal.c +++ b/arch/alpha/kernel/signal.c @@ -65,7 +65,7 @@ SYSCALL_DEFINE3(osf_sigaction, int, sig, if (act) { old_sigset_t mask; - if (!access_ok(VERIFY_READ, act, sizeof(*act)) || + if (!access_ok(act, sizeof(*act)) || __get_user(new_ka.sa.sa_handler, &act->sa_handler) || __get_user(new_ka.sa.sa_flags, &act->sa_flags) || __get_user(mask, &act->sa_mask)) @@ -77,7 +77,7 @@ SYSCALL_DEFINE3(osf_sigaction, int, sig, ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); if (!ret && oact) { - if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) || + if (!access_ok(oact, sizeof(*oact)) || __put_user(old_ka.sa.sa_handler, &oact->sa_handler) || __put_user(old_ka.sa.sa_flags, &oact->sa_flags) || __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask)) @@ -207,7 +207,7 @@ do_sigreturn(struct sigcontext __user *sc) sigset_t set; /* Verify that it's a good sigcontext before using it */ - if (!access_ok(VERIFY_READ, sc, sizeof(*sc))) + if (!access_ok(sc, sizeof(*sc))) goto give_sigsegv; if (__get_user(set.sig[0], &sc->sc_mask)) goto give_sigsegv; @@ -235,7 +235,7 @@ do_rt_sigreturn(struct rt_sigframe __user *frame) sigset_t set; /* Verify that it's a good ucontext_t before using it */ - if (!access_ok(VERIFY_READ, &frame->uc, sizeof(frame->uc))) + if (!access_ok(&frame->uc, sizeof(frame->uc))) goto give_sigsegv; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) goto give_sigsegv; @@ -332,7 +332,7 @@ setup_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) oldsp = rdusp(); frame = get_sigframe(ksig, oldsp, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; err |= setup_sigcontext(&frame->sc, regs, set->sig[0], oldsp); @@ -377,7 +377,7 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) oldsp = rdusp(); frame = get_sigframe(ksig, oldsp, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; err |= copy_siginfo_to_user(&frame->info, &ksig->info); diff --git a/arch/alpha/lib/csum_partial_copy.c b/arch/alpha/lib/csum_partial_copy.c index ddb9c2f376fa..e53f96e8aa6d 100644 --- a/arch/alpha/lib/csum_partial_copy.c +++ b/arch/alpha/lib/csum_partial_copy.c @@ -333,7 +333,7 @@ csum_partial_copy_from_user(const void __user *src, void *dst, int len, unsigned long doff = 7 & (unsigned long) dst; if (len) { - if (!access_ok(VERIFY_READ, src, len)) { + if (!access_ok(src, len)) { if (errp) *errp = -EFAULT; memset(dst, 0, len); return sum; diff --git a/arch/arc/include/asm/futex.h b/arch/arc/include/asm/futex.h index eb887dd13e74..c29c3fae6854 100644 --- a/arch/arc/include/asm/futex.h +++ b/arch/arc/include/asm/futex.h @@ -126,7 +126,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 expval, int ret = 0; u32 existval; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; #ifndef CONFIG_ARC_HAS_LLSC diff --git a/arch/arc/kernel/process.c b/arch/arc/kernel/process.c index 8ce6e7235915..641c364fc232 100644 --- a/arch/arc/kernel/process.c +++ b/arch/arc/kernel/process.c @@ -61,7 +61,7 @@ SYSCALL_DEFINE3(arc_usr_cmpxchg, int *, uaddr, int, expected, int, new) /* Z indicates to userspace if operation succeded */ regs->status32 &= ~STATUS_Z_MASK; - ret = access_ok(VERIFY_WRITE, uaddr, sizeof(*uaddr)); + ret = access_ok(uaddr, sizeof(*uaddr)); if (!ret) goto fail; diff --git a/arch/arc/kernel/signal.c b/arch/arc/kernel/signal.c index 48685445002e..1bfb7de696bd 100644 --- a/arch/arc/kernel/signal.c +++ b/arch/arc/kernel/signal.c @@ -169,7 +169,7 @@ SYSCALL_DEFINE0(rt_sigreturn) sf = (struct rt_sigframe __force __user *)(regs->sp); - if (!access_ok(VERIFY_READ, sf, sizeof(*sf))) + if (!access_ok(sf, sizeof(*sf))) goto badframe; if (__get_user(magic, &sf->sigret_magic)) @@ -219,7 +219,7 @@ static inline void __user *get_sigframe(struct ksignal *ksig, frame = (void __user *)((sp - framesize) & ~7); /* Check that we can actually write to the signal frame */ - if (!access_ok(VERIFY_WRITE, frame, framesize)) + if (!access_ok(frame, framesize)) frame = NULL; return frame; diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index ffebe7b7a5b7..0a46676b4245 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -50,7 +50,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, int ret; u32 val; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; smp_mb(); @@ -104,7 +104,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, int ret = 0; u32 val; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; preempt_disable(); diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index c136eef8f690..27ed17ec45fe 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -279,7 +279,7 @@ static inline void set_fs(mm_segment_t fs) #endif /* CONFIG_MMU */ -#define access_ok(type, addr, size) (__range_ok(addr, size) == 0) +#define access_ok(addr, size) (__range_ok(addr, size) == 0) #define user_addr_max() \ (uaccess_kernel() ? ~0UL : get_fs()) @@ -560,7 +560,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) { - if (access_ok(VERIFY_WRITE, to, n)) + if (access_ok(to, n)) n = __clear_user(to, n); return n; } diff --git a/arch/arm/kernel/perf_callchain.c b/arch/arm/kernel/perf_callchain.c index 08e43a32a693..3b69a76d341e 100644 --- a/arch/arm/kernel/perf_callchain.c +++ b/arch/arm/kernel/perf_callchain.c @@ -37,7 +37,7 @@ user_backtrace(struct frame_tail __user *tail, struct frame_tail buftail; unsigned long err; - if (!access_ok(VERIFY_READ, tail, sizeof(buftail))) + if (!access_ok(tail, sizeof(buftail))) return NULL; pagefault_disable(); diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c index b908382b69ff..76bb8de6bf6b 100644 --- a/arch/arm/kernel/signal.c +++ b/arch/arm/kernel/signal.c @@ -241,7 +241,7 @@ asmlinkage int sys_sigreturn(struct pt_regs *regs) frame = (struct sigframe __user *)regs->ARM_sp; - if (!access_ok(VERIFY_READ, frame, sizeof (*frame))) + if (!access_ok(frame, sizeof (*frame))) goto badframe; if (restore_sigframe(regs, frame)) @@ -271,7 +271,7 @@ asmlinkage int sys_rt_sigreturn(struct pt_regs *regs) frame = (struct rt_sigframe __user *)regs->ARM_sp; - if (!access_ok(VERIFY_READ, frame, sizeof (*frame))) + if (!access_ok(frame, sizeof (*frame))) goto badframe; if (restore_sigframe(regs, &frame->sig)) @@ -355,7 +355,7 @@ get_sigframe(struct ksignal *ksig, struct pt_regs *regs, int framesize) /* * Check that we can actually write to the signal frame. */ - if (!access_ok(VERIFY_WRITE, frame, framesize)) + if (!access_ok(frame, framesize)) frame = NULL; return frame; diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c index a188d5e8ab7f..76f6e6a9736c 100644 --- a/arch/arm/kernel/swp_emulate.c +++ b/arch/arm/kernel/swp_emulate.c @@ -198,7 +198,7 @@ static int swp_handler(struct pt_regs *regs, unsigned int instr) destreg, EXTRACT_REG_NUM(instr, RT2_OFFSET), data); /* Check access in reasonable access range for both SWP and SWPB */ - if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) { + if (!access_ok((address & ~3), 4)) { pr_debug("SWP{B} emulation: access to %p not allowed!\n", (void *)address); res = -EFAULT; diff --git a/arch/arm/kernel/sys_oabi-compat.c b/arch/arm/kernel/sys_oabi-compat.c index 40da0872170f..92ab36f38795 100644 --- a/arch/arm/kernel/sys_oabi-compat.c +++ b/arch/arm/kernel/sys_oabi-compat.c @@ -285,7 +285,7 @@ asmlinkage long sys_oabi_epoll_wait(int epfd, maxevents > (INT_MAX/sizeof(*kbuf)) || maxevents > (INT_MAX/sizeof(*events))) return -EINVAL; - if (!access_ok(VERIFY_WRITE, events, sizeof(*events) * maxevents)) + if (!access_ok(events, sizeof(*events) * maxevents)) return -EFAULT; kbuf = kmalloc_array(maxevents, sizeof(*kbuf), GFP_KERNEL); if (!kbuf) @@ -326,7 +326,7 @@ asmlinkage long sys_oabi_semtimedop(int semid, if (nsops < 1 || nsops > SEMOPM) return -EINVAL; - if (!access_ok(VERIFY_READ, tsops, sizeof(*tsops) * nsops)) + if (!access_ok(tsops, sizeof(*tsops) * nsops)) return -EFAULT; sops = kmalloc_array(nsops, sizeof(*sops), GFP_KERNEL); if (!sops) diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 2d668cff8ef4..33af097c454b 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -582,7 +582,7 @@ do_cache_op(unsigned long start, unsigned long end, int flags) if (end < start || flags) return -EINVAL; - if (!access_ok(VERIFY_READ, start, end - start)) + if (!access_ok(start, end - start)) return -EFAULT; return __do_cache_op(start, end); diff --git a/arch/arm/oprofile/common.c b/arch/arm/oprofile/common.c index cc649a1e46da..7cb3e0453fcd 100644 --- a/arch/arm/oprofile/common.c +++ b/arch/arm/oprofile/common.c @@ -88,7 +88,7 @@ static struct frame_tail* user_backtrace(struct frame_tail *tail) struct frame_tail buftail[2]; /* Also check accessibility of one struct frame_tail beyond */ - if (!access_ok(VERIFY_READ, tail, sizeof(buftail))) + if (!access_ok(tail, sizeof(buftail))) return NULL; if (__copy_from_user_inatomic(buftail, tail, sizeof(buftail))) return NULL; diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index 07fe2479d310..cccb83ad7fa8 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -96,7 +96,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr, u32 val, tmp; u32 __user *uaddr; - if (!access_ok(VERIFY_WRITE, _uaddr, sizeof(u32))) + if (!access_ok(_uaddr, sizeof(u32))) return -EFAULT; uaddr = __uaccess_mask_ptr(_uaddr); diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index ed252435fd92..547d7a0c9d05 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -95,7 +95,7 @@ static inline unsigned long __range_ok(const void __user *addr, unsigned long si return ret; } -#define access_ok(type, addr, size) __range_ok(addr, size) +#define access_ok(addr, size) __range_ok(addr, size) #define user_addr_max get_fs #define _ASM_EXTABLE(from, to) \ @@ -301,7 +301,7 @@ do { \ ({ \ __typeof__(*(ptr)) __user *__p = (ptr); \ might_fault(); \ - if (access_ok(VERIFY_READ, __p, sizeof(*__p))) { \ + if (access_ok(__p, sizeof(*__p))) { \ __p = uaccess_mask_ptr(__p); \ __get_user_err((x), __p, (err)); \ } else { \ @@ -370,7 +370,7 @@ do { \ ({ \ __typeof__(*(ptr)) __user *__p = (ptr); \ might_fault(); \ - if (access_ok(VERIFY_WRITE, __p, sizeof(*__p))) { \ + if (access_ok(__p, sizeof(*__p))) { \ __p = uaccess_mask_ptr(__p); \ __put_user_err((x), __p, (err)); \ } else { \ @@ -418,7 +418,7 @@ extern unsigned long __must_check __arch_copy_in_user(void __user *to, const voi extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n); static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n) { - if (access_ok(VERIFY_WRITE, to, n)) + if (access_ok(to, n)) n = __arch_clear_user(__uaccess_mask_ptr(to), n); return n; } diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c index 92be1d12d590..e52e7280884a 100644 --- a/arch/arm64/kernel/armv8_deprecated.c +++ b/arch/arm64/kernel/armv8_deprecated.c @@ -402,7 +402,7 @@ static int swp_handler(struct pt_regs *regs, u32 instr) /* Check access in reasonable access range for both SWP and SWPB */ user_ptr = (const void __user *)(unsigned long)(address & ~3); - if (!access_ok(VERIFY_WRITE, user_ptr, 4)) { + if (!access_ok(user_ptr, 4)) { pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n", address); goto fault; diff --git a/arch/arm64/kernel/perf_callchain.c b/arch/arm64/kernel/perf_callchain.c index a34c26afacb0..61d983f5756f 100644 --- a/arch/arm64/kernel/perf_callchain.c +++ b/arch/arm64/kernel/perf_callchain.c @@ -39,7 +39,7 @@ user_backtrace(struct frame_tail __user *tail, unsigned long lr; /* Also check accessibility of one struct frame_tail beyond */ - if (!access_ok(VERIFY_READ, tail, sizeof(buftail))) + if (!access_ok(tail, sizeof(buftail))) return NULL; pagefault_disable(); @@ -86,7 +86,7 @@ compat_user_backtrace(struct compat_frame_tail __user *tail, unsigned long err; /* Also check accessibility of one struct frame_tail beyond */ - if (!access_ok(VERIFY_READ, tail, sizeof(buftail))) + if (!access_ok(tail, sizeof(buftail))) return NULL; pagefault_disable(); diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 5dcc942906db..867a7cea70e5 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -470,7 +470,7 @@ static int parse_user_sigframe(struct user_ctxs *user, offset = 0; limit = extra_size; - if (!access_ok(VERIFY_READ, base, limit)) + if (!access_ok(base, limit)) goto invalid; continue; @@ -556,7 +556,7 @@ SYSCALL_DEFINE0(rt_sigreturn) frame = (struct rt_sigframe __user *)regs->sp; - if (!access_ok(VERIFY_READ, frame, sizeof (*frame))) + if (!access_ok(frame, sizeof (*frame))) goto badframe; if (restore_sigframe(regs, frame)) @@ -730,7 +730,7 @@ static int get_sigframe(struct rt_sigframe_user_layout *user, /* * Check that we can actually write to the signal frame. */ - if (!access_ok(VERIFY_WRITE, user->sigframe, sp_top - sp)) + if (!access_ok(user->sigframe, sp_top - sp)) return -EFAULT; return 0; diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c index 24b09003f821..cb7800acd19f 100644 --- a/arch/arm64/kernel/signal32.c +++ b/arch/arm64/kernel/signal32.c @@ -303,7 +303,7 @@ COMPAT_SYSCALL_DEFINE0(sigreturn) frame = (struct compat_sigframe __user *)regs->compat_sp; - if (!access_ok(VERIFY_READ, frame, sizeof (*frame))) + if (!access_ok(frame, sizeof (*frame))) goto badframe; if (compat_restore_sigframe(regs, frame)) @@ -334,7 +334,7 @@ COMPAT_SYSCALL_DEFINE0(rt_sigreturn) frame = (struct compat_rt_sigframe __user *)regs->compat_sp; - if (!access_ok(VERIFY_READ, frame, sizeof (*frame))) + if (!access_ok(frame, sizeof (*frame))) goto badframe; if (compat_restore_sigframe(regs, &frame->sig)) @@ -365,7 +365,7 @@ static void __user *compat_get_sigframe(struct ksignal *ksig, /* * Check that we can actually write to the signal frame. */ - if (!access_ok(VERIFY_WRITE, frame, framesize)) + if (!access_ok(frame, framesize)) frame = NULL; return frame; diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c index 32653d156747..21005dfe8406 100644 --- a/arch/arm64/kernel/sys_compat.c +++ b/arch/arm64/kernel/sys_compat.c @@ -58,7 +58,7 @@ do_compat_cache_op(unsigned long start, unsigned long end, int flags) if (end < start || flags) return -EINVAL; - if (!access_ok(VERIFY_READ, (const void __user *)start, end - start)) + if (!access_ok((const void __user *)start, end - start)) return -EFAULT; return __do_compat_cache_op(start, end); diff --git a/arch/c6x/kernel/signal.c b/arch/c6x/kernel/signal.c index 3c4bb5a5c382..33b9f69c38f7 100644 --- a/arch/c6x/kernel/signal.c +++ b/arch/c6x/kernel/signal.c @@ -80,7 +80,7 @@ asmlinkage int do_rt_sigreturn(struct pt_regs *regs) frame = (struct rt_sigframe __user *) ((unsigned long) regs->sp + 8); - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; @@ -149,7 +149,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, frame = get_sigframe(ksig, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; err |= __put_user(&frame->info, &frame->pinfo); diff --git a/arch/csky/abiv1/alignment.c b/arch/csky/abiv1/alignment.c index 60205e98fb87..d789be36eb4f 100644 --- a/arch/csky/abiv1/alignment.c +++ b/arch/csky/abiv1/alignment.c @@ -32,7 +32,7 @@ static int ldb_asm(uint32_t addr, uint32_t *valp) uint32_t val; int err; - if (!access_ok(VERIFY_READ, (void *)addr, 1)) + if (!access_ok((void *)addr, 1)) return 1; asm volatile ( @@ -67,7 +67,7 @@ static int stb_asm(uint32_t addr, uint32_t val) { int err; - if (!access_ok(VERIFY_WRITE, (void *)addr, 1)) + if (!access_ok((void *)addr, 1)) return 1; asm volatile ( diff --git a/arch/csky/include/asm/uaccess.h b/arch/csky/include/asm/uaccess.h index acaf0e210d81..eaa1c3403a42 100644 --- a/arch/csky/include/asm/uaccess.h +++ b/arch/csky/include/asm/uaccess.h @@ -16,10 +16,7 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - -static inline int access_ok(int type, const void *addr, unsigned long size) +static inline int access_ok(const void *addr, unsigned long size) { unsigned long limit = current_thread_info()->addr_limit.seg; @@ -27,12 +24,7 @@ static inline int access_ok(int type, const void *addr, unsigned long size) ((unsigned long)(addr + size) < limit)); } -static inline int verify_area(int type, const void *addr, unsigned long size) -{ - return access_ok(type, addr, size) ? 0 : -EFAULT; -} - -#define __addr_ok(addr) (access_ok(VERIFY_READ, addr, 0)) +#define __addr_ok(addr) (access_ok(addr, 0)) extern int __put_user_bad(void); @@ -91,7 +83,7 @@ extern int __put_user_bad(void); long __pu_err = -EFAULT; \ typeof(*(ptr)) *__pu_addr = (ptr); \ typeof(*(ptr)) __pu_val = (typeof(*(ptr)))(x); \ - if (access_ok(VERIFY_WRITE, __pu_addr, size) && __pu_addr) \ + if (access_ok(__pu_addr, size) && __pu_addr) \ __put_user_size(__pu_val, __pu_addr, (size), __pu_err); \ __pu_err; \ }) @@ -217,7 +209,7 @@ do { \ ({ \ int __gu_err = -EFAULT; \ const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \ - if (access_ok(VERIFY_READ, __gu_ptr, size) && __gu_ptr) \ + if (access_ok(__gu_ptr, size) && __gu_ptr) \ __get_user_size(x, __gu_ptr, size, __gu_err); \ __gu_err; \ }) diff --git a/arch/csky/kernel/signal.c b/arch/csky/kernel/signal.c index 66e1b729b10b..9967c10eee2b 100644 --- a/arch/csky/kernel/signal.c +++ b/arch/csky/kernel/signal.c @@ -88,7 +88,7 @@ do_rt_sigreturn(void) struct pt_regs *regs = current_pt_regs(); struct rt_sigframe *frame = (struct rt_sigframe *)(regs->usp); - if (verify_area(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; diff --git a/arch/csky/lib/usercopy.c b/arch/csky/lib/usercopy.c index ac9170e2cbb8..647a23986fb5 100644 --- a/arch/csky/lib/usercopy.c +++ b/arch/csky/lib/usercopy.c @@ -7,7 +7,7 @@ unsigned long raw_copy_from_user(void *to, const void *from, unsigned long n) { - if (access_ok(VERIFY_READ, from, n)) + if (access_ok(from, n)) __copy_user_zeroing(to, from, n); else memset(to, 0, n); @@ -18,7 +18,7 @@ EXPORT_SYMBOL(raw_copy_from_user); unsigned long raw_copy_to_user(void *to, const void *from, unsigned long n) { - if (access_ok(VERIFY_WRITE, to, n)) + if (access_ok(to, n)) __copy_user(to, from, n); return n; } @@ -113,7 +113,7 @@ long strncpy_from_user(char *dst, const char *src, long count) { long res = -EFAULT; - if (access_ok(VERIFY_READ, src, 1)) + if (access_ok(src, 1)) __do_strncpy_from_user(dst, src, count, res); return res; } @@ -236,7 +236,7 @@ do { \ unsigned long clear_user(void __user *to, unsigned long n) { - if (access_ok(VERIFY_WRITE, to, n)) + if (access_ok(to, n)) __do_clear_user(to, n); return n; } diff --git a/arch/h8300/kernel/signal.c b/arch/h8300/kernel/signal.c index 1e8070d08770..e0f2b708e5d9 100644 --- a/arch/h8300/kernel/signal.c +++ b/arch/h8300/kernel/signal.c @@ -110,7 +110,7 @@ asmlinkage int sys_rt_sigreturn(void) sigset_t set; int er0; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; @@ -165,7 +165,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, frame = get_sigframe(ksig, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; if (ksig->ka.sa.sa_flags & SA_SIGINFO) diff --git a/arch/hexagon/include/asm/futex.h b/arch/hexagon/include/asm/futex.h index c889f5993ecd..cb635216a732 100644 --- a/arch/hexagon/include/asm/futex.h +++ b/arch/hexagon/include/asm/futex.h @@ -77,7 +77,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, int prev; int ret; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; __asm__ __volatile__ ( diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h index 458b69886b34..a30e58d5f351 100644 --- a/arch/hexagon/include/asm/uaccess.h +++ b/arch/hexagon/include/asm/uaccess.h @@ -29,9 +29,6 @@ /* * access_ok: - Checks if a user space pointer is valid - * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that - * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe - * to write to a block, it is always safe to read from it. * @addr: User space pointer to start of block to check * @size: Size of block to check * diff --git a/arch/hexagon/kernel/signal.c b/arch/hexagon/kernel/signal.c index 78aa7304a5c9..31e2cf95f189 100644 --- a/arch/hexagon/kernel/signal.c +++ b/arch/hexagon/kernel/signal.c @@ -115,7 +115,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, frame = get_sigframe(ksig, regs, sizeof(struct rt_sigframe)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(struct rt_sigframe))) + if (!access_ok(frame, sizeof(struct rt_sigframe))) return -EFAULT; if (copy_siginfo_to_user(&frame->info, &ksig->info)) @@ -244,7 +244,7 @@ asmlinkage int sys_rt_sigreturn(void) current->restart_block.fn = do_no_restart_syscall; frame = (struct rt_sigframe __user *)pt_psp(regs); - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&blocked, &frame->uc.uc_sigmask, sizeof(blocked))) goto badframe; diff --git a/arch/hexagon/mm/uaccess.c b/arch/hexagon/mm/uaccess.c index c599eb126c9e..6f9c4697552c 100644 --- a/arch/hexagon/mm/uaccess.c +++ b/arch/hexagon/mm/uaccess.c @@ -51,7 +51,7 @@ __kernel_size_t __clear_user_hexagon(void __user *dest, unsigned long count) unsigned long clear_user_hexagon(void __user *dest, unsigned long count) { - if (!access_ok(VERIFY_WRITE, dest, count)) + if (!access_ok(dest, count)) return count; else return __clear_user_hexagon(dest, count); diff --git a/arch/ia64/include/asm/futex.h b/arch/ia64/include/asm/futex.h index db2dd85918c2..2e106d462196 100644 --- a/arch/ia64/include/asm/futex.h +++ b/arch/ia64/include/asm/futex.h @@ -86,7 +86,7 @@ static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) { - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; { diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index a74524f2d625..306d469e43da 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -67,7 +67,7 @@ static inline int __access_ok(const void __user *p, unsigned long size) return likely(addr <= seg) && (seg == KERNEL_DS.seg || likely(REGION_OFFSET(addr) < RGN_MAP_LIMIT)); } -#define access_ok(type, addr, size) __access_ok((addr), (size)) +#define access_ok(addr, size) __access_ok((addr), (size)) /* * These are the main single-value transfer routines. They automatically diff --git a/arch/ia64/kernel/ptrace.c b/arch/ia64/kernel/ptrace.c index 427cd565fd61..6d50ede0ed69 100644 --- a/arch/ia64/kernel/ptrace.c +++ b/arch/ia64/kernel/ptrace.c @@ -836,7 +836,7 @@ ptrace_getregs (struct task_struct *child, struct pt_all_user_regs __user *ppr) char nat = 0; int i; - if (!access_ok(VERIFY_WRITE, ppr, sizeof(struct pt_all_user_regs))) + if (!access_ok(ppr, sizeof(struct pt_all_user_regs))) return -EIO; pt = task_pt_regs(child); @@ -981,7 +981,7 @@ ptrace_setregs (struct task_struct *child, struct pt_all_user_regs __user *ppr) memset(&fpval, 0, sizeof(fpval)); - if (!access_ok(VERIFY_READ, ppr, sizeof(struct pt_all_user_regs))) + if (!access_ok(ppr, sizeof(struct pt_all_user_regs))) return -EIO; pt = task_pt_regs(child); diff --git a/arch/ia64/kernel/signal.c b/arch/ia64/kernel/signal.c index 99099f73b207..6062fd14e34e 100644 --- a/arch/ia64/kernel/signal.c +++ b/arch/ia64/kernel/signal.c @@ -132,7 +132,7 @@ ia64_rt_sigreturn (struct sigscratch *scr) */ retval = (long) &ia64_strace_leave_kernel; - if (!access_ok(VERIFY_READ, sc, sizeof(*sc))) + if (!access_ok(sc, sizeof(*sc))) goto give_sigsegv; if (GET_SIGSET(&set, &sc->sc_mask)) @@ -264,7 +264,7 @@ setup_frame(struct ksignal *ksig, sigset_t *set, struct sigscratch *scr) } frame = (void __user *) ((new_sp - sizeof(*frame)) & -STACK_ALIGN); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) { + if (!access_ok(frame, sizeof(*frame))) { force_sigsegv(ksig->sig, current); return 1; } diff --git a/arch/m68k/include/asm/uaccess_mm.h b/arch/m68k/include/asm/uaccess_mm.h index c4cb889660aa..7e85de984df1 100644 --- a/arch/m68k/include/asm/uaccess_mm.h +++ b/arch/m68k/include/asm/uaccess_mm.h @@ -10,7 +10,7 @@ #include /* We let the MMU do all checking */ -static inline int access_ok(int type, const void __user *addr, +static inline int access_ok(const void __user *addr, unsigned long size) { return 1; diff --git a/arch/m68k/include/asm/uaccess_no.h b/arch/m68k/include/asm/uaccess_no.h index 892efb56beef..0134008bf539 100644 --- a/arch/m68k/include/asm/uaccess_no.h +++ b/arch/m68k/include/asm/uaccess_no.h @@ -10,7 +10,7 @@ #include -#define access_ok(type,addr,size) _access_ok((unsigned long)(addr),(size)) +#define access_ok(addr,size) _access_ok((unsigned long)(addr),(size)) /* * It is not enough to just have access_ok check for a real RAM address. diff --git a/arch/m68k/kernel/signal.c b/arch/m68k/kernel/signal.c index 72850b85ecf8..e2a9421c5797 100644 --- a/arch/m68k/kernel/signal.c +++ b/arch/m68k/kernel/signal.c @@ -787,7 +787,7 @@ asmlinkage int do_sigreturn(struct pt_regs *regs, struct switch_stack *sw) struct sigframe __user *frame = (struct sigframe __user *)(usp - 4); sigset_t set; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__get_user(set.sig[0], &frame->sc.sc_mask) || (_NSIG_WORDS > 1 && @@ -812,7 +812,7 @@ asmlinkage int do_rt_sigreturn(struct pt_regs *regs, struct switch_stack *sw) struct rt_sigframe __user *frame = (struct rt_sigframe __user *)(usp - 4); sigset_t set; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; diff --git a/arch/microblaze/include/asm/futex.h b/arch/microblaze/include/asm/futex.h index 2572077b04ea..8c90357e5983 100644 --- a/arch/microblaze/include/asm/futex.h +++ b/arch/microblaze/include/asm/futex.h @@ -71,7 +71,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, int ret = 0, cmp; u32 prev; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; __asm__ __volatile__ ("1: lwx %1, %3, r0; \ diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index 81f16aadbf9e..dbfea093a7c7 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -60,26 +60,25 @@ static inline int ___range_ok(unsigned long addr, unsigned long size) #define __range_ok(addr, size) \ ___range_ok((unsigned long)(addr), (unsigned long)(size)) -#define access_ok(type, addr, size) (__range_ok((addr), (size)) == 0) +#define access_ok(addr, size) (__range_ok((addr), (size)) == 0) #else -static inline int access_ok(int type, const void __user *addr, - unsigned long size) +static inline int access_ok(const void __user *addr, unsigned long size) { if (!size) goto ok; if ((get_fs().seg < ((unsigned long)addr)) || (get_fs().seg < ((unsigned long)addr + size - 1))) { - pr_devel("ACCESS fail: %s at 0x%08x (size 0x%x), seg 0x%08x\n", - type ? "WRITE" : "READ ", (__force u32)addr, (u32)size, + pr_devel("ACCESS fail at 0x%08x (size 0x%x), seg 0x%08x\n", + (__force u32)addr, (u32)size, (u32)get_fs().seg); return 0; } ok: - pr_devel("ACCESS OK: %s at 0x%08x (size 0x%x), seg 0x%08x\n", - type ? "WRITE" : "READ ", (__force u32)addr, (u32)size, + pr_devel("ACCESS OK at 0x%08x (size 0x%x), seg 0x%08x\n", + (__force u32)addr, (u32)size, (u32)get_fs().seg); return 1; } @@ -120,7 +119,7 @@ static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) { might_fault(); - if (unlikely(!access_ok(VERIFY_WRITE, to, n))) + if (unlikely(!access_ok(to, n))) return n; return __clear_user(to, n); @@ -174,7 +173,7 @@ extern long __user_bad(void); const typeof(*(ptr)) __user *__gu_addr = (ptr); \ int __gu_err = 0; \ \ - if (access_ok(VERIFY_READ, __gu_addr, size)) { \ + if (access_ok(__gu_addr, size)) { \ switch (size) { \ case 1: \ __get_user_asm("lbu", __gu_addr, __gu_val, \ @@ -286,7 +285,7 @@ extern long __user_bad(void); typeof(*(ptr)) __user *__pu_addr = (ptr); \ int __pu_err = 0; \ \ - if (access_ok(VERIFY_WRITE, __pu_addr, size)) { \ + if (access_ok(__pu_addr, size)) { \ switch (size) { \ case 1: \ __put_user_asm("sb", __pu_addr, __pu_val, \ @@ -358,7 +357,7 @@ extern int __strncpy_user(char *to, const char __user *from, int len); static inline long strncpy_from_user(char *dst, const char __user *src, long count) { - if (!access_ok(VERIFY_READ, src, 1)) + if (!access_ok(src, 1)) return -EFAULT; return __strncpy_user(dst, src, count); } @@ -372,7 +371,7 @@ extern int __strnlen_user(const char __user *sstr, int len); static inline long strnlen_user(const char __user *src, long n) { - if (!access_ok(VERIFY_READ, src, 1)) + if (!access_ok(src, 1)) return 0; return __strnlen_user(src, n); } diff --git a/arch/microblaze/kernel/signal.c b/arch/microblaze/kernel/signal.c index 97001524ca2d..0685696349bb 100644 --- a/arch/microblaze/kernel/signal.c +++ b/arch/microblaze/kernel/signal.c @@ -91,7 +91,7 @@ asmlinkage long sys_rt_sigreturn(struct pt_regs *regs) /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) @@ -166,7 +166,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, frame = get_sigframe(ksig, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; if (ksig->ka.sa.sa_flags & SA_SIGINFO) diff --git a/arch/mips/include/asm/checksum.h b/arch/mips/include/asm/checksum.h index e8161e4dfde7..dcebaaf8c862 100644 --- a/arch/mips/include/asm/checksum.h +++ b/arch/mips/include/asm/checksum.h @@ -63,7 +63,7 @@ static inline __wsum csum_and_copy_from_user(const void __user *src, void *dst, int len, __wsum sum, int *err_ptr) { - if (access_ok(VERIFY_READ, src, len)) + if (access_ok(src, len)) return csum_partial_copy_from_user(src, dst, len, sum, err_ptr); if (len) @@ -81,7 +81,7 @@ __wsum csum_and_copy_to_user(const void *src, void __user *dst, int len, __wsum sum, int *err_ptr) { might_fault(); - if (access_ok(VERIFY_WRITE, dst, len)) { + if (access_ok(dst, len)) { if (uaccess_kernel()) return __csum_partial_copy_kernel(src, (__force void *)dst, diff --git a/arch/mips/include/asm/futex.h b/arch/mips/include/asm/futex.h index 8eff134b3a43..c14d798f3888 100644 --- a/arch/mips/include/asm/futex.h +++ b/arch/mips/include/asm/futex.h @@ -129,7 +129,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, int ret = 0; u32 val; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; if (cpu_has_llsc && R10000_LLSC_WAR) { diff --git a/arch/mips/include/asm/termios.h b/arch/mips/include/asm/termios.h index ce2d72e34274..bc29eeacc55a 100644 --- a/arch/mips/include/asm/termios.h +++ b/arch/mips/include/asm/termios.h @@ -32,7 +32,7 @@ static inline int user_termio_to_kernel_termios(struct ktermios *termios, unsigned short iflag, oflag, cflag, lflag; unsigned int err; - if (!access_ok(VERIFY_READ, termio, sizeof(struct termio))) + if (!access_ok(termio, sizeof(struct termio))) return -EFAULT; err = __get_user(iflag, &termio->c_iflag); @@ -61,7 +61,7 @@ static inline int kernel_termios_to_user_termio(struct termio __user *termio, { int err; - if (!access_ok(VERIFY_WRITE, termio, sizeof(struct termio))) + if (!access_ok(termio, sizeof(struct termio))) return -EFAULT; err = __put_user(termios->c_iflag, &termio->c_iflag); diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index 06629011a434..d43c1dc6ef15 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -109,9 +109,6 @@ static inline bool eva_kernel_access(void) /* * access_ok: - Checks if a user space pointer is valid - * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that - * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe - * to write to a block, it is always safe to read from it. * @addr: User space pointer to start of block to check * @size: Size of block to check * @@ -134,7 +131,7 @@ static inline int __access_ok(const void __user *p, unsigned long size) return (get_fs().seg & (addr | (addr + size) | __ua_size(size))) == 0; } -#define access_ok(type, addr, size) \ +#define access_ok(addr, size) \ likely(__access_ok((addr), (size))) /* @@ -304,7 +301,7 @@ do { \ const __typeof__(*(ptr)) __user * __gu_ptr = (ptr); \ \ might_fault(); \ - if (likely(access_ok(VERIFY_READ, __gu_ptr, size))) { \ + if (likely(access_ok( __gu_ptr, size))) { \ if (eva_kernel_access()) \ __get_kernel_common((x), size, __gu_ptr); \ else \ @@ -446,7 +443,7 @@ do { \ int __pu_err = -EFAULT; \ \ might_fault(); \ - if (likely(access_ok(VERIFY_WRITE, __pu_addr, size))) { \ + if (likely(access_ok( __pu_addr, size))) { \ if (eva_kernel_access()) \ __put_kernel_common(__pu_addr, size); \ else \ @@ -691,8 +688,7 @@ __clear_user(void __user *addr, __kernel_size_t size) ({ \ void __user * __cl_addr = (addr); \ unsigned long __cl_size = (n); \ - if (__cl_size && access_ok(VERIFY_WRITE, \ - __cl_addr, __cl_size)) \ + if (__cl_size && access_ok(__cl_addr, __cl_size)) \ __cl_size = __clear_user(__cl_addr, __cl_size); \ __cl_size; \ }) diff --git a/arch/mips/kernel/mips-r2-to-r6-emul.c b/arch/mips/kernel/mips-r2-to-r6-emul.c index cb22a558431e..c50c89a978f1 100644 --- a/arch/mips/kernel/mips-r2-to-r6-emul.c +++ b/arch/mips/kernel/mips-r2-to-r6-emul.c @@ -1205,7 +1205,7 @@ fpu_emul: case lwl_op: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_READ, (void __user *)vaddr, 4)) { + if (!access_ok((void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1278,7 +1278,7 @@ fpu_emul: case lwr_op: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_READ, (void __user *)vaddr, 4)) { + if (!access_ok((void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1352,7 +1352,7 @@ fpu_emul: case swl_op: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 4)) { + if (!access_ok((void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1422,7 +1422,7 @@ fpu_emul: case swr_op: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 4)) { + if (!access_ok((void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1497,7 +1497,7 @@ fpu_emul: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_READ, (void __user *)vaddr, 8)) { + if (!access_ok((void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1616,7 +1616,7 @@ fpu_emul: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_READ, (void __user *)vaddr, 8)) { + if (!access_ok((void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1735,7 +1735,7 @@ fpu_emul: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 8)) { + if (!access_ok((void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1853,7 +1853,7 @@ fpu_emul: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 8)) { + if (!access_ok((void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1970,7 +1970,7 @@ fpu_emul: err = SIGBUS; break; } - if (!access_ok(VERIFY_READ, (void __user *)vaddr, 4)) { + if (!access_ok((void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGBUS; break; @@ -2026,7 +2026,7 @@ fpu_emul: err = SIGBUS; break; } - if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 4)) { + if (!access_ok((void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGBUS; break; @@ -2089,7 +2089,7 @@ fpu_emul: err = SIGBUS; break; } - if (!access_ok(VERIFY_READ, (void __user *)vaddr, 8)) { + if (!access_ok((void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGBUS; break; @@ -2150,7 +2150,7 @@ fpu_emul: err = SIGBUS; break; } - if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 8)) { + if (!access_ok((void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGBUS; break; diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c index ea54575255ea..0057c910bc2f 100644 --- a/arch/mips/kernel/ptrace.c +++ b/arch/mips/kernel/ptrace.c @@ -71,7 +71,7 @@ int ptrace_getregs(struct task_struct *child, struct user_pt_regs __user *data) struct pt_regs *regs; int i; - if (!access_ok(VERIFY_WRITE, data, 38 * 8)) + if (!access_ok(data, 38 * 8)) return -EIO; regs = task_pt_regs(child); @@ -98,7 +98,7 @@ int ptrace_setregs(struct task_struct *child, struct user_pt_regs __user *data) struct pt_regs *regs; int i; - if (!access_ok(VERIFY_READ, data, 38 * 8)) + if (!access_ok(data, 38 * 8)) return -EIO; regs = task_pt_regs(child); @@ -125,7 +125,7 @@ int ptrace_get_watch_regs(struct task_struct *child, if (!cpu_has_watch || boot_cpu_data.watch_reg_use_cnt == 0) return -EIO; - if (!access_ok(VERIFY_WRITE, addr, sizeof(struct pt_watch_regs))) + if (!access_ok(addr, sizeof(struct pt_watch_regs))) return -EIO; #ifdef CONFIG_32BIT @@ -167,7 +167,7 @@ int ptrace_set_watch_regs(struct task_struct *child, if (!cpu_has_watch || boot_cpu_data.watch_reg_use_cnt == 0) return -EIO; - if (!access_ok(VERIFY_READ, addr, sizeof(struct pt_watch_regs))) + if (!access_ok(addr, sizeof(struct pt_watch_regs))) return -EIO; /* Check the values. */ for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { @@ -359,7 +359,7 @@ int ptrace_getfpregs(struct task_struct *child, __u32 __user *data) { int i; - if (!access_ok(VERIFY_WRITE, data, 33 * 8)) + if (!access_ok(data, 33 * 8)) return -EIO; if (tsk_used_math(child)) { @@ -385,7 +385,7 @@ int ptrace_setfpregs(struct task_struct *child, __u32 __user *data) u32 value; int i; - if (!access_ok(VERIFY_READ, data, 33 * 8)) + if (!access_ok(data, 33 * 8)) return -EIO; init_fp_ctx(child); diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c index d3a23758592c..d75337974ee9 100644 --- a/arch/mips/kernel/signal.c +++ b/arch/mips/kernel/signal.c @@ -590,7 +590,7 @@ SYSCALL_DEFINE3(sigaction, int, sig, const struct sigaction __user *, act, if (act) { old_sigset_t mask; - if (!access_ok(VERIFY_READ, act, sizeof(*act))) + if (!access_ok(act, sizeof(*act))) return -EFAULT; err |= __get_user(new_ka.sa.sa_handler, &act->sa_handler); err |= __get_user(new_ka.sa.sa_flags, &act->sa_flags); @@ -604,7 +604,7 @@ SYSCALL_DEFINE3(sigaction, int, sig, const struct sigaction __user *, act, ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); if (!ret && oact) { - if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact))) + if (!access_ok(oact, sizeof(*oact))) return -EFAULT; err |= __put_user(old_ka.sa.sa_flags, &oact->sa_flags); err |= __put_user(old_ka.sa.sa_handler, &oact->sa_handler); @@ -630,7 +630,7 @@ asmlinkage void sys_sigreturn(void) regs = current_pt_regs(); frame = (struct sigframe __user *)regs->regs[29]; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&blocked, &frame->sf_mask, sizeof(blocked))) goto badframe; @@ -667,7 +667,7 @@ asmlinkage void sys_rt_sigreturn(void) regs = current_pt_regs(); frame = (struct rt_sigframe __user *)regs->regs[29]; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->rs_uc.uc_sigmask, sizeof(set))) goto badframe; @@ -705,7 +705,7 @@ static int setup_frame(void *sig_return, struct ksignal *ksig, int err = 0; frame = get_sigframe(ksig, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) + if (!access_ok(frame, sizeof (*frame))) return -EFAULT; err |= setup_sigcontext(regs, &frame->sf_sc); @@ -744,7 +744,7 @@ static int setup_rt_frame(void *sig_return, struct ksignal *ksig, int err = 0; frame = get_sigframe(ksig, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) + if (!access_ok(frame, sizeof (*frame))) return -EFAULT; /* Create siginfo. */ diff --git a/arch/mips/kernel/signal32.c b/arch/mips/kernel/signal32.c index b5d9e1784aff..59b8965433c2 100644 --- a/arch/mips/kernel/signal32.c +++ b/arch/mips/kernel/signal32.c @@ -46,7 +46,7 @@ SYSCALL_DEFINE3(32_sigaction, long, sig, const struct compat_sigaction __user *, old_sigset_t mask; s32 handler; - if (!access_ok(VERIFY_READ, act, sizeof(*act))) + if (!access_ok(act, sizeof(*act))) return -EFAULT; err |= __get_user(handler, &act->sa_handler); new_ka.sa.sa_handler = (void __user *)(s64)handler; @@ -61,7 +61,7 @@ SYSCALL_DEFINE3(32_sigaction, long, sig, const struct compat_sigaction __user *, ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); if (!ret && oact) { - if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact))) + if (!access_ok(oact, sizeof(*oact))) return -EFAULT; err |= __put_user(old_ka.sa.sa_flags, &oact->sa_flags); err |= __put_user((u32)(u64)old_ka.sa.sa_handler, diff --git a/arch/mips/kernel/signal_n32.c b/arch/mips/kernel/signal_n32.c index 8f65aaf9206d..c498b027823e 100644 --- a/arch/mips/kernel/signal_n32.c +++ b/arch/mips/kernel/signal_n32.c @@ -73,7 +73,7 @@ asmlinkage void sysn32_rt_sigreturn(void) regs = current_pt_regs(); frame = (struct rt_sigframe_n32 __user *)regs->regs[29]; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_conv_sigset_from_user(&set, &frame->rs_uc.uc_sigmask)) goto badframe; @@ -110,7 +110,7 @@ static int setup_rt_frame_n32(void *sig_return, struct ksignal *ksig, int err = 0; frame = get_sigframe(ksig, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) + if (!access_ok(frame, sizeof (*frame))) return -EFAULT; /* Create siginfo. */ diff --git a/arch/mips/kernel/signal_o32.c b/arch/mips/kernel/signal_o32.c index b6e3ddef48a0..df259618e834 100644 --- a/arch/mips/kernel/signal_o32.c +++ b/arch/mips/kernel/signal_o32.c @@ -118,7 +118,7 @@ static int setup_frame_32(void *sig_return, struct ksignal *ksig, int err = 0; frame = get_sigframe(ksig, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) + if (!access_ok(frame, sizeof (*frame))) return -EFAULT; err |= setup_sigcontext32(regs, &frame->sf_sc); @@ -160,7 +160,7 @@ asmlinkage void sys32_rt_sigreturn(void) regs = current_pt_regs(); frame = (struct rt_sigframe32 __user *)regs->regs[29]; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_conv_sigset_from_user(&set, &frame->rs_uc.uc_sigmask)) goto badframe; @@ -197,7 +197,7 @@ static int setup_rt_frame_32(void *sig_return, struct ksignal *ksig, int err = 0; frame = get_sigframe(ksig, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) + if (!access_ok(frame, sizeof (*frame))) return -EFAULT; /* Convert (siginfo_t -> compat_siginfo_t) and copy to user. */ @@ -262,7 +262,7 @@ asmlinkage void sys32_sigreturn(void) regs = current_pt_regs(); frame = (struct sigframe32 __user *)regs->regs[29]; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_conv_sigset_from_user(&blocked, &frame->sf_mask)) goto badframe; diff --git a/arch/mips/kernel/syscall.c b/arch/mips/kernel/syscall.c index 41a0db08cd37..b6dc78ad5d8c 100644 --- a/arch/mips/kernel/syscall.c +++ b/arch/mips/kernel/syscall.c @@ -101,7 +101,7 @@ static inline int mips_atomic_set(unsigned long addr, unsigned long new) if (unlikely(addr & 3)) return -EINVAL; - if (unlikely(!access_ok(VERIFY_WRITE, (const void __user *)addr, 4))) + if (unlikely(!access_ok((const void __user *)addr, 4))) return -EINVAL; if (cpu_has_llsc && R10000_LLSC_WAR) { diff --git a/arch/mips/kernel/unaligned.c b/arch/mips/kernel/unaligned.c index c60e7719ef77..595ca9c85111 100644 --- a/arch/mips/kernel/unaligned.c +++ b/arch/mips/kernel/unaligned.c @@ -936,7 +936,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, if (insn.dsp_format.func == lx_op) { switch (insn.dsp_format.op) { case lwx_op: - if (!access_ok(VERIFY_READ, addr, 4)) + if (!access_ok(addr, 4)) goto sigbus; LoadW(addr, value, res); if (res) @@ -945,7 +945,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, regs->regs[insn.dsp_format.rd] = value; break; case lhx_op: - if (!access_ok(VERIFY_READ, addr, 2)) + if (!access_ok(addr, 2)) goto sigbus; LoadHW(addr, value, res); if (res) @@ -968,7 +968,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, set_fs(USER_DS); switch (insn.spec3_format.func) { case lhe_op: - if (!access_ok(VERIFY_READ, addr, 2)) { + if (!access_ok(addr, 2)) { set_fs(seg); goto sigbus; } @@ -981,7 +981,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, regs->regs[insn.spec3_format.rt] = value; break; case lwe_op: - if (!access_ok(VERIFY_READ, addr, 4)) { + if (!access_ok(addr, 4)) { set_fs(seg); goto sigbus; } @@ -994,7 +994,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, regs->regs[insn.spec3_format.rt] = value; break; case lhue_op: - if (!access_ok(VERIFY_READ, addr, 2)) { + if (!access_ok(addr, 2)) { set_fs(seg); goto sigbus; } @@ -1007,7 +1007,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, regs->regs[insn.spec3_format.rt] = value; break; case she_op: - if (!access_ok(VERIFY_WRITE, addr, 2)) { + if (!access_ok(addr, 2)) { set_fs(seg); goto sigbus; } @@ -1020,7 +1020,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, } break; case swe_op: - if (!access_ok(VERIFY_WRITE, addr, 4)) { + if (!access_ok(addr, 4)) { set_fs(seg); goto sigbus; } @@ -1041,7 +1041,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, #endif break; case lh_op: - if (!access_ok(VERIFY_READ, addr, 2)) + if (!access_ok(addr, 2)) goto sigbus; if (IS_ENABLED(CONFIG_EVA)) { @@ -1060,7 +1060,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, break; case lw_op: - if (!access_ok(VERIFY_READ, addr, 4)) + if (!access_ok(addr, 4)) goto sigbus; if (IS_ENABLED(CONFIG_EVA)) { @@ -1079,7 +1079,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, break; case lhu_op: - if (!access_ok(VERIFY_READ, addr, 2)) + if (!access_ok(addr, 2)) goto sigbus; if (IS_ENABLED(CONFIG_EVA)) { @@ -1106,7 +1106,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 4)) + if (!access_ok(addr, 4)) goto sigbus; LoadWU(addr, value, res); @@ -1129,7 +1129,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 8)) + if (!access_ok(addr, 8)) goto sigbus; LoadDW(addr, value, res); @@ -1144,7 +1144,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, goto sigill; case sh_op: - if (!access_ok(VERIFY_WRITE, addr, 2)) + if (!access_ok(addr, 2)) goto sigbus; compute_return_epc(regs); @@ -1164,7 +1164,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, break; case sw_op: - if (!access_ok(VERIFY_WRITE, addr, 4)) + if (!access_ok(addr, 4)) goto sigbus; compute_return_epc(regs); @@ -1192,7 +1192,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_WRITE, addr, 8)) + if (!access_ok(addr, 8)) goto sigbus; compute_return_epc(regs); @@ -1254,7 +1254,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, switch (insn.msa_mi10_format.func) { case msa_ld_op: - if (!access_ok(VERIFY_READ, addr, sizeof(*fpr))) + if (!access_ok(addr, sizeof(*fpr))) goto sigbus; do { @@ -1290,7 +1290,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, break; case msa_st_op: - if (!access_ok(VERIFY_WRITE, addr, sizeof(*fpr))) + if (!access_ok(addr, sizeof(*fpr))) goto sigbus; /* @@ -1463,7 +1463,7 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if (reg == 31) goto sigbus; - if (!access_ok(VERIFY_READ, addr, 8)) + if (!access_ok(addr, 8)) goto sigbus; LoadW(addr, value, res); @@ -1482,7 +1482,7 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if (reg == 31) goto sigbus; - if (!access_ok(VERIFY_WRITE, addr, 8)) + if (!access_ok(addr, 8)) goto sigbus; value = regs->regs[reg]; @@ -1502,7 +1502,7 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if (reg == 31) goto sigbus; - if (!access_ok(VERIFY_READ, addr, 16)) + if (!access_ok(addr, 16)) goto sigbus; LoadDW(addr, value, res); @@ -1525,7 +1525,7 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if (reg == 31) goto sigbus; - if (!access_ok(VERIFY_WRITE, addr, 16)) + if (!access_ok(addr, 16)) goto sigbus; value = regs->regs[reg]; @@ -1548,11 +1548,10 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { - if (!access_ok - (VERIFY_READ, addr, 4 * (rvar + 1))) + if (!access_ok(addr, 4 * (rvar + 1))) goto sigbus; } else { - if (!access_ok(VERIFY_READ, addr, 4 * rvar)) + if (!access_ok(addr, 4 * rvar)) goto sigbus; } if (rvar == 9) @@ -1585,11 +1584,10 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { - if (!access_ok - (VERIFY_WRITE, addr, 4 * (rvar + 1))) + if (!access_ok(addr, 4 * (rvar + 1))) goto sigbus; } else { - if (!access_ok(VERIFY_WRITE, addr, 4 * rvar)) + if (!access_ok(addr, 4 * rvar)) goto sigbus; } if (rvar == 9) @@ -1623,11 +1621,10 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { - if (!access_ok - (VERIFY_READ, addr, 8 * (rvar + 1))) + if (!access_ok(addr, 8 * (rvar + 1))) goto sigbus; } else { - if (!access_ok(VERIFY_READ, addr, 8 * rvar)) + if (!access_ok(addr, 8 * rvar)) goto sigbus; } if (rvar == 9) @@ -1665,11 +1662,10 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { - if (!access_ok - (VERIFY_WRITE, addr, 8 * (rvar + 1))) + if (!access_ok(addr, 8 * (rvar + 1))) goto sigbus; } else { - if (!access_ok(VERIFY_WRITE, addr, 8 * rvar)) + if (!access_ok(addr, 8 * rvar)) goto sigbus; } if (rvar == 9) @@ -1788,7 +1784,7 @@ fpu_emul: case mm_lwm16_op: reg = insn.mm16_m_format.rlist; rvar = reg + 1; - if (!access_ok(VERIFY_READ, addr, 4 * rvar)) + if (!access_ok(addr, 4 * rvar)) goto sigbus; for (i = 16; rvar; rvar--, i++) { @@ -1808,7 +1804,7 @@ fpu_emul: case mm_swm16_op: reg = insn.mm16_m_format.rlist; rvar = reg + 1; - if (!access_ok(VERIFY_WRITE, addr, 4 * rvar)) + if (!access_ok(addr, 4 * rvar)) goto sigbus; for (i = 16; rvar; rvar--, i++) { @@ -1862,7 +1858,7 @@ fpu_emul: } loadHW: - if (!access_ok(VERIFY_READ, addr, 2)) + if (!access_ok(addr, 2)) goto sigbus; LoadHW(addr, value, res); @@ -1872,7 +1868,7 @@ loadHW: goto success; loadHWU: - if (!access_ok(VERIFY_READ, addr, 2)) + if (!access_ok(addr, 2)) goto sigbus; LoadHWU(addr, value, res); @@ -1882,7 +1878,7 @@ loadHWU: goto success; loadW: - if (!access_ok(VERIFY_READ, addr, 4)) + if (!access_ok(addr, 4)) goto sigbus; LoadW(addr, value, res); @@ -1900,7 +1896,7 @@ loadWU: * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 4)) + if (!access_ok(addr, 4)) goto sigbus; LoadWU(addr, value, res); @@ -1922,7 +1918,7 @@ loadDW: * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 8)) + if (!access_ok(addr, 8)) goto sigbus; LoadDW(addr, value, res); @@ -1936,7 +1932,7 @@ loadDW: goto sigill; storeHW: - if (!access_ok(VERIFY_WRITE, addr, 2)) + if (!access_ok(addr, 2)) goto sigbus; value = regs->regs[reg]; @@ -1946,7 +1942,7 @@ storeHW: goto success; storeW: - if (!access_ok(VERIFY_WRITE, addr, 4)) + if (!access_ok(addr, 4)) goto sigbus; value = regs->regs[reg]; @@ -1964,7 +1960,7 @@ storeDW: * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_WRITE, addr, 8)) + if (!access_ok(addr, 8)) goto sigbus; value = regs->regs[reg]; @@ -2122,7 +2118,7 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) goto sigbus; case MIPS16e_lh_op: - if (!access_ok(VERIFY_READ, addr, 2)) + if (!access_ok(addr, 2)) goto sigbus; LoadHW(addr, value, res); @@ -2133,7 +2129,7 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) break; case MIPS16e_lhu_op: - if (!access_ok(VERIFY_READ, addr, 2)) + if (!access_ok(addr, 2)) goto sigbus; LoadHWU(addr, value, res); @@ -2146,7 +2142,7 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) case MIPS16e_lw_op: case MIPS16e_lwpc_op: case MIPS16e_lwsp_op: - if (!access_ok(VERIFY_READ, addr, 4)) + if (!access_ok(addr, 4)) goto sigbus; LoadW(addr, value, res); @@ -2165,7 +2161,7 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 4)) + if (!access_ok(addr, 4)) goto sigbus; LoadWU(addr, value, res); @@ -2189,7 +2185,7 @@ loadDW: * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 8)) + if (!access_ok(addr, 8)) goto sigbus; LoadDW(addr, value, res); @@ -2204,7 +2200,7 @@ loadDW: goto sigill; case MIPS16e_sh_op: - if (!access_ok(VERIFY_WRITE, addr, 2)) + if (!access_ok(addr, 2)) goto sigbus; MIPS16e_compute_return_epc(regs, &oldinst); @@ -2217,7 +2213,7 @@ loadDW: case MIPS16e_sw_op: case MIPS16e_swsp_op: case MIPS16e_i8_op: /* actually - MIPS16e_swrasp_func */ - if (!access_ok(VERIFY_WRITE, addr, 4)) + if (!access_ok(addr, 4)) goto sigbus; MIPS16e_compute_return_epc(regs, &oldinst); @@ -2237,7 +2233,7 @@ writeDW: * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_WRITE, addr, 8)) + if (!access_ok(addr, 8)) goto sigbus; MIPS16e_compute_return_epc(regs, &oldinst); diff --git a/arch/mips/math-emu/cp1emu.c b/arch/mips/math-emu/cp1emu.c index 82e2993c1a2c..e60e29078ef5 100644 --- a/arch/mips/math-emu/cp1emu.c +++ b/arch/mips/math-emu/cp1emu.c @@ -1063,7 +1063,7 @@ emul: MIPSInst_SIMM(ir)); MIPS_FPU_EMU_INC_STATS(loads); - if (!access_ok(VERIFY_READ, dva, sizeof(u64))) { + if (!access_ok(dva, sizeof(u64))) { MIPS_FPU_EMU_INC_STATS(errors); *fault_addr = dva; return SIGBUS; @@ -1081,7 +1081,7 @@ emul: MIPSInst_SIMM(ir)); MIPS_FPU_EMU_INC_STATS(stores); DIFROMREG(dval, MIPSInst_RT(ir)); - if (!access_ok(VERIFY_WRITE, dva, sizeof(u64))) { + if (!access_ok(dva, sizeof(u64))) { MIPS_FPU_EMU_INC_STATS(errors); *fault_addr = dva; return SIGBUS; @@ -1097,7 +1097,7 @@ emul: wva = (u32 __user *) (xcp->regs[MIPSInst_RS(ir)] + MIPSInst_SIMM(ir)); MIPS_FPU_EMU_INC_STATS(loads); - if (!access_ok(VERIFY_READ, wva, sizeof(u32))) { + if (!access_ok(wva, sizeof(u32))) { MIPS_FPU_EMU_INC_STATS(errors); *fault_addr = wva; return SIGBUS; @@ -1115,7 +1115,7 @@ emul: MIPSInst_SIMM(ir)); MIPS_FPU_EMU_INC_STATS(stores); SIFROMREG(wval, MIPSInst_RT(ir)); - if (!access_ok(VERIFY_WRITE, wva, sizeof(u32))) { + if (!access_ok(wva, sizeof(u32))) { MIPS_FPU_EMU_INC_STATS(errors); *fault_addr = wva; return SIGBUS; @@ -1493,7 +1493,7 @@ static int fpux_emu(struct pt_regs *xcp, struct mips_fpu_struct *ctx, xcp->regs[MIPSInst_FT(ir)]); MIPS_FPU_EMU_INC_STATS(loads); - if (!access_ok(VERIFY_READ, va, sizeof(u32))) { + if (!access_ok(va, sizeof(u32))) { MIPS_FPU_EMU_INC_STATS(errors); *fault_addr = va; return SIGBUS; @@ -1513,7 +1513,7 @@ static int fpux_emu(struct pt_regs *xcp, struct mips_fpu_struct *ctx, MIPS_FPU_EMU_INC_STATS(stores); SIFROMREG(val, MIPSInst_FS(ir)); - if (!access_ok(VERIFY_WRITE, va, sizeof(u32))) { + if (!access_ok(va, sizeof(u32))) { MIPS_FPU_EMU_INC_STATS(errors); *fault_addr = va; return SIGBUS; @@ -1590,7 +1590,7 @@ static int fpux_emu(struct pt_regs *xcp, struct mips_fpu_struct *ctx, xcp->regs[MIPSInst_FT(ir)]); MIPS_FPU_EMU_INC_STATS(loads); - if (!access_ok(VERIFY_READ, va, sizeof(u64))) { + if (!access_ok(va, sizeof(u64))) { MIPS_FPU_EMU_INC_STATS(errors); *fault_addr = va; return SIGBUS; @@ -1609,7 +1609,7 @@ static int fpux_emu(struct pt_regs *xcp, struct mips_fpu_struct *ctx, MIPS_FPU_EMU_INC_STATS(stores); DIFROMREG(val, MIPSInst_FS(ir)); - if (!access_ok(VERIFY_WRITE, va, sizeof(u64))) { + if (!access_ok(va, sizeof(u64))) { MIPS_FPU_EMU_INC_STATS(errors); *fault_addr = va; return SIGBUS; diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index 70a523151ff3..55099fbff4e6 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -76,7 +76,7 @@ SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes, { if (bytes == 0) return 0; - if (!access_ok(VERIFY_WRITE, (void __user *) addr, bytes)) + if (!access_ok((void __user *) addr, bytes)) return -EFAULT; __flush_icache_user_range(addr, addr + bytes); diff --git a/arch/mips/mm/gup.c b/arch/mips/mm/gup.c index 5a4875cac1ec..0d14e0d8eacf 100644 --- a/arch/mips/mm/gup.c +++ b/arch/mips/mm/gup.c @@ -195,8 +195,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write, addr = start; len = (unsigned long) nr_pages << PAGE_SHIFT; end = start + len; - if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ, - (void __user *)start, len))) + if (unlikely(!access_ok((void __user *)start, len))) return 0; /* diff --git a/arch/mips/oprofile/backtrace.c b/arch/mips/oprofile/backtrace.c index 806fb798091f..07d98ba7f49e 100644 --- a/arch/mips/oprofile/backtrace.c +++ b/arch/mips/oprofile/backtrace.c @@ -19,7 +19,7 @@ struct stackframe { static inline int get_mem(unsigned long addr, unsigned long *result) { unsigned long *address = (unsigned long *) addr; - if (!access_ok(VERIFY_READ, address, sizeof(unsigned long))) + if (!access_ok(address, sizeof(unsigned long))) return -1; if (__copy_from_user_inatomic(result, address, sizeof(unsigned long))) return -3; diff --git a/arch/mips/sibyte/common/sb_tbprof.c b/arch/mips/sibyte/common/sb_tbprof.c index 99c720be72d2..9ff26b0cd3b6 100644 --- a/arch/mips/sibyte/common/sb_tbprof.c +++ b/arch/mips/sibyte/common/sb_tbprof.c @@ -458,7 +458,7 @@ static ssize_t sbprof_tb_read(struct file *filp, char *buf, char *dest = buf; long cur_off = *offp; - if (!access_ok(VERIFY_WRITE, buf, size)) + if (!access_ok(buf, size)) return -EFAULT; mutex_lock(&sbp.lock); diff --git a/arch/nds32/include/asm/futex.h b/arch/nds32/include/asm/futex.h index cb6cb91cfdf8..baf178bf1d0b 100644 --- a/arch/nds32/include/asm/futex.h +++ b/arch/nds32/include/asm/futex.h @@ -40,7 +40,7 @@ futex_atomic_cmpxchg_inatomic(u32 * uval, u32 __user * uaddr, int ret = 0; u32 val, tmp, flags; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; smp_mb(); diff --git a/arch/nds32/include/asm/uaccess.h b/arch/nds32/include/asm/uaccess.h index 362a32d9bd16..53dcb49b0b12 100644 --- a/arch/nds32/include/asm/uaccess.h +++ b/arch/nds32/include/asm/uaccess.h @@ -13,9 +13,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define __asmeq(x, y) ".ifnc " x "," y " ; .err ; .endif\n\t" /* @@ -53,7 +50,7 @@ static inline void set_fs(mm_segment_t fs) #define __range_ok(addr, size) (size <= get_fs() && addr <= (get_fs() -size)) -#define access_ok(type, addr, size) \ +#define access_ok(addr, size) \ __range_ok((unsigned long)addr, (unsigned long)size) /* * Single-value transfer routines. They automatically use the right @@ -94,7 +91,7 @@ static inline void set_fs(mm_segment_t fs) ({ \ const __typeof__(*(ptr)) __user *__p = (ptr); \ might_fault(); \ - if (access_ok(VERIFY_READ, __p, sizeof(*__p))) { \ + if (access_ok(__p, sizeof(*__p))) { \ __get_user_err((x), __p, (err)); \ } else { \ (x) = 0; (err) = -EFAULT; \ @@ -189,7 +186,7 @@ do { \ ({ \ __typeof__(*(ptr)) __user *__p = (ptr); \ might_fault(); \ - if (access_ok(VERIFY_WRITE, __p, sizeof(*__p))) { \ + if (access_ok(__p, sizeof(*__p))) { \ __put_user_err((x), __p, (err)); \ } else { \ (err) = -EFAULT; \ @@ -279,7 +276,7 @@ extern unsigned long __arch_copy_to_user(void __user * to, const void *from, #define INLINE_COPY_TO_USER static inline unsigned long clear_user(void __user * to, unsigned long n) { - if (access_ok(VERIFY_WRITE, to, n)) + if (access_ok(to, n)) n = __arch_clear_user(to, n); return n; } diff --git a/arch/nds32/kernel/perf_event_cpu.c b/arch/nds32/kernel/perf_event_cpu.c index 5e00ce54d0ff..334c2a6cec23 100644 --- a/arch/nds32/kernel/perf_event_cpu.c +++ b/arch/nds32/kernel/perf_event_cpu.c @@ -1306,7 +1306,7 @@ user_backtrace(struct perf_callchain_entry_ctx *entry, unsigned long fp) (unsigned long *)(fp - (unsigned long)sizeof(buftail)); /* Check accessibility of one struct frame_tail beyond */ - if (!access_ok(VERIFY_READ, user_frame_tail, sizeof(buftail))) + if (!access_ok(user_frame_tail, sizeof(buftail))) return 0; if (__copy_from_user_inatomic (&buftail, user_frame_tail, sizeof(buftail))) @@ -1332,7 +1332,7 @@ user_backtrace_opt_size(struct perf_callchain_entry_ctx *entry, (unsigned long *)(fp - (unsigned long)sizeof(buftail)); /* Check accessibility of one struct frame_tail beyond */ - if (!access_ok(VERIFY_READ, user_frame_tail, sizeof(buftail))) + if (!access_ok(user_frame_tail, sizeof(buftail))) return 0; if (__copy_from_user_inatomic (&buftail, user_frame_tail, sizeof(buftail))) @@ -1386,7 +1386,7 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, user_frame_tail = (unsigned long *)(fp - (unsigned long)sizeof(fp)); - if (!access_ok(VERIFY_READ, user_frame_tail, sizeof(fp))) + if (!access_ok(user_frame_tail, sizeof(fp))) return; if (__copy_from_user_inatomic @@ -1406,8 +1406,7 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, (unsigned long *)(fp - (unsigned long)sizeof(buftail)); - if (!access_ok - (VERIFY_READ, user_frame_tail, sizeof(buftail))) + if (!access_ok(user_frame_tail, sizeof(buftail))) return; if (__copy_from_user_inatomic @@ -1424,7 +1423,7 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, (unsigned long *)(fp - (unsigned long) sizeof(buftail_opt_size)); - if (!access_ok(VERIFY_READ, user_frame_tail, + if (!access_ok(user_frame_tail, sizeof(buftail_opt_size))) return; diff --git a/arch/nds32/kernel/signal.c b/arch/nds32/kernel/signal.c index 5b5be082cfa4..5f7660aa2d68 100644 --- a/arch/nds32/kernel/signal.c +++ b/arch/nds32/kernel/signal.c @@ -151,7 +151,7 @@ asmlinkage long sys_rt_sigreturn(struct pt_regs *regs) frame = (struct rt_sigframe __user *)regs->sp; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (restore_sigframe(regs, frame)) @@ -275,7 +275,7 @@ setup_rt_frame(struct ksignal *ksig, sigset_t * set, struct pt_regs *regs) get_sigframe(ksig, regs, sizeof(*frame)); int err = 0; - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; __put_user_error(0, &frame->uc.uc_flags, err); diff --git a/arch/nds32/mm/alignment.c b/arch/nds32/mm/alignment.c index e1aed9dc692d..c8b9061a2ee3 100644 --- a/arch/nds32/mm/alignment.c +++ b/arch/nds32/mm/alignment.c @@ -289,13 +289,13 @@ static inline int do_16(unsigned long inst, struct pt_regs *regs) unaligned_addr += shift; if (load) { - if (!access_ok(VERIFY_READ, (void *)unaligned_addr, len)) + if (!access_ok((void *)unaligned_addr, len)) return -EACCES; get_data(unaligned_addr, &target_val, len); *idx_to_addr(regs, target_idx) = target_val; } else { - if (!access_ok(VERIFY_WRITE, (void *)unaligned_addr, len)) + if (!access_ok((void *)unaligned_addr, len)) return -EACCES; target_val = *idx_to_addr(regs, target_idx); set_data((void *)unaligned_addr, target_val, len); @@ -479,7 +479,7 @@ static inline int do_32(unsigned long inst, struct pt_regs *regs) if (load) { - if (!access_ok(VERIFY_READ, (void *)unaligned_addr, len)) + if (!access_ok((void *)unaligned_addr, len)) return -EACCES; get_data(unaligned_addr, &target_val, len); @@ -491,7 +491,7 @@ static inline int do_32(unsigned long inst, struct pt_regs *regs) *idx_to_addr(regs, RT(inst)) = target_val; } else { - if (!access_ok(VERIFY_WRITE, (void *)unaligned_addr, len)) + if (!access_ok((void *)unaligned_addr, len)) return -EACCES; target_val = *idx_to_addr(regs, RT(inst)); diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h index dfa3c7cb30b4..e0ea10806491 100644 --- a/arch/nios2/include/asm/uaccess.h +++ b/arch/nios2/include/asm/uaccess.h @@ -37,7 +37,7 @@ (((signed long)(((long)get_fs().seg) & \ ((long)(addr) | (((long)(addr)) + (len)) | (len)))) == 0) -#define access_ok(type, addr, len) \ +#define access_ok(addr, len) \ likely(__access_ok((unsigned long)(addr), (unsigned long)(len))) # define __EX_TABLE_SECTION ".section __ex_table,\"a\"\n" @@ -70,7 +70,7 @@ static inline unsigned long __must_check __clear_user(void __user *to, static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) { - if (!access_ok(VERIFY_WRITE, to, n)) + if (!access_ok(to, n)) return n; return __clear_user(to, n); } @@ -142,7 +142,7 @@ do { \ long __gu_err = -EFAULT; \ const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \ unsigned long __gu_val = 0; \ - if (access_ok(VERIFY_READ, __gu_ptr, sizeof(*__gu_ptr))) \ + if (access_ok( __gu_ptr, sizeof(*__gu_ptr))) \ __get_user_common(__gu_val, sizeof(*__gu_ptr), \ __gu_ptr, __gu_err); \ (x) = (__force __typeof__(x))__gu_val; \ @@ -168,7 +168,7 @@ do { \ long __pu_err = -EFAULT; \ __typeof__(*(ptr)) __user *__pu_ptr = (ptr); \ __typeof__(*(ptr)) __pu_val = (__typeof(*ptr))(x); \ - if (access_ok(VERIFY_WRITE, __pu_ptr, sizeof(*__pu_ptr))) { \ + if (access_ok(__pu_ptr, sizeof(*__pu_ptr))) { \ switch (sizeof(*__pu_ptr)) { \ case 1: \ __put_user_asm(__pu_val, "stb", __pu_ptr, __pu_err); \ diff --git a/arch/nios2/kernel/signal.c b/arch/nios2/kernel/signal.c index 20662b0f6c9e..4a81876b6086 100644 --- a/arch/nios2/kernel/signal.c +++ b/arch/nios2/kernel/signal.c @@ -106,7 +106,7 @@ asmlinkage int do_rt_sigreturn(struct switch_stack *sw) sigset_t set; int rval; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) diff --git a/arch/openrisc/include/asm/futex.h b/arch/openrisc/include/asm/futex.h index 618da4a1bffb..fe894e6331ae 100644 --- a/arch/openrisc/include/asm/futex.h +++ b/arch/openrisc/include/asm/futex.h @@ -72,7 +72,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, int ret = 0; u32 prev; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; __asm__ __volatile__ ( \ diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h index bbf5c79cce7a..bc8191a34db7 100644 --- a/arch/openrisc/include/asm/uaccess.h +++ b/arch/openrisc/include/asm/uaccess.h @@ -58,7 +58,7 @@ /* Ensure that addr is below task's addr_limit */ #define __addr_ok(addr) ((unsigned long) addr < get_fs()) -#define access_ok(type, addr, size) \ +#define access_ok(addr, size) \ __range_ok((unsigned long)addr, (unsigned long)size) /* @@ -102,7 +102,7 @@ extern long __put_user_bad(void); ({ \ long __pu_err = -EFAULT; \ __typeof__(*(ptr)) *__pu_addr = (ptr); \ - if (access_ok(VERIFY_WRITE, __pu_addr, size)) \ + if (access_ok(__pu_addr, size)) \ __put_user_size((x), __pu_addr, (size), __pu_err); \ __pu_err; \ }) @@ -175,7 +175,7 @@ struct __large_struct { ({ \ long __gu_err = -EFAULT, __gu_val = 0; \ const __typeof__(*(ptr)) * __gu_addr = (ptr); \ - if (access_ok(VERIFY_READ, __gu_addr, size)) \ + if (access_ok(__gu_addr, size)) \ __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ (x) = (__force __typeof__(*(ptr)))__gu_val; \ __gu_err; \ @@ -254,7 +254,7 @@ extern unsigned long __clear_user(void *addr, unsigned long size); static inline __must_check unsigned long clear_user(void *addr, unsigned long size) { - if (likely(access_ok(VERIFY_WRITE, addr, size))) + if (likely(access_ok(addr, size))) size = __clear_user(addr, size); return size; } diff --git a/arch/openrisc/kernel/signal.c b/arch/openrisc/kernel/signal.c index 265f10fb3930..5ac9d3b1d615 100644 --- a/arch/openrisc/kernel/signal.c +++ b/arch/openrisc/kernel/signal.c @@ -50,7 +50,7 @@ static int restore_sigcontext(struct pt_regs *regs, /* * Restore the regs from &sc->regs. - * (sc is already checked for VERIFY_READ since the sigframe was + * (sc is already checked since the sigframe was * checked in sys_sigreturn previously) */ err |= __copy_from_user(regs, sc->regs.gpr, 32 * sizeof(unsigned long)); @@ -83,7 +83,7 @@ asmlinkage long _sys_rt_sigreturn(struct pt_regs *regs) if (((long)frame) & 3) goto badframe; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; @@ -161,7 +161,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, frame = get_sigframe(ksig, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; /* Create siginfo. */ diff --git a/arch/parisc/include/asm/futex.h b/arch/parisc/include/asm/futex.h index cf7ba058f619..d2c3e4106851 100644 --- a/arch/parisc/include/asm/futex.h +++ b/arch/parisc/include/asm/futex.h @@ -95,7 +95,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, if (uaccess_kernel() && !uaddr) return -EFAULT; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; /* HPPA has no cmpxchg in hardware and therefore the diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h index ea70e36ce6af..30ac2865ea73 100644 --- a/arch/parisc/include/asm/uaccess.h +++ b/arch/parisc/include/asm/uaccess.h @@ -27,7 +27,7 @@ * that put_user is the same as __put_user, etc. */ -#define access_ok(type, uaddr, size) \ +#define access_ok(uaddr, size) \ ( (uaddr) == (uaddr) ) #define put_user __put_user diff --git a/arch/powerpc/include/asm/futex.h b/arch/powerpc/include/asm/futex.h index 94542776a62d..88b38b37c21b 100644 --- a/arch/powerpc/include/asm/futex.h +++ b/arch/powerpc/include/asm/futex.h @@ -72,7 +72,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, int ret = 0; u32 prev; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; __asm__ __volatile__ ( diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index ebc0b916dcf9..b31bf45eebd4 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -62,7 +62,7 @@ static inline int __access_ok(unsigned long addr, unsigned long size, #endif -#define access_ok(type, addr, size) \ +#define access_ok(addr, size) \ (__chk_user_ptr(addr), (void)(type), \ __access_ok((__force unsigned long)(addr), (size), get_fs())) @@ -166,7 +166,7 @@ do { \ long __pu_err = -EFAULT; \ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ might_fault(); \ - if (access_ok(VERIFY_WRITE, __pu_addr, size)) \ + if (access_ok(__pu_addr, size)) \ __put_user_size((x), __pu_addr, (size), __pu_err); \ __pu_err; \ }) @@ -276,7 +276,7 @@ do { \ __long_type(*(ptr)) __gu_val = 0; \ __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ might_fault(); \ - if (access_ok(VERIFY_READ, __gu_addr, (size))) { \ + if (access_ok(__gu_addr, (size))) { \ barrier_nospec(); \ __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ } \ @@ -374,7 +374,7 @@ extern unsigned long __clear_user(void __user *addr, unsigned long size); static inline unsigned long clear_user(void __user *addr, unsigned long size) { might_fault(); - if (likely(access_ok(VERIFY_WRITE, addr, size))) + if (likely(access_ok(addr, size))) return __clear_user(addr, size); return size; } diff --git a/arch/powerpc/kernel/align.c b/arch/powerpc/kernel/align.c index 11550a3d1ac2..0d1b6370bae0 100644 --- a/arch/powerpc/kernel/align.c +++ b/arch/powerpc/kernel/align.c @@ -131,8 +131,7 @@ static int emulate_spe(struct pt_regs *regs, unsigned int reg, /* Verify the address of the operand */ if (unlikely(user_mode(regs) && - !access_ok((flags & ST ? VERIFY_WRITE : VERIFY_READ), - addr, nb))) + !access_ok(addr, nb))) return -EFAULT; /* userland only */ diff --git a/arch/powerpc/kernel/rtas_flash.c b/arch/powerpc/kernel/rtas_flash.c index 10fabae2574d..8246f437bbc6 100644 --- a/arch/powerpc/kernel/rtas_flash.c +++ b/arch/powerpc/kernel/rtas_flash.c @@ -523,7 +523,7 @@ static ssize_t validate_flash_write(struct file *file, const char __user *buf, args_buf->status = VALIDATE_INCOMPLETE; } - if (!access_ok(VERIFY_READ, buf, count)) { + if (!access_ok(buf, count)) { rc = -EFAULT; goto done; } diff --git a/arch/powerpc/kernel/rtasd.c b/arch/powerpc/kernel/rtasd.c index 38cadae4ca4f..8a1746d755c9 100644 --- a/arch/powerpc/kernel/rtasd.c +++ b/arch/powerpc/kernel/rtasd.c @@ -335,7 +335,7 @@ static ssize_t rtas_log_read(struct file * file, char __user * buf, count = rtas_error_log_buffer_max; - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; tmp = kmalloc(count, GFP_KERNEL); diff --git a/arch/powerpc/kernel/signal.c b/arch/powerpc/kernel/signal.c index b3e8db376ecd..e6c30cee6abf 100644 --- a/arch/powerpc/kernel/signal.c +++ b/arch/powerpc/kernel/signal.c @@ -44,7 +44,7 @@ void __user *get_sigframe(struct ksignal *ksig, unsigned long sp, newsp = (oldsp - frame_size) & ~0xFUL; /* Check access */ - if (!access_ok(VERIFY_WRITE, (void __user *)newsp, oldsp - newsp)) + if (!access_ok((void __user *)newsp, oldsp - newsp)) return NULL; return (void __user *)newsp; diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c index 2d47cc79e5b3..ede4f04281ae 100644 --- a/arch/powerpc/kernel/signal_32.c +++ b/arch/powerpc/kernel/signal_32.c @@ -1017,7 +1017,7 @@ static int do_setcontext(struct ucontext __user *ucp, struct pt_regs *regs, int #else if (__get_user(mcp, &ucp->uc_regs)) return -EFAULT; - if (!access_ok(VERIFY_READ, mcp, sizeof(*mcp))) + if (!access_ok(mcp, sizeof(*mcp))) return -EFAULT; #endif set_current_blocked(&set); @@ -1120,7 +1120,7 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx, */ mctx = (struct mcontext __user *) ((unsigned long) &old_ctx->uc_mcontext & ~0xfUL); - if (!access_ok(VERIFY_WRITE, old_ctx, ctx_size) + if (!access_ok(old_ctx, ctx_size) || save_user_regs(regs, mctx, NULL, 0, ctx_has_vsx_region) || put_sigset_t(&old_ctx->uc_sigmask, ¤t->blocked) || __put_user(to_user_ptr(mctx), &old_ctx->uc_regs)) @@ -1128,7 +1128,7 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx, } if (new_ctx == NULL) return 0; - if (!access_ok(VERIFY_READ, new_ctx, ctx_size) || + if (!access_ok(new_ctx, ctx_size) || fault_in_pages_readable((u8 __user *)new_ctx, ctx_size)) return -EFAULT; @@ -1169,7 +1169,7 @@ SYSCALL_DEFINE0(rt_sigreturn) rt_sf = (struct rt_sigframe __user *) (regs->gpr[1] + __SIGNAL_FRAMESIZE + 16); - if (!access_ok(VERIFY_READ, rt_sf, sizeof(*rt_sf))) + if (!access_ok(rt_sf, sizeof(*rt_sf))) goto bad; #ifdef CONFIG_PPC_TRANSACTIONAL_MEM @@ -1315,7 +1315,7 @@ SYSCALL_DEFINE3(debug_setcontext, struct ucontext __user *, ctx, current->thread.debug.dbcr0 = new_dbcr0; #endif - if (!access_ok(VERIFY_READ, ctx, sizeof(*ctx)) || + if (!access_ok(ctx, sizeof(*ctx)) || fault_in_pages_readable((u8 __user *)ctx, sizeof(*ctx))) return -EFAULT; @@ -1500,7 +1500,7 @@ SYSCALL_DEFINE0(sigreturn) { sr = (struct mcontext __user *)from_user_ptr(sigctx.regs); addr = sr; - if (!access_ok(VERIFY_READ, sr, sizeof(*sr)) + if (!access_ok(sr, sizeof(*sr)) || restore_user_regs(regs, sr, 1)) goto badframe; } diff --git a/arch/powerpc/kernel/signal_64.c b/arch/powerpc/kernel/signal_64.c index 0935fe6c282a..bd5e6834ca69 100644 --- a/arch/powerpc/kernel/signal_64.c +++ b/arch/powerpc/kernel/signal_64.c @@ -383,7 +383,7 @@ static long restore_sigcontext(struct task_struct *tsk, sigset_t *set, int sig, err |= __get_user(v_regs, &sc->v_regs); if (err) return err; - if (v_regs && !access_ok(VERIFY_READ, v_regs, 34 * sizeof(vector128))) + if (v_regs && !access_ok(v_regs, 34 * sizeof(vector128))) return -EFAULT; /* Copy 33 vec registers (vr0..31 and vscr) from the stack */ if (v_regs != NULL && (msr & MSR_VEC) != 0) { @@ -502,10 +502,9 @@ static long restore_tm_sigcontexts(struct task_struct *tsk, err |= __get_user(tm_v_regs, &tm_sc->v_regs); if (err) return err; - if (v_regs && !access_ok(VERIFY_READ, v_regs, 34 * sizeof(vector128))) + if (v_regs && !access_ok(v_regs, 34 * sizeof(vector128))) return -EFAULT; - if (tm_v_regs && !access_ok(VERIFY_READ, - tm_v_regs, 34 * sizeof(vector128))) + if (tm_v_regs && !access_ok(tm_v_regs, 34 * sizeof(vector128))) return -EFAULT; /* Copy 33 vec registers (vr0..31 and vscr) from the stack */ if (v_regs != NULL && tm_v_regs != NULL && (msr & MSR_VEC) != 0) { @@ -671,7 +670,7 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx, ctx_has_vsx_region = 1; if (old_ctx != NULL) { - if (!access_ok(VERIFY_WRITE, old_ctx, ctx_size) + if (!access_ok(old_ctx, ctx_size) || setup_sigcontext(&old_ctx->uc_mcontext, current, 0, NULL, 0, ctx_has_vsx_region) || __copy_to_user(&old_ctx->uc_sigmask, @@ -680,7 +679,7 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx, } if (new_ctx == NULL) return 0; - if (!access_ok(VERIFY_READ, new_ctx, ctx_size) + if (!access_ok(new_ctx, ctx_size) || __get_user(tmp, (u8 __user *) new_ctx) || __get_user(tmp, (u8 __user *) new_ctx + ctx_size - 1)) return -EFAULT; @@ -725,7 +724,7 @@ SYSCALL_DEFINE0(rt_sigreturn) /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; - if (!access_ok(VERIFY_READ, uc, sizeof(*uc))) + if (!access_ok(uc, sizeof(*uc))) goto badframe; if (__copy_from_user(&set, &uc->uc_sigmask, sizeof(set))) diff --git a/arch/powerpc/kernel/syscalls.c b/arch/powerpc/kernel/syscalls.c index 466216506eb2..e6982ab21816 100644 --- a/arch/powerpc/kernel/syscalls.c +++ b/arch/powerpc/kernel/syscalls.c @@ -89,7 +89,7 @@ ppc_select(int n, fd_set __user *inp, fd_set __user *outp, fd_set __user *exp, s if ( (unsigned long)n >= 4096 ) { unsigned long __user *buffer = (unsigned long __user *)n; - if (!access_ok(VERIFY_READ, buffer, 5*sizeof(unsigned long)) + if (!access_ok(buffer, 5*sizeof(unsigned long)) || __get_user(n, buffer) || __get_user(inp, ((fd_set __user * __user *)(buffer+1))) || __get_user(outp, ((fd_set __user * __user *)(buffer+2))) diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index 00af2c4febf4..64936b60d521 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -837,7 +837,7 @@ static void p9_hmi_special_emu(struct pt_regs *regs) addr = (__force const void __user *)ea; /* Check it */ - if (!access_ok(VERIFY_READ, addr, 16)) { + if (!access_ok(addr, 16)) { pr_devel("HMI vec emu: bad access %i:%s[%d] nip=%016lx" " instr=%08x addr=%016lx\n", smp_processor_id(), current->comm, current->pid, diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index 6f2d2fb4e098..bd2dcfbf00cd 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -1744,7 +1744,7 @@ static ssize_t kvm_htab_read(struct file *file, char __user *buf, int first_pass; unsigned long hpte[2]; - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; if (kvm_is_radix(kvm)) return 0; @@ -1844,7 +1844,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf, int mmu_ready; int pshift; - if (!access_ok(VERIFY_READ, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; if (kvm_is_radix(kvm)) return -EINVAL; diff --git a/arch/powerpc/lib/checksum_wrappers.c b/arch/powerpc/lib/checksum_wrappers.c index a0cb63fb76a1..890d4ddd91d6 100644 --- a/arch/powerpc/lib/checksum_wrappers.c +++ b/arch/powerpc/lib/checksum_wrappers.c @@ -37,7 +37,7 @@ __wsum csum_and_copy_from_user(const void __user *src, void *dst, goto out; } - if (unlikely((len < 0) || !access_ok(VERIFY_READ, src, len))) { + if (unlikely((len < 0) || !access_ok(src, len))) { *err_ptr = -EFAULT; csum = (__force unsigned int)sum; goto out; @@ -78,7 +78,7 @@ __wsum csum_and_copy_to_user(const void *src, void __user *dst, int len, goto out; } - if (unlikely((len < 0) || !access_ok(VERIFY_WRITE, dst, len))) { + if (unlikely((len < 0) || !access_ok(dst, len))) { *err_ptr = -EFAULT; csum = -1; /* invalid checksum */ goto out; diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index a6dcfda3e11e..887f11bcf330 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -274,7 +274,7 @@ static bool bad_stack_expansion(struct pt_regs *regs, unsigned long address, return false; if ((flags & FAULT_FLAG_WRITE) && (flags & FAULT_FLAG_USER) && - access_ok(VERIFY_READ, nip, sizeof(*nip))) { + access_ok(nip, sizeof(*nip))) { unsigned int inst; int res; diff --git a/arch/powerpc/mm/subpage-prot.c b/arch/powerpc/mm/subpage-prot.c index 3327551c8b47..5e4178790dee 100644 --- a/arch/powerpc/mm/subpage-prot.c +++ b/arch/powerpc/mm/subpage-prot.c @@ -214,7 +214,7 @@ SYSCALL_DEFINE3(subpage_prot, unsigned long, addr, return 0; } - if (!access_ok(VERIFY_READ, map, (len >> PAGE_SHIFT) * sizeof(u32))) + if (!access_ok(map, (len >> PAGE_SHIFT) * sizeof(u32))) return -EFAULT; down_write(&mm->mmap_sem); diff --git a/arch/powerpc/oprofile/backtrace.c b/arch/powerpc/oprofile/backtrace.c index 5df6290d1ccc..260c53700978 100644 --- a/arch/powerpc/oprofile/backtrace.c +++ b/arch/powerpc/oprofile/backtrace.c @@ -31,7 +31,7 @@ static unsigned int user_getsp32(unsigned int sp, int is_first) unsigned int stack_frame[2]; void __user *p = compat_ptr(sp); - if (!access_ok(VERIFY_READ, p, sizeof(stack_frame))) + if (!access_ok(p, sizeof(stack_frame))) return 0; /* @@ -57,7 +57,7 @@ static unsigned long user_getsp64(unsigned long sp, int is_first) { unsigned long stack_frame[3]; - if (!access_ok(VERIFY_READ, (void __user *)sp, sizeof(stack_frame))) + if (!access_ok((void __user *)sp, sizeof(stack_frame))) return 0; if (__copy_from_user_inatomic(stack_frame, (void __user *)sp, diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index 43e7b93f27c7..ae8123edddc6 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -609,7 +609,7 @@ static ssize_t spufs_mbox_read(struct file *file, char __user *buf, if (len < 4) return -EINVAL; - if (!access_ok(VERIFY_WRITE, buf, len)) + if (!access_ok(buf, len)) return -EFAULT; udata = (void __user *)buf; @@ -717,7 +717,7 @@ static ssize_t spufs_ibox_read(struct file *file, char __user *buf, if (len < 4) return -EINVAL; - if (!access_ok(VERIFY_WRITE, buf, len)) + if (!access_ok(buf, len)) return -EFAULT; udata = (void __user *)buf; @@ -856,7 +856,7 @@ static ssize_t spufs_wbox_write(struct file *file, const char __user *buf, return -EINVAL; udata = (void __user *)buf; - if (!access_ok(VERIFY_READ, buf, len)) + if (!access_ok(buf, len)) return -EFAULT; if (__get_user(wbox_data, udata)) @@ -1994,7 +1994,7 @@ static ssize_t spufs_mbox_info_read(struct file *file, char __user *buf, int ret; struct spu_context *ctx = file->private_data; - if (!access_ok(VERIFY_WRITE, buf, len)) + if (!access_ok(buf, len)) return -EFAULT; ret = spu_acquire_saved(ctx); @@ -2034,7 +2034,7 @@ static ssize_t spufs_ibox_info_read(struct file *file, char __user *buf, struct spu_context *ctx = file->private_data; int ret; - if (!access_ok(VERIFY_WRITE, buf, len)) + if (!access_ok(buf, len)) return -EFAULT; ret = spu_acquire_saved(ctx); @@ -2077,7 +2077,7 @@ static ssize_t spufs_wbox_info_read(struct file *file, char __user *buf, struct spu_context *ctx = file->private_data; int ret; - if (!access_ok(VERIFY_WRITE, buf, len)) + if (!access_ok(buf, len)) return -EFAULT; ret = spu_acquire_saved(ctx); @@ -2129,7 +2129,7 @@ static ssize_t spufs_dma_info_read(struct file *file, char __user *buf, struct spu_context *ctx = file->private_data; int ret; - if (!access_ok(VERIFY_WRITE, buf, len)) + if (!access_ok(buf, len)) return -EFAULT; ret = spu_acquire_saved(ctx); @@ -2160,7 +2160,7 @@ static ssize_t __spufs_proxydma_info_read(struct spu_context *ctx, if (len < ret) return -EINVAL; - if (!access_ok(VERIFY_WRITE, buf, len)) + if (!access_ok(buf, len)) return -EFAULT; info.proxydma_info_type = ctx->csa.prob.dma_querytype_RW; diff --git a/arch/powerpc/platforms/powernv/opal-lpc.c b/arch/powerpc/platforms/powernv/opal-lpc.c index 6c7ad1d8b32e..2623996a193a 100644 --- a/arch/powerpc/platforms/powernv/opal-lpc.c +++ b/arch/powerpc/platforms/powernv/opal-lpc.c @@ -192,7 +192,7 @@ static ssize_t lpc_debug_read(struct file *filp, char __user *ubuf, u32 data, pos, len, todo; int rc; - if (!access_ok(VERIFY_WRITE, ubuf, count)) + if (!access_ok(ubuf, count)) return -EFAULT; todo = count; @@ -283,7 +283,7 @@ static ssize_t lpc_debug_write(struct file *filp, const char __user *ubuf, u32 data, pos, len, todo; int rc; - if (!access_ok(VERIFY_READ, ubuf, count)) + if (!access_ok(ubuf, count)) return -EFAULT; todo = count; diff --git a/arch/powerpc/platforms/pseries/scanlog.c b/arch/powerpc/platforms/pseries/scanlog.c index 054ce7a16fc3..24b157e1e890 100644 --- a/arch/powerpc/platforms/pseries/scanlog.c +++ b/arch/powerpc/platforms/pseries/scanlog.c @@ -63,7 +63,7 @@ static ssize_t scanlog_read(struct file *file, char __user *buf, return -EINVAL; } - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; for (;;) { diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h index 3b19eba1bc8e..66641624d8a5 100644 --- a/arch/riscv/include/asm/futex.h +++ b/arch/riscv/include/asm/futex.h @@ -95,7 +95,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 val; uintptr_t tmp; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; __enable_user_access(); diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 8c3e3e3c8be1..637b896894fc 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -54,14 +54,8 @@ static inline void set_fs(mm_segment_t fs) #define user_addr_max() (get_fs()) -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /** * access_ok: - Checks if a user space pointer is valid - * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that - * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe - * to write to a block, it is always safe to read from it. * @addr: User space pointer to start of block to check * @size: Size of block to check * @@ -76,7 +70,7 @@ static inline void set_fs(mm_segment_t fs) * checks that the pointer is in the user space range - after calling * this function, memory access functions may still return -EFAULT. */ -#define access_ok(type, addr, size) ({ \ +#define access_ok(addr, size) ({ \ __chk_user_ptr(addr); \ likely(__access_ok((unsigned long __force)(addr), (size))); \ }) @@ -258,7 +252,7 @@ do { \ ({ \ const __typeof__(*(ptr)) __user *__p = (ptr); \ might_fault(); \ - access_ok(VERIFY_READ, __p, sizeof(*__p)) ? \ + access_ok(__p, sizeof(*__p)) ? \ __get_user((x), __p) : \ ((x) = 0, -EFAULT); \ }) @@ -386,7 +380,7 @@ do { \ ({ \ __typeof__(*(ptr)) __user *__p = (ptr); \ might_fault(); \ - access_ok(VERIFY_WRITE, __p, sizeof(*__p)) ? \ + access_ok(__p, sizeof(*__p)) ? \ __put_user((x), __p) : \ -EFAULT; \ }) @@ -421,7 +415,7 @@ static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) { might_fault(); - return access_ok(VERIFY_WRITE, to, n) ? + return access_ok(to, n) ? __clear_user(to, n) : n; } diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index f9b5e7e352ef..837e1646091a 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -115,7 +115,7 @@ SYSCALL_DEFINE0(rt_sigreturn) frame = (struct rt_sigframe __user *)regs->sp; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) @@ -187,7 +187,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, long err = 0; frame = get_sigframe(ksig, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; err |= copy_siginfo_to_user(&frame->info, &ksig->info); diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index ad6b91013a05..bd2545977ad3 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -48,7 +48,7 @@ static inline int __range_ok(unsigned long addr, unsigned long size) __range_ok((unsigned long)(addr), (size)); \ }) -#define access_ok(type, addr, size) __access_ok(addr, size) +#define access_ok(addr, size) __access_ok(addr, size) unsigned long __must_check raw_copy_from_user(void *to, const void __user *from, unsigned long n); diff --git a/arch/sh/include/asm/checksum_32.h b/arch/sh/include/asm/checksum_32.h index b58f3d95dc19..36b84cfd3f67 100644 --- a/arch/sh/include/asm/checksum_32.h +++ b/arch/sh/include/asm/checksum_32.h @@ -197,7 +197,7 @@ static inline __wsum csum_and_copy_to_user(const void *src, int len, __wsum sum, int *err_ptr) { - if (access_ok(VERIFY_WRITE, dst, len)) + if (access_ok(dst, len)) return csum_partial_copy_generic((__force const void *)src, dst, len, sum, NULL, err_ptr); diff --git a/arch/sh/include/asm/futex.h b/arch/sh/include/asm/futex.h index 6d192f4908a7..3190ec89df81 100644 --- a/arch/sh/include/asm/futex.h +++ b/arch/sh/include/asm/futex.h @@ -22,7 +22,7 @@ static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) { - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; return atomic_futex_op_cmpxchg_inatomic(uval, uaddr, oldval, newval); diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h index 32eb56e00c11..deebbfab5342 100644 --- a/arch/sh/include/asm/uaccess.h +++ b/arch/sh/include/asm/uaccess.h @@ -18,7 +18,7 @@ */ #define __access_ok(addr, size) \ (__addr_ok((addr) + (size))) -#define access_ok(type, addr, size) \ +#define access_ok(addr, size) \ (__chk_user_ptr(addr), \ __access_ok((unsigned long __force)(addr), (size))) @@ -66,7 +66,7 @@ struct __large_struct { unsigned long buf[100]; }; long __gu_err = -EFAULT; \ unsigned long __gu_val = 0; \ const __typeof__(*(ptr)) *__gu_addr = (ptr); \ - if (likely(access_ok(VERIFY_READ, __gu_addr, (size)))) \ + if (likely(access_ok(__gu_addr, (size)))) \ __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ (x) = (__force __typeof__(*(ptr)))__gu_val; \ __gu_err; \ @@ -87,7 +87,7 @@ struct __large_struct { unsigned long buf[100]; }; long __pu_err = -EFAULT; \ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ __typeof__(*(ptr)) __pu_val = x; \ - if (likely(access_ok(VERIFY_WRITE, __pu_addr, size))) \ + if (likely(access_ok(__pu_addr, size))) \ __put_user_size(__pu_val, __pu_addr, (size), \ __pu_err); \ __pu_err; \ @@ -132,8 +132,7 @@ __kernel_size_t __clear_user(void *addr, __kernel_size_t size); void __user * __cl_addr = (addr); \ unsigned long __cl_size = (n); \ \ - if (__cl_size && access_ok(VERIFY_WRITE, \ - ((unsigned long)(__cl_addr)), __cl_size)) \ + if (__cl_size && access_ok(__cl_addr, __cl_size)) \ __cl_size = __clear_user(__cl_addr, __cl_size); \ \ __cl_size; \ diff --git a/arch/sh/kernel/signal_32.c b/arch/sh/kernel/signal_32.c index c46c0020ff55..2a2121ba8ebe 100644 --- a/arch/sh/kernel/signal_32.c +++ b/arch/sh/kernel/signal_32.c @@ -160,7 +160,7 @@ asmlinkage int sys_sigreturn(void) /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__get_user(set.sig[0], &frame->sc.oldmask) @@ -190,7 +190,7 @@ asmlinkage int sys_rt_sigreturn(void) /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) @@ -272,7 +272,7 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, frame = get_sigframe(&ksig->ka, regs->regs[15], sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; err |= setup_sigcontext(&frame->sc, regs, set->sig[0]); @@ -338,7 +338,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, frame = get_sigframe(&ksig->ka, regs->regs[15], sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; err |= copy_siginfo_to_user(&frame->info, &ksig->info); diff --git a/arch/sh/kernel/signal_64.c b/arch/sh/kernel/signal_64.c index 76661dee3c65..f1f1598879c2 100644 --- a/arch/sh/kernel/signal_64.c +++ b/arch/sh/kernel/signal_64.c @@ -259,7 +259,7 @@ asmlinkage int sys_sigreturn(unsigned long r2, unsigned long r3, /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__get_user(set.sig[0], &frame->sc.oldmask) @@ -293,7 +293,7 @@ asmlinkage int sys_rt_sigreturn(unsigned long r2, unsigned long r3, /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) @@ -379,7 +379,7 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs frame = get_sigframe(&ksig->ka, regs->regs[REG_SP], sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; err |= setup_sigcontext(&frame->sc, regs, set->sig[0]); @@ -465,7 +465,7 @@ static int setup_rt_frame(struct ksignal *kig, sigset_t *set, frame = get_sigframe(&ksig->ka, regs->regs[REG_SP], sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; err |= __put_user(&frame->info, &frame->pinfo); diff --git a/arch/sh/kernel/traps_64.c b/arch/sh/kernel/traps_64.c index c52bda4d2574..8ce90a7da67d 100644 --- a/arch/sh/kernel/traps_64.c +++ b/arch/sh/kernel/traps_64.c @@ -40,7 +40,7 @@ static int read_opcode(reg_size_t pc, insn_size_t *result_opcode, int from_user_ /* SHmedia */ aligned_pc = pc & ~3; if (from_user_mode) { - if (!access_ok(VERIFY_READ, aligned_pc, sizeof(insn_size_t))) { + if (!access_ok(aligned_pc, sizeof(insn_size_t))) { get_user_error = -EFAULT; } else { get_user_error = __get_user(opcode, (insn_size_t *)aligned_pc); @@ -180,7 +180,7 @@ static int misaligned_load(struct pt_regs *regs, if (user_mode(regs)) { __u64 buffer; - if (!access_ok(VERIFY_READ, (unsigned long) address, 1UL<thread.float_regs[0], &fpu->si_float_regs[0], diff --git a/arch/sparc/kernel/unaligned_32.c b/arch/sparc/kernel/unaligned_32.c index 64ac8c0c1429..83db94c0b431 100644 --- a/arch/sparc/kernel/unaligned_32.c +++ b/arch/sparc/kernel/unaligned_32.c @@ -278,7 +278,6 @@ static inline int ok_for_user(struct pt_regs *regs, unsigned int insn, enum direction dir) { unsigned int reg; - int check = (dir == load) ? VERIFY_READ : VERIFY_WRITE; int size = ((insn >> 19) & 3) == 3 ? 8 : 4; if ((regs->pc | regs->npc) & 3) @@ -290,18 +289,18 @@ static inline int ok_for_user(struct pt_regs *regs, unsigned int insn, reg = (insn >> 25) & 0x1f; if (reg >= 16) { - if (!access_ok(check, WINREG_ADDR(reg - 16), size)) + if (!access_ok(WINREG_ADDR(reg - 16), size)) return -EFAULT; } reg = (insn >> 14) & 0x1f; if (reg >= 16) { - if (!access_ok(check, WINREG_ADDR(reg - 16), size)) + if (!access_ok(WINREG_ADDR(reg - 16), size)) return -EFAULT; } if (!(insn & 0x2000)) { reg = (insn & 0x1f); if (reg >= 16) { - if (!access_ok(check, WINREG_ADDR(reg - 16), size)) + if (!access_ok(WINREG_ADDR(reg - 16), size)) return -EFAULT; } } diff --git a/arch/um/kernel/ptrace.c b/arch/um/kernel/ptrace.c index 1a1d88a4d940..5f47422401e1 100644 --- a/arch/um/kernel/ptrace.c +++ b/arch/um/kernel/ptrace.c @@ -66,7 +66,7 @@ long arch_ptrace(struct task_struct *child, long request, #ifdef PTRACE_GETREGS case PTRACE_GETREGS: { /* Get all gp regs from the child. */ - if (!access_ok(VERIFY_WRITE, p, MAX_REG_OFFSET)) { + if (!access_ok(p, MAX_REG_OFFSET)) { ret = -EIO; break; } @@ -81,7 +81,7 @@ long arch_ptrace(struct task_struct *child, long request, #ifdef PTRACE_SETREGS case PTRACE_SETREGS: { /* Set all gp regs in the child. */ unsigned long tmp = 0; - if (!access_ok(VERIFY_READ, p, MAX_REG_OFFSET)) { + if (!access_ok(p, MAX_REG_OFFSET)) { ret = -EIO; break; } diff --git a/arch/unicore32/kernel/signal.c b/arch/unicore32/kernel/signal.c index 4ae51cf15ade..63be04809d40 100644 --- a/arch/unicore32/kernel/signal.c +++ b/arch/unicore32/kernel/signal.c @@ -117,7 +117,7 @@ asmlinkage int __sys_rt_sigreturn(struct pt_regs *regs) frame = (struct rt_sigframe __user *)regs->UCreg_sp; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (restore_sigframe(regs, &frame->sig)) @@ -205,7 +205,7 @@ static inline void __user *get_sigframe(struct k_sigaction *ka, /* * Check that we can actually write to the signal frame. */ - if (!access_ok(VERIFY_WRITE, frame, framesize)) + if (!access_ok(frame, framesize)) frame = NULL; return frame; diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c index d78bcc03e60e..d9d81ad7a400 100644 --- a/arch/x86/entry/vsyscall/vsyscall_64.c +++ b/arch/x86/entry/vsyscall/vsyscall_64.c @@ -99,7 +99,7 @@ static bool write_ok_or_segv(unsigned long ptr, size_t size) * sig_on_uaccess_err, this could go away. */ - if (!access_ok(VERIFY_WRITE, (void __user *)ptr, size)) { + if (!access_ok((void __user *)ptr, size)) { struct thread_struct *thread = ¤t->thread; thread->error_code = X86_PF_USER | X86_PF_WRITE; diff --git a/arch/x86/ia32/ia32_aout.c b/arch/x86/ia32/ia32_aout.c index 8e02b30cf08e..f65b78d32f5e 100644 --- a/arch/x86/ia32/ia32_aout.c +++ b/arch/x86/ia32/ia32_aout.c @@ -176,10 +176,10 @@ static int aout_core_dump(struct coredump_params *cprm) /* make sure we actually have a data and stack area to dump */ set_fs(USER_DS); - if (!access_ok(VERIFY_READ, (void *) (unsigned long)START_DATA(dump), + if (!access_ok((void *) (unsigned long)START_DATA(dump), dump.u_dsize << PAGE_SHIFT)) dump.u_dsize = 0; - if (!access_ok(VERIFY_READ, (void *) (unsigned long)START_STACK(dump), + if (!access_ok((void *) (unsigned long)START_STACK(dump), dump.u_ssize << PAGE_SHIFT)) dump.u_ssize = 0; diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c index 86b1341cba9a..321fe5f5d0e9 100644 --- a/arch/x86/ia32/ia32_signal.c +++ b/arch/x86/ia32/ia32_signal.c @@ -119,7 +119,7 @@ asmlinkage long sys32_sigreturn(void) struct sigframe_ia32 __user *frame = (struct sigframe_ia32 __user *)(regs->sp-8); sigset_t set; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__get_user(set.sig[0], &frame->sc.oldmask) || (_COMPAT_NSIG_WORDS > 1 @@ -147,7 +147,7 @@ asmlinkage long sys32_rt_sigreturn(void) frame = (struct rt_sigframe_ia32 __user *)(regs->sp - 4); - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; @@ -269,7 +269,7 @@ int ia32_setup_frame(int sig, struct ksignal *ksig, frame = get_sigframe(ksig, regs, sizeof(*frame), &fpstate); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; if (__put_user(sig, &frame->sig)) @@ -349,7 +349,7 @@ int ia32_setup_rt_frame(int sig, struct ksignal *ksig, frame = get_sigframe(ksig, regs, sizeof(*frame), &fpstate); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; put_user_try { diff --git a/arch/x86/ia32/sys_ia32.c b/arch/x86/ia32/sys_ia32.c index 11ef7b7c9cc8..a43212036257 100644 --- a/arch/x86/ia32/sys_ia32.c +++ b/arch/x86/ia32/sys_ia32.c @@ -75,7 +75,7 @@ static int cp_stat64(struct stat64 __user *ubuf, struct kstat *stat) typeof(ubuf->st_gid) gid = 0; SET_UID(uid, from_kuid_munged(current_user_ns(), stat->uid)); SET_GID(gid, from_kgid_munged(current_user_ns(), stat->gid)); - if (!access_ok(VERIFY_WRITE, ubuf, sizeof(struct stat64)) || + if (!access_ok(ubuf, sizeof(struct stat64)) || __put_user(huge_encode_dev(stat->dev), &ubuf->st_dev) || __put_user(stat->ino, &ubuf->__st_ino) || __put_user(stat->ino, &ubuf->st_ino) || diff --git a/arch/x86/include/asm/checksum_32.h b/arch/x86/include/asm/checksum_32.h index 7a659c74cd03..f57b94e02c57 100644 --- a/arch/x86/include/asm/checksum_32.h +++ b/arch/x86/include/asm/checksum_32.h @@ -182,7 +182,7 @@ static inline __wsum csum_and_copy_to_user(const void *src, __wsum ret; might_sleep(); - if (access_ok(VERIFY_WRITE, dst, len)) { + if (access_ok(dst, len)) { stac(); ret = csum_partial_copy_generic(src, (__force void *)dst, len, sum, NULL, err_ptr); diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h index b3ec519e3982..4fe9e7fc74d3 100644 --- a/arch/x86/include/asm/pgtable_32.h +++ b/arch/x86/include/asm/pgtable_32.h @@ -37,7 +37,7 @@ void sync_initial_page_table(void); /* * Define this if things work differently on an i386 and an i486: * it will (on an i486) warn about kernel memory accesses that are - * done without a 'access_ok(VERIFY_WRITE,..)' + * done without a 'access_ok( ..)' */ #undef TEST_ACCESS_OK diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index b5e58cc0c5e7..3920f456db79 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -77,9 +77,6 @@ static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, un /** * access_ok: - Checks if a user space pointer is valid - * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that - * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe - * to write to a block, it is always safe to read from it. * @addr: User space pointer to start of block to check * @size: Size of block to check * @@ -95,7 +92,7 @@ static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, un * checks that the pointer is in the user space range - after calling * this function, memory access functions may still return -EFAULT. */ -#define access_ok(type, addr, size) \ +#define access_ok(addr, size) \ ({ \ WARN_ON_IN_IRQ(); \ likely(!__range_not_ok(addr, size, user_addr_max())); \ @@ -670,7 +667,7 @@ extern void __cmpxchg_wrong_size(void) #define user_atomic_cmpxchg_inatomic(uval, ptr, old, new) \ ({ \ - access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ? \ + access_ok((ptr), sizeof(*(ptr))) ? \ __user_atomic_cmpxchg_inatomic((uval), (ptr), \ (old), (new), sizeof(*(ptr))) : \ -EFAULT; \ diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c index d99a8ee9e185..f6a1d299627c 100644 --- a/arch/x86/kernel/fpu/signal.c +++ b/arch/x86/kernel/fpu/signal.c @@ -164,7 +164,7 @@ int copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size) ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) || IS_ENABLED(CONFIG_IA32_EMULATION)); - if (!access_ok(VERIFY_WRITE, buf, size)) + if (!access_ok(buf, size)) return -EACCES; if (!static_cpu_has(X86_FEATURE_FPU)) @@ -281,7 +281,7 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size) return 0; } - if (!access_ok(VERIFY_READ, buf, size)) + if (!access_ok(buf, size)) return -EACCES; fpu__initialize(fpu); diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index 92a3b312a53c..08dfd4c1a4f9 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -322,7 +322,7 @@ __setup_frame(int sig, struct ksignal *ksig, sigset_t *set, frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fpstate); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; if (__put_user(sig, &frame->sig)) @@ -385,7 +385,7 @@ static int __setup_rt_frame(int sig, struct ksignal *ksig, frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fpstate); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; put_user_try { @@ -465,7 +465,7 @@ static int __setup_rt_frame(int sig, struct ksignal *ksig, frame = get_sigframe(&ksig->ka, regs, sizeof(struct rt_sigframe), &fp); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; if (ksig->ka.sa.sa_flags & SA_SIGINFO) { @@ -547,7 +547,7 @@ static int x32_setup_rt_frame(struct ksignal *ksig, frame = get_sigframe(&ksig->ka, regs, sizeof(*frame), &fpstate); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return -EFAULT; if (ksig->ka.sa.sa_flags & SA_SIGINFO) { @@ -610,7 +610,7 @@ SYSCALL_DEFINE0(sigreturn) frame = (struct sigframe __user *)(regs->sp - 8); - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__get_user(set.sig[0], &frame->sc.oldmask) || (_NSIG_WORDS > 1 && __copy_from_user(&set.sig[1], &frame->extramask, @@ -642,7 +642,7 @@ SYSCALL_DEFINE0(rt_sigreturn) unsigned long uc_flags; frame = (struct rt_sigframe __user *)(regs->sp - sizeof(long)); - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; @@ -871,7 +871,7 @@ asmlinkage long sys32_x32_rt_sigreturn(void) frame = (struct rt_sigframe_x32 __user *)(regs->sp - 8); - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c index 7627455047c2..5c2d71a1dc06 100644 --- a/arch/x86/kernel/stacktrace.c +++ b/arch/x86/kernel/stacktrace.c @@ -177,7 +177,7 @@ copy_stack_frame(const void __user *fp, struct stack_frame_user *frame) { int ret; - if (!access_ok(VERIFY_READ, fp, sizeof(*frame))) + if (!access_ok(fp, sizeof(*frame))) return 0; ret = 1; diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c index c2fd39752da8..a092b6b40c6b 100644 --- a/arch/x86/kernel/vm86_32.c +++ b/arch/x86/kernel/vm86_32.c @@ -114,7 +114,7 @@ void save_v86_state(struct kernel_vm86_regs *regs, int retval) set_flags(regs->pt.flags, VEFLAGS, X86_EFLAGS_VIF | vm86->veflags_mask); user = vm86->user_vm86; - if (!access_ok(VERIFY_WRITE, user, vm86->vm86plus.is_vm86pus ? + if (!access_ok(user, vm86->vm86plus.is_vm86pus ? sizeof(struct vm86plus_struct) : sizeof(struct vm86_struct))) { pr_alert("could not access userspace vm86 info\n"); @@ -278,7 +278,7 @@ static long do_sys_vm86(struct vm86plus_struct __user *user_vm86, bool plus) if (vm86->saved_sp0) return -EPERM; - if (!access_ok(VERIFY_READ, user_vm86, plus ? + if (!access_ok(user_vm86, plus ? sizeof(struct vm86_struct) : sizeof(struct vm86plus_struct))) return -EFAULT; diff --git a/arch/x86/lib/csum-wrappers_64.c b/arch/x86/lib/csum-wrappers_64.c index 8bd53589ecfb..a6a2b7dccbff 100644 --- a/arch/x86/lib/csum-wrappers_64.c +++ b/arch/x86/lib/csum-wrappers_64.c @@ -27,7 +27,7 @@ csum_partial_copy_from_user(const void __user *src, void *dst, might_sleep(); *errp = 0; - if (!likely(access_ok(VERIFY_READ, src, len))) + if (!likely(access_ok(src, len))) goto out_err; /* @@ -89,7 +89,7 @@ csum_partial_copy_to_user(const void *src, void __user *dst, might_sleep(); - if (unlikely(!access_ok(VERIFY_WRITE, dst, len))) { + if (unlikely(!access_ok(dst, len))) { *errp = -EFAULT; return 0; } diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c index 71fb58d44d58..bfd94e7812fc 100644 --- a/arch/x86/lib/usercopy_32.c +++ b/arch/x86/lib/usercopy_32.c @@ -67,7 +67,7 @@ unsigned long clear_user(void __user *to, unsigned long n) { might_fault(); - if (access_ok(VERIFY_WRITE, to, n)) + if (access_ok(to, n)) __do_clear_user(to, n); return n; } diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c index 1bd837cdc4b1..ee42bb0cbeb3 100644 --- a/arch/x86/lib/usercopy_64.c +++ b/arch/x86/lib/usercopy_64.c @@ -48,7 +48,7 @@ EXPORT_SYMBOL(__clear_user); unsigned long clear_user(void __user *to, unsigned long n) { - if (access_ok(VERIFY_WRITE, to, n)) + if (access_ok(to, n)) return __clear_user(to, n); return n; } diff --git a/arch/x86/math-emu/fpu_system.h b/arch/x86/math-emu/fpu_system.h index c8b1b31ed7c4..f98a0c956764 100644 --- a/arch/x86/math-emu/fpu_system.h +++ b/arch/x86/math-emu/fpu_system.h @@ -104,7 +104,7 @@ static inline bool seg_writable(struct desc_struct *d) #define instruction_address (*(struct address *)&I387->soft.fip) #define operand_address (*(struct address *)&I387->soft.foo) -#define FPU_access_ok(x,y,z) if ( !access_ok(x,y,z) ) \ +#define FPU_access_ok(y,z) if ( !access_ok(y,z) ) \ math_abort(FPU_info,SIGSEGV) #define FPU_abort math_abort(FPU_info, SIGSEGV) @@ -119,7 +119,7 @@ static inline bool seg_writable(struct desc_struct *d) /* A simpler test than access_ok() can probably be done for FPU_code_access_ok() because the only possible error is to step past the upper boundary of a legal code area. */ -#define FPU_code_access_ok(z) FPU_access_ok(VERIFY_READ,(void __user *)FPU_EIP,z) +#define FPU_code_access_ok(z) FPU_access_ok((void __user *)FPU_EIP,z) #endif #define FPU_get_user(x,y) get_user((x),(y)) diff --git a/arch/x86/math-emu/load_store.c b/arch/x86/math-emu/load_store.c index f821a9cd7753..f15263e158e8 100644 --- a/arch/x86/math-emu/load_store.c +++ b/arch/x86/math-emu/load_store.c @@ -251,7 +251,7 @@ int FPU_load_store(u_char type, fpu_addr_modes addr_modes, break; case 024: /* fldcw */ RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_READ, data_address, 2); + FPU_access_ok(data_address, 2); FPU_get_user(control_word, (unsigned short __user *)data_address); RE_ENTRANT_CHECK_ON; @@ -291,7 +291,7 @@ int FPU_load_store(u_char type, fpu_addr_modes addr_modes, break; case 034: /* fstcw m16int */ RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, data_address, 2); + FPU_access_ok(data_address, 2); FPU_put_user(control_word, (unsigned short __user *)data_address); RE_ENTRANT_CHECK_ON; @@ -305,7 +305,7 @@ int FPU_load_store(u_char type, fpu_addr_modes addr_modes, break; case 036: /* fstsw m2byte */ RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, data_address, 2); + FPU_access_ok(data_address, 2); FPU_put_user(status_word(), (unsigned short __user *)data_address); RE_ENTRANT_CHECK_ON; diff --git a/arch/x86/math-emu/reg_ld_str.c b/arch/x86/math-emu/reg_ld_str.c index d40ff45497b9..f3779743d15e 100644 --- a/arch/x86/math-emu/reg_ld_str.c +++ b/arch/x86/math-emu/reg_ld_str.c @@ -84,7 +84,7 @@ int FPU_load_extended(long double __user *s, int stnr) FPU_REG *sti_ptr = &st(stnr); RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_READ, s, 10); + FPU_access_ok(s, 10); __copy_from_user(sti_ptr, s, 10); RE_ENTRANT_CHECK_ON; @@ -98,7 +98,7 @@ int FPU_load_double(double __user *dfloat, FPU_REG *loaded_data) unsigned m64, l64; RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_READ, dfloat, 8); + FPU_access_ok(dfloat, 8); FPU_get_user(m64, 1 + (unsigned long __user *)dfloat); FPU_get_user(l64, (unsigned long __user *)dfloat); RE_ENTRANT_CHECK_ON; @@ -159,7 +159,7 @@ int FPU_load_single(float __user *single, FPU_REG *loaded_data) int exp, tag, negative; RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_READ, single, 4); + FPU_access_ok(single, 4); FPU_get_user(m32, (unsigned long __user *)single); RE_ENTRANT_CHECK_ON; @@ -214,7 +214,7 @@ int FPU_load_int64(long long __user *_s) FPU_REG *st0_ptr = &st(0); RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_READ, _s, 8); + FPU_access_ok(_s, 8); if (copy_from_user(&s, _s, 8)) FPU_abort; RE_ENTRANT_CHECK_ON; @@ -243,7 +243,7 @@ int FPU_load_int32(long __user *_s, FPU_REG *loaded_data) int negative; RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_READ, _s, 4); + FPU_access_ok(_s, 4); FPU_get_user(s, _s); RE_ENTRANT_CHECK_ON; @@ -271,7 +271,7 @@ int FPU_load_int16(short __user *_s, FPU_REG *loaded_data) int s, negative; RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_READ, _s, 2); + FPU_access_ok(_s, 2); /* Cast as short to get the sign extended. */ FPU_get_user(s, _s); RE_ENTRANT_CHECK_ON; @@ -304,7 +304,7 @@ int FPU_load_bcd(u_char __user *s) int sign; RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_READ, s, 10); + FPU_access_ok(s, 10); RE_ENTRANT_CHECK_ON; for (pos = 8; pos >= 0; pos--) { l *= 10; @@ -345,7 +345,7 @@ int FPU_store_extended(FPU_REG *st0_ptr, u_char st0_tag, if (st0_tag != TAG_Empty) { RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, d, 10); + FPU_access_ok(d, 10); FPU_put_user(st0_ptr->sigl, (unsigned long __user *)d); FPU_put_user(st0_ptr->sigh, @@ -364,7 +364,7 @@ int FPU_store_extended(FPU_REG *st0_ptr, u_char st0_tag, /* The masked response */ /* Put out the QNaN indefinite */ RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, d, 10); + FPU_access_ok(d, 10); FPU_put_user(0, (unsigned long __user *)d); FPU_put_user(0xc0000000, 1 + (unsigned long __user *)d); FPU_put_user(0xffff, 4 + (short __user *)d); @@ -539,7 +539,7 @@ denormal_arg: /* The masked response */ /* Put out the QNaN indefinite */ RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, dfloat, 8); + FPU_access_ok(dfloat, 8); FPU_put_user(0, (unsigned long __user *)dfloat); FPU_put_user(0xfff80000, 1 + (unsigned long __user *)dfloat); @@ -552,7 +552,7 @@ denormal_arg: l[1] |= 0x80000000; RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, dfloat, 8); + FPU_access_ok(dfloat, 8); FPU_put_user(l[0], (unsigned long __user *)dfloat); FPU_put_user(l[1], 1 + (unsigned long __user *)dfloat); RE_ENTRANT_CHECK_ON; @@ -724,7 +724,7 @@ int FPU_store_single(FPU_REG *st0_ptr, u_char st0_tag, float __user *single) /* The masked response */ /* Put out the QNaN indefinite */ RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, single, 4); + FPU_access_ok(single, 4); FPU_put_user(0xffc00000, (unsigned long __user *)single); RE_ENTRANT_CHECK_ON; @@ -742,7 +742,7 @@ int FPU_store_single(FPU_REG *st0_ptr, u_char st0_tag, float __user *single) templ |= 0x80000000; RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, single, 4); + FPU_access_ok(single, 4); FPU_put_user(templ, (unsigned long __user *)single); RE_ENTRANT_CHECK_ON; @@ -791,7 +791,7 @@ int FPU_store_int64(FPU_REG *st0_ptr, u_char st0_tag, long long __user *d) } RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, d, 8); + FPU_access_ok(d, 8); if (copy_to_user(d, &tll, 8)) FPU_abort; RE_ENTRANT_CHECK_ON; @@ -838,7 +838,7 @@ int FPU_store_int32(FPU_REG *st0_ptr, u_char st0_tag, long __user *d) } RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, d, 4); + FPU_access_ok(d, 4); FPU_put_user(t.sigl, (unsigned long __user *)d); RE_ENTRANT_CHECK_ON; @@ -884,7 +884,7 @@ int FPU_store_int16(FPU_REG *st0_ptr, u_char st0_tag, short __user *d) } RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, d, 2); + FPU_access_ok(d, 2); FPU_put_user((short)t.sigl, d); RE_ENTRANT_CHECK_ON; @@ -925,7 +925,7 @@ int FPU_store_bcd(FPU_REG *st0_ptr, u_char st0_tag, u_char __user *d) if (control_word & CW_Invalid) { /* Produce the QNaN "indefinite" */ RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, d, 10); + FPU_access_ok(d, 10); for (i = 0; i < 7; i++) FPU_put_user(0, d + i); /* These bytes "undefined" */ FPU_put_user(0xc0, d + 7); /* This byte "undefined" */ @@ -941,7 +941,7 @@ int FPU_store_bcd(FPU_REG *st0_ptr, u_char st0_tag, u_char __user *d) } RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, d, 10); + FPU_access_ok(d, 10); RE_ENTRANT_CHECK_ON; for (i = 0; i < 9; i++) { b = FPU_div_small(&ll, 10); @@ -1034,7 +1034,7 @@ u_char __user *fldenv(fpu_addr_modes addr_modes, u_char __user *s) ((addr_modes.default_mode == PM16) ^ (addr_modes.override.operand_size == OP_SIZE_PREFIX))) { RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_READ, s, 0x0e); + FPU_access_ok(s, 0x0e); FPU_get_user(control_word, (unsigned short __user *)s); FPU_get_user(partial_status, (unsigned short __user *)(s + 2)); FPU_get_user(tag_word, (unsigned short __user *)(s + 4)); @@ -1056,7 +1056,7 @@ u_char __user *fldenv(fpu_addr_modes addr_modes, u_char __user *s) } } else { RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_READ, s, 0x1c); + FPU_access_ok(s, 0x1c); FPU_get_user(control_word, (unsigned short __user *)s); FPU_get_user(partial_status, (unsigned short __user *)(s + 4)); FPU_get_user(tag_word, (unsigned short __user *)(s + 8)); @@ -1125,7 +1125,7 @@ void frstor(fpu_addr_modes addr_modes, u_char __user *data_address) /* Copy all registers in stack order. */ RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_READ, s, 80); + FPU_access_ok(s, 80); __copy_from_user(register_base + offset, s, other); if (offset) __copy_from_user(register_base, s + other, offset); @@ -1146,7 +1146,7 @@ u_char __user *fstenv(fpu_addr_modes addr_modes, u_char __user *d) ((addr_modes.default_mode == PM16) ^ (addr_modes.override.operand_size == OP_SIZE_PREFIX))) { RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, d, 14); + FPU_access_ok(d, 14); #ifdef PECULIAR_486 FPU_put_user(control_word & ~0xe080, (unsigned long __user *)d); #else @@ -1174,7 +1174,7 @@ u_char __user *fstenv(fpu_addr_modes addr_modes, u_char __user *d) d += 0x0e; } else { RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, d, 7 * 4); + FPU_access_ok(d, 7 * 4); #ifdef PECULIAR_486 control_word &= ~0xe080; /* An 80486 sets nearly all of the reserved bits to 1. */ @@ -1204,7 +1204,7 @@ void fsave(fpu_addr_modes addr_modes, u_char __user *data_address) d = fstenv(addr_modes, data_address); RE_ENTRANT_CHECK_OFF; - FPU_access_ok(VERIFY_WRITE, d, 80); + FPU_access_ok(d, 80); /* Copy all registers in stack order. */ if (__copy_to_user(d, register_base + offset, other)) diff --git a/arch/x86/mm/mpx.c b/arch/x86/mm/mpx.c index 2385538e8065..de1851d15699 100644 --- a/arch/x86/mm/mpx.c +++ b/arch/x86/mm/mpx.c @@ -495,7 +495,7 @@ static int get_bt_addr(struct mm_struct *mm, unsigned long bd_entry; unsigned long bt_addr; - if (!access_ok(VERIFY_READ, (bd_entry_ptr), sizeof(*bd_entry_ptr))) + if (!access_ok((bd_entry_ptr), sizeof(*bd_entry_ptr))) return -EFAULT; while (1) { diff --git a/arch/x86/um/asm/checksum_32.h b/arch/x86/um/asm/checksum_32.h index 83a75f8a1233..b9ac7c9eb72c 100644 --- a/arch/x86/um/asm/checksum_32.h +++ b/arch/x86/um/asm/checksum_32.h @@ -43,7 +43,7 @@ static __inline__ __wsum csum_and_copy_to_user(const void *src, void __user *dst, int len, __wsum sum, int *err_ptr) { - if (access_ok(VERIFY_WRITE, dst, len)) { + if (access_ok(dst, len)) { if (copy_to_user(dst, src, len)) { *err_ptr = -EFAULT; return (__force __wsum)-1; diff --git a/arch/x86/um/signal.c b/arch/x86/um/signal.c index 727ed442e0a5..8b4a71efe7ee 100644 --- a/arch/x86/um/signal.c +++ b/arch/x86/um/signal.c @@ -367,7 +367,7 @@ int setup_signal_stack_sc(unsigned long stack_top, struct ksignal *ksig, /* This is the same calculation as i386 - ((sp + 4) & 15) == 0 */ stack_top = ((stack_top + 4) & -16UL) - 4; frame = (struct sigframe __user *) stack_top - 1; - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return 1; restorer = frame->retcode; @@ -412,7 +412,7 @@ int setup_signal_stack_si(unsigned long stack_top, struct ksignal *ksig, stack_top &= -8UL; frame = (struct rt_sigframe __user *) stack_top - 1; - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) return 1; restorer = frame->retcode; @@ -497,7 +497,7 @@ int setup_signal_stack_si(unsigned long stack_top, struct ksignal *ksig, /* Subtract 128 for a red zone and 8 for proper alignment */ frame = (struct rt_sigframe __user *) ((unsigned long) frame - 128 - 8); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto out; if (ksig->ka.sa.sa_flags & SA_SIGINFO) { diff --git a/arch/xtensa/include/asm/checksum.h b/arch/xtensa/include/asm/checksum.h index 3ae74d7e074b..f302ef57973a 100644 --- a/arch/xtensa/include/asm/checksum.h +++ b/arch/xtensa/include/asm/checksum.h @@ -243,7 +243,7 @@ static __inline__ __wsum csum_and_copy_to_user(const void *src, void __user *dst, int len, __wsum sum, int *err_ptr) { - if (access_ok(VERIFY_WRITE, dst, len)) + if (access_ok(dst, len)) return csum_partial_copy_generic(src,dst,len,sum,NULL,err_ptr); if (len) diff --git a/arch/xtensa/include/asm/futex.h b/arch/xtensa/include/asm/futex.h index fd0eef6b8e7c..505d09eff184 100644 --- a/arch/xtensa/include/asm/futex.h +++ b/arch/xtensa/include/asm/futex.h @@ -93,7 +93,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, { int ret = 0; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; #if !XCHAL_HAVE_S32C1I diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index d11ef2939652..4b2480304bc3 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -42,7 +42,7 @@ #define __user_ok(addr, size) \ (((size) <= TASK_SIZE)&&((addr) <= TASK_SIZE-(size))) #define __access_ok(addr, size) (__kernel_ok || __user_ok((addr), (size))) -#define access_ok(type, addr, size) __access_ok((unsigned long)(addr), (size)) +#define access_ok(addr, size) __access_ok((unsigned long)(addr), (size)) #define user_addr_max() (uaccess_kernel() ? ~0UL : TASK_SIZE) @@ -86,7 +86,7 @@ extern long __put_user_bad(void); ({ \ long __pu_err = -EFAULT; \ __typeof__(*(ptr)) *__pu_addr = (ptr); \ - if (access_ok(VERIFY_WRITE, __pu_addr, size)) \ + if (access_ok(__pu_addr, size)) \ __put_user_size((x), __pu_addr, (size), __pu_err); \ __pu_err; \ }) @@ -183,7 +183,7 @@ __asm__ __volatile__( \ ({ \ long __gu_err = -EFAULT, __gu_val = 0; \ const __typeof__(*(ptr)) *__gu_addr = (ptr); \ - if (access_ok(VERIFY_READ, __gu_addr, size)) \ + if (access_ok(__gu_addr, size)) \ __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ (x) = (__force __typeof__(*(ptr)))__gu_val; \ __gu_err; \ @@ -269,7 +269,7 @@ __xtensa_clear_user(void *addr, unsigned long size) static inline unsigned long clear_user(void *addr, unsigned long size) { - if (access_ok(VERIFY_WRITE, addr, size)) + if (access_ok(addr, size)) return __xtensa_clear_user(addr, size); return size ? -EFAULT : 0; } @@ -284,7 +284,7 @@ extern long __strncpy_user(char *, const char *, long); static inline long strncpy_from_user(char *dst, const char *src, long count) { - if (access_ok(VERIFY_READ, src, 1)) + if (access_ok(src, 1)) return __strncpy_user(dst, src, count); return -EFAULT; } diff --git a/arch/xtensa/kernel/signal.c b/arch/xtensa/kernel/signal.c index 74e1682876ac..dc22a238ed9c 100644 --- a/arch/xtensa/kernel/signal.c +++ b/arch/xtensa/kernel/signal.c @@ -251,7 +251,7 @@ asmlinkage long xtensa_rt_sigreturn(long a0, long a1, long a2, long a3, frame = (struct rt_sigframe __user *) regs->areg[1]; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set))) @@ -348,7 +348,7 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, if (regs->depc > 64) panic ("Double exception sys_sigreturn\n"); - if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) { + if (!access_ok(frame, sizeof(*frame))) { return -EFAULT; } diff --git a/arch/xtensa/kernel/stacktrace.c b/arch/xtensa/kernel/stacktrace.c index 0df4080fa20f..174c11f13bba 100644 --- a/arch/xtensa/kernel/stacktrace.c +++ b/arch/xtensa/kernel/stacktrace.c @@ -91,7 +91,7 @@ void xtensa_backtrace_user(struct pt_regs *regs, unsigned int depth, pc = MAKE_PC_FROM_RA(a0, pc); /* Check if the region is OK to access. */ - if (!access_ok(VERIFY_READ, &SPILL_SLOT(a1, 0), 8)) + if (!access_ok(&SPILL_SLOT(a1, 0), 8)) return; /* Copy a1, a0 from user space stack frame. */ if (__get_user(a0, &SPILL_SLOT(a1, 0)) || diff --git a/drivers/acpi/acpi_dbg.c b/drivers/acpi/acpi_dbg.c index f21c99ec46ee..a2dcd62ea32f 100644 --- a/drivers/acpi/acpi_dbg.c +++ b/drivers/acpi/acpi_dbg.c @@ -614,7 +614,7 @@ static ssize_t acpi_aml_read(struct file *file, char __user *buf, if (!count) return 0; - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; while (count > 0) { @@ -684,7 +684,7 @@ static ssize_t acpi_aml_write(struct file *file, const char __user *buf, if (!count) return 0; - if (!access_ok(VERIFY_READ, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; while (count > 0) { diff --git a/drivers/char/generic_nvram.c b/drivers/char/generic_nvram.c index 14e728fbb8a0..ff5394f47587 100644 --- a/drivers/char/generic_nvram.c +++ b/drivers/char/generic_nvram.c @@ -44,7 +44,7 @@ static ssize_t read_nvram(struct file *file, char __user *buf, unsigned int i; char __user *p = buf; - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; if (*ppos >= nvram_len) return 0; @@ -62,7 +62,7 @@ static ssize_t write_nvram(struct file *file, const char __user *buf, const char __user *p = buf; char c; - if (!access_ok(VERIFY_READ, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; if (*ppos >= nvram_len) return 0; diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 7b4e4de778e4..b08dc50f9f26 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -609,7 +609,7 @@ static ssize_t read_port(struct file *file, char __user *buf, unsigned long i = *ppos; char __user *tmp = buf; - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; while (count-- > 0 && i < 65536) { if (__put_user(inb(i), tmp) < 0) @@ -627,7 +627,7 @@ static ssize_t write_port(struct file *file, const char __user *buf, unsigned long i = *ppos; const char __user *tmp = buf; - if (!access_ok(VERIFY_READ, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; while (count-- > 0 && i < 65536) { char c; diff --git a/drivers/char/nwflash.c b/drivers/char/nwflash.c index a284ae25e69a..76fb434068d4 100644 --- a/drivers/char/nwflash.c +++ b/drivers/char/nwflash.c @@ -167,7 +167,7 @@ static ssize_t flash_write(struct file *file, const char __user *buf, if (count > gbFlashSize - p) count = gbFlashSize - p; - if (!access_ok(VERIFY_READ, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; /* diff --git a/drivers/char/pcmcia/cm4000_cs.c b/drivers/char/pcmcia/cm4000_cs.c index 809507bf8f1c..7a4eb86aedac 100644 --- a/drivers/char/pcmcia/cm4000_cs.c +++ b/drivers/char/pcmcia/cm4000_cs.c @@ -1445,11 +1445,11 @@ static long cmm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) _IOC_DIR(cmd), _IOC_READ, _IOC_WRITE, size, cmd); if (_IOC_DIR(cmd) & _IOC_READ) { - if (!access_ok(VERIFY_WRITE, argp, size)) + if (!access_ok(argp, size)) goto out; } if (_IOC_DIR(cmd) & _IOC_WRITE) { - if (!access_ok(VERIFY_READ, argp, size)) + if (!access_ok(argp, size)) goto out; } rc = 0; diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c index d64a78ccc03e..b16be8a11d92 100644 --- a/drivers/crypto/ccp/psp-dev.c +++ b/drivers/crypto/ccp/psp-dev.c @@ -364,7 +364,7 @@ static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp) goto cmd; /* allocate a physically contiguous buffer to store the CSR blob */ - if (!access_ok(VERIFY_WRITE, input.address, input.length) || + if (!access_ok(input.address, input.length) || input.length > SEV_FW_BLOB_MAX_SIZE) { ret = -EFAULT; goto e_free; @@ -644,14 +644,14 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp) /* Allocate a physically contiguous buffer to store the PDH blob. */ if ((input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE) || - !access_ok(VERIFY_WRITE, input.pdh_cert_address, input.pdh_cert_len)) { + !access_ok(input.pdh_cert_address, input.pdh_cert_len)) { ret = -EFAULT; goto e_free; } /* Allocate a physically contiguous buffer to store the cert chain blob. */ if ((input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE) || - !access_ok(VERIFY_WRITE, input.cert_chain_address, input.cert_chain_len)) { + !access_ok(input.cert_chain_address, input.cert_chain_len)) { ret = -EFAULT; goto e_free; } diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c index d8e185582642..16a7045736a9 100644 --- a/drivers/firewire/core-cdev.c +++ b/drivers/firewire/core-cdev.c @@ -1094,7 +1094,7 @@ static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg) return -EINVAL; p = (struct fw_cdev_iso_packet __user *)u64_to_uptr(a->packets); - if (!access_ok(VERIFY_READ, p, a->size)) + if (!access_ok(p, a->size)) return -EFAULT; end = (void __user *)p + a->size; diff --git a/drivers/firmware/efi/test/efi_test.c b/drivers/firmware/efi/test/efi_test.c index 769640940c9f..51ecf7d6da48 100644 --- a/drivers/firmware/efi/test/efi_test.c +++ b/drivers/firmware/efi/test/efi_test.c @@ -68,7 +68,7 @@ copy_ucs2_from_user_len(efi_char16_t **dst, efi_char16_t __user *src, return 0; } - if (!access_ok(VERIFY_READ, src, 1)) + if (!access_ok(src, 1)) return -EFAULT; buf = memdup_user(src, len); @@ -89,7 +89,7 @@ copy_ucs2_from_user_len(efi_char16_t **dst, efi_char16_t __user *src, static inline int get_ucs2_strsize_from_user(efi_char16_t __user *src, size_t *len) { - if (!access_ok(VERIFY_READ, src, 1)) + if (!access_ok(src, 1)) return -EFAULT; *len = user_ucs2_strsize(src); @@ -116,7 +116,7 @@ copy_ucs2_from_user(efi_char16_t **dst, efi_char16_t __user *src) { size_t len; - if (!access_ok(VERIFY_READ, src, 1)) + if (!access_ok(src, 1)) return -EFAULT; len = user_ucs2_strsize(src); @@ -140,7 +140,7 @@ copy_ucs2_to_user_len(efi_char16_t __user *dst, efi_char16_t *src, size_t len) if (!src) return 0; - if (!access_ok(VERIFY_WRITE, dst, 1)) + if (!access_ok(dst, 1)) return -EFAULT; return copy_to_user(dst, src, len); diff --git a/drivers/fpga/dfl-afu-dma-region.c b/drivers/fpga/dfl-afu-dma-region.c index 025aba3ea76c..e18a786fc943 100644 --- a/drivers/fpga/dfl-afu-dma-region.c +++ b/drivers/fpga/dfl-afu-dma-region.c @@ -369,7 +369,7 @@ int afu_dma_map_region(struct dfl_feature_platform_data *pdata, if (user_addr + length < user_addr) return -EINVAL; - if (!access_ok(VERIFY_WRITE, (void __user *)(unsigned long)user_addr, + if (!access_ok((void __user *)(unsigned long)user_addr, length)) return -EINVAL; diff --git a/drivers/fpga/dfl-fme-pr.c b/drivers/fpga/dfl-fme-pr.c index fe5a5578fbf7..d9ca9554844a 100644 --- a/drivers/fpga/dfl-fme-pr.c +++ b/drivers/fpga/dfl-fme-pr.c @@ -99,8 +99,7 @@ static int fme_pr(struct platform_device *pdev, unsigned long arg) return -EINVAL; } - if (!access_ok(VERIFY_READ, - (void __user *)(unsigned long)port_pr.buffer_address, + if (!access_ok((void __user *)(unsigned long)port_pr.buffer_address, port_pr.buffer_size)) return -EFAULT; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 3623538baf6f..be68752c3469 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -158,8 +158,7 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties, } if ((args->ring_base_address) && - (!access_ok(VERIFY_WRITE, - (const void __user *) args->ring_base_address, + (!access_ok((const void __user *) args->ring_base_address, sizeof(uint64_t)))) { pr_err("Can't access ring base address\n"); return -EFAULT; @@ -170,31 +169,27 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties, return -EINVAL; } - if (!access_ok(VERIFY_WRITE, - (const void __user *) args->read_pointer_address, + if (!access_ok((const void __user *) args->read_pointer_address, sizeof(uint32_t))) { pr_err("Can't access read pointer\n"); return -EFAULT; } - if (!access_ok(VERIFY_WRITE, - (const void __user *) args->write_pointer_address, + if (!access_ok((const void __user *) args->write_pointer_address, sizeof(uint32_t))) { pr_err("Can't access write pointer\n"); return -EFAULT; } if (args->eop_buffer_address && - !access_ok(VERIFY_WRITE, - (const void __user *) args->eop_buffer_address, + !access_ok((const void __user *) args->eop_buffer_address, sizeof(uint32_t))) { pr_debug("Can't access eop buffer"); return -EFAULT; } if (args->ctx_save_restore_address && - !access_ok(VERIFY_WRITE, - (const void __user *) args->ctx_save_restore_address, + !access_ok((const void __user *) args->ctx_save_restore_address, sizeof(uint32_t))) { pr_debug("Can't access ctx save restore buffer"); return -EFAULT; @@ -365,8 +360,7 @@ static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, } if ((args->ring_base_address) && - (!access_ok(VERIFY_WRITE, - (const void __user *) args->ring_base_address, + (!access_ok((const void __user *) args->ring_base_address, sizeof(uint64_t)))) { pr_err("Can't access ring base address\n"); return -EFAULT; diff --git a/drivers/gpu/drm/armada/armada_gem.c b/drivers/gpu/drm/armada/armada_gem.c index 892c1d9304bb..642d0e70d0f8 100644 --- a/drivers/gpu/drm/armada/armada_gem.c +++ b/drivers/gpu/drm/armada/armada_gem.c @@ -334,7 +334,7 @@ int armada_gem_pwrite_ioctl(struct drm_device *dev, void *data, ptr = (char __user *)(uintptr_t)args->ptr; - if (!access_ok(VERIFY_READ, ptr, args->size)) + if (!access_ok(ptr, args->size)) return -EFAULT; ret = fault_in_pages_readable(ptr, args->size); diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c index ffa8dc35515f..46f48f245eb5 100644 --- a/drivers/gpu/drm/drm_file.c +++ b/drivers/gpu/drm/drm_file.c @@ -525,7 +525,7 @@ ssize_t drm_read(struct file *filp, char __user *buffer, struct drm_device *dev = file_priv->minor->dev; ssize_t ret; - if (!access_ok(VERIFY_WRITE, buffer, count)) + if (!access_ok(buffer, count)) return -EFAULT; ret = mutex_lock_interruptible(&file_priv->event_read_lock); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c index 96efc84396bf..18c27f795cf6 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c @@ -339,7 +339,6 @@ static int etnaviv_ioctl_gem_userptr(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_etnaviv_gem_userptr *args = data; - int access; if (args->flags & ~(ETNA_USERPTR_READ|ETNA_USERPTR_WRITE) || args->flags == 0) @@ -351,12 +350,7 @@ static int etnaviv_ioctl_gem_userptr(struct drm_device *dev, void *data, args->user_ptr & ~PAGE_MASK) return -EINVAL; - if (args->flags & ETNA_USERPTR_WRITE) - access = VERIFY_WRITE; - else - access = VERIFY_READ; - - if (!access_ok(access, (void __user *)(unsigned long)args->user_ptr, + if (!access_ok((void __user *)(unsigned long)args->user_ptr, args->user_size)) return -EFAULT; diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index a9de07bb72c8..216f52b744a6 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -1282,8 +1282,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, if (args->size == 0) return 0; - if (!access_ok(VERIFY_WRITE, - u64_to_user_ptr(args->data_ptr), + if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size)) return -EFAULT; @@ -1609,9 +1608,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, if (args->size == 0) return 0; - if (!access_ok(VERIFY_READ, - u64_to_user_ptr(args->data_ptr), - args->size)) + if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size)) return -EFAULT; obj = i915_gem_object_lookup(file, args->handle); diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index 8ff6b581cf1c..fee66ccebed6 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -1447,7 +1447,7 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma) * to read. However, if the array is not writable the user loses * the updated relocation values. */ - if (unlikely(!access_ok(VERIFY_READ, urelocs, remain*sizeof(*urelocs)))) + if (unlikely(!access_ok(urelocs, remain*sizeof(*urelocs)))) return -EFAULT; do { @@ -1554,7 +1554,7 @@ static int check_relocations(const struct drm_i915_gem_exec_object2 *entry) addr = u64_to_user_ptr(entry->relocs_ptr); size *= sizeof(struct drm_i915_gem_relocation_entry); - if (!access_ok(VERIFY_READ, addr, size)) + if (!access_ok(addr, size)) return -EFAULT; end = addr + size; @@ -2090,7 +2090,7 @@ get_fence_array(struct drm_i915_gem_execbuffer2 *args, return ERR_PTR(-EINVAL); user = u64_to_user_ptr(args->cliprects_ptr); - if (!access_ok(VERIFY_READ, user, nfences * sizeof(*user))) + if (!access_ok(user, nfences * sizeof(*user))) return ERR_PTR(-EFAULT); fences = kvmalloc_array(nfences, sizeof(*fences), diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c index 3df77020aada..9558582c105e 100644 --- a/drivers/gpu/drm/i915/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/i915_gem_userptr.c @@ -789,8 +789,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev, if (offset_in_page(args->user_ptr | args->user_size)) return -EINVAL; - if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE, - (char __user *)(unsigned long)args->user_ptr, args->user_size)) + if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size)) return -EFAULT; if (args->flags & I915_USERPTR_READ_ONLY) { diff --git a/drivers/gpu/drm/i915/i915_ioc32.c b/drivers/gpu/drm/i915/i915_ioc32.c index 0e5c580d117c..e869daf9c8a9 100644 --- a/drivers/gpu/drm/i915/i915_ioc32.c +++ b/drivers/gpu/drm/i915/i915_ioc32.c @@ -52,7 +52,7 @@ static int compat_i915_getparam(struct file *file, unsigned int cmd, return -EFAULT; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) || + if (!access_ok(request, sizeof(*request)) || __put_user(req32.param, &request->param) || __put_user((void __user *)(unsigned long)req32.value, &request->value)) diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index 4529edfdcfc8..2b2eb57ca71f 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -3052,7 +3052,7 @@ static struct i915_oa_reg *alloc_oa_regs(struct drm_i915_private *dev_priv, if (!n_regs) return NULL; - if (!access_ok(VERIFY_READ, regs, n_regs * sizeof(u32) * 2)) + if (!access_ok(regs, n_regs * sizeof(u32) * 2)) return ERR_PTR(-EFAULT); /* No is_valid function means we're not allowing any register to be programmed. */ diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c index 6fc4b8eeab42..fe56465cdfd6 100644 --- a/drivers/gpu/drm/i915/i915_query.c +++ b/drivers/gpu/drm/i915/i915_query.c @@ -46,7 +46,7 @@ static int query_topology_info(struct drm_i915_private *dev_priv, if (topo.flags != 0) return -EINVAL; - if (!access_ok(VERIFY_WRITE, u64_to_user_ptr(query_item->data_ptr), + if (!access_ok(u64_to_user_ptr(query_item->data_ptr), total_length)) return -EFAULT; diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c index a28465d90529..12b983fc0b56 100644 --- a/drivers/gpu/drm/msm/msm_gem_submit.c +++ b/drivers/gpu/drm/msm/msm_gem_submit.c @@ -77,7 +77,7 @@ void msm_gem_submit_free(struct msm_gem_submit *submit) static inline unsigned long __must_check copy_from_user_inatomic(void *to, const void __user *from, unsigned long n) { - if (access_ok(VERIFY_READ, from, n)) + if (access_ok(from, n)) return __copy_from_user_inatomic(to, from, n); return -EFAULT; } diff --git a/drivers/gpu/drm/qxl/qxl_ioctl.c b/drivers/gpu/drm/qxl/qxl_ioctl.c index 6e828158bcb0..d410e2925162 100644 --- a/drivers/gpu/drm/qxl/qxl_ioctl.c +++ b/drivers/gpu/drm/qxl/qxl_ioctl.c @@ -163,8 +163,7 @@ static int qxl_process_single_command(struct qxl_device *qdev, if (cmd->command_size > PAGE_SIZE - sizeof(union qxl_release_info)) return -EINVAL; - if (!access_ok(VERIFY_READ, - u64_to_user_ptr(cmd->command), + if (!access_ok(u64_to_user_ptr(cmd->command), cmd->command_size)) return -EFAULT; diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 9f9172eb1512..fb0007aa0c27 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -611,8 +611,7 @@ static ssize_t verify_hdr(struct ib_uverbs_cmd_hdr *hdr, if (hdr->out_words * 8 < method_elm->resp_size) return -ENOSPC; - if (!access_ok(VERIFY_WRITE, - u64_to_user_ptr(ex_hdr->response), + if (!access_ok(u64_to_user_ptr(ex_hdr->response), (hdr->out_words + ex_hdr->provider_out_words) * 8)) return -EFAULT; } else { diff --git a/drivers/infiniband/hw/hfi1/user_exp_rcv.c b/drivers/infiniband/hw/hfi1/user_exp_rcv.c index dbe7d14a5c76..0cd71ce7cc71 100644 --- a/drivers/infiniband/hw/hfi1/user_exp_rcv.c +++ b/drivers/infiniband/hw/hfi1/user_exp_rcv.c @@ -232,7 +232,7 @@ static int pin_rcv_pages(struct hfi1_filedata *fd, struct tid_user_buf *tidbuf) } /* Verify that access is OK for the user buffer */ - if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, + if (!access_ok((void __user *)vaddr, npages * PAGE_SIZE)) { dd_dev_err(dd, "Fail vaddr %p, %u pages, !access_ok\n", (void *)vaddr, npages); diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c index 98e1ce14fa2a..78fa634de98a 100644 --- a/drivers/infiniband/hw/qib/qib_file_ops.c +++ b/drivers/infiniband/hw/qib/qib_file_ops.c @@ -343,7 +343,7 @@ static int qib_tid_update(struct qib_ctxtdata *rcd, struct file *fp, /* virtual address of first page in transfer */ vaddr = ti->tidvaddr; - if (!access_ok(VERIFY_WRITE, (void __user *) vaddr, + if (!access_ok((void __user *) vaddr, cnt * PAGE_SIZE)) { ret = -EFAULT; goto done; diff --git a/drivers/macintosh/ans-lcd.c b/drivers/macintosh/ans-lcd.c index ef0c2366cf59..400960cf04d5 100644 --- a/drivers/macintosh/ans-lcd.c +++ b/drivers/macintosh/ans-lcd.c @@ -64,7 +64,7 @@ anslcd_write( struct file * file, const char __user * buf, printk(KERN_DEBUG "LCD: write\n"); #endif - if (!access_ok(VERIFY_READ, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; mutex_lock(&anslcd_mutex); diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c index ac0cf37d6239..21d532a78fa4 100644 --- a/drivers/macintosh/via-pmu.c +++ b/drivers/macintosh/via-pmu.c @@ -2188,7 +2188,7 @@ pmu_read(struct file *file, char __user *buf, if (count < 1 || !pp) return -EINVAL; - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; spin_lock_irqsave(&pp->lock, flags); diff --git a/drivers/media/pci/ivtv/ivtvfb.c b/drivers/media/pci/ivtv/ivtvfb.c index 3e02de02ffdd..8ec2525d8ef5 100644 --- a/drivers/media/pci/ivtv/ivtvfb.c +++ b/drivers/media/pci/ivtv/ivtvfb.c @@ -356,7 +356,7 @@ static int ivtvfb_prep_frame(struct ivtv *itv, int cmd, void __user *source, IVTVFB_WARN("ivtvfb_prep_frame: Count not a multiple of 4 (%d)\n", count); /* Check Source */ - if (!access_ok(VERIFY_READ, source + dest_offset, count)) { + if (!access_ok(source + dest_offset, count)) { IVTVFB_WARN("Invalid userspace pointer %p\n", source); IVTVFB_DEBUG_WARN("access_ok() failed for offset 0x%08lx source %p count %d\n", diff --git a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c index fe4577a46869..73dac1d8d4f6 100644 --- a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c +++ b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c @@ -158,7 +158,7 @@ static int get_v4l2_window32(struct v4l2_window __user *p64, compat_caddr_t p; u32 clipcount; - if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || copy_in_user(&p64->w, &p32->w, sizeof(p32->w)) || assign_in_user(&p64->field, &p32->field) || assign_in_user(&p64->chromakey, &p32->chromakey) || @@ -283,7 +283,7 @@ static int __bufsize_v4l2_format(struct v4l2_format32 __user *p32, u32 *size) static int bufsize_v4l2_format(struct v4l2_format32 __user *p32, u32 *size) { - if (!access_ok(VERIFY_READ, p32, sizeof(*p32))) + if (!access_ok(p32, sizeof(*p32))) return -EFAULT; return __bufsize_v4l2_format(p32, size); } @@ -335,7 +335,7 @@ static int get_v4l2_format32(struct v4l2_format __user *p64, struct v4l2_format32 __user *p32, void __user *aux_buf, u32 aux_space) { - if (!access_ok(VERIFY_READ, p32, sizeof(*p32))) + if (!access_ok(p32, sizeof(*p32))) return -EFAULT; return __get_v4l2_format32(p64, p32, aux_buf, aux_space); } @@ -343,7 +343,7 @@ static int get_v4l2_format32(struct v4l2_format __user *p64, static int bufsize_v4l2_create(struct v4l2_create_buffers32 __user *p32, u32 *size) { - if (!access_ok(VERIFY_READ, p32, sizeof(*p32))) + if (!access_ok(p32, sizeof(*p32))) return -EFAULT; return __bufsize_v4l2_format(&p32->format, size); } @@ -352,7 +352,7 @@ static int get_v4l2_create32(struct v4l2_create_buffers __user *p64, struct v4l2_create_buffers32 __user *p32, void __user *aux_buf, u32 aux_space) { - if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || copy_in_user(p64, p32, offsetof(struct v4l2_create_buffers32, format))) return -EFAULT; @@ -404,7 +404,7 @@ static int __put_v4l2_format32(struct v4l2_format __user *p64, static int put_v4l2_format32(struct v4l2_format __user *p64, struct v4l2_format32 __user *p32) { - if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32))) + if (!access_ok(p32, sizeof(*p32))) return -EFAULT; return __put_v4l2_format32(p64, p32); } @@ -412,7 +412,7 @@ static int put_v4l2_format32(struct v4l2_format __user *p64, static int put_v4l2_create32(struct v4l2_create_buffers __user *p64, struct v4l2_create_buffers32 __user *p32) { - if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || copy_in_user(p32, p64, offsetof(struct v4l2_create_buffers32, format)) || assign_in_user(&p32->capabilities, &p64->capabilities) || @@ -434,7 +434,7 @@ static int get_v4l2_standard32(struct v4l2_standard __user *p64, struct v4l2_standard32 __user *p32) { /* other fields are not set by the user, nor used by the driver */ - if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || assign_in_user(&p64->index, &p32->index)) return -EFAULT; return 0; @@ -443,7 +443,7 @@ static int get_v4l2_standard32(struct v4l2_standard __user *p64, static int put_v4l2_standard32(struct v4l2_standard __user *p64, struct v4l2_standard32 __user *p32) { - if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || assign_in_user(&p32->index, &p64->index) || assign_in_user(&p32->id, &p64->id) || copy_in_user(p32->name, p64->name, sizeof(p32->name)) || @@ -560,7 +560,7 @@ static int bufsize_v4l2_buffer(struct v4l2_buffer32 __user *p32, u32 *size) u32 type; u32 length; - if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || get_user(type, &p32->type) || get_user(length, &p32->length)) return -EFAULT; @@ -593,7 +593,7 @@ static int get_v4l2_buffer32(struct v4l2_buffer __user *p64, compat_caddr_t p; int ret; - if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || assign_in_user(&p64->index, &p32->index) || get_user(type, &p32->type) || put_user(type, &p64->type) || @@ -632,7 +632,7 @@ static int get_v4l2_buffer32(struct v4l2_buffer __user *p64, return -EFAULT; uplane32 = compat_ptr(p); - if (!access_ok(VERIFY_READ, uplane32, + if (!access_ok(uplane32, num_planes * sizeof(*uplane32))) return -EFAULT; @@ -691,7 +691,7 @@ static int put_v4l2_buffer32(struct v4l2_buffer __user *p64, compat_caddr_t p; int ret; - if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || assign_in_user(&p32->index, &p64->index) || get_user(type, &p64->type) || put_user(type, &p32->type) || @@ -781,7 +781,7 @@ static int get_v4l2_framebuffer32(struct v4l2_framebuffer __user *p64, { compat_caddr_t tmp; - if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || get_user(tmp, &p32->base) || put_user_force(compat_ptr(tmp), &p64->base) || assign_in_user(&p64->capability, &p32->capability) || @@ -796,7 +796,7 @@ static int put_v4l2_framebuffer32(struct v4l2_framebuffer __user *p64, { void *base; - if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || get_user(base, &p64->base) || put_user(ptr_to_compat((void __user *)base), &p32->base) || assign_in_user(&p32->capability, &p64->capability) || @@ -893,7 +893,7 @@ static int bufsize_v4l2_ext_controls(struct v4l2_ext_controls32 __user *p32, { u32 count; - if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || get_user(count, &p32->count)) return -EFAULT; if (count > V4L2_CID_MAX_CTRLS) @@ -913,7 +913,7 @@ static int get_v4l2_ext_controls32(struct file *file, u32 n; compat_caddr_t p; - if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || assign_in_user(&p64->which, &p32->which) || get_user(count, &p32->count) || put_user(count, &p64->count) || @@ -929,7 +929,7 @@ static int get_v4l2_ext_controls32(struct file *file, if (get_user(p, &p32->controls)) return -EFAULT; ucontrols = compat_ptr(p); - if (!access_ok(VERIFY_READ, ucontrols, count * sizeof(*ucontrols))) + if (!access_ok(ucontrols, count * sizeof(*ucontrols))) return -EFAULT; if (aux_space < count * sizeof(*kcontrols)) return -EFAULT; @@ -979,7 +979,7 @@ static int put_v4l2_ext_controls32(struct file *file, * with __user causes smatch warnings, so instead declare it * without __user and cast it as a userspace pointer where needed. */ - if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || assign_in_user(&p32->which, &p64->which) || get_user(count, &p64->count) || put_user(count, &p32->count) || @@ -994,7 +994,7 @@ static int put_v4l2_ext_controls32(struct file *file, if (get_user(p, &p32->controls)) return -EFAULT; ucontrols = compat_ptr(p); - if (!access_ok(VERIFY_WRITE, ucontrols, count * sizeof(*ucontrols))) + if (!access_ok(ucontrols, count * sizeof(*ucontrols))) return -EFAULT; for (n = 0; n < count; n++) { @@ -1043,7 +1043,7 @@ struct v4l2_event32 { static int put_v4l2_event32(struct v4l2_event __user *p64, struct v4l2_event32 __user *p32) { - if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || assign_in_user(&p32->type, &p64->type) || copy_in_user(&p32->u, &p64->u, sizeof(p64->u)) || assign_in_user(&p32->pending, &p64->pending) || @@ -1069,7 +1069,7 @@ static int get_v4l2_edid32(struct v4l2_edid __user *p64, { compat_uptr_t tmp; - if (!access_ok(VERIFY_READ, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || assign_in_user(&p64->pad, &p32->pad) || assign_in_user(&p64->start_block, &p32->start_block) || assign_in_user_cast(&p64->blocks, &p32->blocks) || @@ -1085,7 +1085,7 @@ static int put_v4l2_edid32(struct v4l2_edid __user *p64, { void *edid; - if (!access_ok(VERIFY_WRITE, p32, sizeof(*p32)) || + if (!access_ok(p32, sizeof(*p32)) || assign_in_user(&p32->pad, &p64->pad) || assign_in_user(&p32->start_block, &p64->start_block) || assign_in_user(&p32->blocks, &p64->blocks) || diff --git a/drivers/misc/vmw_vmci/vmci_host.c b/drivers/misc/vmw_vmci/vmci_host.c index 5da1f3e3f997..997f92543dd4 100644 --- a/drivers/misc/vmw_vmci/vmci_host.c +++ b/drivers/misc/vmw_vmci/vmci_host.c @@ -236,7 +236,7 @@ static int vmci_host_setup_notify(struct vmci_ctx *context, * about the size. */ BUILD_BUG_ON(sizeof(bool) != sizeof(u8)); - if (!access_ok(VERIFY_WRITE, (void __user *)uva, sizeof(u8))) + if (!access_ok((void __user *)uva, sizeof(u8))) return VMCI_ERROR_GENERIC; /* diff --git a/drivers/pci/proc.c b/drivers/pci/proc.c index 7ac035af39f0..6fa1627ce08d 100644 --- a/drivers/pci/proc.c +++ b/drivers/pci/proc.c @@ -52,7 +52,7 @@ static ssize_t proc_bus_pci_read(struct file *file, char __user *buf, nbytes = size - pos; cnt = nbytes; - if (!access_ok(VERIFY_WRITE, buf, cnt)) + if (!access_ok(buf, cnt)) return -EINVAL; pci_config_pm_runtime_get(dev); @@ -125,7 +125,7 @@ static ssize_t proc_bus_pci_write(struct file *file, const char __user *buf, nbytes = size - pos; cnt = nbytes; - if (!access_ok(VERIFY_READ, buf, cnt)) + if (!access_ok(buf, cnt)) return -EINVAL; pci_config_pm_runtime_get(dev); diff --git a/drivers/platform/goldfish/goldfish_pipe.c b/drivers/platform/goldfish/goldfish_pipe.c index 7c639006252e..321bc673c417 100644 --- a/drivers/platform/goldfish/goldfish_pipe.c +++ b/drivers/platform/goldfish/goldfish_pipe.c @@ -416,8 +416,7 @@ static ssize_t goldfish_pipe_read_write(struct file *filp, if (unlikely(bufflen == 0)) return 0; /* Check the buffer range for access */ - if (unlikely(!access_ok(is_write ? VERIFY_WRITE : VERIFY_READ, - buffer, bufflen))) + if (unlikely(!access_ok(buffer, bufflen))) return -EFAULT; address = (unsigned long)buffer; diff --git a/drivers/pnp/isapnp/proc.c b/drivers/pnp/isapnp/proc.c index 262285e48a09..051613140812 100644 --- a/drivers/pnp/isapnp/proc.c +++ b/drivers/pnp/isapnp/proc.c @@ -47,7 +47,7 @@ static ssize_t isapnp_proc_bus_read(struct file *file, char __user * buf, nbytes = size - pos; cnt = nbytes; - if (!access_ok(VERIFY_WRITE, buf, cnt)) + if (!access_ok(buf, cnt)) return -EINVAL; isapnp_cfg_begin(dev->card->number, dev->number); diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c index 7c4673308f5b..e338d7a4f571 100644 --- a/drivers/scsi/pmcraid.c +++ b/drivers/scsi/pmcraid.c @@ -3600,7 +3600,7 @@ static long pmcraid_ioctl_passthrough( u32 ioasc; int request_size; int buffer_size; - u8 access, direction; + u8 direction; int rc = 0; /* If IOA reset is in progress, wait 10 secs for reset to complete */ @@ -3649,10 +3649,8 @@ static long pmcraid_ioctl_passthrough( request_size = le32_to_cpu(buffer->ioarcb.data_transfer_length); if (buffer->ioarcb.request_flags0 & TRANSFER_DIR_WRITE) { - access = VERIFY_READ; direction = DMA_TO_DEVICE; } else { - access = VERIFY_WRITE; direction = DMA_FROM_DEVICE; } diff --git a/drivers/scsi/scsi_ioctl.c b/drivers/scsi/scsi_ioctl.c index cc30fccc1a2e..840d96fe81bc 100644 --- a/drivers/scsi/scsi_ioctl.c +++ b/drivers/scsi/scsi_ioctl.c @@ -221,7 +221,7 @@ int scsi_ioctl(struct scsi_device *sdev, int cmd, void __user *arg) switch (cmd) { case SCSI_IOCTL_GET_IDLUN: - if (!access_ok(VERIFY_WRITE, arg, sizeof(struct scsi_idlun))) + if (!access_ok(arg, sizeof(struct scsi_idlun))) return -EFAULT; __put_user((sdev->id & 0xff) diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index 4e27460ec926..d3f15319b9b3 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -434,7 +434,7 @@ sg_read(struct file *filp, char __user *buf, size_t count, loff_t * ppos) SCSI_LOG_TIMEOUT(3, sg_printk(KERN_INFO, sdp, "sg_read: count=%d\n", (int) count)); - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; if (sfp->force_packid && (count >= SZ_SG_HEADER)) { old_hdr = kmalloc(SZ_SG_HEADER, GFP_KERNEL); @@ -632,7 +632,7 @@ sg_write(struct file *filp, const char __user *buf, size_t count, loff_t * ppos) scsi_block_when_processing_errors(sdp->device))) return -ENXIO; - if (!access_ok(VERIFY_READ, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; /* protects following copy_from_user()s + get_user()s */ if (count < SZ_SG_HEADER) return -EIO; @@ -729,7 +729,7 @@ sg_new_write(Sg_fd *sfp, struct file *file, const char __user *buf, if (count < SZ_SG_IO_HDR) return -EINVAL; - if (!access_ok(VERIFY_READ, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; /* protects following copy_from_user()s + get_user()s */ sfp->cmd_q = 1; /* when sg_io_hdr seen, set command queuing on */ @@ -768,7 +768,7 @@ sg_new_write(Sg_fd *sfp, struct file *file, const char __user *buf, sg_remove_request(sfp, srp); return -EMSGSIZE; } - if (!access_ok(VERIFY_READ, hp->cmdp, hp->cmd_len)) { + if (!access_ok(hp->cmdp, hp->cmd_len)) { sg_remove_request(sfp, srp); return -EFAULT; /* protects following copy_from_user()s + get_user()s */ } @@ -922,7 +922,7 @@ sg_ioctl(struct file *filp, unsigned int cmd_in, unsigned long arg) return -ENODEV; if (!scsi_block_when_processing_errors(sdp->device)) return -ENXIO; - if (!access_ok(VERIFY_WRITE, p, SZ_SG_IO_HDR)) + if (!access_ok(p, SZ_SG_IO_HDR)) return -EFAULT; result = sg_new_write(sfp, filp, p, SZ_SG_IO_HDR, 1, read_only, 1, &srp); @@ -968,7 +968,7 @@ sg_ioctl(struct file *filp, unsigned int cmd_in, unsigned long arg) case SG_GET_LOW_DMA: return put_user((int) sdp->device->host->unchecked_isa_dma, ip); case SG_GET_SCSI_ID: - if (!access_ok(VERIFY_WRITE, p, sizeof (sg_scsi_id_t))) + if (!access_ok(p, sizeof (sg_scsi_id_t))) return -EFAULT; else { sg_scsi_id_t __user *sg_idp = p; @@ -997,7 +997,7 @@ sg_ioctl(struct file *filp, unsigned int cmd_in, unsigned long arg) sfp->force_packid = val ? 1 : 0; return 0; case SG_GET_PACK_ID: - if (!access_ok(VERIFY_WRITE, ip, sizeof (int))) + if (!access_ok(ip, sizeof (int))) return -EFAULT; read_lock_irqsave(&sfp->rq_list_lock, iflags); list_for_each_entry(srp, &sfp->rq_list, entry) { @@ -1078,7 +1078,7 @@ sg_ioctl(struct file *filp, unsigned int cmd_in, unsigned long arg) val = (sdp->device ? 1 : 0); return put_user(val, ip); case SG_GET_REQUEST_TABLE: - if (!access_ok(VERIFY_WRITE, p, SZ_SG_REQ_INFO * SG_MAX_QUEUE)) + if (!access_ok(p, SZ_SG_REQ_INFO * SG_MAX_QUEUE)) return -EFAULT; else { sg_req_info_t *rinfo; diff --git a/drivers/staging/comedi/comedi_compat32.c b/drivers/staging/comedi/comedi_compat32.c index fa9d239474ee..36a3564ba1fb 100644 --- a/drivers/staging/comedi/comedi_compat32.c +++ b/drivers/staging/comedi/comedi_compat32.c @@ -102,8 +102,8 @@ static int compat_chaninfo(struct file *file, unsigned long arg) chaninfo = compat_alloc_user_space(sizeof(*chaninfo)); /* Copy chaninfo structure. Ignore unused members. */ - if (!access_ok(VERIFY_READ, chaninfo32, sizeof(*chaninfo32)) || - !access_ok(VERIFY_WRITE, chaninfo, sizeof(*chaninfo))) + if (!access_ok(chaninfo32, sizeof(*chaninfo32)) || + !access_ok(chaninfo, sizeof(*chaninfo))) return -EFAULT; err = 0; @@ -136,8 +136,8 @@ static int compat_rangeinfo(struct file *file, unsigned long arg) rangeinfo = compat_alloc_user_space(sizeof(*rangeinfo)); /* Copy rangeinfo structure. */ - if (!access_ok(VERIFY_READ, rangeinfo32, sizeof(*rangeinfo32)) || - !access_ok(VERIFY_WRITE, rangeinfo, sizeof(*rangeinfo))) + if (!access_ok(rangeinfo32, sizeof(*rangeinfo32)) || + !access_ok(rangeinfo, sizeof(*rangeinfo))) return -EFAULT; err = 0; @@ -163,8 +163,8 @@ static int get_compat_cmd(struct comedi_cmd __user *cmd, } temp; /* Copy cmd structure. */ - if (!access_ok(VERIFY_READ, cmd32, sizeof(*cmd32)) || - !access_ok(VERIFY_WRITE, cmd, sizeof(*cmd))) + if (!access_ok(cmd32, sizeof(*cmd32)) || + !access_ok(cmd, sizeof(*cmd))) return -EFAULT; err = 0; @@ -217,8 +217,8 @@ static int put_compat_cmd(struct comedi32_cmd_struct __user *cmd32, * Assume the pointer values are already valid. * (Could use ptr_to_compat() to set them.) */ - if (!access_ok(VERIFY_READ, cmd, sizeof(*cmd)) || - !access_ok(VERIFY_WRITE, cmd32, sizeof(*cmd32))) + if (!access_ok(cmd, sizeof(*cmd)) || + !access_ok(cmd32, sizeof(*cmd32))) return -EFAULT; err = 0; @@ -317,8 +317,8 @@ static int get_compat_insn(struct comedi_insn __user *insn, /* Copy insn structure. Ignore the unused members. */ err = 0; - if (!access_ok(VERIFY_READ, insn32, sizeof(*insn32)) || - !access_ok(VERIFY_WRITE, insn, sizeof(*insn))) + if (!access_ok(insn32, sizeof(*insn32)) || + !access_ok(insn, sizeof(*insn))) return -EFAULT; err |= __get_user(temp.uint, &insn32->insn); @@ -350,7 +350,7 @@ static int compat_insnlist(struct file *file, unsigned long arg) insnlist32 = compat_ptr(arg); /* Get 32-bit insnlist structure. */ - if (!access_ok(VERIFY_READ, insnlist32, sizeof(*insnlist32))) + if (!access_ok(insnlist32, sizeof(*insnlist32))) return -EFAULT; err = 0; @@ -365,7 +365,7 @@ static int compat_insnlist(struct file *file, unsigned long arg) insn[n_insns])); /* Set native insnlist structure. */ - if (!access_ok(VERIFY_WRITE, &s->insnlist, sizeof(s->insnlist))) + if (!access_ok(&s->insnlist, sizeof(s->insnlist))) return -EFAULT; err |= __put_user(n_insns, &s->insnlist.n_insns); diff --git a/drivers/tty/n_hdlc.c b/drivers/tty/n_hdlc.c index 99460af61b77..4164414d4c64 100644 --- a/drivers/tty/n_hdlc.c +++ b/drivers/tty/n_hdlc.c @@ -573,7 +573,7 @@ static ssize_t n_hdlc_tty_read(struct tty_struct *tty, struct file *file, return -EIO; /* verify user access to buffer */ - if (!access_ok(VERIFY_WRITE, buf, nr)) { + if (!access_ok(buf, nr)) { printk(KERN_WARNING "%s(%d) n_hdlc_tty_read() can't verify user " "buffer\n", __FILE__, __LINE__); return -EFAULT; diff --git a/drivers/usb/core/devices.c b/drivers/usb/core/devices.c index 3de3c750b5f6..44f28a114c2b 100644 --- a/drivers/usb/core/devices.c +++ b/drivers/usb/core/devices.c @@ -598,7 +598,7 @@ static ssize_t usb_device_read(struct file *file, char __user *buf, return -EINVAL; if (nbytes <= 0) return 0; - if (!access_ok(VERIFY_WRITE, buf, nbytes)) + if (!access_ok(buf, nbytes)) return -EFAULT; mutex_lock(&usb_bus_idr_lock); diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c index a75bc0b8a50f..d65566341dd1 100644 --- a/drivers/usb/core/devio.c +++ b/drivers/usb/core/devio.c @@ -1094,7 +1094,7 @@ static int proc_control(struct usb_dev_state *ps, void __user *arg) ctrl.bRequestType, ctrl.bRequest, ctrl.wValue, ctrl.wIndex, ctrl.wLength); if (ctrl.bRequestType & 0x80) { - if (ctrl.wLength && !access_ok(VERIFY_WRITE, ctrl.data, + if (ctrl.wLength && !access_ok(ctrl.data, ctrl.wLength)) { ret = -EINVAL; goto done; @@ -1183,7 +1183,7 @@ static int proc_bulk(struct usb_dev_state *ps, void __user *arg) } tmo = bulk.timeout; if (bulk.ep & 0x80) { - if (len1 && !access_ok(VERIFY_WRITE, bulk.data, len1)) { + if (len1 && !access_ok(bulk.data, len1)) { ret = -EINVAL; goto done; } @@ -1584,8 +1584,7 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb } if (uurb->buffer_length > 0 && - !access_ok(is_in ? VERIFY_WRITE : VERIFY_READ, - uurb->buffer, uurb->buffer_length)) { + !access_ok(uurb->buffer, uurb->buffer_length)) { ret = -EFAULT; goto error; } diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c index 54e859dcb25c..75b113a5b25c 100644 --- a/drivers/usb/gadget/function/f_hid.c +++ b/drivers/usb/gadget/function/f_hid.c @@ -252,7 +252,7 @@ static ssize_t f_hidg_read(struct file *file, char __user *buffer, if (!count) return 0; - if (!access_ok(VERIFY_WRITE, buffer, count)) + if (!access_ok(buffer, count)) return -EFAULT; spin_lock_irqsave(&hidg->read_spinlock, flags); @@ -339,7 +339,7 @@ static ssize_t f_hidg_write(struct file *file, const char __user *buffer, unsigned long flags; ssize_t status = -ENOMEM; - if (!access_ok(VERIFY_READ, buffer, count)) + if (!access_ok(buffer, count)) return -EFAULT; spin_lock_irqsave(&hidg->write_spinlock, flags); diff --git a/drivers/usb/gadget/udc/atmel_usba_udc.c b/drivers/usb/gadget/udc/atmel_usba_udc.c index 11247322d587..660712e0bf98 100644 --- a/drivers/usb/gadget/udc/atmel_usba_udc.c +++ b/drivers/usb/gadget/udc/atmel_usba_udc.c @@ -88,7 +88,7 @@ static ssize_t queue_dbg_read(struct file *file, char __user *buf, size_t len, remaining, actual = 0; char tmpbuf[38]; - if (!access_ok(VERIFY_WRITE, buf, nbytes)) + if (!access_ok(buf, nbytes)) return -EFAULT; inode_lock(file_inode(file)); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 55e5aa662ad5..9f7942cbcbb2 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -655,7 +655,7 @@ static bool log_access_ok(void __user *log_base, u64 addr, unsigned long sz) a + (unsigned long)log_base > ULONG_MAX) return false; - return access_ok(VERIFY_WRITE, log_base + a, + return access_ok(log_base + a, (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8); } @@ -681,7 +681,7 @@ static bool vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem, return false; - if (!access_ok(VERIFY_WRITE, (void __user *)a, + if (!access_ok((void __user *)a, node->size)) return false; else if (log_all && !log_access_ok(log_base, @@ -973,10 +973,10 @@ static bool umem_access_ok(u64 uaddr, u64 size, int access) return false; if ((access & VHOST_ACCESS_RO) && - !access_ok(VERIFY_READ, (void __user *)a, size)) + !access_ok((void __user *)a, size)) return false; if ((access & VHOST_ACCESS_WO) && - !access_ok(VERIFY_WRITE, (void __user *)a, size)) + !access_ok((void __user *)a, size)) return false; return true; } @@ -1185,10 +1185,10 @@ static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, { size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; - return access_ok(VERIFY_READ, desc, num * sizeof *desc) && - access_ok(VERIFY_READ, avail, + return access_ok(desc, num * sizeof *desc) && + access_ok(avail, sizeof *avail + num * sizeof *avail->ring + s) && - access_ok(VERIFY_WRITE, used, + access_ok(used, sizeof *used + num * sizeof *used->ring + s); } @@ -1814,7 +1814,7 @@ int vhost_vq_init_access(struct vhost_virtqueue *vq) goto err; vq->signalled_used_valid = false; if (!vq->iotlb && - !access_ok(VERIFY_READ, &vq->used->idx, sizeof vq->used->idx)) { + !access_ok(&vq->used->idx, sizeof vq->used->idx)) { r = -EFAULT; goto err; } diff --git a/drivers/video/fbdev/amifb.c b/drivers/video/fbdev/amifb.c index 0777aff211e5..758457026694 100644 --- a/drivers/video/fbdev/amifb.c +++ b/drivers/video/fbdev/amifb.c @@ -1855,7 +1855,7 @@ static int ami_get_var_cursorinfo(struct fb_var_cursorinfo *var, var->yspot = par->crsr.spot_y; if (size > var->height * var->width) return -ENAMETOOLONG; - if (!access_ok(VERIFY_WRITE, data, size)) + if (!access_ok(data, size)) return -EFAULT; delta = 1 << par->crsr.fmode; lspr = lofsprite + (delta << 1); @@ -1935,7 +1935,7 @@ static int ami_set_var_cursorinfo(struct fb_var_cursorinfo *var, return -EINVAL; if (!var->height) return -EINVAL; - if (!access_ok(VERIFY_READ, data, var->width * var->height)) + if (!access_ok(data, var->width * var->height)) return -EFAULT; delta = 1 << fmode; lofsprite = shfsprite = (u_short *)spritememory; diff --git a/drivers/video/fbdev/omap2/omapfb/omapfb-ioctl.c b/drivers/video/fbdev/omap2/omapfb/omapfb-ioctl.c index a3edb20ea4c3..53f93616c671 100644 --- a/drivers/video/fbdev/omap2/omapfb/omapfb-ioctl.c +++ b/drivers/video/fbdev/omap2/omapfb/omapfb-ioctl.c @@ -493,7 +493,7 @@ static int omapfb_memory_read(struct fb_info *fbi, if (!display || !display->driver->memory_read) return -ENOENT; - if (!access_ok(VERIFY_WRITE, mr->buffer, mr->buffer_size)) + if (!access_ok(mr->buffer, mr->buffer_size)) return -EFAULT; if (mr->w > 4096 || mr->h > 4096) diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c index 7e6e682104dc..b24ddac1604b 100644 --- a/drivers/xen/privcmd.c +++ b/drivers/xen/privcmd.c @@ -459,14 +459,14 @@ static long privcmd_ioctl_mmap_batch( return -EFAULT; /* Returns per-frame error in m.arr. */ m.err = NULL; - if (!access_ok(VERIFY_WRITE, m.arr, m.num * sizeof(*m.arr))) + if (!access_ok(m.arr, m.num * sizeof(*m.arr))) return -EFAULT; break; case 2: if (copy_from_user(&m, udata, sizeof(struct privcmd_mmapbatch_v2))) return -EFAULT; /* Returns per-frame error code in m.err. */ - if (!access_ok(VERIFY_WRITE, m.err, m.num * (sizeof(*m.err)))) + if (!access_ok(m.err, m.num * (sizeof(*m.err)))) return -EFAULT; break; default: @@ -661,7 +661,7 @@ static long privcmd_ioctl_dm_op(struct file *file, void __user *udata) goto out; } - if (!access_ok(VERIFY_WRITE, kbufs[i].uptr, + if (!access_ok(kbufs[i].uptr, kbufs[i].size)) { rc = -EFAULT; goto out; diff --git a/fs/binfmt_aout.c b/fs/binfmt_aout.c index c3deb2e35f20..ca9725f18e00 100644 --- a/fs/binfmt_aout.c +++ b/fs/binfmt_aout.c @@ -78,9 +78,9 @@ static int aout_core_dump(struct coredump_params *cprm) /* make sure we actually have a data and stack area to dump */ set_fs(USER_DS); - if (!access_ok(VERIFY_READ, START_DATA(dump), dump.u_dsize << PAGE_SHIFT)) + if (!access_ok(START_DATA(dump), dump.u_dsize << PAGE_SHIFT)) dump.u_dsize = 0; - if (!access_ok(VERIFY_READ, START_STACK(dump), dump.u_ssize << PAGE_SHIFT)) + if (!access_ok(START_STACK(dump), dump.u_ssize << PAGE_SHIFT)) dump.u_ssize = 0; set_fs(KERNEL_DS); diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 1b15b43905f8..7ea2d6b1f170 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -6646,7 +6646,7 @@ long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args *arg) goto out; } - if (!access_ok(VERIFY_READ, arg->clone_sources, + if (!access_ok(arg->clone_sources, sizeof(*arg->clone_sources) * arg->clone_sources_count)) { ret = -EFAULT; diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 8a5a1010886b..7ebae39fbcb3 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -2172,7 +2172,7 @@ static int do_epoll_wait(int epfd, struct epoll_event __user *events, return -EINVAL; /* Verify that the area passed by the user is writeable */ - if (!access_ok(VERIFY_WRITE, events, maxevents * sizeof(struct epoll_event))) + if (!access_ok(events, maxevents * sizeof(struct epoll_event))) return -EFAULT; /* Get the "struct file *" for the eventpoll file */ diff --git a/fs/fat/dir.c b/fs/fat/dir.c index c8366cb8eccd..0295a095b920 100644 --- a/fs/fat/dir.c +++ b/fs/fat/dir.c @@ -805,7 +805,7 @@ static long fat_dir_ioctl(struct file *filp, unsigned int cmd, return fat_generic_ioctl(filp, cmd, arg); } - if (!access_ok(VERIFY_WRITE, d1, sizeof(struct __fat_dirent[2]))) + if (!access_ok(d1, sizeof(struct __fat_dirent[2]))) return -EFAULT; /* * Yes, we don't need this put_user() absolutely. However old @@ -845,7 +845,7 @@ static long fat_compat_dir_ioctl(struct file *filp, unsigned cmd, return fat_generic_ioctl(filp, cmd, (unsigned long)arg); } - if (!access_ok(VERIFY_WRITE, d1, sizeof(struct compat_dirent[2]))) + if (!access_ok(d1, sizeof(struct compat_dirent[2]))) return -EFAULT; /* * Yes, we don't need this put_user() absolutely. However old diff --git a/fs/ioctl.c b/fs/ioctl.c index d64f622cac8b..fef3a6bf7c78 100644 --- a/fs/ioctl.c +++ b/fs/ioctl.c @@ -203,7 +203,7 @@ static int ioctl_fiemap(struct file *filp, unsigned long arg) fieinfo.fi_extents_start = ufiemap->fm_extents; if (fiemap.fm_extent_count != 0 && - !access_ok(VERIFY_WRITE, fieinfo.fi_extents_start, + !access_ok(fieinfo.fi_extents_start, fieinfo.fi_extents_max * sizeof(struct fiemap_extent))) return -EFAULT; diff --git a/fs/namespace.c b/fs/namespace.c index a7f91265ea67..97b7c7098c3d 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -2651,7 +2651,7 @@ static long exact_copy_from_user(void *to, const void __user * from, const char __user *f = from; char c; - if (!access_ok(VERIFY_READ, from, n)) + if (!access_ok(from, n)) return n; current->kernel_uaccess_faults_ok++; diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c index b8fa1487cd85..8decbe95dcec 100644 --- a/fs/ocfs2/dlmfs/dlmfs.c +++ b/fs/ocfs2/dlmfs/dlmfs.c @@ -254,7 +254,7 @@ static ssize_t dlmfs_file_read(struct file *filp, if (!count) return 0; - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; /* don't read past the lvb */ @@ -302,7 +302,7 @@ static ssize_t dlmfs_file_write(struct file *filp, if (!count) return 0; - if (!access_ok(VERIFY_READ, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; /* don't write past the lvb */ diff --git a/fs/pstore/pmsg.c b/fs/pstore/pmsg.c index 24db02de1787..97fcef74e5af 100644 --- a/fs/pstore/pmsg.c +++ b/fs/pstore/pmsg.c @@ -33,7 +33,7 @@ static ssize_t write_pmsg(struct file *file, const char __user *buf, record.size = count; /* check outside lock, page in any data. write_user also checks */ - if (!access_ok(VERIFY_READ, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; mutex_lock(&pmsg_lock); diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c index c11711c2cc83..f375c0735351 100644 --- a/fs/pstore/ram_core.c +++ b/fs/pstore/ram_core.c @@ -357,7 +357,7 @@ int notrace persistent_ram_write_user(struct persistent_ram_zone *prz, int rem, ret = 0, c = count; size_t start; - if (unlikely(!access_ok(VERIFY_READ, s, count))) + if (unlikely(!access_ok(s, count))) return -EFAULT; if (unlikely(c > prz->buffer_size)) { s += c - prz->buffer_size; diff --git a/fs/read_write.c b/fs/read_write.c index 58f30537c47a..ff3c5e6f87cf 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -442,7 +442,7 @@ ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) return -EBADF; if (!(file->f_mode & FMODE_CAN_READ)) return -EINVAL; - if (unlikely(!access_ok(VERIFY_WRITE, buf, count))) + if (unlikely(!access_ok(buf, count))) return -EFAULT; ret = rw_verify_area(READ, file, pos, count); @@ -538,7 +538,7 @@ ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_ return -EBADF; if (!(file->f_mode & FMODE_CAN_WRITE)) return -EINVAL; - if (unlikely(!access_ok(VERIFY_READ, buf, count))) + if (unlikely(!access_ok(buf, count))) return -EFAULT; ret = rw_verify_area(WRITE, file, pos, count); @@ -718,9 +718,6 @@ static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter, return ret; } -/* A write operation does a read from user space and vice versa */ -#define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ) - /** * rw_copy_check_uvector() - Copy an array of &struct iovec from userspace * into the kernel and check that it is valid. @@ -810,7 +807,7 @@ ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, goto out; } if (type >= 0 - && unlikely(!access_ok(vrfy_dir(type), buf, len))) { + && unlikely(!access_ok(buf, len))) { ret = -EFAULT; goto out; } @@ -856,7 +853,7 @@ ssize_t compat_rw_copy_check_uvector(int type, *ret_pointer = iov; ret = -EFAULT; - if (!access_ok(VERIFY_READ, uvector, nr_segs*sizeof(*uvector))) + if (!access_ok(uvector, nr_segs*sizeof(*uvector))) goto out; /* @@ -881,7 +878,7 @@ ssize_t compat_rw_copy_check_uvector(int type, if (len < 0) /* size_t not fitting in compat_ssize_t .. */ goto out; if (type >= 0 && - !access_ok(vrfy_dir(type), compat_ptr(buf), len)) { + !access_ok(compat_ptr(buf), len)) { ret = -EFAULT; goto out; } diff --git a/fs/readdir.c b/fs/readdir.c index d97f548e6323..2f6a4534e0df 100644 --- a/fs/readdir.c +++ b/fs/readdir.c @@ -105,7 +105,7 @@ static int fillonedir(struct dir_context *ctx, const char *name, int namlen, } buf->result++; dirent = buf->dirent; - if (!access_ok(VERIFY_WRITE, dirent, + if (!access_ok(dirent, (unsigned long)(dirent->d_name + namlen + 1) - (unsigned long)dirent)) goto efault; @@ -221,7 +221,7 @@ SYSCALL_DEFINE3(getdents, unsigned int, fd, }; int error; - if (!access_ok(VERIFY_WRITE, dirent, count)) + if (!access_ok(dirent, count)) return -EFAULT; f = fdget_pos(fd); @@ -304,7 +304,7 @@ int ksys_getdents64(unsigned int fd, struct linux_dirent64 __user *dirent, }; int error; - if (!access_ok(VERIFY_WRITE, dirent, count)) + if (!access_ok(dirent, count)) return -EFAULT; f = fdget_pos(fd); @@ -365,7 +365,7 @@ static int compat_fillonedir(struct dir_context *ctx, const char *name, } buf->result++; dirent = buf->dirent; - if (!access_ok(VERIFY_WRITE, dirent, + if (!access_ok(dirent, (unsigned long)(dirent->d_name + namlen + 1) - (unsigned long)dirent)) goto efault; @@ -475,7 +475,7 @@ COMPAT_SYSCALL_DEFINE3(getdents, unsigned int, fd, }; int error; - if (!access_ok(VERIFY_WRITE, dirent, count)) + if (!access_ok(dirent, count)) return -EFAULT; f = fdget_pos(fd); diff --git a/fs/select.c b/fs/select.c index 4c8652390c94..d0f35dbc0e8f 100644 --- a/fs/select.c +++ b/fs/select.c @@ -381,9 +381,6 @@ typedef struct { #define FDS_BYTES(nr) (FDS_LONGS(nr)*sizeof(long)) /* - * We do a VERIFY_WRITE here even though we are only reading this time: - * we'll write to it eventually.. - * * Use "unsigned long" accesses to let user-mode fd_set's be long-aligned. */ static inline @@ -782,7 +779,7 @@ SYSCALL_DEFINE6(pselect6, int, n, fd_set __user *, inp, fd_set __user *, outp, sigset_t __user *up = NULL; if (sig) { - if (!access_ok(VERIFY_READ, sig, sizeof(void *)+sizeof(size_t)) + if (!access_ok(sig, sizeof(void *)+sizeof(size_t)) || __get_user(up, (sigset_t __user * __user *)sig) || __get_user(sigsetsize, (size_t __user *)(sig+sizeof(void *)))) @@ -802,7 +799,7 @@ SYSCALL_DEFINE6(pselect6_time32, int, n, fd_set __user *, inp, fd_set __user *, sigset_t __user *up = NULL; if (sig) { - if (!access_ok(VERIFY_READ, sig, sizeof(void *)+sizeof(size_t)) + if (!access_ok(sig, sizeof(void *)+sizeof(size_t)) || __get_user(up, (sigset_t __user * __user *)sig) || __get_user(sigsetsize, (size_t __user *)(sig+sizeof(void *)))) @@ -1368,7 +1365,7 @@ COMPAT_SYSCALL_DEFINE6(pselect6_time64, int, n, compat_ulong_t __user *, inp, compat_uptr_t up = 0; if (sig) { - if (!access_ok(VERIFY_READ, sig, + if (!access_ok(sig, sizeof(compat_uptr_t)+sizeof(compat_size_t)) || __get_user(up, (compat_uptr_t __user *)sig) || __get_user(sigsetsize, @@ -1390,7 +1387,7 @@ COMPAT_SYSCALL_DEFINE6(pselect6, int, n, compat_ulong_t __user *, inp, compat_uptr_t up = 0; if (sig) { - if (!access_ok(VERIFY_READ, sig, + if (!access_ok(sig, sizeof(compat_uptr_t)+sizeof(compat_size_t)) || __get_user(up, (compat_uptr_t __user *)sig) || __get_user(sigsetsize, diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index 6b2e63df2739..d82c78a79da5 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -35,7 +35,7 @@ static inline void set_fs(mm_segment_t fs) #define segment_eq(a, b) ((a).seg == (b).seg) #endif -#define access_ok(type, addr, size) __access_ok((unsigned long)(addr),(size)) +#define access_ok(addr, size) __access_ok((unsigned long)(addr),(size)) /* * The architecture should really override this if possible, at least @@ -78,7 +78,7 @@ static inline int __access_ok(unsigned long addr, unsigned long size) ({ \ void __user *__p = (ptr); \ might_fault(); \ - access_ok(VERIFY_WRITE, __p, sizeof(*ptr)) ? \ + access_ok(__p, sizeof(*ptr)) ? \ __put_user((x), ((__typeof__(*(ptr)) __user *)__p)) : \ -EFAULT; \ }) @@ -140,7 +140,7 @@ extern int __put_user_bad(void) __attribute__((noreturn)); ({ \ const void __user *__p = (ptr); \ might_fault(); \ - access_ok(VERIFY_READ, __p, sizeof(*ptr)) ? \ + access_ok(__p, sizeof(*ptr)) ? \ __get_user((x), (__typeof__(*(ptr)) __user *)__p) :\ ((x) = (__typeof__(*(ptr)))0,-EFAULT); \ }) @@ -175,7 +175,7 @@ __strncpy_from_user(char *dst, const char __user *src, long count) static inline long strncpy_from_user(char *dst, const char __user *src, long count) { - if (!access_ok(VERIFY_READ, src, 1)) + if (!access_ok(src, 1)) return -EFAULT; return __strncpy_from_user(dst, src, count); } @@ -196,7 +196,7 @@ strncpy_from_user(char *dst, const char __user *src, long count) */ static inline long strnlen_user(const char __user *src, long n) { - if (!access_ok(VERIFY_READ, src, 1)) + if (!access_ok(src, 1)) return 0; return __strnlen_user(src, n); } @@ -217,7 +217,7 @@ static inline __must_check unsigned long clear_user(void __user *to, unsigned long n) { might_fault(); - if (!access_ok(VERIFY_WRITE, to, n)) + if (!access_ok(to, n)) return n; return __clear_user(to, n); diff --git a/include/linux/regset.h b/include/linux/regset.h index 494cedaafdf2..a85c1707285c 100644 --- a/include/linux/regset.h +++ b/include/linux/regset.h @@ -376,7 +376,7 @@ static inline int copy_regset_to_user(struct task_struct *target, if (!regset->get) return -EOPNOTSUPP; - if (!access_ok(VERIFY_WRITE, data, size)) + if (!access_ok(data, size)) return -EFAULT; return regset->get(target, regset, offset, size, NULL, data); @@ -402,7 +402,7 @@ static inline int copy_regset_from_user(struct task_struct *target, if (!regset->set) return -EOPNOTSUPP; - if (!access_ok(VERIFY_READ, data, size)) + if (!access_ok(data, size)) return -EFAULT; return regset->set(target, regset, offset, size, NULL, data); diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index efe79c1cdd47..bf2523867a02 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -6,9 +6,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define uaccess_kernel() segment_eq(get_fs(), KERNEL_DS) #include @@ -111,7 +108,7 @@ _copy_from_user(void *to, const void __user *from, unsigned long n) { unsigned long res = n; might_fault(); - if (likely(access_ok(VERIFY_READ, from, n))) { + if (likely(access_ok(from, n))) { kasan_check_write(to, n); res = raw_copy_from_user(to, from, n); } @@ -129,7 +126,7 @@ static inline unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) { might_fault(); - if (access_ok(VERIFY_WRITE, to, n)) { + if (access_ok(to, n)) { kasan_check_read(from, n); n = raw_copy_to_user(to, from, n); } @@ -160,7 +157,7 @@ static __always_inline unsigned long __must_check copy_in_user(void __user *to, const void __user *from, unsigned long n) { might_fault(); - if (access_ok(VERIFY_WRITE, to, n) && access_ok(VERIFY_READ, from, n)) + if (access_ok(to, n) && access_ok(from, n)) n = raw_copy_in_user(to, from, n); return n; } diff --git a/include/net/checksum.h b/include/net/checksum.h index aef2b2bb6603..0f319e13be2c 100644 --- a/include/net/checksum.h +++ b/include/net/checksum.h @@ -30,7 +30,7 @@ static inline __wsum csum_and_copy_from_user (const void __user *src, void *dst, int len, __wsum sum, int *err_ptr) { - if (access_ok(VERIFY_READ, src, len)) + if (access_ok(src, len)) return csum_partial_copy_from_user(src, dst, len, sum, err_ptr); if (len) @@ -46,7 +46,7 @@ static __inline__ __wsum csum_and_copy_to_user { sum = csum_partial(src, len, sum); - if (access_ok(VERIFY_WRITE, dst, len)) { + if (access_ok(dst, len)) { if (copy_to_user(dst, src, len) == 0) return sum; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 0607db304def..b155cd17c1bd 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -79,7 +79,7 @@ int bpf_check_uarg_tail_zero(void __user *uaddr, if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ return -E2BIG; - if (unlikely(!access_ok(VERIFY_READ, uaddr, actual_size))) + if (unlikely(!access_ok(uaddr, actual_size))) return -EFAULT; if (actual_size <= expected_size) diff --git a/kernel/compat.c b/kernel/compat.c index 089d00d0da9c..705d4ae6c018 100644 --- a/kernel/compat.c +++ b/kernel/compat.c @@ -95,28 +95,28 @@ int compat_put_timex(struct compat_timex __user *utp, const struct timex *txc) static int __compat_get_timeval(struct timeval *tv, const struct old_timeval32 __user *ctv) { - return (!access_ok(VERIFY_READ, ctv, sizeof(*ctv)) || + return (!access_ok(ctv, sizeof(*ctv)) || __get_user(tv->tv_sec, &ctv->tv_sec) || __get_user(tv->tv_usec, &ctv->tv_usec)) ? -EFAULT : 0; } static int __compat_put_timeval(const struct timeval *tv, struct old_timeval32 __user *ctv) { - return (!access_ok(VERIFY_WRITE, ctv, sizeof(*ctv)) || + return (!access_ok(ctv, sizeof(*ctv)) || __put_user(tv->tv_sec, &ctv->tv_sec) || __put_user(tv->tv_usec, &ctv->tv_usec)) ? -EFAULT : 0; } static int __compat_get_timespec(struct timespec *ts, const struct old_timespec32 __user *cts) { - return (!access_ok(VERIFY_READ, cts, sizeof(*cts)) || + return (!access_ok(cts, sizeof(*cts)) || __get_user(ts->tv_sec, &cts->tv_sec) || __get_user(ts->tv_nsec, &cts->tv_nsec)) ? -EFAULT : 0; } static int __compat_put_timespec(const struct timespec *ts, struct old_timespec32 __user *cts) { - return (!access_ok(VERIFY_WRITE, cts, sizeof(*cts)) || + return (!access_ok(cts, sizeof(*cts)) || __put_user(ts->tv_sec, &cts->tv_sec) || __put_user(ts->tv_nsec, &cts->tv_nsec)) ? -EFAULT : 0; } @@ -335,7 +335,7 @@ int get_compat_sigevent(struct sigevent *event, const struct compat_sigevent __user *u_event) { memset(event, 0, sizeof(*event)); - return (!access_ok(VERIFY_READ, u_event, sizeof(*u_event)) || + return (!access_ok(u_event, sizeof(*u_event)) || __get_user(event->sigev_value.sival_int, &u_event->sigev_value.sival_int) || __get_user(event->sigev_signo, &u_event->sigev_signo) || @@ -354,7 +354,7 @@ long compat_get_bitmap(unsigned long *mask, const compat_ulong_t __user *umask, bitmap_size = ALIGN(bitmap_size, BITS_PER_COMPAT_LONG); nr_compat_longs = BITS_TO_COMPAT_LONGS(bitmap_size); - if (!access_ok(VERIFY_READ, umask, bitmap_size / 8)) + if (!access_ok(umask, bitmap_size / 8)) return -EFAULT; user_access_begin(); @@ -384,7 +384,7 @@ long compat_put_bitmap(compat_ulong_t __user *umask, unsigned long *mask, bitmap_size = ALIGN(bitmap_size, BITS_PER_COMPAT_LONG); nr_compat_longs = BITS_TO_COMPAT_LONGS(bitmap_size); - if (!access_ok(VERIFY_WRITE, umask, bitmap_size / 8)) + if (!access_ok(umask, bitmap_size / 8)) return -EFAULT; user_access_begin(); @@ -438,7 +438,7 @@ void __user *compat_alloc_user_space(unsigned long len) ptr = arch_compat_alloc_user_space(len); - if (unlikely(!access_ok(VERIFY_WRITE, ptr, len))) + if (unlikely(!access_ok(ptr, len))) return NULL; return ptr; diff --git a/kernel/events/core.c b/kernel/events/core.c index 67ecac337374..3cd13a30f732 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -10135,7 +10135,7 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, u32 size; int ret; - if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0)) + if (!access_ok(uattr, PERF_ATTR_SIZE_VER0)) return -EFAULT; /* diff --git a/kernel/exit.c b/kernel/exit.c index 0e21e6d21f35..8a01b671dc1f 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -1604,7 +1604,7 @@ SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *, if (!infop) return err; - if (!access_ok(VERIFY_WRITE, infop, sizeof(*infop))) + if (!access_ok(infop, sizeof(*infop))) return -EFAULT; user_access_begin(); @@ -1732,7 +1732,7 @@ COMPAT_SYSCALL_DEFINE5(waitid, if (!infop) return err; - if (!access_ok(VERIFY_WRITE, infop, sizeof(*infop))) + if (!access_ok(infop, sizeof(*infop))) return -EFAULT; user_access_begin(); diff --git a/kernel/futex.c b/kernel/futex.c index 054105854e0e..be3bff2315ff 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -481,13 +481,18 @@ static void drop_futex_key_refs(union futex_key *key) } } +enum futex_access { + FUTEX_READ, + FUTEX_WRITE +}; + /** * get_futex_key() - Get parameters which are the keys for a futex * @uaddr: virtual address of the futex * @fshared: 0 for a PROCESS_PRIVATE futex, 1 for PROCESS_SHARED * @key: address where result is stored. - * @rw: mapping needs to be read/write (values: VERIFY_READ, - * VERIFY_WRITE) + * @rw: mapping needs to be read/write (values: FUTEX_READ, + * FUTEX_WRITE) * * Return: a negative error code or 0 * @@ -500,7 +505,7 @@ static void drop_futex_key_refs(union futex_key *key) * lock_page() might sleep, the caller should not hold a spinlock. */ static int -get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw) +get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, enum futex_access rw) { unsigned long address = (unsigned long)uaddr; struct mm_struct *mm = current->mm; @@ -516,7 +521,7 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw) return -EINVAL; address -= key->both.offset; - if (unlikely(!access_ok(rw, uaddr, sizeof(u32)))) + if (unlikely(!access_ok(uaddr, sizeof(u32)))) return -EFAULT; if (unlikely(should_fail_futex(fshared))) @@ -546,7 +551,7 @@ again: * If write access is not required (eg. FUTEX_WAIT), try * and get read-only access. */ - if (err == -EFAULT && rw == VERIFY_READ) { + if (err == -EFAULT && rw == FUTEX_READ) { err = get_user_pages_fast(address, 1, 0, &page); ro = 1; } @@ -1583,7 +1588,7 @@ futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) if (!bitset) return -EINVAL; - ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_READ); + ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ); if (unlikely(ret != 0)) goto out; @@ -1642,7 +1647,7 @@ static int futex_atomic_op_inuser(unsigned int encoded_op, u32 __user *uaddr) oparg = 1 << oparg; } - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) + if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; ret = arch_futex_atomic_op_inuser(op, oparg, &oldval, uaddr); @@ -1682,10 +1687,10 @@ futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, DEFINE_WAKE_Q(wake_q); retry: - ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ); + ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ); if (unlikely(ret != 0)) goto out; - ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE); + ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE); if (unlikely(ret != 0)) goto out_put_key1; @@ -1961,11 +1966,11 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, } retry: - ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ); + ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ); if (unlikely(ret != 0)) goto out; ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, - requeue_pi ? VERIFY_WRITE : VERIFY_READ); + requeue_pi ? FUTEX_WRITE : FUTEX_READ); if (unlikely(ret != 0)) goto out_put_key1; @@ -2634,7 +2639,7 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, * while the syscall executes. */ retry: - ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, VERIFY_READ); + ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ); if (unlikely(ret != 0)) return ret; @@ -2793,7 +2798,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags, } retry: - ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, VERIFY_WRITE); + ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, FUTEX_WRITE); if (unlikely(ret != 0)) goto out; @@ -2972,7 +2977,7 @@ retry: if ((uval & FUTEX_TID_MASK) != vpid) return -EPERM; - ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_WRITE); + ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_WRITE); if (ret) return ret; @@ -3199,7 +3204,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, */ rt_mutex_init_waiter(&rt_waiter); - ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE); + ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE); if (unlikely(ret != 0)) goto out; diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 1306fe0c1dc6..d3d170374ceb 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -1466,7 +1466,7 @@ int do_syslog(int type, char __user *buf, int len, int source) return -EINVAL; if (!len) return 0; - if (!access_ok(VERIFY_WRITE, buf, len)) + if (!access_ok(buf, len)) return -EFAULT; error = wait_event_interruptible(log_wait, syslog_seq != log_next_seq); @@ -1484,7 +1484,7 @@ int do_syslog(int type, char __user *buf, int len, int source) return -EINVAL; if (!len) return 0; - if (!access_ok(VERIFY_WRITE, buf, len)) + if (!access_ok(buf, len)) return -EFAULT; error = syslog_print_all(buf, len, clear); break; diff --git a/kernel/ptrace.c b/kernel/ptrace.c index c2cee9db5204..771e93f9c43f 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -1073,7 +1073,7 @@ int ptrace_request(struct task_struct *child, long request, struct iovec kiov; struct iovec __user *uiov = datavp; - if (!access_ok(VERIFY_WRITE, uiov, sizeof(*uiov))) + if (!access_ok(uiov, sizeof(*uiov))) return -EFAULT; if (__get_user(kiov.iov_base, &uiov->iov_base) || @@ -1229,7 +1229,7 @@ int compat_ptrace_request(struct task_struct *child, compat_long_t request, compat_uptr_t ptr; compat_size_t len; - if (!access_ok(VERIFY_WRITE, uiov, sizeof(*uiov))) + if (!access_ok(uiov, sizeof(*uiov))) return -EFAULT; if (__get_user(ptr, &uiov->iov_base) || diff --git a/kernel/rseq.c b/kernel/rseq.c index c6242d8594dc..25e9a7b60eba 100644 --- a/kernel/rseq.c +++ b/kernel/rseq.c @@ -267,7 +267,7 @@ void __rseq_handle_notify_resume(struct ksignal *ksig, struct pt_regs *regs) if (unlikely(t->flags & PF_EXITING)) return; - if (unlikely(!access_ok(VERIFY_WRITE, t->rseq, sizeof(*t->rseq)))) + if (unlikely(!access_ok(t->rseq, sizeof(*t->rseq)))) goto error; ret = rseq_ip_fixup(regs); if (unlikely(ret < 0)) @@ -295,7 +295,7 @@ void rseq_syscall(struct pt_regs *regs) if (!t->rseq) return; - if (!access_ok(VERIFY_READ, t->rseq, sizeof(*t->rseq)) || + if (!access_ok(t->rseq, sizeof(*t->rseq)) || rseq_get_rseq_cs(t, &rseq_cs) || in_rseq_cs(ip, &rseq_cs)) force_sig(SIGSEGV, t); } @@ -351,7 +351,7 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, if (!IS_ALIGNED((unsigned long)rseq, __alignof__(*rseq)) || rseq_len != sizeof(*rseq)) return -EINVAL; - if (!access_ok(VERIFY_WRITE, rseq, rseq_len)) + if (!access_ok(rseq, rseq_len)) return -EFAULT; current->rseq = rseq; current->rseq_len = rseq_len; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f66920173370..1f3e19fd6dc6 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4450,7 +4450,7 @@ static int sched_copy_attr(struct sched_attr __user *uattr, struct sched_attr *a u32 size; int ret; - if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0)) + if (!access_ok(uattr, SCHED_ATTR_SIZE_VER0)) return -EFAULT; /* Zero the full structure, so that a short copy will be nice: */ @@ -4650,7 +4650,7 @@ static int sched_read_attr(struct sched_attr __user *uattr, { int ret; - if (!access_ok(VERIFY_WRITE, uattr, usize)) + if (!access_ok(uattr, usize)) return -EFAULT; /* diff --git a/kernel/signal.c b/kernel/signal.c index 53e07d97ffe0..e1d7ad8e6ab1 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -3997,7 +3997,7 @@ SYSCALL_DEFINE3(sigaction, int, sig, if (act) { old_sigset_t mask; - if (!access_ok(VERIFY_READ, act, sizeof(*act)) || + if (!access_ok(act, sizeof(*act)) || __get_user(new_ka.sa.sa_handler, &act->sa_handler) || __get_user(new_ka.sa.sa_restorer, &act->sa_restorer) || __get_user(new_ka.sa.sa_flags, &act->sa_flags) || @@ -4012,7 +4012,7 @@ SYSCALL_DEFINE3(sigaction, int, sig, ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); if (!ret && oact) { - if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) || + if (!access_ok(oact, sizeof(*oact)) || __put_user(old_ka.sa.sa_handler, &oact->sa_handler) || __put_user(old_ka.sa.sa_restorer, &oact->sa_restorer) || __put_user(old_ka.sa.sa_flags, &oact->sa_flags) || @@ -4034,7 +4034,7 @@ COMPAT_SYSCALL_DEFINE3(sigaction, int, sig, compat_uptr_t handler, restorer; if (act) { - if (!access_ok(VERIFY_READ, act, sizeof(*act)) || + if (!access_ok(act, sizeof(*act)) || __get_user(handler, &act->sa_handler) || __get_user(restorer, &act->sa_restorer) || __get_user(new_ka.sa.sa_flags, &act->sa_flags) || @@ -4052,7 +4052,7 @@ COMPAT_SYSCALL_DEFINE3(sigaction, int, sig, ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); if (!ret && oact) { - if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) || + if (!access_ok(oact, sizeof(*oact)) || __put_user(ptr_to_compat(old_ka.sa.sa_handler), &oact->sa_handler) || __put_user(ptr_to_compat(old_ka.sa.sa_restorer), diff --git a/kernel/sys.c b/kernel/sys.c index 64b5a230f38d..a48cbf1414b8 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2627,7 +2627,7 @@ COMPAT_SYSCALL_DEFINE1(sysinfo, struct compat_sysinfo __user *, info) s.freehigh >>= bitcount; } - if (!access_ok(VERIFY_WRITE, info, sizeof(struct compat_sysinfo)) || + if (!access_ok(info, sizeof(struct compat_sysinfo)) || __put_user(s.uptime, &info->uptime) || __put_user(s.loads[0], &info->loads[0]) || __put_user(s.loads[1], &info->loads[1]) || diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 9ddb6fddb4e0..8b068adb9da1 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -170,7 +170,7 @@ BPF_CALL_3(bpf_probe_write_user, void *, unsafe_ptr, const void *, src, return -EPERM; if (unlikely(uaccess_kernel())) return -EPERM; - if (!access_ok(VERIFY_WRITE, unsafe_ptr, size)) + if (!access_ok(unsafe_ptr, size)) return -EPERM; return probe_kernel_write(unsafe_ptr, src, size); diff --git a/lib/bitmap.c b/lib/bitmap.c index eead55aa7170..98872e9025da 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -443,7 +443,7 @@ int bitmap_parse_user(const char __user *ubuf, unsigned int ulen, unsigned long *maskp, int nmaskbits) { - if (!access_ok(VERIFY_READ, ubuf, ulen)) + if (!access_ok(ubuf, ulen)) return -EFAULT; return __bitmap_parse((const char __force *)ubuf, ulen, 1, maskp, nmaskbits); @@ -641,7 +641,7 @@ int bitmap_parselist_user(const char __user *ubuf, unsigned int ulen, unsigned long *maskp, int nmaskbits) { - if (!access_ok(VERIFY_READ, ubuf, ulen)) + if (!access_ok(ubuf, ulen)) return -EFAULT; return __bitmap_parselist((const char __force *)ubuf, ulen, 1, maskp, nmaskbits); diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 1928009f506e..c93870987b58 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -136,7 +136,7 @@ static int copyout(void __user *to, const void *from, size_t n) { - if (access_ok(VERIFY_WRITE, to, n)) { + if (access_ok(to, n)) { kasan_check_read(from, n); n = raw_copy_to_user(to, from, n); } @@ -145,7 +145,7 @@ static int copyout(void __user *to, const void *from, size_t n) static int copyin(void *to, const void __user *from, size_t n) { - if (access_ok(VERIFY_READ, from, n)) { + if (access_ok(from, n)) { kasan_check_write(to, n); n = raw_copy_from_user(to, from, n); } @@ -614,7 +614,7 @@ EXPORT_SYMBOL(_copy_to_iter); #ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE static int copyout_mcsafe(void __user *to, const void *from, size_t n) { - if (access_ok(VERIFY_WRITE, to, n)) { + if (access_ok(to, n)) { kasan_check_read(from, n); n = copy_to_user_mcsafe((__force void *) to, from, n); } @@ -1663,7 +1663,7 @@ int import_single_range(int rw, void __user *buf, size_t len, { if (len > MAX_RW_COUNT) len = MAX_RW_COUNT; - if (unlikely(!access_ok(!rw, buf, len))) + if (unlikely(!access_ok(buf, len))) return -EFAULT; iov->iov_base = buf; diff --git a/lib/usercopy.c b/lib/usercopy.c index 3744b2a8e591..c2bfbcaeb3dc 100644 --- a/lib/usercopy.c +++ b/lib/usercopy.c @@ -8,7 +8,7 @@ unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n { unsigned long res = n; might_fault(); - if (likely(access_ok(VERIFY_READ, from, n))) { + if (likely(access_ok(from, n))) { kasan_check_write(to, n); res = raw_copy_from_user(to, from, n); } @@ -23,7 +23,7 @@ EXPORT_SYMBOL(_copy_from_user); unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) { might_fault(); - if (likely(access_ok(VERIFY_WRITE, to, n))) { + if (likely(access_ok(to, n))) { kasan_check_read(from, n); n = raw_copy_to_user(to, from, n); } diff --git a/mm/gup.c b/mm/gup.c index 8cb68a50dbdf..6f591ccb8eca 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1813,8 +1813,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write, len = (unsigned long) nr_pages << PAGE_SHIFT; end = start + len; - if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ, - (void __user *)start, len))) + if (unlikely(!access_ok((void __user *)start, len))) return 0; /* @@ -1868,8 +1867,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, if (nr_pages <= 0) return 0; - if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ, - (void __user *)start, len))) + if (unlikely(!access_ok((void __user *)start, len))) return -EFAULT; if (gup_fast_permitted(start, nr_pages, write)) { diff --git a/mm/mincore.c b/mm/mincore.c index 4985965aa20a..218099b5ed31 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -233,14 +233,14 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, return -EINVAL; /* ..and we need to be passed a valid user-space range */ - if (!access_ok(VERIFY_READ, (void __user *) start, len)) + if (!access_ok((void __user *) start, len)) return -ENOMEM; /* This also avoids any overflows on PAGE_ALIGN */ pages = len >> PAGE_SHIFT; pages += (offset_in_page(len)) != 0; - if (!access_ok(VERIFY_WRITE, vec, pages)) + if (!access_ok(vec, pages)) return -EFAULT; tmp = (void *) __get_free_page(GFP_USER); diff --git a/net/batman-adv/icmp_socket.c b/net/batman-adv/icmp_socket.c index d70f363c52ae..6d5859714f52 100644 --- a/net/batman-adv/icmp_socket.c +++ b/net/batman-adv/icmp_socket.c @@ -147,7 +147,7 @@ static ssize_t batadv_socket_read(struct file *file, char __user *buf, if (!buf || count < sizeof(struct batadv_icmp_packet)) return -EINVAL; - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; error = wait_event_interruptible(socket_client->queue_wait, diff --git a/net/batman-adv/log.c b/net/batman-adv/log.c index 02e55b78132f..75f602e1ce94 100644 --- a/net/batman-adv/log.c +++ b/net/batman-adv/log.c @@ -136,7 +136,7 @@ static ssize_t batadv_log_read(struct file *file, char __user *buf, if (count == 0) return 0; - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; error = wait_event_interruptible(debug_log->queue_wait, diff --git a/net/compat.c b/net/compat.c index c3a2f868e8af..959d1c51826d 100644 --- a/net/compat.c +++ b/net/compat.c @@ -358,7 +358,7 @@ static int do_set_sock_timeout(struct socket *sock, int level, if (optlen < sizeof(*up)) return -EINVAL; - if (!access_ok(VERIFY_READ, up, sizeof(*up)) || + if (!access_ok(up, sizeof(*up)) || __get_user(ktime.tv_sec, &up->tv_sec) || __get_user(ktime.tv_usec, &up->tv_usec)) return -EFAULT; @@ -438,7 +438,7 @@ static int do_get_sock_timeout(struct socket *sock, int level, int optname, if (!err) { if (put_user(sizeof(*up), optlen) || - !access_ok(VERIFY_WRITE, up, sizeof(*up)) || + !access_ok(up, sizeof(*up)) || __put_user(ktime.tv_sec, &up->tv_sec) || __put_user(ktime.tv_usec, &up->tv_usec)) err = -EFAULT; @@ -590,8 +590,8 @@ int compat_mc_setsockopt(struct sock *sock, int level, int optname, compat_alloc_user_space(sizeof(struct group_req)); u32 interface; - if (!access_ok(VERIFY_READ, gr32, sizeof(*gr32)) || - !access_ok(VERIFY_WRITE, kgr, sizeof(struct group_req)) || + if (!access_ok(gr32, sizeof(*gr32)) || + !access_ok(kgr, sizeof(struct group_req)) || __get_user(interface, &gr32->gr_interface) || __put_user(interface, &kgr->gr_interface) || copy_in_user(&kgr->gr_group, &gr32->gr_group, @@ -611,8 +611,8 @@ int compat_mc_setsockopt(struct sock *sock, int level, int optname, sizeof(struct group_source_req)); u32 interface; - if (!access_ok(VERIFY_READ, gsr32, sizeof(*gsr32)) || - !access_ok(VERIFY_WRITE, kgsr, + if (!access_ok(gsr32, sizeof(*gsr32)) || + !access_ok(kgsr, sizeof(struct group_source_req)) || __get_user(interface, &gsr32->gsr_interface) || __put_user(interface, &kgsr->gsr_interface) || @@ -631,7 +631,7 @@ int compat_mc_setsockopt(struct sock *sock, int level, int optname, struct group_filter __user *kgf; u32 interface, fmode, numsrc; - if (!access_ok(VERIFY_READ, gf32, __COMPAT_GF0_SIZE) || + if (!access_ok(gf32, __COMPAT_GF0_SIZE) || __get_user(interface, &gf32->gf_interface) || __get_user(fmode, &gf32->gf_fmode) || __get_user(numsrc, &gf32->gf_numsrc)) @@ -641,7 +641,7 @@ int compat_mc_setsockopt(struct sock *sock, int level, int optname, if (koptlen < GROUP_FILTER_SIZE(numsrc)) return -EINVAL; kgf = compat_alloc_user_space(koptlen); - if (!access_ok(VERIFY_WRITE, kgf, koptlen) || + if (!access_ok(kgf, koptlen) || __put_user(interface, &kgf->gf_interface) || __put_user(fmode, &kgf->gf_fmode) || __put_user(numsrc, &kgf->gf_numsrc) || @@ -675,7 +675,7 @@ int compat_mc_getsockopt(struct sock *sock, int level, int optname, return getsockopt(sock, level, optname, optval, optlen); koptlen = compat_alloc_user_space(sizeof(*koptlen)); - if (!access_ok(VERIFY_READ, optlen, sizeof(*optlen)) || + if (!access_ok(optlen, sizeof(*optlen)) || __get_user(ulen, optlen)) return -EFAULT; @@ -685,14 +685,14 @@ int compat_mc_getsockopt(struct sock *sock, int level, int optname, if (klen < GROUP_FILTER_SIZE(0)) return -EINVAL; - if (!access_ok(VERIFY_WRITE, koptlen, sizeof(*koptlen)) || + if (!access_ok(koptlen, sizeof(*koptlen)) || __put_user(klen, koptlen)) return -EFAULT; /* have to allow space for previous compat_alloc_user_space, too */ kgf = compat_alloc_user_space(klen+sizeof(*optlen)); - if (!access_ok(VERIFY_READ, gf32, __COMPAT_GF0_SIZE) || + if (!access_ok(gf32, __COMPAT_GF0_SIZE) || __get_user(interface, &gf32->gf_interface) || __get_user(fmode, &gf32->gf_fmode) || __get_user(numsrc, &gf32->gf_numsrc) || @@ -706,18 +706,18 @@ int compat_mc_getsockopt(struct sock *sock, int level, int optname, if (err) return err; - if (!access_ok(VERIFY_READ, koptlen, sizeof(*koptlen)) || + if (!access_ok(koptlen, sizeof(*koptlen)) || __get_user(klen, koptlen)) return -EFAULT; ulen = klen - (sizeof(*kgf)-sizeof(*gf32)); - if (!access_ok(VERIFY_WRITE, optlen, sizeof(*optlen)) || + if (!access_ok(optlen, sizeof(*optlen)) || __put_user(ulen, optlen)) return -EFAULT; - if (!access_ok(VERIFY_READ, kgf, klen) || - !access_ok(VERIFY_WRITE, gf32, ulen) || + if (!access_ok(kgf, klen) || + !access_ok(gf32, ulen) || __get_user(interface, &kgf->gf_interface) || __get_user(fmode, &kgf->gf_fmode) || __get_user(numsrc, &kgf->gf_numsrc) || diff --git a/net/sunrpc/sysctl.c b/net/sunrpc/sysctl.c index 8c3936403fea..0bea8ff8b0d3 100644 --- a/net/sunrpc/sysctl.c +++ b/net/sunrpc/sysctl.c @@ -89,7 +89,7 @@ proc_dodebug(struct ctl_table *table, int write, left = *lenp; if (write) { - if (!access_ok(VERIFY_READ, buffer, left)) + if (!access_ok(buffer, left)) return -EFAULT; p = buffer; while (left && __get_user(c, p) >= 0 && isspace(c)) diff --git a/security/tomoyo/common.c b/security/tomoyo/common.c index 9b38f94b5dd0..c598aa00d5e3 100644 --- a/security/tomoyo/common.c +++ b/security/tomoyo/common.c @@ -2591,7 +2591,7 @@ ssize_t tomoyo_write_control(struct tomoyo_io_buffer *head, int idx; if (!head->write) return -ENOSYS; - if (!access_ok(VERIFY_READ, buffer, buffer_len)) + if (!access_ok(buffer, buffer_len)) return -EFAULT; if (mutex_lock_interruptible(&head->io_sem)) return -EINTR; diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index 92e6524a3a9d..7d4640d1fe9f 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -393,7 +393,7 @@ static ssize_t snd_seq_read(struct file *file, char __user *buf, size_t count, if (!(snd_seq_file_flags(file) & SNDRV_SEQ_LFLG_INPUT)) return -ENXIO; - if (!access_ok(VERIFY_WRITE, buf, count)) + if (!access_ok(buf, count)) return -EFAULT; /* check client structures are in place */ diff --git a/sound/isa/sb/emu8000_patch.c b/sound/isa/sb/emu8000_patch.c index d45a6b9d6437..3d44c358c4b3 100644 --- a/sound/isa/sb/emu8000_patch.c +++ b/sound/isa/sb/emu8000_patch.c @@ -183,10 +183,10 @@ snd_emu8000_sample_new(struct snd_emux *rec, struct snd_sf_sample *sp, } if (sp->v.mode_flags & SNDRV_SFNT_SAMPLE_8BITS) { - if (!access_ok(VERIFY_READ, data, sp->v.size)) + if (!access_ok(data, sp->v.size)) return -EFAULT; } else { - if (!access_ok(VERIFY_READ, data, sp->v.size * 2)) + if (!access_ok(data, sp->v.size * 2)) return -EFAULT; } diff --git a/tools/perf/util/include/asm/uaccess.h b/tools/perf/util/include/asm/uaccess.h index 6a6f4b990547..548100315710 100644 --- a/tools/perf/util/include/asm/uaccess.h +++ b/tools/perf/util/include/asm/uaccess.h @@ -10,6 +10,6 @@ #define get_user __get_user -#define access_ok(type, addr, size) 1 +#define access_ok(addr, size) 1 #endif diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 666d0155662d..1f888a103f78 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -939,8 +939,7 @@ int __kvm_set_memory_region(struct kvm *kvm, /* We can read the guest memory with __xxx_user() later on. */ if ((id < KVM_USER_MEM_SLOTS) && ((mem->userspace_addr & (PAGE_SIZE - 1)) || - !access_ok(VERIFY_WRITE, - (void __user *)(unsigned long)mem->userspace_addr, + !access_ok((void __user *)(unsigned long)mem->userspace_addr, mem->memory_size))) goto out; if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) -- cgit v1.2.3 From 594cc251fdd0d231d342d88b2fdff4bc42fb0690 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Fri, 4 Jan 2019 12:56:09 -0800 Subject: make 'user_access_begin()' do 'access_ok()' Originally, the rule used to be that you'd have to do access_ok() separately, and then user_access_begin() before actually doing the direct (optimized) user access. But experience has shown that people then decide not to do access_ok() at all, and instead rely on it being implied by other operations or similar. Which makes it very hard to verify that the access has actually been range-checked. If you use the unsafe direct user accesses, hardware features (either SMAP - Supervisor Mode Access Protection - on x86, or PAN - Privileged Access Never - on ARM) do force you to use user_access_begin(). But nothing really forces the range check. By putting the range check into user_access_begin(), we actually force people to do the right thing (tm), and the range check vill be visible near the actual accesses. We have way too long a history of people trying to avoid them. Signed-off-by: Linus Torvalds --- arch/x86/include/asm/uaccess.h | 9 ++++++++- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 15 +++++++++++++-- include/linux/uaccess.h | 2 +- kernel/compat.c | 6 ++---- kernel/exit.c | 6 ++---- lib/strncpy_from_user.c | 9 +++++---- lib/strnlen_user.c | 9 +++++---- 7 files changed, 36 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 3920f456db79..a87ab5290ab4 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -705,7 +705,14 @@ extern struct movsl_mask { * checking before using them, but you have to surround them with the * user_access_begin/end() pair. */ -#define user_access_begin() __uaccess_begin() +static __must_check inline bool user_access_begin(const void __user *ptr, size_t len) +{ + if (unlikely(!access_ok(ptr,len))) + return 0; + __uaccess_begin(); + return 1; +} +#define user_access_begin(a,b) user_access_begin(a,b) #define user_access_end() __uaccess_end() #define unsafe_put_user(x, ptr, err_label) \ diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index 55d8f9b8777f..485b259127c3 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -1624,7 +1624,9 @@ end_user: * happened we would make the mistake of assuming that the * relocations were valid. */ - user_access_begin(); + if (!user_access_begin(urelocs, size)) + goto end_user; + for (copied = 0; copied < nreloc; copied++) unsafe_put_user(-1, &urelocs[copied].presumed_offset, @@ -2606,7 +2608,16 @@ i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, unsigned int i; /* Copy the new buffer offsets back to the user's exec list. */ - user_access_begin(); + /* + * Note: count * sizeof(*user_exec_list) does not overflow, + * because we checked 'count' in check_buffer_count(). + * + * And this range already got effectively checked earlier + * when we did the "copy_from_user()" above. + */ + if (!user_access_begin(user_exec_list, count * sizeof(*user_exec_list))) + goto end_user; + for (i = 0; i < args->buffer_count; i++) { if (!(exec2_list[i].offset & UPDATE)) continue; diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index bf2523867a02..37b226e8df13 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -264,7 +264,7 @@ extern long strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count); probe_kernel_read(&retval, addr, sizeof(retval)) #ifndef user_access_begin -#define user_access_begin() do { } while (0) +#define user_access_begin(ptr,len) access_ok(ptr, len) #define user_access_end() do { } while (0) #define unsafe_get_user(x, ptr, err) do { if (unlikely(__get_user(x, ptr))) goto err; } while (0) #define unsafe_put_user(x, ptr, err) do { if (unlikely(__put_user(x, ptr))) goto err; } while (0) diff --git a/kernel/compat.c b/kernel/compat.c index 705d4ae6c018..f01affa17e22 100644 --- a/kernel/compat.c +++ b/kernel/compat.c @@ -354,10 +354,9 @@ long compat_get_bitmap(unsigned long *mask, const compat_ulong_t __user *umask, bitmap_size = ALIGN(bitmap_size, BITS_PER_COMPAT_LONG); nr_compat_longs = BITS_TO_COMPAT_LONGS(bitmap_size); - if (!access_ok(umask, bitmap_size / 8)) + if (!user_access_begin(umask, bitmap_size / 8)) return -EFAULT; - user_access_begin(); while (nr_compat_longs > 1) { compat_ulong_t l1, l2; unsafe_get_user(l1, umask++, Efault); @@ -384,10 +383,9 @@ long compat_put_bitmap(compat_ulong_t __user *umask, unsigned long *mask, bitmap_size = ALIGN(bitmap_size, BITS_PER_COMPAT_LONG); nr_compat_longs = BITS_TO_COMPAT_LONGS(bitmap_size); - if (!access_ok(umask, bitmap_size / 8)) + if (!user_access_begin(umask, bitmap_size / 8)) return -EFAULT; - user_access_begin(); while (nr_compat_longs > 1) { unsigned long m = *mask++; unsafe_put_user((compat_ulong_t)m, umask++, Efault); diff --git a/kernel/exit.c b/kernel/exit.c index 8a01b671dc1f..2d14979577ee 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -1604,10 +1604,9 @@ SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *, if (!infop) return err; - if (!access_ok(infop, sizeof(*infop))) + if (!user_access_begin(infop, sizeof(*infop))) return -EFAULT; - user_access_begin(); unsafe_put_user(signo, &infop->si_signo, Efault); unsafe_put_user(0, &infop->si_errno, Efault); unsafe_put_user(info.cause, &infop->si_code, Efault); @@ -1732,10 +1731,9 @@ COMPAT_SYSCALL_DEFINE5(waitid, if (!infop) return err; - if (!access_ok(infop, sizeof(*infop))) + if (!user_access_begin(infop, sizeof(*infop))) return -EFAULT; - user_access_begin(); unsafe_put_user(signo, &infop->si_signo, Efault); unsafe_put_user(0, &infop->si_errno, Efault); unsafe_put_user(info.cause, &infop->si_code, Efault); diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c index b53e1b5d80f4..58eacd41526c 100644 --- a/lib/strncpy_from_user.c +++ b/lib/strncpy_from_user.c @@ -114,10 +114,11 @@ long strncpy_from_user(char *dst, const char __user *src, long count) kasan_check_write(dst, count); check_object_size(dst, count, false); - user_access_begin(); - retval = do_strncpy_from_user(dst, src, count, max); - user_access_end(); - return retval; + if (user_access_begin(src, max)) { + retval = do_strncpy_from_user(dst, src, count, max); + user_access_end(); + return retval; + } } return -EFAULT; } diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c index 60d0bbda8f5e..1c1a1b0e38a5 100644 --- a/lib/strnlen_user.c +++ b/lib/strnlen_user.c @@ -114,10 +114,11 @@ long strnlen_user(const char __user *str, long count) unsigned long max = max_addr - src_addr; long retval; - user_access_begin(); - retval = do_strnlen_user(str, count, max); - user_access_end(); - return retval; + if (user_access_begin(str, max)) { + retval = do_strnlen_user(str, count, max); + user_access_end(); + return retval; + } } return 0; } -- cgit v1.2.3 From 52fbf1134d479234d7e64ba9dcbaea23405f229e Mon Sep 17 00:00:00 2001 From: Alexey Skidanov Date: Thu, 3 Jan 2019 15:26:44 -0800 Subject: lib/genalloc.c: fix allocation of aligned buffer from non-aligned chunk gen_pool_alloc_algo() uses different allocation functions implementing different allocation algorithms. With gen_pool_first_fit_align() allocation function, the returned address should be aligned on the requested boundary. If chunk start address isn't aligned on the requested boundary, the returned address isn't aligned too. The only way to get properly aligned address is to initialize the pool with chunks aligned on the requested boundary. If want to have an ability to allocate buffers aligned on different boundaries (for example, 4K, 1MB, ...), the chunk start address should be aligned on the max possible alignment. This happens because gen_pool_first_fit_align() looks for properly aligned memory block without taking into account the chunk start address alignment. To fix this, we provide chunk start address to gen_pool_first_fit_align() and change its implementation such that it starts looking for properly aligned block with appropriate offset (exactly as is done in CMA). Link: https://lkml.kernel.org/lkml/a170cf65-6884-3592-1de9-4c235888cc8a@intel.com Link: http://lkml.kernel.org/r/1541690953-4623-1-git-send-email-alexey.skidanov@intel.com Signed-off-by: Alexey Skidanov Reviewed-by: Andrew Morton Cc: Logan Gunthorpe Cc: Daniel Mentz Cc: Mathieu Desnoyers Cc: Laura Abbott Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/genalloc.h | 13 +++++++------ lib/genalloc.c | 20 ++++++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/include/linux/genalloc.h b/include/linux/genalloc.h index 872f930f1b06..dd0a452373e7 100644 --- a/include/linux/genalloc.h +++ b/include/linux/genalloc.h @@ -51,7 +51,8 @@ typedef unsigned long (*genpool_algo_t)(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, - void *data, struct gen_pool *pool); + void *data, struct gen_pool *pool, + unsigned long start_addr); /* * General purpose special memory pool descriptor. @@ -131,24 +132,24 @@ extern void gen_pool_set_algo(struct gen_pool *pool, genpool_algo_t algo, extern unsigned long gen_pool_first_fit(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, void *data, - struct gen_pool *pool); + struct gen_pool *pool, unsigned long start_addr); extern unsigned long gen_pool_fixed_alloc(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, - void *data, struct gen_pool *pool); + void *data, struct gen_pool *pool, unsigned long start_addr); extern unsigned long gen_pool_first_fit_align(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, - void *data, struct gen_pool *pool); + void *data, struct gen_pool *pool, unsigned long start_addr); extern unsigned long gen_pool_first_fit_order_align(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, - void *data, struct gen_pool *pool); + void *data, struct gen_pool *pool, unsigned long start_addr); extern unsigned long gen_pool_best_fit(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, void *data, - struct gen_pool *pool); + struct gen_pool *pool, unsigned long start_addr); extern struct gen_pool *devm_gen_pool_create(struct device *dev, diff --git a/lib/genalloc.c b/lib/genalloc.c index ca06adc4f445..5deb25c40a5a 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -311,7 +311,7 @@ unsigned long gen_pool_alloc_algo(struct gen_pool *pool, size_t size, end_bit = chunk_size(chunk) >> order; retry: start_bit = algo(chunk->bits, end_bit, start_bit, - nbits, data, pool); + nbits, data, pool, chunk->start_addr); if (start_bit >= end_bit) continue; remain = bitmap_set_ll(chunk->bits, start_bit, nbits); @@ -525,7 +525,7 @@ EXPORT_SYMBOL(gen_pool_set_algo); */ unsigned long gen_pool_first_fit(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, void *data, - struct gen_pool *pool) + struct gen_pool *pool, unsigned long start_addr) { return bitmap_find_next_zero_area(map, size, start, nr, 0); } @@ -543,16 +543,19 @@ EXPORT_SYMBOL(gen_pool_first_fit); */ unsigned long gen_pool_first_fit_align(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, void *data, - struct gen_pool *pool) + struct gen_pool *pool, unsigned long start_addr) { struct genpool_data_align *alignment; - unsigned long align_mask; + unsigned long align_mask, align_off; int order; alignment = data; order = pool->min_alloc_order; align_mask = ((alignment->align + (1UL << order) - 1) >> order) - 1; - return bitmap_find_next_zero_area(map, size, start, nr, align_mask); + align_off = (start_addr & (alignment->align - 1)) >> order; + + return bitmap_find_next_zero_area_off(map, size, start, nr, + align_mask, align_off); } EXPORT_SYMBOL(gen_pool_first_fit_align); @@ -567,7 +570,7 @@ EXPORT_SYMBOL(gen_pool_first_fit_align); */ unsigned long gen_pool_fixed_alloc(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, void *data, - struct gen_pool *pool) + struct gen_pool *pool, unsigned long start_addr) { struct genpool_data_fixed *fixed_data; int order; @@ -601,7 +604,8 @@ EXPORT_SYMBOL(gen_pool_fixed_alloc); */ unsigned long gen_pool_first_fit_order_align(unsigned long *map, unsigned long size, unsigned long start, - unsigned int nr, void *data, struct gen_pool *pool) + unsigned int nr, void *data, struct gen_pool *pool, + unsigned long start_addr) { unsigned long align_mask = roundup_pow_of_two(nr) - 1; @@ -624,7 +628,7 @@ EXPORT_SYMBOL(gen_pool_first_fit_order_align); */ unsigned long gen_pool_best_fit(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, void *data, - struct gen_pool *pool) + struct gen_pool *pool, unsigned long start_addr) { unsigned long start_bit = size; unsigned long len = size + 1; -- cgit v1.2.3 From 439e00b76a5fb1662e3ae49fc48bd3f950575b9d Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Thu, 3 Jan 2019 15:26:48 -0800 Subject: lib/find_bit_benchmark.c: align test_find_next_and_bit with others Contrary to other tests, test_find_next_and_bit() test uses tab formatting in output and get_cycles() instead of ktime_get(). get_cycles() is not supported by some arches, so ktime_get() fits better in generic code. Fix it and minor style issues, so the output looks like this: Start testing find_bit() with random-filled bitmap find_next_bit: 7142816 ns, 163282 iterations find_next_zero_bit: 8545712 ns, 164399 iterations find_last_bit: 6332032 ns, 163282 iterations find_first_bit: 20509424 ns, 16606 iterations find_next_and_bit: 4060016 ns, 73424 iterations Start testing find_bit() with sparse bitmap find_next_bit: 55984 ns, 656 iterations find_next_zero_bit: 19197536 ns, 327025 iterations find_last_bit: 65088 ns, 656 iterations find_first_bit: 5923712 ns, 656 iterations find_next_and_bit: 29088 ns, 1 iterations Link: http://lkml.kernel.org/r/20181123174803.10916-1-ynorov@caviumnetworks.com Signed-off-by: Yury Norov Reviewed-by: Andrew Morton Cc: "Norov, Yuri" Cc: Clement Courbet Cc: Geert Uytterhoeven Cc: Alexey Dobriyan Cc: Matthew Wilcox Cc: Rasmus Villemoes Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/find_bit_benchmark.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/find_bit_benchmark.c b/lib/find_bit_benchmark.c index 5367ffa5c18f..f0e394dd2beb 100644 --- a/lib/find_bit_benchmark.c +++ b/lib/find_bit_benchmark.c @@ -108,14 +108,13 @@ static int __init test_find_next_and_bit(const void *bitmap, const void *bitmap2, unsigned long len) { unsigned long i, cnt; - cycles_t cycles; + ktime_t time; - cycles = get_cycles(); + time = ktime_get(); for (cnt = i = 0; i < BITMAP_LEN; cnt++) - i = find_next_and_bit(bitmap, bitmap2, BITMAP_LEN, i+1); - cycles = get_cycles() - cycles; - pr_err("find_next_and_bit:\t\t%llu cycles, %ld iterations\n", - (u64)cycles, cnt); + i = find_next_and_bit(bitmap, bitmap2, BITMAP_LEN, i + 1); + time = ktime_get() - time; + pr_err("find_next_and_bit: %18llu ns, %6ld iterations\n", time, cnt); return 0; } -- cgit v1.2.3 From 6862d2fc81859f88c1f3f660886427893f2b4f3f Mon Sep 17 00:00:00 2001 From: Huang Shijie Date: Thu, 3 Jan 2019 15:26:51 -0800 Subject: lib/genalloc.c: use vzalloc_node() to allocate the bitmap Some devices may have big memory on chip, such as over 1G. In some cases, the nbytes maybe bigger then 4M which is the bounday of the memory buddy system (4K default). So use vzalloc_node() to allocate the bitmap. Also use vfree to free it. Link: http://lkml.kernel.org/r/20181225015701.6289-1-sjhuang@iluvatar.ai Signed-off-by: Huang Shijie Reviewed-by: Andrew Morton Cc: Alexey Skidanov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/genalloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/genalloc.c b/lib/genalloc.c index 5deb25c40a5a..f365d71cdc77 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -187,7 +187,7 @@ int gen_pool_add_virt(struct gen_pool *pool, unsigned long virt, phys_addr_t phy int nbytes = sizeof(struct gen_pool_chunk) + BITS_TO_LONGS(nbits) * sizeof(long); - chunk = kzalloc_node(nbytes, GFP_KERNEL, nid); + chunk = vzalloc_node(nbytes, nid); if (unlikely(chunk == NULL)) return -ENOMEM; @@ -251,7 +251,7 @@ void gen_pool_destroy(struct gen_pool *pool) bit = find_next_bit(chunk->bits, end_bit, 0); BUG_ON(bit < end_bit); - kfree(chunk); + vfree(chunk); } kfree_const(pool->name); kfree(pool); -- cgit v1.2.3 From 35004f2e55807a1a1491db24ab512dd2f770a130 Mon Sep 17 00:00:00 2001 From: Olof Johansson Date: Sat, 5 Jan 2019 13:21:18 -0800 Subject: lib/genalloc.c: include vmalloc.h Fixes build break on most ARM/ARM64 defconfigs: lib/genalloc.c: In function 'gen_pool_add_virt': lib/genalloc.c:190:10: error: implicit declaration of function 'vzalloc_node'; did you mean 'kzalloc_node'? lib/genalloc.c:190:8: warning: assignment to 'struct gen_pool_chunk *' from 'int' makes pointer from integer without a cast [-Wint-conversion] lib/genalloc.c: In function 'gen_pool_destroy': lib/genalloc.c:254:3: error: implicit declaration of function 'vfree'; did you mean 'kfree'? Fixes: 6862d2fc8185 ('lib/genalloc.c: use vzalloc_node() to allocate the bitmap') Cc: Huang Shijie Cc: Andrew Morton Cc: Alexey Skidanov Signed-off-by: Olof Johansson Signed-off-by: Linus Torvalds --- lib/genalloc.c | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/genalloc.c b/lib/genalloc.c index f365d71cdc77..7e85d1e37a6e 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -35,6 +35,7 @@ #include #include #include +#include static inline size_t chunk_size(const struct gen_pool_chunk *chunk) { -- cgit v1.2.3 From e9666d10a5677a494260d60d1fa0b73cc7646eb3 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 31 Dec 2018 00:14:15 +0900 Subject: jump_label: move 'asm goto' support test to Kconfig Currently, CONFIG_JUMP_LABEL just means "I _want_ to use jump label". The jump label is controlled by HAVE_JUMP_LABEL, which is defined like this: #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) # define HAVE_JUMP_LABEL #endif We can improve this by testing 'asm goto' support in Kconfig, then make JUMP_LABEL depend on CC_HAS_ASM_GOTO. Ugly #ifdef HAVE_JUMP_LABEL will go away, and CONFIG_JUMP_LABEL will match to the real kernel capability. Signed-off-by: Masahiro Yamada Acked-by: Michael Ellerman (powerpc) Tested-by: Sedat Dilek --- Makefile | 7 ------- arch/Kconfig | 1 + arch/arm/kernel/jump_label.c | 4 ---- arch/arm64/kernel/jump_label.c | 4 ---- arch/mips/kernel/jump_label.c | 4 ---- arch/powerpc/include/asm/asm-prototypes.h | 2 +- arch/powerpc/kernel/jump_label.c | 2 -- arch/powerpc/platforms/powernv/opal-tracepoints.c | 2 +- arch/powerpc/platforms/powernv/opal-wrappers.S | 2 +- arch/powerpc/platforms/pseries/hvCall.S | 4 ++-- arch/powerpc/platforms/pseries/lpar.c | 2 +- arch/s390/kernel/Makefile | 3 ++- arch/s390/kernel/jump_label.c | 4 ---- arch/sparc/kernel/Makefile | 2 +- arch/sparc/kernel/jump_label.c | 4 ---- arch/x86/Makefile | 2 +- arch/x86/entry/calling.h | 2 +- arch/x86/include/asm/cpufeature.h | 2 +- arch/x86/include/asm/jump_label.h | 13 ------------- arch/x86/include/asm/rmwcc.h | 6 +++--- arch/x86/kernel/Makefile | 3 ++- arch/x86/kernel/jump_label.c | 4 ---- arch/x86/kvm/emulate.c | 2 +- arch/xtensa/kernel/jump_label.c | 4 ---- include/linux/dynamic_debug.h | 6 +++--- include/linux/jump_label.h | 22 +++++++++------------- include/linux/jump_label_ratelimit.h | 8 +++----- include/linux/module.h | 2 +- include/linux/netfilter.h | 4 ++-- include/linux/netfilter_ingress.h | 2 +- init/Kconfig | 3 +++ kernel/jump_label.c | 10 +++------- kernel/module.c | 2 +- kernel/sched/core.c | 2 +- kernel/sched/debug.c | 4 ++-- kernel/sched/fair.c | 6 +++--- kernel/sched/sched.h | 6 +++--- lib/dynamic_debug.c | 2 +- net/core/dev.c | 6 +++--- net/netfilter/core.c | 6 +++--- scripts/gcc-goto.sh | 2 +- tools/arch/x86/include/asm/rmwcc.h | 6 +++--- 42 files changed, 65 insertions(+), 119 deletions(-) (limited to 'lib') diff --git a/Makefile b/Makefile index 60a473247657..04a857817f77 100644 --- a/Makefile +++ b/Makefile @@ -514,13 +514,6 @@ RETPOLINE_VDSO_CFLAGS := $(call cc-option,$(RETPOLINE_VDSO_CFLAGS_GCC),$(call cc export RETPOLINE_CFLAGS export RETPOLINE_VDSO_CFLAGS -# check for 'asm goto' -ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC) $(KBUILD_CFLAGS)), y) - CC_HAVE_ASM_GOTO := 1 - KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO - KBUILD_AFLAGS += -DCC_HAVE_ASM_GOTO -endif - # The expansion should be delayed until arch/$(SRCARCH)/Makefile is included. # Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile. # CC_VERSION_TEXT is referenced from Kconfig (so it needs export), diff --git a/arch/Kconfig b/arch/Kconfig index b70c952ac838..4cfb6de48f79 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -71,6 +71,7 @@ config KPROBES config JUMP_LABEL bool "Optimize very unlikely/likely branches" depends on HAVE_ARCH_JUMP_LABEL + depends on CC_HAS_ASM_GOTO help This option enables a transparent branch optimization that makes certain almost-always-true or almost-always-false branch diff --git a/arch/arm/kernel/jump_label.c b/arch/arm/kernel/jump_label.c index 90bce3d9928e..303b3ab87f7e 100644 --- a/arch/arm/kernel/jump_label.c +++ b/arch/arm/kernel/jump_label.c @@ -4,8 +4,6 @@ #include #include -#ifdef HAVE_JUMP_LABEL - static void __arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type, bool is_static) @@ -35,5 +33,3 @@ void arch_jump_label_transform_static(struct jump_entry *entry, { __arch_jump_label_transform(entry, type, true); } - -#endif diff --git a/arch/arm64/kernel/jump_label.c b/arch/arm64/kernel/jump_label.c index 646b9562ee64..1eff270e8861 100644 --- a/arch/arm64/kernel/jump_label.c +++ b/arch/arm64/kernel/jump_label.c @@ -20,8 +20,6 @@ #include #include -#ifdef HAVE_JUMP_LABEL - void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type) { @@ -49,5 +47,3 @@ void arch_jump_label_transform_static(struct jump_entry *entry, * NOP needs to be replaced by a branch. */ } - -#endif /* HAVE_JUMP_LABEL */ diff --git a/arch/mips/kernel/jump_label.c b/arch/mips/kernel/jump_label.c index 32e3168316cd..ab943927f97a 100644 --- a/arch/mips/kernel/jump_label.c +++ b/arch/mips/kernel/jump_label.c @@ -16,8 +16,6 @@ #include #include -#ifdef HAVE_JUMP_LABEL - /* * Define parameters for the standard MIPS and the microMIPS jump * instruction encoding respectively: @@ -70,5 +68,3 @@ void arch_jump_label_transform(struct jump_entry *e, mutex_unlock(&text_mutex); } - -#endif /* HAVE_JUMP_LABEL */ diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h index 6f201b199c02..1d911f68a23b 100644 --- a/arch/powerpc/include/asm/asm-prototypes.h +++ b/arch/powerpc/include/asm/asm-prototypes.h @@ -38,7 +38,7 @@ extern struct static_key hcall_tracepoint_key; void __trace_hcall_entry(unsigned long opcode, unsigned long *args); void __trace_hcall_exit(long opcode, long retval, unsigned long *retbuf); /* OPAL tracing */ -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL extern struct static_key opal_tracepoint_key; #endif diff --git a/arch/powerpc/kernel/jump_label.c b/arch/powerpc/kernel/jump_label.c index 6472472093d0..0080c5fbd225 100644 --- a/arch/powerpc/kernel/jump_label.c +++ b/arch/powerpc/kernel/jump_label.c @@ -11,7 +11,6 @@ #include #include -#ifdef HAVE_JUMP_LABEL void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type) { @@ -22,4 +21,3 @@ void arch_jump_label_transform(struct jump_entry *entry, else patch_instruction(addr, PPC_INST_NOP); } -#endif diff --git a/arch/powerpc/platforms/powernv/opal-tracepoints.c b/arch/powerpc/platforms/powernv/opal-tracepoints.c index 1ab7d26c0a2c..f16a43540e30 100644 --- a/arch/powerpc/platforms/powernv/opal-tracepoints.c +++ b/arch/powerpc/platforms/powernv/opal-tracepoints.c @@ -4,7 +4,7 @@ #include #include -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL struct static_key opal_tracepoint_key = STATIC_KEY_INIT; int opal_tracepoint_regfunc(void) diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S index 251528231a9e..f4875fe3f8ff 100644 --- a/arch/powerpc/platforms/powernv/opal-wrappers.S +++ b/arch/powerpc/platforms/powernv/opal-wrappers.S @@ -20,7 +20,7 @@ .section ".text" #ifdef CONFIG_TRACEPOINTS -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL #define OPAL_BRANCH(LABEL) \ ARCH_STATIC_BRANCH(LABEL, opal_tracepoint_key) #else diff --git a/arch/powerpc/platforms/pseries/hvCall.S b/arch/powerpc/platforms/pseries/hvCall.S index d91412c591ef..50dc9426d0be 100644 --- a/arch/powerpc/platforms/pseries/hvCall.S +++ b/arch/powerpc/platforms/pseries/hvCall.S @@ -19,7 +19,7 @@ #ifdef CONFIG_TRACEPOINTS -#ifndef HAVE_JUMP_LABEL +#ifndef CONFIG_JUMP_LABEL .section ".toc","aw" .globl hcall_tracepoint_refcount @@ -79,7 +79,7 @@ hcall_tracepoint_refcount: mr r5,BUFREG; \ __HCALL_INST_POSTCALL -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL #define HCALL_BRANCH(LABEL) \ ARCH_STATIC_BRANCH(LABEL, hcall_tracepoint_key) #else diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c index 32d4452973e7..f2a9f0adc2d3 100644 --- a/arch/powerpc/platforms/pseries/lpar.c +++ b/arch/powerpc/platforms/pseries/lpar.c @@ -1040,7 +1040,7 @@ EXPORT_SYMBOL(arch_free_page); #endif /* CONFIG_PPC_BOOK3S_64 */ #ifdef CONFIG_TRACEPOINTS -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL struct static_key hcall_tracepoint_key = STATIC_KEY_INIT; int hcall_tracepoint_regfunc(void) diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 386b1abb217b..e216e116a9a9 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -48,7 +48,7 @@ CFLAGS_ptrace.o += -DUTS_MACHINE='"$(UTS_MACHINE)"' obj-y := traps.o time.o process.o base.o early.o setup.o idle.o vtime.o obj-y += processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o early_nobss.o -obj-y += sysinfo.o jump_label.o lgr.o os_info.o machine_kexec.o pgm_check.o +obj-y += sysinfo.o lgr.o os_info.o machine_kexec.o pgm_check.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o obj-y += nospec-branch.o ipl_vmparm.o @@ -72,6 +72,7 @@ obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_UPROBES) += uprobes.o +obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c index 50a1798604a8..3f10b56bd5a3 100644 --- a/arch/s390/kernel/jump_label.c +++ b/arch/s390/kernel/jump_label.c @@ -10,8 +10,6 @@ #include #include -#ifdef HAVE_JUMP_LABEL - struct insn { u16 opcode; s32 offset; @@ -103,5 +101,3 @@ void arch_jump_label_transform_static(struct jump_entry *entry, { __jump_label_transform(entry, type, 1); } - -#endif diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile index cf8640841b7a..97c0e19263d1 100644 --- a/arch/sparc/kernel/Makefile +++ b/arch/sparc/kernel/Makefile @@ -118,4 +118,4 @@ pc--$(CONFIG_PERF_EVENTS) := perf_event.o obj-$(CONFIG_SPARC64) += $(pc--y) obj-$(CONFIG_UPROBES) += uprobes.o -obj-$(CONFIG_SPARC64) += jump_label.o +obj-$(CONFIG_JUMP_LABEL) += jump_label.o diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c index 7f8eac51df33..a4cfaeecaf5e 100644 --- a/arch/sparc/kernel/jump_label.c +++ b/arch/sparc/kernel/jump_label.c @@ -9,8 +9,6 @@ #include -#ifdef HAVE_JUMP_LABEL - void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type) { @@ -47,5 +45,3 @@ void arch_jump_label_transform(struct jump_entry *entry, flushi(insn); mutex_unlock(&text_mutex); } - -#endif diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 16c3145c0a5f..9c5a67d1b9c1 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -289,7 +289,7 @@ vdso_install: archprepare: checkbin checkbin: -ifndef CC_HAVE_ASM_GOTO +ifndef CONFIG_CC_HAS_ASM_GOTO @echo Compiler lacks asm-goto support. @exit 1 endif diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h index 20d0885b00fb..efb0d1b1f15f 100644 --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -351,7 +351,7 @@ For 32-bit we have the following conventions - kernel is built with */ .macro CALL_enter_from_user_mode #ifdef CONFIG_CONTEXT_TRACKING -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0 #endif call enter_from_user_mode diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index aced6c9290d6..ce95b8cbd229 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h @@ -140,7 +140,7 @@ extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit); #define setup_force_cpu_bug(bit) setup_force_cpu_cap(bit) -#if defined(__clang__) && !defined(CC_HAVE_ASM_GOTO) +#if defined(__clang__) && !defined(CONFIG_CC_HAS_ASM_GOTO) /* * Workaround for the sake of BPF compilation which utilizes kernel diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h index 21efc9d07ed9..65191ce8e1cf 100644 --- a/arch/x86/include/asm/jump_label.h +++ b/arch/x86/include/asm/jump_label.h @@ -2,19 +2,6 @@ #ifndef _ASM_X86_JUMP_LABEL_H #define _ASM_X86_JUMP_LABEL_H -#ifndef HAVE_JUMP_LABEL -/* - * For better or for worse, if jump labels (the gcc extension) are missing, - * then the entire static branch patching infrastructure is compiled out. - * If that happens, the code in here will malfunction. Raise a compiler - * error instead. - * - * In theory, jump labels and the static branch patching infrastructure - * could be decoupled to fix this. - */ -#error asm/jump_label.h included on a non-jump-label kernel -#endif - #define JUMP_LABEL_NOP_SIZE 5 #ifdef CONFIG_X86_64 diff --git a/arch/x86/include/asm/rmwcc.h b/arch/x86/include/asm/rmwcc.h index 46ac84b506f5..8a9eba191516 100644 --- a/arch/x86/include/asm/rmwcc.h +++ b/arch/x86/include/asm/rmwcc.h @@ -11,7 +11,7 @@ #define __CLOBBERS_MEM(clb...) "memory", ## clb -#if !defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(CC_HAVE_ASM_GOTO) +#if !defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(CONFIG_CC_HAS_ASM_GOTO) /* Use asm goto */ @@ -27,7 +27,7 @@ cc_label: c = true; \ c; \ }) -#else /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */ +#else /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CONFIG_CC_HAS_ASM_GOTO) */ /* Use flags output or a set instruction */ @@ -40,7 +40,7 @@ cc_label: c = true; \ c; \ }) -#endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */ +#endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CONFIG_CC_HAS_ASM_GOTO) */ #define GEN_UNARY_RMWcc_4(op, var, cc, arg0) \ __GEN_RMWcc(op " " arg0, var, cc, __CLOBBERS_MEM()) diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index eb51b0e1189c..00b7e27bc2b7 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -49,7 +49,8 @@ obj-$(CONFIG_COMPAT) += signal_compat.o obj-y += traps.o idt.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o dumpstack.o nmi.o obj-$(CONFIG_MODIFY_LDT_SYSCALL) += ldt.o -obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o +obj-y += setup.o x86_init.o i8259.o irqinit.o +obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-y += probe_roms.o obj-$(CONFIG_X86_64) += sys_x86_64.o diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index aac0c1f7e354..f99bd26bd3f1 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -16,8 +16,6 @@ #include #include -#ifdef HAVE_JUMP_LABEL - union jump_code_union { char code[JUMP_LABEL_NOP_SIZE]; struct { @@ -130,5 +128,3 @@ __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, if (jlstate == JL_STATE_UPDATE) __jump_label_transform(entry, type, text_poke_early, 1); } - -#endif diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 78e430f4e15c..c338984c850d 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -456,7 +456,7 @@ FOP_END; /* * XXX: inoutclob user must know where the argument is being expanded. - * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault. + * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault. */ #define asm_safe(insn, inoutclob...) \ ({ \ diff --git a/arch/xtensa/kernel/jump_label.c b/arch/xtensa/kernel/jump_label.c index d108f721c116..61cf6497a646 100644 --- a/arch/xtensa/kernel/jump_label.c +++ b/arch/xtensa/kernel/jump_label.c @@ -10,8 +10,6 @@ #include -#ifdef HAVE_JUMP_LABEL - #define J_OFFSET_MASK 0x0003ffff #define J_SIGN_MASK (~(J_OFFSET_MASK >> 1)) @@ -95,5 +93,3 @@ void arch_jump_label_transform(struct jump_entry *e, patch_text(jump_entry_code(e), &insn, JUMP_LABEL_NOP_SIZE); } - -#endif /* HAVE_JUMP_LABEL */ diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h index 2fd8006153c3..b3419da1a776 100644 --- a/include/linux/dynamic_debug.h +++ b/include/linux/dynamic_debug.h @@ -2,7 +2,7 @@ #ifndef _DYNAMIC_DEBUG_H #define _DYNAMIC_DEBUG_H -#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) +#if defined(CONFIG_JUMP_LABEL) #include #endif @@ -38,7 +38,7 @@ struct _ddebug { #define _DPRINTK_FLAGS_DEFAULT 0 #endif unsigned int flags:8; -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL union { struct static_key_true dd_key_true; struct static_key_false dd_key_false; @@ -83,7 +83,7 @@ void __dynamic_netdev_dbg(struct _ddebug *descriptor, dd_key_init(key, init) \ } -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL #define dd_key_init(key, init) key = (init) diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 5df6a621e464..3e113a1fa0f1 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -71,10 +71,6 @@ * Additional babbling in: Documentation/static-keys.txt */ -#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) -# define HAVE_JUMP_LABEL -#endif - #ifndef __ASSEMBLY__ #include @@ -86,7 +82,7 @@ extern bool static_key_initialized; "%s(): static key '%pS' used before call to jump_label_init()", \ __func__, (key)) -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL struct static_key { atomic_t enabled; @@ -114,10 +110,10 @@ struct static_key { struct static_key { atomic_t enabled; }; -#endif /* HAVE_JUMP_LABEL */ +#endif /* CONFIG_JUMP_LABEL */ #endif /* __ASSEMBLY__ */ -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL #include #ifndef __ASSEMBLY__ @@ -192,7 +188,7 @@ enum jump_label_type { struct module; -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL #define JUMP_TYPE_FALSE 0UL #define JUMP_TYPE_TRUE 1UL @@ -245,7 +241,7 @@ extern void static_key_disable_cpuslocked(struct static_key *key); { .enabled = { 0 }, \ { .entries = (void *)JUMP_TYPE_FALSE } } -#else /* !HAVE_JUMP_LABEL */ +#else /* !CONFIG_JUMP_LABEL */ #include #include @@ -330,7 +326,7 @@ static inline void static_key_disable(struct static_key *key) #define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) } #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) } -#endif /* HAVE_JUMP_LABEL */ +#endif /* CONFIG_JUMP_LABEL */ #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE #define jump_label_enabled static_key_enabled @@ -394,7 +390,7 @@ extern bool ____wrong_branch_error(void); static_key_count((struct static_key *)x) > 0; \ }) -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL /* * Combine the right initial value (type) with the right branch order @@ -476,12 +472,12 @@ extern bool ____wrong_branch_error(void); unlikely(branch); \ }) -#else /* !HAVE_JUMP_LABEL */ +#else /* !CONFIG_JUMP_LABEL */ #define static_branch_likely(x) likely(static_key_enabled(&(x)->key)) #define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key)) -#endif /* HAVE_JUMP_LABEL */ +#endif /* CONFIG_JUMP_LABEL */ /* * Advanced usage; refcount, branch is enabled when: count != 0 diff --git a/include/linux/jump_label_ratelimit.h b/include/linux/jump_label_ratelimit.h index baa8eabbaa56..a49f2b45b3f0 100644 --- a/include/linux/jump_label_ratelimit.h +++ b/include/linux/jump_label_ratelimit.h @@ -5,21 +5,19 @@ #include #include -#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) +#if defined(CONFIG_JUMP_LABEL) struct static_key_deferred { struct static_key key; unsigned long timeout; struct delayed_work work; }; -#endif -#ifdef HAVE_JUMP_LABEL extern void static_key_slow_dec_deferred(struct static_key_deferred *key); extern void static_key_deferred_flush(struct static_key_deferred *key); extern void jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl); -#else /* !HAVE_JUMP_LABEL */ +#else /* !CONFIG_JUMP_LABEL */ struct static_key_deferred { struct static_key key; }; @@ -38,5 +36,5 @@ jump_label_rate_limit(struct static_key_deferred *key, { STATIC_KEY_CHECK_USE(key); } -#endif /* HAVE_JUMP_LABEL */ +#endif /* CONFIG_JUMP_LABEL */ #endif /* _LINUX_JUMP_LABEL_RATELIMIT_H */ diff --git a/include/linux/module.h b/include/linux/module.h index d5453eb5a68b..9a21fe3509af 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -436,7 +436,7 @@ struct module { unsigned int num_bpf_raw_events; struct bpf_raw_event_map *bpf_raw_events; #endif -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL struct jump_entry *jump_entries; unsigned int num_jump_entries; #endif diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h index bbe99d2b28b4..72cb19c3db6a 100644 --- a/include/linux/netfilter.h +++ b/include/linux/netfilter.h @@ -176,7 +176,7 @@ void nf_unregister_net_hooks(struct net *net, const struct nf_hook_ops *reg, int nf_register_sockopt(struct nf_sockopt_ops *reg); void nf_unregister_sockopt(struct nf_sockopt_ops *reg); -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; #endif @@ -198,7 +198,7 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net, struct nf_hook_entries *hook_head = NULL; int ret = 1; -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL if (__builtin_constant_p(pf) && __builtin_constant_p(hook) && !static_key_false(&nf_hooks_needed[pf][hook])) diff --git a/include/linux/netfilter_ingress.h b/include/linux/netfilter_ingress.h index 554c920691dd..a13774be2eb5 100644 --- a/include/linux/netfilter_ingress.h +++ b/include/linux/netfilter_ingress.h @@ -8,7 +8,7 @@ #ifdef CONFIG_NETFILTER_INGRESS static inline bool nf_hook_ingress_active(const struct sk_buff *skb) { -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL if (!static_key_false(&nf_hooks_needed[NFPROTO_NETDEV][NF_NETDEV_INGRESS])) return false; #endif diff --git a/init/Kconfig b/init/Kconfig index 3e6be1694766..d47cb77a220e 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -23,6 +23,9 @@ config CLANG_VERSION int default $(shell,$(srctree)/scripts/clang-version.sh $(CC)) +config CC_HAS_ASM_GOTO + def_bool $(success,$(srctree)/scripts/gcc-goto.sh $(CC)) + config CONSTRUCTORS bool depends on !UML diff --git a/kernel/jump_label.c b/kernel/jump_label.c index b28028b08d44..bad96b476eb6 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -18,8 +18,6 @@ #include #include -#ifdef HAVE_JUMP_LABEL - /* mutex to protect coming/going of the the jump_label table */ static DEFINE_MUTEX(jump_label_mutex); @@ -80,13 +78,13 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) static void jump_label_update(struct static_key *key); /* - * There are similar definitions for the !HAVE_JUMP_LABEL case in jump_label.h. + * There are similar definitions for the !CONFIG_JUMP_LABEL case in jump_label.h. * The use of 'atomic_read()' requires atomic.h and its problematic for some * kernel headers such as kernel.h and others. Since static_key_count() is not - * used in the branch statements as it is for the !HAVE_JUMP_LABEL case its ok + * used in the branch statements as it is for the !CONFIG_JUMP_LABEL case its ok * to have it be a function here. Similarly, for 'static_key_enable()' and * 'static_key_disable()', which require bug.h. This should allow jump_label.h - * to be included from most/all places for HAVE_JUMP_LABEL. + * to be included from most/all places for CONFIG_JUMP_LABEL. */ int static_key_count(struct static_key *key) { @@ -791,5 +789,3 @@ static __init int jump_label_test(void) } early_initcall(jump_label_test); #endif /* STATIC_KEYS_SELFTEST */ - -#endif /* HAVE_JUMP_LABEL */ diff --git a/kernel/module.c b/kernel/module.c index fcbc0128810b..2ad1b5239910 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -3102,7 +3102,7 @@ static int find_module_sections(struct module *mod, struct load_info *info) sizeof(*mod->bpf_raw_events), &mod->num_bpf_raw_events); #endif -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL mod->jump_entries = section_objs(info, "__jump_table", sizeof(*mod->jump_entries), &mod->num_jump_entries); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 223f78d5c111..a674c7db2f29 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -24,7 +24,7 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); -#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL) +#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_JUMP_LABEL) /* * Debugging: various feature bits * diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 02bd5f969b21..de3de997e245 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -73,7 +73,7 @@ static int sched_feat_show(struct seq_file *m, void *v) return 0; } -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL #define jump_label_key__true STATIC_KEY_INIT_TRUE #define jump_label_key__false STATIC_KEY_INIT_FALSE @@ -99,7 +99,7 @@ static void sched_feat_enable(int i) #else static void sched_feat_disable(int i) { }; static void sched_feat_enable(int i) { }; -#endif /* HAVE_JUMP_LABEL */ +#endif /* CONFIG_JUMP_LABEL */ static int sched_feat_set(char *cmp) { diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 6483834f1278..50aa2aba69bd 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4217,7 +4217,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) #ifdef CONFIG_CFS_BANDWIDTH -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL static struct static_key __cfs_bandwidth_used; static inline bool cfs_bandwidth_used(void) @@ -4234,7 +4234,7 @@ void cfs_bandwidth_usage_dec(void) { static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); } -#else /* HAVE_JUMP_LABEL */ +#else /* CONFIG_JUMP_LABEL */ static bool cfs_bandwidth_used(void) { return true; @@ -4242,7 +4242,7 @@ static bool cfs_bandwidth_used(void) void cfs_bandwidth_usage_inc(void) {} void cfs_bandwidth_usage_dec(void) {} -#endif /* HAVE_JUMP_LABEL */ +#endif /* CONFIG_JUMP_LABEL */ /* * default period for cfs group bandwidth. diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 0ba08924e017..d04530bf251f 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1488,7 +1488,7 @@ enum { #undef SCHED_FEAT -#if defined(CONFIG_SCHED_DEBUG) && defined(HAVE_JUMP_LABEL) +#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_JUMP_LABEL) /* * To support run-time toggling of sched features, all the translation units @@ -1508,7 +1508,7 @@ static __always_inline bool static_branch_##name(struct static_key *key) \ extern struct static_key sched_feat_keys[__SCHED_FEAT_NR]; #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x])) -#else /* !(SCHED_DEBUG && HAVE_JUMP_LABEL) */ +#else /* !(SCHED_DEBUG && CONFIG_JUMP_LABEL) */ /* * Each translation unit has its own copy of sysctl_sched_features to allow @@ -1524,7 +1524,7 @@ static const_debug __maybe_unused unsigned int sysctl_sched_features = #define sched_feat(x) !!(sysctl_sched_features & (1UL << __SCHED_FEAT_##x)) -#endif /* SCHED_DEBUG && HAVE_JUMP_LABEL */ +#endif /* SCHED_DEBUG && CONFIG_JUMP_LABEL */ extern struct static_key_false sched_numa_balancing; extern struct static_key_false sched_schedstats; diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index c7c96bc7654a..dbf2b457e47e 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -188,7 +188,7 @@ static int ddebug_change(const struct ddebug_query *query, newflags = (dp->flags & mask) | flags; if (newflags == dp->flags) continue; -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL if (dp->flags & _DPRINTK_FLAGS_PRINT) { if (!(flags & _DPRINTK_FLAGS_PRINT)) static_branch_disable(&dp->key.dd_key_true); diff --git a/net/core/dev.c b/net/core/dev.c index 1b5a4410be0e..82f20022259d 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1821,7 +1821,7 @@ EXPORT_SYMBOL_GPL(net_dec_egress_queue); #endif static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key); -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL static atomic_t netstamp_needed_deferred; static atomic_t netstamp_wanted; static void netstamp_clear(struct work_struct *work) @@ -1840,7 +1840,7 @@ static DECLARE_WORK(netstamp_work, netstamp_clear); void net_enable_timestamp(void) { -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL int wanted; while (1) { @@ -1860,7 +1860,7 @@ EXPORT_SYMBOL(net_enable_timestamp); void net_disable_timestamp(void) { -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL int wanted; while (1) { diff --git a/net/netfilter/core.c b/net/netfilter/core.c index dc240cb47ddf..93aaec3a54ec 100644 --- a/net/netfilter/core.c +++ b/net/netfilter/core.c @@ -33,7 +33,7 @@ EXPORT_SYMBOL_GPL(nf_ipv6_ops); DEFINE_PER_CPU(bool, nf_skb_duplicated); EXPORT_SYMBOL_GPL(nf_skb_duplicated); -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; EXPORT_SYMBOL(nf_hooks_needed); #endif @@ -347,7 +347,7 @@ static int __nf_register_net_hook(struct net *net, int pf, if (pf == NFPROTO_NETDEV && reg->hooknum == NF_NETDEV_INGRESS) net_inc_ingress_queue(); #endif -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL static_key_slow_inc(&nf_hooks_needed[pf][reg->hooknum]); #endif BUG_ON(p == new_hooks); @@ -405,7 +405,7 @@ static void __nf_unregister_net_hook(struct net *net, int pf, if (pf == NFPROTO_NETDEV && reg->hooknum == NF_NETDEV_INGRESS) net_dec_ingress_queue(); #endif -#ifdef HAVE_JUMP_LABEL +#ifdef CONFIG_JUMP_LABEL static_key_slow_dec(&nf_hooks_needed[pf][reg->hooknum]); #endif } else { diff --git a/scripts/gcc-goto.sh b/scripts/gcc-goto.sh index 083c526073ef..8b980fb2270a 100755 --- a/scripts/gcc-goto.sh +++ b/scripts/gcc-goto.sh @@ -3,7 +3,7 @@ # Test for gcc 'asm goto' support # Copyright (C) 2010, Jason Baron -cat << "END" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y" +cat << "END" | $@ -x c - -fno-PIE -c -o /dev/null int main(void) { #if defined(__arm__) || defined(__aarch64__) diff --git a/tools/arch/x86/include/asm/rmwcc.h b/tools/arch/x86/include/asm/rmwcc.h index dc90c0c2fae3..fee7983a90b4 100644 --- a/tools/arch/x86/include/asm/rmwcc.h +++ b/tools/arch/x86/include/asm/rmwcc.h @@ -2,7 +2,7 @@ #ifndef _TOOLS_LINUX_ASM_X86_RMWcc #define _TOOLS_LINUX_ASM_X86_RMWcc -#ifdef CC_HAVE_ASM_GOTO +#ifdef CONFIG_CC_HAS_ASM_GOTO #define __GEN_RMWcc(fullop, var, cc, ...) \ do { \ @@ -20,7 +20,7 @@ cc_label: \ #define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ __GEN_RMWcc(op " %1, " arg0, var, cc, vcon (val)) -#else /* !CC_HAVE_ASM_GOTO */ +#else /* !CONFIG_CC_HAS_ASM_GOTO */ #define __GEN_RMWcc(fullop, var, cc, ...) \ do { \ @@ -37,6 +37,6 @@ do { \ #define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ __GEN_RMWcc(op " %2, " arg0, var, cc, vcon (val)) -#endif /* CC_HAVE_ASM_GOTO */ +#endif /* CONFIG_CC_HAS_ASM_GOTO */ #endif /* _TOOLS_LINUX_ASM_X86_RMWcc */ -- cgit v1.2.3 From 172caf1993b7a6503a9f7faf589e2cf26eb1f219 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 31 Dec 2018 17:24:08 +0900 Subject: kbuild: remove redundant target cleaning on failure Since commit 9c2af1c7377a ("kbuild: add .DELETE_ON_ERROR special target"), the target file is automatically deleted on failure. The boilerplate code ... || { rm -f $@; false; } is unneeded. Signed-off-by: Masahiro Yamada --- arch/arm/boot/Makefile | 2 +- arch/arm/boot/compressed/Makefile | 2 +- arch/arm/tools/Makefile | 3 +-- arch/sh/tools/Makefile | 2 +- arch/x86/boot/compressed/Makefile | 2 +- arch/x86/lib/Makefile | 2 +- certs/Makefile | 2 +- lib/raid6/Makefile | 5 ++--- scripts/Makefile.lib | 21 +++++++-------------- 9 files changed, 16 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/arch/arm/boot/Makefile b/arch/arm/boot/Makefile index a3af4dc08c3e..0b3cd7a33a26 100644 --- a/arch/arm/boot/Makefile +++ b/arch/arm/boot/Makefile @@ -32,7 +32,7 @@ targets := Image zImage xipImage bootpImage uImage ifeq ($(CONFIG_XIP_KERNEL),y) cmd_deflate_xip_data = $(CONFIG_SHELL) -c \ - '$(srctree)/$(src)/deflate_xip_data.sh $< $@ || { rm -f $@; false; }' + '$(srctree)/$(src)/deflate_xip_data.sh $< $@' ifeq ($(CONFIG_XIP_DEFLATED_DATA),y) quiet_cmd_mkxip = XIPZ $@ diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index 01bf2585a0fa..6114ae6ea466 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -170,7 +170,7 @@ check_for_bad_syms = \ bad_syms=$$($(CROSS_COMPILE)nm $@ | sed -n 's/^.\{8\} [bc] \(.*\)/\1/p') && \ [ -z "$$bad_syms" ] || \ ( echo "following symbols must have non local/private scope:" >&2; \ - echo "$$bad_syms" >&2; rm -f $@; false ) + echo "$$bad_syms" >&2; false ) check_for_multiple_zreladdr = \ if [ $(words $(ZRELADDR)) -gt 1 -a "$(CONFIG_AUTO_ZRELADDR)" = "" ]; then \ diff --git a/arch/arm/tools/Makefile b/arch/arm/tools/Makefile index ddb89a7db36f..27d8beb7c941 100644 --- a/arch/arm/tools/Makefile +++ b/arch/arm/tools/Makefile @@ -35,8 +35,7 @@ _dummy := $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') \ quiet_cmd_gen_mach = GEN $@ cmd_gen_mach = mkdir -p $(dir $@) && \ - $(AWK) -f $(filter-out $(PHONY),$^) > $@ || \ - { rm -f $@; /bin/false; } + $(AWK) -f $(filter-out $(PHONY),$^) > $@ $(kapi)/mach-types.h: $(src)/gen-mach-types $(src)/mach-types FORCE $(call if_changed,gen_mach) diff --git a/arch/sh/tools/Makefile b/arch/sh/tools/Makefile index 2082af1f3fef..e5ba31c79fe0 100644 --- a/arch/sh/tools/Makefile +++ b/arch/sh/tools/Makefile @@ -13,4 +13,4 @@ include/generated/machtypes.h: $(src)/gen-mach-types $(src)/mach-types @echo ' Generating $@' $(Q)mkdir -p $(dir $@) - $(Q)LC_ALL=C $(AWK) -f $^ > $@ || { rm -f $@; /bin/false; } + $(Q)LC_ALL=C $(AWK) -f $^ > $@ diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 466f66c8a7f8..f0515ac895a4 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -151,7 +151,7 @@ suffix-$(CONFIG_KERNEL_LZO) := lzo suffix-$(CONFIG_KERNEL_LZ4) := lz4 quiet_cmd_mkpiggy = MKPIGGY $@ - cmd_mkpiggy = $(obj)/mkpiggy $< > $@ || ( rm -f $@ ; false ) + cmd_mkpiggy = $(obj)/mkpiggy $< > $@ targets += piggy.S $(obj)/piggy.S: $(obj)/vmlinux.bin.$(suffix-y) $(obj)/mkpiggy FORCE diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index ce28829f1281..140e61843a07 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -9,7 +9,7 @@ KCOV_INSTRUMENT_delay.o := n inat_tables_script = $(srctree)/arch/x86/tools/gen-insn-attr-x86.awk inat_tables_maps = $(srctree)/arch/x86/lib/x86-opcode-map.txt quiet_cmd_inat_tables = GEN $@ - cmd_inat_tables = $(AWK) -f $(inat_tables_script) $(inat_tables_maps) > $@ || rm -f $@ + cmd_inat_tables = $(AWK) -f $(inat_tables_script) $(inat_tables_maps) > $@ $(obj)/inat-tables.c: $(inat_tables_script) $(inat_tables_maps) $(call cmd,inat_tables) diff --git a/certs/Makefile b/certs/Makefile index 5d0999b9e21b..f4c25b67aad9 100644 --- a/certs/Makefile +++ b/certs/Makefile @@ -22,7 +22,7 @@ $(obj)/system_certificates.o: $(obj)/x509_certificate_list AFLAGS_system_certificates.o := -I$(srctree) quiet_cmd_extract_certs = EXTRACT_CERTS $(patsubst "%",%,$(2)) - cmd_extract_certs = scripts/extract-cert $(2) $@ || ( rm $@; exit 1) + cmd_extract_certs = scripts/extract-cert $(2) $@ targets += x509_certificate_list $(obj)/x509_certificate_list: scripts/extract-cert $(SYSTEM_TRUSTED_KEYS_SRCPREFIX)$(SYSTEM_TRUSTED_KEYS_FILENAME) FORCE diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile index 7ed43eaa02ef..4e90d443d1b0 100644 --- a/lib/raid6/Makefile +++ b/lib/raid6/Makefile @@ -13,8 +13,7 @@ raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o hostprogs-y += mktables quiet_cmd_unroll = UNROLL $@ - cmd_unroll = $(AWK) -f$(srctree)/$(src)/unroll.awk -vN=$(UNROLL) \ - < $< > $@ || ( rm -f $@ && exit 1 ) + cmd_unroll = $(AWK) -f$(srctree)/$(src)/unroll.awk -vN=$(UNROLL) < $< > $@ ifeq ($(CONFIG_ALTIVEC),y) altivec_flags := -maltivec $(call cc-option,-mabi=altivec) @@ -160,7 +159,7 @@ $(obj)/s390vx8.c: $(src)/s390vx.uc $(src)/unroll.awk FORCE $(call if_changed,unroll) quiet_cmd_mktable = TABLE $@ - cmd_mktable = $(obj)/mktables > $@ || ( rm -f $@ && exit 1 ) + cmd_mktable = $(obj)/mktables > $@ targets += tables.c $(obj)/tables.c: $(obj)/mktables FORCE diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index d10c61e5c6b9..390957f9306f 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -242,8 +242,7 @@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@ # --------------------------------------------------------------------------- quiet_cmd_gzip = GZIP $@ -cmd_gzip = (cat $(filter-out FORCE,$^) | gzip -n -f -9 > $@) || \ - (rm -f $@ ; false) + cmd_gzip = cat $(filter-out FORCE,$^) | gzip -n -f -9 > $@ # DTC # --------------------------------------------------------------------------- @@ -336,26 +335,22 @@ printf "%08x\n" $$dec_size | \ quiet_cmd_bzip2 = BZIP2 $@ cmd_bzip2 = (cat $(filter-out FORCE,$^) | \ - bzip2 -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \ - (rm -f $@ ; false) + bzip2 -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ # Lzma # --------------------------------------------------------------------------- quiet_cmd_lzma = LZMA $@ cmd_lzma = (cat $(filter-out FORCE,$^) | \ - lzma -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \ - (rm -f $@ ; false) + lzma -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ quiet_cmd_lzo = LZO $@ cmd_lzo = (cat $(filter-out FORCE,$^) | \ - lzop -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ || \ - (rm -f $@ ; false) + lzop -9 && $(call size_append, $(filter-out FORCE,$^))) > $@ quiet_cmd_lz4 = LZ4 $@ cmd_lz4 = (cat $(filter-out FORCE,$^) | \ - lz4c -l -c1 stdin stdout && $(call size_append, $(filter-out FORCE,$^))) > $@ || \ - (rm -f $@ ; false) + lz4c -l -c1 stdin stdout && $(call size_append, $(filter-out FORCE,$^))) > $@ # U-Boot mkimage # --------------------------------------------------------------------------- @@ -399,13 +394,11 @@ quiet_cmd_uimage = UIMAGE $@ quiet_cmd_xzkern = XZKERN $@ cmd_xzkern = (cat $(filter-out FORCE,$^) | \ sh $(srctree)/scripts/xz_wrap.sh && \ - $(call size_append, $(filter-out FORCE,$^))) > $@ || \ - (rm -f $@ ; false) + $(call size_append, $(filter-out FORCE,$^))) > $@ quiet_cmd_xzmisc = XZMISC $@ cmd_xzmisc = (cat $(filter-out FORCE,$^) | \ - xz --check=crc32 --lzma2=dict=1MiB) > $@ || \ - (rm -f $@ ; false) + xz --check=crc32 --lzma2=dict=1MiB) > $@ # ASM offsets # --------------------------------------------------------------------------- -- cgit v1.2.3 From 490fd30f859572ac97a51faa31860869744ba97b Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Mon, 17 Dec 2018 17:37:25 -0500 Subject: XArray tests: Add RCU locking 0day picked up that I'd forgotten to add locking to this new test. Signed-off-by: Matthew Wilcox --- lib/test_xarray.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 4676c0a1eeca..a885afde0aef 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -839,6 +839,7 @@ static noinline void check_find_3(struct xarray *xa) for (i = 0; i < 100; i++) { for (j = 0; j < 100; j++) { + rcu_read_lock(); for (k = 0; k < 100; k++) { xas_set(&xas, j); xas_for_each_marked(&xas, entry, k, XA_MARK_0) @@ -847,6 +848,7 @@ static noinline void check_find_3(struct xarray *xa) XA_BUG_ON(xa, xas.xa_node != XAS_RESTART); } + rcu_read_unlock(); } xa_store_index(xa, i, GFP_KERNEL); xa_set_mark(xa, i, XA_MARK_0); -- cgit v1.2.3 From 02669b17a433c242a40f01f14b691c9c9d1f8a13 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Wed, 5 Dec 2018 16:37:03 -0500 Subject: XArray: Turn xa_init_flags into a static inline A regular xa_init_flags() put all dynamically-initialised XArrays into the same locking class. That leads to lockdep believing that taking one XArray lock while holding another is a deadlock. It's possible to work around some of these situations with separate locking classes for irq/bh/regular XArrays, and SINGLE_DEPTH_NESTING, but that's ugly, and it doesn't work for all situations (where we have completely unrelated XArrays). Signed-off-by: Matthew Wilcox --- include/linux/xarray.h | 19 ++++++++++++++++++- lib/xarray.c | 29 ----------------------------- 2 files changed, 18 insertions(+), 30 deletions(-) (limited to 'lib') diff --git a/include/linux/xarray.h b/include/linux/xarray.h index f492e21c4aa2..4cf3cd128689 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -286,7 +286,6 @@ struct xarray { */ #define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC) -void xa_init_flags(struct xarray *, gfp_t flags); void *xa_load(struct xarray *, unsigned long index); void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); void *xa_erase(struct xarray *, unsigned long index); @@ -303,6 +302,24 @@ unsigned int xa_extract(struct xarray *, void **dst, unsigned long start, unsigned long max, unsigned int n, xa_mark_t); void xa_destroy(struct xarray *); +/** + * xa_init_flags() - Initialise an empty XArray with flags. + * @xa: XArray. + * @flags: XA_FLAG values. + * + * If you need to initialise an XArray with special flags (eg you need + * to take the lock from interrupt context), use this function instead + * of xa_init(). + * + * Context: Any context. + */ +static inline void xa_init_flags(struct xarray *xa, gfp_t flags) +{ + spin_lock_init(&xa->xa_lock); + xa->xa_flags = flags; + xa->xa_head = NULL; +} + /** * xa_init() - Initialise an empty XArray. * @xa: XArray. diff --git a/lib/xarray.c b/lib/xarray.c index 5f3f9311de89..dda6026d202e 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1250,35 +1250,6 @@ void *xas_find_conflict(struct xa_state *xas) } EXPORT_SYMBOL_GPL(xas_find_conflict); -/** - * xa_init_flags() - Initialise an empty XArray with flags. - * @xa: XArray. - * @flags: XA_FLAG values. - * - * If you need to initialise an XArray with special flags (eg you need - * to take the lock from interrupt context), use this function instead - * of xa_init(). - * - * Context: Any context. - */ -void xa_init_flags(struct xarray *xa, gfp_t flags) -{ - unsigned int lock_type; - static struct lock_class_key xa_lock_irq; - static struct lock_class_key xa_lock_bh; - - spin_lock_init(&xa->xa_lock); - xa->xa_flags = flags; - xa->xa_head = NULL; - - lock_type = xa_lock_type(xa); - if (lock_type == XA_LOCK_IRQ) - lockdep_set_class(&xa->xa_lock, &xa_lock_irq); - else if (lock_type == XA_LOCK_BH) - lockdep_set_class(&xa->xa_lock, &xa_lock_bh); -} -EXPORT_SYMBOL(xa_init_flags); - /** * xa_load() - Load an entry from an XArray. * @xa: XArray. -- cgit v1.2.3 From 4a31896c5b5a2715ecf4033426aa0a35066d92d6 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Mon, 17 Dec 2018 14:45:36 -0500 Subject: XArray: Change xa_for_each iterator There were three problems with this API: 1. It took too many arguments; almost all users wanted to iterate over every element in the array rather than a subset. 2. It required that 'index' be initialised before use, and there's no realistic way to make GCC catch that. 3. 'index' and 'entry' were the opposite way round from every other member of the XArray APIs. So split it into three different APIs: xa_for_each(xa, index, entry) xa_for_each_start(xa, index, entry, start) xa_for_each_marked(xa, index, entry, filter) Signed-off-by: Matthew Wilcox --- include/linux/xarray.h | 78 +++++++++++++++++++++++++++++++++++++++++--------- lib/test_xarray.c | 11 ++++--- 2 files changed, 70 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/include/linux/xarray.h b/include/linux/xarray.h index 4cf3cd128689..3d0ce8b267e3 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -359,20 +359,45 @@ static inline bool xa_marked(const struct xarray *xa, xa_mark_t mark) } /** - * xa_for_each() - Iterate over a portion of an XArray. + * xa_for_each_start() - Iterate over a portion of an XArray. * @xa: XArray. + * @index: Index of @entry. * @entry: Entry retrieved from array. + * @start: First index to retrieve from array. + * + * During the iteration, @entry will have the value of the entry stored + * in @xa at @index. You may modify @index during the iteration if you + * want to skip or reprocess indices. It is safe to modify the array + * during the iteration. At the end of the iteration, @entry will be set + * to NULL and @index will have a value less than or equal to max. + * + * xa_for_each_start() is O(n.log(n)) while xas_for_each() is O(n). You have + * to handle your own locking with xas_for_each(), and if you have to unlock + * after each iteration, it will also end up being O(n.log(n)). + * xa_for_each_start() will spin if it hits a retry entry; if you intend to + * see retry entries, you should use the xas_for_each() iterator instead. + * The xas_for_each() iterator will expand into more inline code than + * xa_for_each_start(). + * + * Context: Any context. Takes and releases the RCU lock. + */ +#define xa_for_each_start(xa, index, entry, start) \ + for (index = start, \ + entry = xa_find(xa, &index, ULONG_MAX, XA_PRESENT); \ + entry; \ + entry = xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT)) + +/** + * xa_for_each() - Iterate over present entries in an XArray. + * @xa: XArray. * @index: Index of @entry. - * @max: Maximum index to retrieve from array. - * @filter: Selection criterion. + * @entry: Entry retrieved from array. * - * Initialise @index to the lowest index you want to retrieve from the - * array. During the iteration, @entry will have the value of the entry - * stored in @xa at @index. The iteration will skip all entries in the - * array which do not match @filter. You may modify @index during the - * iteration if you want to skip or reprocess indices. It is safe to modify - * the array during the iteration. At the end of the iteration, @entry will - * be set to NULL and @index will have a value less than or equal to max. + * During the iteration, @entry will have the value of the entry stored + * in @xa at @index. You may modify @index during the iteration if you want + * to skip or reprocess indices. It is safe to modify the array during the + * iteration. At the end of the iteration, @entry will be set to NULL and + * @index will have a value less than or equal to max. * * xa_for_each() is O(n.log(n)) while xas_for_each() is O(n). You have * to handle your own locking with xas_for_each(), and if you have to unlock @@ -383,9 +408,36 @@ static inline bool xa_marked(const struct xarray *xa, xa_mark_t mark) * * Context: Any context. Takes and releases the RCU lock. */ -#define xa_for_each(xa, entry, index, max, filter) \ - for (entry = xa_find(xa, &index, max, filter); entry; \ - entry = xa_find_after(xa, &index, max, filter)) +#define xa_for_each(xa, index, entry) \ + xa_for_each_start(xa, index, entry, 0) + +/** + * xa_for_each_marked() - Iterate over marked entries in an XArray. + * @xa: XArray. + * @index: Index of @entry. + * @entry: Entry retrieved from array. + * @filter: Selection criterion. + * + * During the iteration, @entry will have the value of the entry stored + * in @xa at @index. The iteration will skip all entries in the array + * which do not match @filter. You may modify @index during the iteration + * if you want to skip or reprocess indices. It is safe to modify the array + * during the iteration. At the end of the iteration, @entry will be set to + * NULL and @index will have a value less than or equal to max. + * + * xa_for_each_marked() is O(n.log(n)) while xas_for_each_marked() is O(n). + * You have to handle your own locking with xas_for_each(), and if you have + * to unlock after each iteration, it will also end up being O(n.log(n)). + * xa_for_each_marked() will spin if it hits a retry entry; if you intend to + * see retry entries, you should use the xas_for_each_marked() iterator + * instead. The xas_for_each_marked() iterator will expand into more inline + * code than xa_for_each_marked(). + * + * Context: Any context. Takes and releases the RCU lock. + */ +#define xa_for_each_marked(xa, index, entry, filter) \ + for (index = 0, entry = xa_find(xa, &index, ULONG_MAX, filter); \ + entry; entry = xa_find_after(xa, &index, ULONG_MAX, filter)) #define xa_trylock(xa) spin_trylock(&(xa)->xa_lock) #define xa_lock(xa) spin_lock(&(xa)->xa_lock) diff --git a/lib/test_xarray.c b/lib/test_xarray.c index a885afde0aef..dc02eff562b8 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -357,7 +357,7 @@ static noinline void check_cmpxchg(struct xarray *xa) static noinline void check_reserve(struct xarray *xa) { void *entry; - unsigned long index = 0; + unsigned long index; /* An array with a reserved entry is not empty */ XA_BUG_ON(xa, !xa_empty(xa)); @@ -393,7 +393,7 @@ static noinline void check_reserve(struct xarray *xa) xa_reserve(xa, 6, GFP_KERNEL); xa_store_index(xa, 7, GFP_KERNEL); - xa_for_each(xa, entry, index, ULONG_MAX, XA_PRESENT) { + xa_for_each(xa, index, entry) { XA_BUG_ON(xa, index != 5 && index != 7); } xa_destroy(xa); @@ -812,17 +812,16 @@ static noinline void check_find_1(struct xarray *xa) static noinline void check_find_2(struct xarray *xa) { void *entry; - unsigned long i, j, index = 0; + unsigned long i, j, index; - xa_for_each(xa, entry, index, ULONG_MAX, XA_PRESENT) { + xa_for_each(xa, index, entry) { XA_BUG_ON(xa, true); } for (i = 0; i < 1024; i++) { xa_store_index(xa, index, GFP_KERNEL); j = 0; - index = 0; - xa_for_each(xa, entry, index, ULONG_MAX, XA_PRESENT) { + xa_for_each(xa, index, entry) { XA_BUG_ON(xa, xa_mk_index(index) != entry); XA_BUG_ON(xa, index != j++); } -- cgit v1.2.3 From 76b4e52995654af260f14558e0e07b5b039ae202 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Fri, 28 Dec 2018 23:20:44 -0500 Subject: XArray: Permit storing 2-byte-aligned pointers On m68k, statically allocated pointers may only be two-byte aligned. This clashes with the XArray's method for tagging internal pointers. Permit storing these pointers in single slots (ie not in multislots). Signed-off-by: Matthew Wilcox --- include/linux/xarray.h | 18 +++++++++++++++--- lib/test_xarray.c | 30 ++++++++++++++++++++++++++++++ lib/xarray.c | 22 +++++++++++++--------- 3 files changed, 58 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/include/linux/xarray.h b/include/linux/xarray.h index 3d0ce8b267e3..435c25b29079 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -176,7 +176,8 @@ static inline bool xa_is_internal(const void *entry) */ static inline bool xa_is_err(const void *entry) { - return unlikely(xa_is_internal(entry)); + return unlikely(xa_is_internal(entry) && + (unsigned long)entry >= -((MAX_ERRNO << 2) + 2)); } /** @@ -1039,8 +1040,8 @@ static inline bool xa_is_sibling(const void *entry) (entry < xa_mk_sibling(XA_CHUNK_SIZE - 1)); } -#define XA_ZERO_ENTRY xa_mk_internal(256) -#define XA_RETRY_ENTRY xa_mk_internal(257) +#define XA_RETRY_ENTRY xa_mk_internal(256) +#define XA_ZERO_ENTRY xa_mk_internal(257) /** * xa_is_zero() - Is the entry a zero entry? @@ -1064,6 +1065,17 @@ static inline bool xa_is_retry(const void *entry) return unlikely(entry == XA_RETRY_ENTRY); } +/** + * xa_is_advanced() - Is the entry only permitted for the advanced API? + * @entry: Entry to be stored in the XArray. + * + * Return: %true if the entry cannot be stored by the normal API. + */ +static inline bool xa_is_advanced(const void *entry) +{ + return xa_is_internal(entry) && (entry <= XA_RETRY_ENTRY); +} + /** * typedef xa_update_node_t - A callback function from the XArray. * @node: The node which is being processed diff --git a/lib/test_xarray.c b/lib/test_xarray.c index dc02eff562b8..6e0212a60b08 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -1184,6 +1184,35 @@ static noinline void check_store_range(struct xarray *xa) } } +static void check_align_1(struct xarray *xa, char *name) +{ + int i; + unsigned int id; + unsigned long index; + void *entry; + + for (i = 0; i < 8; i++) { + id = 0; + XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, name + i, GFP_KERNEL) + != 0); + XA_BUG_ON(xa, id != i); + } + xa_for_each(xa, index, entry) + XA_BUG_ON(xa, xa_is_err(entry)); + xa_destroy(xa); +} + +static noinline void check_align(struct xarray *xa) +{ + char name[] = "Motorola 68000"; + + check_align_1(xa, name); + check_align_1(xa, name + 1); + check_align_1(xa, name + 2); + check_align_1(xa, name + 3); +// check_align_2(xa, name); +} + static LIST_HEAD(shadow_nodes); static void test_update_node(struct xa_node *node) @@ -1333,6 +1362,7 @@ static int xarray_checks(void) check_create_range(&array); check_store_range(&array); check_store_iter(&array); + check_align(&xa0); check_workingset(&array, 0); check_workingset(&array, 64); diff --git a/lib/xarray.c b/lib/xarray.c index dda6026d202e..bffa26b1f0d6 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -232,6 +232,8 @@ void *xas_load(struct xa_state *xas) if (xas->xa_shift > node->shift) break; entry = xas_descend(xas, node); + if (node->shift == 0) + break; } return entry; } @@ -506,7 +508,7 @@ static void xas_free_nodes(struct xa_state *xas, struct xa_node *top) for (;;) { void *entry = xa_entry_locked(xas->xa, node, offset); - if (xa_is_node(entry)) { + if (node->shift && xa_is_node(entry)) { node = xa_to_node(entry); offset = 0; continue; @@ -604,6 +606,7 @@ static int xas_expand(struct xa_state *xas, void *head) /* * xas_create() - Create a slot to store an entry in. * @xas: XArray operation state. + * @allow_root: %true if we can store the entry in the root directly * * Most users will not need to call this function directly, as it is called * by xas_store(). It is useful for doing conditional store operations @@ -613,7 +616,7 @@ static int xas_expand(struct xa_state *xas, void *head) * If the slot was newly created, returns %NULL. If it failed to create the * slot, returns %NULL and indicates the error in @xas. */ -static void *xas_create(struct xa_state *xas) +static void *xas_create(struct xa_state *xas, bool allow_root) { struct xarray *xa = xas->xa; void *entry; @@ -628,6 +631,8 @@ static void *xas_create(struct xa_state *xas) shift = xas_expand(xas, entry); if (shift < 0) return NULL; + if (!shift && !allow_root) + shift = XA_CHUNK_SHIFT; entry = xa_head_locked(xa); slot = &xa->xa_head; } else if (xas_error(xas)) { @@ -687,7 +692,7 @@ void xas_create_range(struct xa_state *xas) xas->xa_sibs = 0; for (;;) { - xas_create(xas); + xas_create(xas, true); if (xas_error(xas)) goto restore; if (xas->xa_index <= (index | XA_CHUNK_MASK)) @@ -754,7 +759,7 @@ void *xas_store(struct xa_state *xas, void *entry) bool value = xa_is_value(entry); if (entry) - first = xas_create(xas); + first = xas_create(xas, !xa_is_node(entry)); else first = xas_load(xas); @@ -1279,7 +1284,6 @@ static void *xas_result(struct xa_state *xas, void *curr) { if (xa_is_zero(curr)) return NULL; - XA_NODE_BUG_ON(xas->xa_node, xa_is_internal(curr)); if (xas_error(xas)) curr = xas->xa_node; return curr; @@ -1349,7 +1353,7 @@ void *__xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) XA_STATE(xas, xa, index); void *curr; - if (WARN_ON_ONCE(xa_is_internal(entry))) + if (WARN_ON_ONCE(xa_is_advanced(entry))) return XA_ERROR(-EINVAL); if (xa_track_free(xa) && !entry) entry = XA_ZERO_ENTRY; @@ -1415,7 +1419,7 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, XA_STATE(xas, xa, index); void *curr; - if (WARN_ON_ONCE(xa_is_internal(entry))) + if (WARN_ON_ONCE(xa_is_advanced(entry))) return XA_ERROR(-EINVAL); if (xa_track_free(xa) && !entry) entry = XA_ZERO_ENTRY; @@ -1538,7 +1542,7 @@ void *xa_store_range(struct xarray *xa, unsigned long first, if (last + 1) order = __ffs(last + 1); xas_set_order(&xas, last, order); - xas_create(&xas); + xas_create(&xas, true); if (xas_error(&xas)) goto unlock; } @@ -1580,7 +1584,7 @@ int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp) XA_STATE(xas, xa, 0); int err; - if (WARN_ON_ONCE(xa_is_internal(entry))) + if (WARN_ON_ONCE(xa_is_advanced(entry))) return -EINVAL; if (WARN_ON_ONCE(!xa_track_free(xa))) return -EINVAL; -- cgit v1.2.3 From b0606fed6eece16a421034eca0bbea9a08b90e91 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Wed, 2 Jan 2019 13:57:03 -0500 Subject: XArray: Honour reserved entries in xa_insert xa_insert() should treat reserved entries as occupied, not as available. Also, it should treat requests to insert a NULL pointer as a request to reserve the slot. Add xa_insert_bh() and xa_insert_irq() for completeness. Signed-off-by: Matthew Wilcox --- Documentation/core-api/xarray.rst | 15 +++--- include/linux/xarray.h | 110 ++++++++++++++++++++++++-------------- lib/test_xarray.c | 8 +-- lib/xarray.c | 41 ++++++++++++++ 4 files changed, 126 insertions(+), 48 deletions(-) (limited to 'lib') diff --git a/Documentation/core-api/xarray.rst b/Documentation/core-api/xarray.rst index 6a6d67acaf69..5d54b27c6eba 100644 --- a/Documentation/core-api/xarray.rst +++ b/Documentation/core-api/xarray.rst @@ -108,12 +108,13 @@ some, but not all of the other indices changing. Sometimes you need to ensure that a subsequent call to :c:func:`xa_store` will not need to allocate memory. The :c:func:`xa_reserve` function -will store a reserved entry at the indicated index. Users of the normal -API will see this entry as containing ``NULL``. If you do not need to -use the reserved entry, you can call :c:func:`xa_release` to remove the -unused entry. If another user has stored to the entry in the meantime, -:c:func:`xa_release` will do nothing; if instead you want the entry to -become ``NULL``, you should use :c:func:`xa_erase`. +will store a reserved entry at the indicated index. Users of the +normal API will see this entry as containing ``NULL``. If you do +not need to use the reserved entry, you can call :c:func:`xa_release` +to remove the unused entry. If another user has stored to the entry +in the meantime, :c:func:`xa_release` will do nothing; if instead you +want the entry to become ``NULL``, you should use :c:func:`xa_erase`. +Using :c:func:`xa_insert` on a reserved entry will fail. If all entries in the array are ``NULL``, the :c:func:`xa_empty` function will return ``true``. @@ -183,6 +184,8 @@ Takes xa_lock internally: * :c:func:`xa_store_bh` * :c:func:`xa_store_irq` * :c:func:`xa_insert` + * :c:func:`xa_insert_bh` + * :c:func:`xa_insert_irq` * :c:func:`xa_erase` * :c:func:`xa_erase_bh` * :c:func:`xa_erase_irq` diff --git a/include/linux/xarray.h b/include/linux/xarray.h index 435c25b29079..12244aa98a69 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -463,39 +463,12 @@ void *__xa_erase(struct xarray *, unsigned long index); void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old, void *entry, gfp_t); +int __xa_insert(struct xarray *, unsigned long index, void *entry, gfp_t); int __xa_alloc(struct xarray *, u32 *id, u32 max, void *entry, gfp_t); int __xa_reserve(struct xarray *, unsigned long index, gfp_t); void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); -/** - * __xa_insert() - Store this entry in the XArray unless another entry is - * already present. - * @xa: XArray. - * @index: Index into array. - * @entry: New entry. - * @gfp: Memory allocation flags. - * - * If you would rather see the existing entry in the array, use __xa_cmpxchg(). - * This function is for users who don't care what the entry is, only that - * one is present. - * - * Context: Any context. Expects xa_lock to be held on entry. May - * release and reacquire xa_lock if the @gfp flags permit. - * Return: 0 if the store succeeded. -EEXIST if another entry was present. - * -ENOMEM if memory could not be allocated. - */ -static inline int __xa_insert(struct xarray *xa, unsigned long index, - void *entry, gfp_t gfp) -{ - void *curr = __xa_cmpxchg(xa, index, NULL, entry, gfp); - if (!curr) - return 0; - if (xa_is_err(curr)) - return xa_err(curr); - return -EEXIST; -} - /** * xa_store_bh() - Store this entry in the XArray. * @xa: XArray. @@ -685,24 +658,83 @@ static inline void *xa_cmpxchg_irq(struct xarray *xa, unsigned long index, * @entry: New entry. * @gfp: Memory allocation flags. * - * If you would rather see the existing entry in the array, use xa_cmpxchg(). - * This function is for users who don't care what the entry is, only that - * one is present. + * Inserting a NULL entry will store a reserved entry (like xa_reserve()) + * if no entry is present. Inserting will fail if a reserved entry is + * present, even though loading from this index will return NULL. * - * Context: Process context. Takes and releases the xa_lock. - * May sleep if the @gfp flags permit. + * Context: Any context. Takes and releases the xa_lock. May sleep if + * the @gfp flags permit. * Return: 0 if the store succeeded. -EEXIST if another entry was present. * -ENOMEM if memory could not be allocated. */ static inline int xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) { - void *curr = xa_cmpxchg(xa, index, NULL, entry, gfp); - if (!curr) - return 0; - if (xa_is_err(curr)) - return xa_err(curr); - return -EEXIST; + int err; + + xa_lock(xa); + err = __xa_insert(xa, index, entry, gfp); + xa_unlock(xa); + + return err; +} + +/** + * xa_insert_bh() - Store this entry in the XArray unless another entry is + * already present. + * @xa: XArray. + * @index: Index into array. + * @entry: New entry. + * @gfp: Memory allocation flags. + * + * Inserting a NULL entry will store a reserved entry (like xa_reserve()) + * if no entry is present. Inserting will fail if a reserved entry is + * present, even though loading from this index will return NULL. + * + * Context: Any context. Takes and releases the xa_lock while + * disabling softirqs. May sleep if the @gfp flags permit. + * Return: 0 if the store succeeded. -EEXIST if another entry was present. + * -ENOMEM if memory could not be allocated. + */ +static inline int xa_insert_bh(struct xarray *xa, unsigned long index, + void *entry, gfp_t gfp) +{ + int err; + + xa_lock_bh(xa); + err = __xa_insert(xa, index, entry, gfp); + xa_unlock_bh(xa); + + return err; +} + +/** + * xa_insert_irq() - Store this entry in the XArray unless another entry is + * already present. + * @xa: XArray. + * @index: Index into array. + * @entry: New entry. + * @gfp: Memory allocation flags. + * + * Inserting a NULL entry will store a reserved entry (like xa_reserve()) + * if no entry is present. Inserting will fail if a reserved entry is + * present, even though loading from this index will return NULL. + * + * Context: Process context. Takes and releases the xa_lock while + * disabling interrupts. May sleep if the @gfp flags permit. + * Return: 0 if the store succeeded. -EEXIST if another entry was present. + * -ENOMEM if memory could not be allocated. + */ +static inline int xa_insert_irq(struct xarray *xa, unsigned long index, + void *entry, gfp_t gfp) +{ + int err; + + xa_lock_irq(xa); + err = __xa_insert(xa, index, entry, gfp); + xa_unlock_irq(xa); + + return err; } /** diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 6e0212a60b08..3cf17338b0a4 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -382,10 +382,12 @@ static noinline void check_reserve(struct xarray *xa) xa_erase_index(xa, 12345678); XA_BUG_ON(xa, !xa_empty(xa)); - /* And so does xa_insert */ + /* But xa_insert does not */ xa_reserve(xa, 12345678, GFP_KERNEL); - XA_BUG_ON(xa, xa_insert(xa, 12345678, xa_mk_value(12345678), 0) != 0); - xa_erase_index(xa, 12345678); + XA_BUG_ON(xa, xa_insert(xa, 12345678, xa_mk_value(12345678), 0) != + -EEXIST); + XA_BUG_ON(xa, xa_empty(xa)); + XA_BUG_ON(xa, xa_erase(xa, 12345678) != NULL); XA_BUG_ON(xa, !xa_empty(xa)); /* Can iterate through a reserved entry */ diff --git a/lib/xarray.c b/lib/xarray.c index bffa26b1f0d6..81c3171ddde9 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1439,6 +1439,47 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, } EXPORT_SYMBOL(__xa_cmpxchg); +/** + * __xa_insert() - Store this entry in the XArray if no entry is present. + * @xa: XArray. + * @index: Index into array. + * @entry: New entry. + * @gfp: Memory allocation flags. + * + * Inserting a NULL entry will store a reserved entry (like xa_reserve()) + * if no entry is present. Inserting will fail if a reserved entry is + * present, even though loading from this index will return NULL. + * + * Context: Any context. Expects xa_lock to be held on entry. May + * release and reacquire xa_lock if @gfp flags permit. + * Return: 0 if the store succeeded. -EEXIST if another entry was present. + * -ENOMEM if memory could not be allocated. + */ +int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) +{ + XA_STATE(xas, xa, index); + void *curr; + + if (WARN_ON_ONCE(xa_is_advanced(entry))) + return -EINVAL; + if (!entry) + entry = XA_ZERO_ENTRY; + + do { + curr = xas_load(&xas); + if (!curr) { + xas_store(&xas, entry); + if (xa_track_free(xa)) + xas_clear_mark(&xas, XA_FREE_MARK); + } else { + xas_set_err(&xas, -EEXIST); + } + } while (__xas_nomem(&xas, gfp)); + + return xas_error(&xas); +} +EXPORT_SYMBOL(__xa_insert); + /** * __xa_reserve() - Reserve this index in the XArray. * @xa: XArray. -- cgit v1.2.3 From 3719876809e745b9db5293d418600c194bbf5c23 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 14 Jan 2019 12:25:40 -0500 Subject: sbitmap: Protect swap_lock from softirqs The swap_lock used by sbitmap has a chain with locks taken from softirq, but the swap_lock is not protected from being preempted by softirqs. A chain exists of: sbq->ws[i].wait -> dispatch_wait_lock -> swap_lock Where the sbq->ws[i].wait lock can be taken from softirq context, which means all locks below it in the chain must also be protected from softirqs. Reported-by: Clark Williams Fixes: 58ab5e32e6fd ("sbitmap: silence bogus lockdep IRQ warning") Fixes: ea86ea2cdced ("sbitmap: amortize cost of clearing bits") Cc: Jens Axboe Cc: Ming Lei Cc: Guenter Roeck Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Linus Torvalds --- lib/sbitmap.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 65c2d06250a6..864354000e04 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -26,14 +26,9 @@ static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index) { unsigned long mask, val; - unsigned long __maybe_unused flags; bool ret = false; - /* Silence bogus lockdep warning */ -#if defined(CONFIG_LOCKDEP) - local_irq_save(flags); -#endif - spin_lock(&sb->map[index].swap_lock); + spin_lock_bh(&sb->map[index].swap_lock); if (!sb->map[index].cleared) goto out_unlock; @@ -54,10 +49,7 @@ static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index) ret = true; out_unlock: - spin_unlock(&sb->map[index].swap_lock); -#if defined(CONFIG_LOCKDEP) - local_irq_restore(flags); -#endif + spin_unlock_bh(&sb->map[index].swap_lock); return ret; } -- cgit v1.2.3 From d69d287a9002b70bdbe2975660b97241ccefc071 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Mon, 14 Jan 2019 13:57:31 -0500 Subject: XArray tests: Check mark 2 gets squashed We do not currently check that the loop in xas_squash_marks() doesn't have an off-by-one error in it. It didn't, but a patch which introduced an off-by-one error wasn't caught by any existing test. Switch the roles of XA_MARK_1 and XA_MARK_2 to catch that bug. Reported-by: Cyrill Gorcunov Signed-off-by: Matthew Wilcox --- lib/test_xarray.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 3cf17338b0a4..c596a957f764 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -199,7 +199,7 @@ static noinline void check_xa_mark_1(struct xarray *xa, unsigned long index) XA_BUG_ON(xa, xa_store_index(xa, index + 1, GFP_KERNEL)); xa_set_mark(xa, index + 1, XA_MARK_0); XA_BUG_ON(xa, xa_store_index(xa, index + 2, GFP_KERNEL)); - xa_set_mark(xa, index + 2, XA_MARK_1); + xa_set_mark(xa, index + 2, XA_MARK_2); XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL)); xa_store_order(xa, index, order, xa_mk_index(index), GFP_KERNEL); @@ -209,8 +209,8 @@ static noinline void check_xa_mark_1(struct xarray *xa, unsigned long index) void *entry; XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_0)); - XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_1)); - XA_BUG_ON(xa, xa_get_mark(xa, i, XA_MARK_2)); + XA_BUG_ON(xa, xa_get_mark(xa, i, XA_MARK_1)); + XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_2)); /* We should see two elements in the array */ rcu_read_lock(); -- cgit v1.2.3 From fe76fc6aaf538df27708ffa3e5d549a6c8e16142 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Tue, 15 Jan 2019 11:59:52 +0800 Subject: sbitmap: Protect swap_lock from hardirq Because we may call blk_mq_get_driver_tag() directly from blk_mq_dispatch_rq_list() without holding any lock, then HARDIRQ may come and the above DEADLOCK is triggered. Commit ab53dcfb3e7b ("sbitmap: Protect swap_lock from hardirq") tries to fix this issue by using 'spin_lock_bh', which isn't enough because we complete request from hardirq context direclty in case of multiqueue. Cc: Clark Williams Fixes: ab53dcfb3e7b ("sbitmap: Protect swap_lock from hardirq") Cc: Jens Axboe Cc: Ming Lei Cc: Guenter Roeck Cc: Steven Rostedt (VMware) Signed-off-by: Ming Lei Signed-off-by: Linus Torvalds --- lib/sbitmap.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 864354000e04..5b382c1244ed 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -27,8 +27,9 @@ static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index) { unsigned long mask, val; bool ret = false; + unsigned long flags; - spin_lock_bh(&sb->map[index].swap_lock); + spin_lock_irqsave(&sb->map[index].swap_lock, flags); if (!sb->map[index].cleared) goto out_unlock; @@ -49,7 +50,7 @@ static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index) ret = true; out_unlock: - spin_unlock_bh(&sb->map[index].swap_lock); + spin_unlock_irqrestore(&sb->map[index].swap_lock, flags); return ret; } -- cgit v1.2.3 From fbfaf851902cd9293f392f3a1735e0543016d530 Mon Sep 17 00:00:00 2001 From: Florian La Roche Date: Sat, 19 Jan 2019 16:14:50 +0100 Subject: fix int_sqrt64() for very large numbers If an input number x for int_sqrt64() has the highest bit set, then fls64(x) is 64. (1UL << 64) is an overflow and breaks the algorithm. Subtracting 1 is a better guess for the initial value of m anyway and that's what also done in int_sqrt() implicitly [*]. [*] Note how int_sqrt() uses __fls() with two underscores, which already returns the proper raw bit number. In contrast, int_sqrt64() used fls64(), and that returns bit numbers illogically starting at 1, because of error handling for the "no bits set" case. Will points out that he bug probably is due to a copy-and-paste error from the regular int_sqrt() case. Signed-off-by: Florian La Roche Acked-by: Will Deacon Signed-off-by: Linus Torvalds --- lib/int_sqrt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/int_sqrt.c b/lib/int_sqrt.c index 14436f4ca6bd..30e0f9770f88 100644 --- a/lib/int_sqrt.c +++ b/lib/int_sqrt.c @@ -52,7 +52,7 @@ u32 int_sqrt64(u64 x) if (x <= ULONG_MAX) return int_sqrt((unsigned long) x); - m = 1ULL << (fls64(x) & ~1ULL); + m = 1ULL << ((fls64(x) - 1) & ~1ULL); while (m != 0) { b = y + m; y >>= 1; -- cgit v1.2.3 From db7ddeab3ce5d64c9696e70d61f45ea9909cd196 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 1 Feb 2019 14:20:58 -0800 Subject: lib/test_kmod.c: potential double free in error handling There is a copy and paste bug so we set "config->test_driver" to NULL twice instead of setting "config->test_fs". Smatch complains that it leads to a double free: lib/test_kmod.c:840 __kmod_config_init() warn: 'config->test_fs' double freed Link: http://lkml.kernel.org/r/20190121140011.GA14283@kadam Fixes: d9c6a72d6fa2 ("kmod: add test driver to stress test the module loader") Signed-off-by: Dan Carpenter Acked-by: Luis Chamberlain Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- lib/test_kmod.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/test_kmod.c b/lib/test_kmod.c index d82d022111e0..9cf77628fc91 100644 --- a/lib/test_kmod.c +++ b/lib/test_kmod.c @@ -632,7 +632,7 @@ static void __kmod_config_free(struct test_config *config) config->test_driver = NULL; kfree_const(config->test_fs); - config->test_driver = NULL; + config->test_fs = NULL; } static void kmod_config_free(struct kmod_test_device *test_dev) -- cgit v1.2.3