From d5f7638eb5fed0eb12e45a127764c4111b11c50e Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 24 Oct 2022 05:34:14 -0700 Subject: Input: matrix_keypad - replace header inclusions by forward declarations When the data structure is only referred by pointer, compiler may not need to see the contents of the data type. Thus, we may replace header inclusions by respective forward declarations. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20220923184632.2157-2-andriy.shevchenko@linux.intel.com Signed-off-by: Dmitry Torokhov --- include/linux/input/matrix_keypad.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/input/matrix_keypad.h b/include/linux/input/matrix_keypad.h index 9476768c3b90..b8d8d69eba29 100644 --- a/include/linux/input/matrix_keypad.h +++ b/include/linux/input/matrix_keypad.h @@ -3,8 +3,9 @@ #define _MATRIX_KEYPAD_H #include -#include -#include + +struct device; +struct input_dev; #define MATRIX_MAX_ROWS 32 #define MATRIX_MAX_COLS 32 -- cgit v1.2.3 From 721da5cee9d43901105f5b8bd33fcb9101b12fc3 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 23 Feb 2023 08:33:26 +0100 Subject: driver core: remove CONFIG_SYSFS_DEPRECATED and CONFIG_SYSFS_DEPRECATED_V2 CONFIG_SYSFS_DEPRECATED was added in commit 88a22c985e35 ("CONFIG_SYSFS_DEPRECATED") in 2006 to allow systems with older versions of some tools (i.e. Fedora 3's version of udev) to boot properly. Four years later, in 2010, the option was attempted to be removed as most of userspace should have been fixed up properly by then, but some kernel developers clung to those old systems and refused to update, so we added CONFIG_SYSFS_DEPRECATED_V2 in commit e52eec13cd6b ("SYSFS: Allow boot time switching between deprecated and modern sysfs layout") to allow them to continue to boot properly, and we allowed a boot time parameter to be used to switch back to the old format if needed. Over time, the logic that was covered under these config options was slowly removed from individual driver subsystems successfully, removed, and the only thing that is now left in the kernel are some changes in the block layer's representation in sysfs where real directories are used instead of symlinks like normal. Because the original changes were done to userspace tools in 2006, and all distros that use those tools are long end-of-life, and older non-udev-based systems do not care about the block layer's sysfs representation, it is time to finally remove this old logic and the config entries from the kernel. Cc: Jonathan Corbet Cc: "Rafael J. Wysocki" Cc: linux-block@vger.kernel.org Cc: linux-doc@vger.kernel.org Acked-by: Jens Axboe Link: https://lore.kernel.org/r/20230223073326.2073220-1-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- Documentation/admin-guide/kernel-parameters.txt | 9 ------ block/genhd.c | 19 +++++-------- drivers/base/class.c | 2 +- drivers/base/core.c | 37 ------------------------ include/linux/device.h | 6 ---- init/Kconfig | 38 ------------------------- 6 files changed, 8 insertions(+), 103 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 6221a1d057dd..207ca1d6e3d3 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6130,15 +6130,6 @@ later by a loaded module cannot be set this way. Example: sysctl.vm.swappiness=40 - sysfs.deprecated=0|1 [KNL] - Enable/disable old style sysfs layout for old udev - on older distributions. When this option is enabled - very new udev will not work anymore. When this option - is disabled (or CONFIG_SYSFS_DEPRECATED not compiled) - in older udev will not work anymore. - Default depends on CONFIG_SYSFS_DEPRECATED_V2 set in - the kernel configuration. - sysrq_always_enabled [KNL] Ignore sysrq setting - this boot parameter will diff --git a/block/genhd.c b/block/genhd.c index 3ee5577e1586..e1e1230b1b9f 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -466,12 +466,10 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk, if (ret) goto out_device_del; - if (!sysfs_deprecated) { - ret = sysfs_create_link(block_depr, &ddev->kobj, - kobject_name(&ddev->kobj)); - if (ret) - goto out_device_del; - } + ret = sysfs_create_link(block_depr, &ddev->kobj, + kobject_name(&ddev->kobj)); + if (ret) + goto out_device_del; /* * avoid probable deadlock caused by allocating memory with @@ -554,8 +552,7 @@ out_put_holder_dir: out_del_integrity: blk_integrity_del(disk); out_del_block_link: - if (!sysfs_deprecated) - sysfs_remove_link(block_depr, dev_name(ddev)); + sysfs_remove_link(block_depr, dev_name(ddev)); out_device_del: device_del(ddev); out_free_ext_minor: @@ -657,8 +654,7 @@ void del_gendisk(struct gendisk *disk) part_stat_set_all(disk->part0, 0); disk->part0->bd_stamp = 0; - if (!sysfs_deprecated) - sysfs_remove_link(block_depr, dev_name(disk_to_dev(disk))); + sysfs_remove_link(block_depr, dev_name(disk_to_dev(disk))); pm_runtime_set_memalloc_noio(disk_to_dev(disk), false); device_del(disk_to_dev(disk)); @@ -912,8 +908,7 @@ static int __init genhd_device_init(void) register_blkdev(BLOCK_EXT_MAJOR, "blkext"); /* create top-level block dir */ - if (!sysfs_deprecated) - block_depr = kobject_create_and_add("block", NULL); + block_depr = kobject_create_and_add("block", NULL); return 0; } diff --git a/drivers/base/class.c b/drivers/base/class.c index 2373b3e210d8..fb8f2a1e1c19 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -180,7 +180,7 @@ int __class_register(struct class *cls, struct lock_class_key *key) #if defined(CONFIG_BLOCK) /* let the block class directory show up in the root of sysfs */ - if (!sysfs_deprecated || cls != &block_class) + if (cls != &block_class) cp->subsys.kobj.kset = class_kset; #else cp->subsys.kobj.kset = class_kset; diff --git a/drivers/base/core.c b/drivers/base/core.c index 6878dfcbf0d6..ba11b51ce4ee 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -36,19 +36,6 @@ #include "physical_location.h" #include "power/power.h" -#ifdef CONFIG_SYSFS_DEPRECATED -#ifdef CONFIG_SYSFS_DEPRECATED_V2 -long sysfs_deprecated = 1; -#else -long sysfs_deprecated = 0; -#endif -static int __init sysfs_deprecated_setup(char *arg) -{ - return kstrtol(arg, 10, &sysfs_deprecated); -} -early_param("sysfs.deprecated", sysfs_deprecated_setup); -#endif - /* Device links support. */ static LIST_HEAD(deferred_sync); static unsigned int defer_sync_state_count = 1; @@ -3137,15 +3124,6 @@ static struct kobject *get_device_parent(struct device *dev, struct kobject *parent_kobj; struct kobject *k; -#ifdef CONFIG_BLOCK - /* block disks show up in /sys/block */ - if (sysfs_deprecated && dev->class == &block_class) { - if (parent && parent->class == &block_class) - return &parent->kobj; - return &block_class.p->subsys.kobj; - } -#endif - /* * If we have no parent, we live in "virtual". * Class-devices with a non class-device as parent, live @@ -3324,12 +3302,6 @@ static int device_add_class_symlinks(struct device *dev) goto out_subsys; } -#ifdef CONFIG_BLOCK - /* /sys/block has directories and does not need symlinks */ - if (sysfs_deprecated && dev->class == &block_class) - return 0; -#endif - /* link in the class directory pointing to the device */ error = sysfs_create_link(&dev->class->p->subsys.kobj, &dev->kobj, dev_name(dev)); @@ -3359,10 +3331,6 @@ static void device_remove_class_symlinks(struct device *dev) if (dev->parent && device_is_not_partition(dev)) sysfs_remove_link(&dev->kobj, "device"); sysfs_remove_link(&dev->kobj, "subsystem"); -#ifdef CONFIG_BLOCK - if (sysfs_deprecated && dev->class == &block_class) - return; -#endif sysfs_delete_link(&dev->class->p->subsys.kobj, &dev->kobj, dev_name(dev)); } @@ -4652,11 +4620,6 @@ int device_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid) if (error) goto out; -#ifdef CONFIG_BLOCK - if (sysfs_deprecated && dev->class == &block_class) - goto out; -#endif - /* * Change the owner of the symlink located in the class directory of * the device class associated with @dev which points to the actual diff --git a/include/linux/device.h b/include/linux/device.h index 1508e637bb26..19b6ba478fbf 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1092,10 +1092,4 @@ int dev_err_probe(const struct device *dev, int err, const char *fmt, ...); #define MODULE_ALIAS_CHARDEV_MAJOR(major) \ MODULE_ALIAS("char-major-" __stringify(major) "-*") -#ifdef CONFIG_SYSFS_DEPRECATED -extern long sysfs_deprecated; -#else -#define sysfs_deprecated 0 -#endif - #endif /* _DEVICE_H_ */ diff --git a/init/Kconfig b/init/Kconfig index 1fb5f313d18f..4464ef98ea0b 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1297,44 +1297,6 @@ config SCHED_AUTOGROUP desktop applications. Task group autogeneration is currently based upon task session. -config SYSFS_DEPRECATED - bool "Enable deprecated sysfs features to support old userspace tools" - depends on SYSFS - default n - help - This option adds code that switches the layout of the "block" class - devices, to not show up in /sys/class/block/, but only in - /sys/block/. - - This switch is only active when the sysfs.deprecated=1 boot option is - passed or the SYSFS_DEPRECATED_V2 option is set. - - This option allows new kernels to run on old distributions and tools, - which might get confused by /sys/class/block/. Since 2007/2008 all - major distributions and tools handle this just fine. - - Recent distributions and userspace tools after 2009/2010 depend on - the existence of /sys/class/block/, and will not work with this - option enabled. - - Only if you are using a new kernel on an old distribution, you might - need to say Y here. - -config SYSFS_DEPRECATED_V2 - bool "Enable deprecated sysfs features by default" - default n - depends on SYSFS - depends on SYSFS_DEPRECATED - help - Enable deprecated sysfs by default. - - See the CONFIG_SYSFS_DEPRECATED option for more details about this - option. - - Only if you are using a new kernel on an old distribution, you might - need to say Y here. Even then, odds are you would not need it - enabled, you can always pass the boot option if absolutely necessary. - config RELAY bool "Kernel->user space relay support (formerly relayfs)" select IRQ_WORK -- cgit v1.2.3 From 5da094ac80cd4f6d4114ad220468f602f86d9980 Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Fri, 24 Feb 2023 17:31:47 +0530 Subject: bus: mhi: host: Remove mhi_poll() API mhi_poll() API is not used within the MHI stack and also not by any client drivers in mainline. So let's remove it until any consumer is available. Reviewed-by: Jeffrey Hugo Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/host/main.c | 15 --------------- include/linux/mhi.h | 7 ------- 2 files changed, 22 deletions(-) (limited to 'include/linux') diff --git a/drivers/bus/mhi/host/main.c b/drivers/bus/mhi/host/main.c index df0fbfee7b78..4fa0969472fc 100644 --- a/drivers/bus/mhi/host/main.c +++ b/drivers/bus/mhi/host/main.c @@ -1679,18 +1679,3 @@ void mhi_unprepare_from_transfer(struct mhi_device *mhi_dev) } } EXPORT_SYMBOL_GPL(mhi_unprepare_from_transfer); - -int mhi_poll(struct mhi_device *mhi_dev, u32 budget) -{ - struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; - struct mhi_chan *mhi_chan = mhi_dev->dl_chan; - struct mhi_event *mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index]; - int ret; - - spin_lock_bh(&mhi_event->lock); - ret = mhi_event->process_event(mhi_cntrl, mhi_event, budget); - spin_unlock_bh(&mhi_event->lock); - - return ret; -} -EXPORT_SYMBOL_GPL(mhi_poll); diff --git a/include/linux/mhi.h b/include/linux/mhi.h index a5441ad33c74..f6de4b6ecfc7 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -765,13 +765,6 @@ int mhi_prepare_for_transfer_autoqueue(struct mhi_device *mhi_dev); */ void mhi_unprepare_from_transfer(struct mhi_device *mhi_dev); -/** - * mhi_poll - Poll for any available data in DL direction - * @mhi_dev: Device associated with the channels - * @budget: # of events to process - */ -int mhi_poll(struct mhi_device *mhi_dev, u32 budget); - /** * mhi_queue_dma - Send or receive DMA mapped buffers from client device * over MHI channel -- cgit v1.2.3 From 071c44e4278156f18a6a56958617223b6bffa6ab Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Mon, 13 Feb 2023 23:05:58 -0800 Subject: sched/idle: Mark arch_cpu_idle_dead() __noreturn Before commit 076cbf5d2163 ("x86/xen: don't let xen_pv_play_dead() return"), in Xen, when a previously offlined CPU was brought back online, it unexpectedly resumed execution where it left off in the middle of the idle loop. There were some hacks to make that work, but the behavior was surprising as do_idle() doesn't expect an offlined CPU to return from the dead (in arch_cpu_idle_dead()). Now that Xen has been fixed, and the arch-specific implementations of arch_cpu_idle_dead() also don't return, give it a __noreturn attribute. This will cause the compiler to complain if an arch-specific implementation might return. It also improves code generation for both caller and callee. Also fixes the following warning: vmlinux.o: warning: objtool: do_idle+0x25f: unreachable instruction Reported-by: Paul E. McKenney Tested-by: Paul E. McKenney Link: https://lore.kernel.org/r/60d527353da8c99d4cf13b6473131d46719ed16d.1676358308.git.jpoimboe@kernel.org Signed-off-by: Josh Poimboeuf --- arch/alpha/kernel/process.c | 2 +- arch/arm/kernel/smp.c | 2 +- arch/arm64/kernel/process.c | 2 +- arch/csky/kernel/smp.c | 2 +- arch/ia64/kernel/process.c | 2 +- arch/loongarch/kernel/process.c | 2 +- arch/mips/kernel/process.c | 2 +- arch/parisc/kernel/process.c | 2 +- arch/powerpc/kernel/smp.c | 2 +- arch/riscv/kernel/cpu-hotplug.c | 2 +- arch/s390/kernel/idle.c | 2 +- arch/sh/kernel/idle.c | 2 +- arch/sparc/kernel/process_64.c | 2 +- arch/x86/kernel/process.c | 2 +- arch/xtensa/kernel/smp.c | 2 +- include/linux/cpu.h | 2 +- kernel/sched/idle.c | 2 +- tools/objtool/check.c | 1 + 18 files changed, 18 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c index a82fefa31bdb..582d96548385 100644 --- a/arch/alpha/kernel/process.c +++ b/arch/alpha/kernel/process.c @@ -60,7 +60,7 @@ void arch_cpu_idle(void) wtint(0); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { wtint(INT_MAX); BUG(); diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 441ea5cff390..d6be4507d22d 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -320,7 +320,7 @@ void __cpu_die(unsigned int cpu) * of the other hotplug-cpu capable cores, so presumably coming * out of idle fixes this. */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { unsigned int cpu = smp_processor_id(); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 71d59b5abede..089ced6d6bd6 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -69,7 +69,7 @@ void (*pm_power_off)(void); EXPORT_SYMBOL_GPL(pm_power_off); #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { cpu_die(); } diff --git a/arch/csky/kernel/smp.c b/arch/csky/kernel/smp.c index 0ec20efaf5fd..9c7a20b73ac6 100644 --- a/arch/csky/kernel/smp.c +++ b/arch/csky/kernel/smp.c @@ -300,7 +300,7 @@ void __cpu_die(unsigned int cpu) pr_notice("CPU%u: shutdown\n", cpu); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { idle_task_exit(); diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c index 78f5794b2dde..9a5cd9fad3a9 100644 --- a/arch/ia64/kernel/process.c +++ b/arch/ia64/kernel/process.c @@ -225,7 +225,7 @@ static inline void __noreturn play_dead(void) } #endif /* CONFIG_HOTPLUG_CPU */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c index fa2443c7afb2..b71e17c1cc0c 100644 --- a/arch/loongarch/kernel/process.c +++ b/arch/loongarch/kernel/process.c @@ -62,7 +62,7 @@ unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE; EXPORT_SYMBOL(boot_option_idle_override); #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index 093dbbd6b843..a3225912c862 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -40,7 +40,7 @@ #include #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index c064719b49b0..97c6f875bd0e 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -159,7 +159,7 @@ EXPORT_SYMBOL(running_on_qemu); /* * Called from the idle thread for the CPU which has been shutdown. */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { #ifdef CONFIG_HOTPLUG_CPU idle_task_exit(); diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 6b90f10a6c81..f62e5e651bcd 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -1752,7 +1752,7 @@ void __cpu_die(unsigned int cpu) smp_ops->cpu_die(cpu); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { /* * Disable on the down path. This will be re-enabled by diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index f7a832e3a1d1..59b80211c25f 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -71,7 +71,7 @@ void __cpu_die(unsigned int cpu) /* * Called from the idle thread for the CPU which has been shutdown. */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { idle_task_exit(); diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 38e267c7bff7..e7239aaf428b 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -88,7 +88,7 @@ void arch_cpu_idle_exit(void) { } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { cpu_die(); } diff --git a/arch/sh/kernel/idle.c b/arch/sh/kernel/idle.c index 114f0c4abeac..d662503b0665 100644 --- a/arch/sh/kernel/idle.c +++ b/arch/sh/kernel/idle.c @@ -30,7 +30,7 @@ void default_idle(void) clear_bl_bit(); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index 91c2b8124527..b51d8fb0ecdc 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -95,7 +95,7 @@ void arch_cpu_idle(void) } #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { sched_preempt_enable_no_resched(); cpu_play_dead(); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index f1ec36caf1d8..3e30147a537e 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -727,7 +727,7 @@ void arch_cpu_idle_enter(void) local_touch_nmi(); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/xtensa/kernel/smp.c b/arch/xtensa/kernel/smp.c index 7bad78495536..054bd64eab19 100644 --- a/arch/xtensa/kernel/smp.c +++ b/arch/xtensa/kernel/smp.c @@ -322,7 +322,7 @@ void __cpu_die(unsigned int cpu) pr_err("CPU%u: unable to kill\n", cpu); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { cpu_die(); } diff --git a/include/linux/cpu.h b/include/linux/cpu.h index f83e4519c5f0..8582a7142623 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -182,7 +182,7 @@ void arch_cpu_idle(void); void arch_cpu_idle_prepare(void); void arch_cpu_idle_enter(void); void arch_cpu_idle_exit(void); -void arch_cpu_idle_dead(void); +void __noreturn arch_cpu_idle_dead(void); int cpu_report_state(int cpu); int cpu_check_up_prepare(int cpu); diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 56e152f06d0f..342f58a329f5 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -75,7 +75,7 @@ static noinline int __cpuidle cpu_idle_poll(void) void __weak arch_cpu_idle_prepare(void) { } void __weak arch_cpu_idle_enter(void) { } void __weak arch_cpu_idle_exit(void) { } -void __weak arch_cpu_idle_dead(void) { while (1); } +void __weak __noreturn arch_cpu_idle_dead(void) { while (1); } void __weak arch_cpu_idle(void) { cpu_idle_force_poll = 1; diff --git a/tools/objtool/check.c b/tools/objtool/check.c index f937be1afe65..37c36dc1d868 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -202,6 +202,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "__reiserfs_panic", "__stack_chk_fail", "__ubsan_handle_builtin_unreachable", + "arch_cpu_idle_dead", "cpu_bringup_and_idle", "cpu_startup_entry", "do_exit", -- cgit v1.2.3 From 85bce38a7e1f392f5051e67fd5fcfbffe71e9b81 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Fri, 17 Feb 2023 21:22:57 +0100 Subject: serial: 8250: Reorder fields in 'struct plat_serial8250_port' Group some variables based on their sizes to reduce hole and avoid padding. On x86_64, this shrinks the size of 'struct plat_serial8250_port' from 144 to 128 bytes. It saves a few bytes of memory. Signed-off-by: Christophe JAILLET Link: https://lore.kernel.org/r/f3cb1efe1454e0615840fd331ee335bc441589a9.1676665358.git.christophe.jaillet@wanadoo.fr Signed-off-by: Greg Kroah-Hartman --- include/linux/serial_8250.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h index 19376bee9667..741ed4807a9c 100644 --- a/include/linux/serial_8250.h +++ b/include/linux/serial_8250.h @@ -18,16 +18,16 @@ struct plat_serial8250_port { unsigned long iobase; /* io base address */ void __iomem *membase; /* ioremap cookie or NULL */ resource_size_t mapbase; /* resource base */ + unsigned int uartclk; /* UART clock rate */ unsigned int irq; /* interrupt number */ unsigned long irqflags; /* request_irq flags */ - unsigned int uartclk; /* UART clock rate */ void *private_data; unsigned char regshift; /* register shift */ unsigned char iotype; /* UPIO_* */ unsigned char hub6; unsigned char has_sysrq; /* supports magic SysRq */ - upf_t flags; /* UPF_* flags */ unsigned int type; /* If UPF_FIXED_TYPE */ + upf_t flags; /* UPF_* flags */ unsigned int (*serial_in)(struct uart_port *, int); void (*serial_out)(struct uart_port *, int, int); void (*set_termios)(struct uart_port *, -- cgit v1.2.3 From 9b12f050c76f090cc6d0aebe0ef76fed79ec3f15 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Wed, 22 Feb 2023 10:23:02 +0100 Subject: char: pcmcia: remove all the drivers These char PCMCIA drivers are buggy[1] and receive only minimal care. It was concluded[2], that we should try to remove most pcmcia drivers completely. Let's start with these char broken one. Note that I also removed a UAPI header: include/uapi/linux/cm4000_cs.h. I found only coccinelle tests mentioning some ioctl constants from that file. But they are not actually used. Anyway, should someone complain, we may reintroduce the header (or its parts). [1] https://lore.kernel.org/all/f41c2765-80e0-48bc-b1e4-8cfd3230fd4a@www.fastmail.com/ [2] https://lore.kernel.org/all/c5b39544-a4fb-4796-a046-0b9be9853787@app.fastmail.com/ Signed-off-by: Jiri Slaby (SUSE) Cc: "Hyunwoo Kim" Cc: Harald Welte Cc: Lubomir Rintel Cc: Arnd Bergmann Cc: Greg Kroah-Hartman Acked-by: Dominik Brodowski Reviewed-by: Arnd Bergmann Link: https://lore.kernel.org/r/20230222092302.6348-2-jirislaby@kernel.org Signed-off-by: Greg Kroah-Hartman --- Documentation/process/magic-number.rst | 1 - .../translations/it_IT/process/magic-number.rst | 1 - .../translations/sp_SP/process/magic-number.rst | 1 - .../translations/zh_CN/process/magic-number.rst | 1 - .../translations/zh_TW/process/magic-number.rst | 1 - Documentation/userspace-api/ioctl/ioctl-number.rst | 1 - MAINTAINERS | 17 - arch/powerpc/configs/ppc6xx_defconfig | 2 - drivers/char/Kconfig | 2 - drivers/char/Makefile | 1 - drivers/char/pcmcia/Kconfig | 59 - drivers/char/pcmcia/Makefile | 11 - drivers/char/pcmcia/cm4000_cs.c | 1912 --------- drivers/char/pcmcia/cm4040_cs.c | 684 ---- drivers/char/pcmcia/cm4040_cs.h | 48 - drivers/char/pcmcia/scr24x_cs.c | 359 -- drivers/char/pcmcia/synclink_cs.c | 4290 -------------------- include/linux/cm4000_cs.h | 11 - include/uapi/linux/cm4000_cs.h | 64 - 19 files changed, 7466 deletions(-) delete mode 100644 drivers/char/pcmcia/Kconfig delete mode 100644 drivers/char/pcmcia/Makefile delete mode 100644 drivers/char/pcmcia/cm4000_cs.c delete mode 100644 drivers/char/pcmcia/cm4040_cs.c delete mode 100644 drivers/char/pcmcia/cm4040_cs.h delete mode 100644 drivers/char/pcmcia/scr24x_cs.c delete mode 100644 drivers/char/pcmcia/synclink_cs.c delete mode 100644 include/linux/cm4000_cs.h delete mode 100644 include/uapi/linux/cm4000_cs.h (limited to 'include/linux') diff --git a/Documentation/process/magic-number.rst b/Documentation/process/magic-number.rst index 64b5948fc1d4..7029c3c084ee 100644 --- a/Documentation/process/magic-number.rst +++ b/Documentation/process/magic-number.rst @@ -72,7 +72,6 @@ PG_MAGIC 'P' pg_{read,write}_hdr ``include/linux/ APM_BIOS_MAGIC 0x4101 apm_user ``arch/x86/kernel/apm_32.c`` FASYNC_MAGIC 0x4601 fasync_struct ``include/linux/fs.h`` SLIP_MAGIC 0x5302 slip ``drivers/net/slip.h`` -MGSLPC_MAGIC 0x5402 mgslpc_info ``drivers/char/pcmcia/synclink_cs.c`` BAYCOM_MAGIC 0x19730510 baycom_state ``drivers/net/baycom_epp.c`` HDLCDRV_MAGIC 0x5ac6e778 hdlcdrv_state ``include/linux/hdlcdrv.h`` KV_MAGIC 0x5f4b565f kernel_vars_s ``arch/mips/include/asm/sn/klkernvars.h`` diff --git a/Documentation/translations/it_IT/process/magic-number.rst b/Documentation/translations/it_IT/process/magic-number.rst index 02eb7eb2448e..ae92ab633c16 100644 --- a/Documentation/translations/it_IT/process/magic-number.rst +++ b/Documentation/translations/it_IT/process/magic-number.rst @@ -78,7 +78,6 @@ PG_MAGIC 'P' pg_{read,write}_hdr ``include/linux/ APM_BIOS_MAGIC 0x4101 apm_user ``arch/x86/kernel/apm_32.c`` FASYNC_MAGIC 0x4601 fasync_struct ``include/linux/fs.h`` SLIP_MAGIC 0x5302 slip ``drivers/net/slip.h`` -MGSLPC_MAGIC 0x5402 mgslpc_info ``drivers/char/pcmcia/synclink_cs.c`` BAYCOM_MAGIC 0x19730510 baycom_state ``drivers/net/baycom_epp.c`` HDLCDRV_MAGIC 0x5ac6e778 hdlcdrv_state ``include/linux/hdlcdrv.h`` KV_MAGIC 0x5f4b565f kernel_vars_s ``arch/mips/include/asm/sn/klkernvars.h`` diff --git a/Documentation/translations/sp_SP/process/magic-number.rst b/Documentation/translations/sp_SP/process/magic-number.rst index 2b62cec34e8e..7c7dfb4ba80b 100644 --- a/Documentation/translations/sp_SP/process/magic-number.rst +++ b/Documentation/translations/sp_SP/process/magic-number.rst @@ -77,7 +77,6 @@ PG_MAGIC 'P' pg_{read,write}_hdr ``include/linux/ APM_BIOS_MAGIC 0x4101 apm_user ``arch/x86/kernel/apm_32.c`` FASYNC_MAGIC 0x4601 fasync_struct ``include/linux/fs.h`` SLIP_MAGIC 0x5302 slip ``drivers/net/slip.h`` -MGSLPC_MAGIC 0x5402 mgslpc_info ``drivers/char/pcmcia/synclink_cs.c`` BAYCOM_MAGIC 0x19730510 baycom_state ``drivers/net/baycom_epp.c`` HDLCDRV_MAGIC 0x5ac6e778 hdlcdrv_state ``include/linux/hdlcdrv.h`` KV_MAGIC 0x5f4b565f kernel_vars_s ``arch/mips/include/asm/sn/klkernvars.h`` diff --git a/Documentation/translations/zh_CN/process/magic-number.rst b/Documentation/translations/zh_CN/process/magic-number.rst index 0617ce125e12..37ed33034134 100644 --- a/Documentation/translations/zh_CN/process/magic-number.rst +++ b/Documentation/translations/zh_CN/process/magic-number.rst @@ -61,7 +61,6 @@ PG_MAGIC 'P' pg_{read,write}_hdr ``include/linux/ APM_BIOS_MAGIC 0x4101 apm_user ``arch/x86/kernel/apm_32.c`` FASYNC_MAGIC 0x4601 fasync_struct ``include/linux/fs.h`` SLIP_MAGIC 0x5302 slip ``drivers/net/slip.h`` -MGSLPC_MAGIC 0x5402 mgslpc_info ``drivers/char/pcmcia/synclink_cs.c`` BAYCOM_MAGIC 0x19730510 baycom_state ``drivers/net/baycom_epp.c`` HDLCDRV_MAGIC 0x5ac6e778 hdlcdrv_state ``include/linux/hdlcdrv.h`` KV_MAGIC 0x5f4b565f kernel_vars_s ``arch/mips/include/asm/sn/klkernvars.h`` diff --git a/Documentation/translations/zh_TW/process/magic-number.rst b/Documentation/translations/zh_TW/process/magic-number.rst index f3f7082e17c6..1d48e1bbf5f2 100644 --- a/Documentation/translations/zh_TW/process/magic-number.rst +++ b/Documentation/translations/zh_TW/process/magic-number.rst @@ -64,7 +64,6 @@ PG_MAGIC 'P' pg_{read,write}_hdr ``include/linux/ APM_BIOS_MAGIC 0x4101 apm_user ``arch/x86/kernel/apm_32.c`` FASYNC_MAGIC 0x4601 fasync_struct ``include/linux/fs.h`` SLIP_MAGIC 0x5302 slip ``drivers/net/slip.h`` -MGSLPC_MAGIC 0x5402 mgslpc_info ``drivers/char/pcmcia/synclink_cs.c`` BAYCOM_MAGIC 0x19730510 baycom_state ``drivers/net/baycom_epp.c`` HDLCDRV_MAGIC 0x5ac6e778 hdlcdrv_state ``include/linux/hdlcdrv.h`` KV_MAGIC 0x5f4b565f kernel_vars_s ``arch/mips/include/asm/sn/klkernvars.h`` diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 0a1882e296ae..176e8fc3f31b 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -222,7 +222,6 @@ Code Seq# Include File Comments 'b' 00-FF conflict! bit3 vme host bridge 'b' 00-0F linux/dma-buf.h conflict! -'c' all linux/cm4000_cs.h conflict! 'c' 00-7F linux/comstats.h conflict! 'c' 00-7F linux/coda.h conflict! 'c' 00-1F linux/chio.h conflict! diff --git a/MAINTAINERS b/MAINTAINERS index 8d5bc223f305..64068a601e59 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15322,18 +15322,6 @@ S: Maintained F: Documentation/filesystems/omfs.rst F: fs/omfs/ -OMNIKEY CARDMAN 4000 DRIVER -M: Harald Welte -S: Maintained -F: drivers/char/pcmcia/cm4000_cs.c -F: include/linux/cm4000_cs.h -F: include/uapi/linux/cm4000_cs.h - -OMNIKEY CARDMAN 4040 DRIVER -M: Harald Welte -S: Maintained -F: drivers/char/pcmcia/cm4040_cs.* - OMNIVISION OG01A1B SENSOR DRIVER M: Shawn Tu L: linux-media@vger.kernel.org @@ -18609,11 +18597,6 @@ F: include/linux/wait.h F: include/uapi/linux/sched.h F: kernel/sched/ -SCR24X CHIP CARD INTERFACE DRIVER -M: Lubomir Rintel -S: Supported -F: drivers/char/pcmcia/scr24x_cs.c - SCSI RDMA PROTOCOL (SRP) INITIATOR M: Bart Van Assche L: linux-rdma@vger.kernel.org diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig index 110258277959..d8729b94400e 100644 --- a/arch/powerpc/configs/ppc6xx_defconfig +++ b/arch/powerpc/configs/ppc6xx_defconfig @@ -614,8 +614,6 @@ CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=m CONFIG_NVRAM=y CONFIG_DTLK=m -CONFIG_CARDMAN_4000=m -CONFIG_CARDMAN_4040=m CONFIG_IPWIRELESS=m CONFIG_I2C_CHARDEV=m CONFIG_I2C_HYDRA=m diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 30fe9848dac1..801d6c83f896 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -247,8 +247,6 @@ config SONYPI To compile this driver as a module, choose M here: the module will be called sonypi. -source "drivers/char/pcmcia/Kconfig" - config MWAVE tristate "ACP Modem (Mwave) support" depends on X86 && TTY diff --git a/drivers/char/Makefile b/drivers/char/Makefile index 1b35d1724565..c5f532e412f1 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -35,7 +35,6 @@ obj-$(CONFIG_TELCLOCK) += tlclk.o obj-$(CONFIG_MWAVE) += mwave/ obj-y += agp/ -obj-$(CONFIG_PCMCIA) += pcmcia/ obj-$(CONFIG_HANGCHECK_TIMER) += hangcheck-timer.o obj-$(CONFIG_TCG_TPM) += tpm/ diff --git a/drivers/char/pcmcia/Kconfig b/drivers/char/pcmcia/Kconfig deleted file mode 100644 index 26724990074c..000000000000 --- a/drivers/char/pcmcia/Kconfig +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# PCMCIA character device configuration -# - -menu "PCMCIA character devices" - depends on PCMCIA!=n - -config SYNCLINK_CS - tristate "SyncLink PC Card support" - depends on PCMCIA && TTY - help - Enable support for the SyncLink PC Card serial adapter, running - asynchronous and HDLC communications up to 512Kbps. The port is - selectable for RS-232, V.35, RS-449, RS-530, and X.21 - - This driver may be built as a module ( = code which can be - inserted in and removed from the running kernel whenever you want). - The module will be called synclink_cs. If you want to do that, say M - here. - -config CARDMAN_4000 - tristate "Omnikey Cardman 4000 support" - depends on PCMCIA - select BITREVERSE - help - Enable support for the Omnikey Cardman 4000 PCMCIA Smartcard - reader. - - This kernel driver requires additional userspace support, either - by the vendor-provided PC/SC ifd_handler (http://www.omnikey.com/), - or via the cm4000 backend of OpenCT (http://www.opensc-project.org/opensc). - -config CARDMAN_4040 - tristate "Omnikey CardMan 4040 support" - depends on PCMCIA - help - Enable support for the Omnikey CardMan 4040 PCMCIA Smartcard - reader. - - This card is basically a USB CCID device connected to a FIFO - in I/O space. To use the kernel driver, you will need either the - PC/SC ifdhandler provided from the Omnikey homepage - (http://www.omnikey.com/), or a current development version of OpenCT - (http://www.opensc-project.org/opensc). - -config SCR24X - tristate "SCR24x Chip Card Interface support" - depends on PCMCIA - help - Enable support for the SCR24x PCMCIA Chip Card Interface. - - To compile this driver as a module, choose M here. - The module will be called scr24x_cs.. - - If unsure say N. - -endmenu - diff --git a/drivers/char/pcmcia/Makefile b/drivers/char/pcmcia/Makefile deleted file mode 100644 index 024eed1c4ca5..000000000000 --- a/drivers/char/pcmcia/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# drivers/char/pcmcia/Makefile -# -# Makefile for the Linux PCMCIA char device drivers. -# - -obj-$(CONFIG_SYNCLINK_CS) += synclink_cs.o -obj-$(CONFIG_CARDMAN_4000) += cm4000_cs.o -obj-$(CONFIG_CARDMAN_4040) += cm4040_cs.o -obj-$(CONFIG_SCR24X) += scr24x_cs.o diff --git a/drivers/char/pcmcia/cm4000_cs.c b/drivers/char/pcmcia/cm4000_cs.c deleted file mode 100644 index e656f42a28ac..000000000000 --- a/drivers/char/pcmcia/cm4000_cs.c +++ /dev/null @@ -1,1912 +0,0 @@ - /* - * A driver for the PCMCIA Smartcard Reader "Omnikey CardMan Mobile 4000" - * - * cm4000_cs.c support.linux@omnikey.com - * - * Tue Oct 23 11:32:43 GMT 2001 herp - cleaned up header files - * Sun Jan 20 10:11:15 MET 2002 herp - added modversion header files - * Thu Nov 14 16:34:11 GMT 2002 mh - added PPS functionality - * Tue Nov 19 16:36:27 GMT 2002 mh - added SUSPEND/RESUME functionailty - * Wed Jul 28 12:55:01 CEST 2004 mh - kernel 2.6 adjustments - * - * current version: 2.4.0gm4 - * - * (C) 2000,2001,2002,2003,2004 Omnikey AG - * - * (C) 2005-2006 Harald Welte - * - Adhere to Kernel process/coding-style.rst - * - Port to 2.6.13 "new" style PCMCIA - * - Check for copy_{from,to}_user return values - * - Use nonseekable_open() - * - add class interface for udev device creation - * - * All rights reserved. Licensed under dual BSD/GPL license. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include - -/* #define ATR_CSUM */ - -#define reader_to_dev(x) (&x->p_dev->dev) - -/* n (debug level) is ignored */ -/* additional debug output may be enabled by re-compiling with - * CM4000_DEBUG set */ -/* #define CM4000_DEBUG */ -#define DEBUGP(n, rdr, x, args...) do { \ - dev_dbg(reader_to_dev(rdr), "%s:" x, \ - __func__ , ## args); \ - } while (0) - -static DEFINE_MUTEX(cmm_mutex); - -#define T_1SEC (HZ) -#define T_10MSEC msecs_to_jiffies(10) -#define T_20MSEC msecs_to_jiffies(20) -#define T_40MSEC msecs_to_jiffies(40) -#define T_50MSEC msecs_to_jiffies(50) -#define T_100MSEC msecs_to_jiffies(100) -#define T_500MSEC msecs_to_jiffies(500) - -static void cm4000_release(struct pcmcia_device *link); - -static int major; /* major number we get from the kernel */ - -/* note: the first state has to have number 0 always */ - -#define M_FETCH_ATR 0 -#define M_TIMEOUT_WAIT 1 -#define M_READ_ATR_LEN 2 -#define M_READ_ATR 3 -#define M_ATR_PRESENT 4 -#define M_BAD_CARD 5 -#define M_CARDOFF 6 - -#define LOCK_IO 0 -#define LOCK_MONITOR 1 - -#define IS_AUTOPPS_ACT 6 -#define IS_PROCBYTE_PRESENT 7 -#define IS_INVREV 8 -#define IS_ANY_T0 9 -#define IS_ANY_T1 10 -#define IS_ATR_PRESENT 11 -#define IS_ATR_VALID 12 -#define IS_CMM_ABSENT 13 -#define IS_BAD_LENGTH 14 -#define IS_BAD_CSUM 15 -#define IS_BAD_CARD 16 - -#define REG_FLAGS0(x) (x + 0) -#define REG_FLAGS1(x) (x + 1) -#define REG_NUM_BYTES(x) (x + 2) -#define REG_BUF_ADDR(x) (x + 3) -#define REG_BUF_DATA(x) (x + 4) -#define REG_NUM_SEND(x) (x + 5) -#define REG_BAUDRATE(x) (x + 6) -#define REG_STOPBITS(x) (x + 7) - -struct cm4000_dev { - struct pcmcia_device *p_dev; - - unsigned char atr[MAX_ATR]; - unsigned char rbuf[512]; - unsigned char sbuf[512]; - - wait_queue_head_t devq; /* when removing cardman must not be - zeroed! */ - - wait_queue_head_t ioq; /* if IO is locked, wait on this Q */ - wait_queue_head_t atrq; /* wait for ATR valid */ - wait_queue_head_t readq; /* used by write to wake blk.read */ - - /* warning: do not move this struct group. - * initialising to zero depends on it - see ZERO_DEV below. */ - struct_group(init, - unsigned char atr_csum; - unsigned char atr_len_retry; - unsigned short atr_len; - unsigned short rlen; /* bytes avail. after write */ - unsigned short rpos; /* latest read pos. write zeroes */ - unsigned char procbyte; /* T=0 procedure byte */ - unsigned char mstate; /* state of card monitor */ - unsigned char cwarn; /* slow down warning */ - unsigned char flags0; /* cardman IO-flags 0 */ - unsigned char flags1; /* cardman IO-flags 1 */ - unsigned int mdelay; /* variable monitor speeds, in jiffies */ - - unsigned int baudv; /* baud value for speed */ - unsigned char ta1; - unsigned char proto; /* T=0, T=1, ... */ - unsigned long flags; /* lock+flags (MONITOR,IO,ATR) * for concurrent - access */ - - unsigned char pts[4]; - - struct timer_list timer; /* used to keep monitor running */ - int monitor_running; - ); -}; - -#define ZERO_DEV(dev) memset(&((dev)->init), 0, sizeof((dev)->init)) - -static struct pcmcia_device *dev_table[CM4000_MAX_DEV]; -static struct class *cmm_class; - -/* This table doesn't use spaces after the comma between fields and thus - * violates process/coding-style.rst. However, I don't really think wrapping it around will - * make it any clearer to read -HW */ -static unsigned char fi_di_table[10][14] = { -/*FI 00 01 02 03 04 05 06 07 08 09 10 11 12 13 */ -/*DI */ -/* 0 */ {0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11}, -/* 1 */ {0x01,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x91,0x11,0x11,0x11,0x11}, -/* 2 */ {0x02,0x12,0x22,0x32,0x11,0x11,0x11,0x11,0x11,0x92,0xA2,0xB2,0x11,0x11}, -/* 3 */ {0x03,0x13,0x23,0x33,0x43,0x53,0x63,0x11,0x11,0x93,0xA3,0xB3,0xC3,0xD3}, -/* 4 */ {0x04,0x14,0x24,0x34,0x44,0x54,0x64,0x11,0x11,0x94,0xA4,0xB4,0xC4,0xD4}, -/* 5 */ {0x00,0x15,0x25,0x35,0x45,0x55,0x65,0x11,0x11,0x95,0xA5,0xB5,0xC5,0xD5}, -/* 6 */ {0x06,0x16,0x26,0x36,0x46,0x56,0x66,0x11,0x11,0x96,0xA6,0xB6,0xC6,0xD6}, -/* 7 */ {0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11}, -/* 8 */ {0x08,0x11,0x28,0x38,0x48,0x58,0x68,0x11,0x11,0x98,0xA8,0xB8,0xC8,0xD8}, -/* 9 */ {0x09,0x19,0x29,0x39,0x49,0x59,0x69,0x11,0x11,0x99,0xA9,0xB9,0xC9,0xD9} -}; - -#ifndef CM4000_DEBUG -#define xoutb outb -#define xinb inb -#else -static inline void xoutb(unsigned char val, unsigned short port) -{ - pr_debug("outb(val=%.2x,port=%.4x)\n", val, port); - outb(val, port); -} -static inline unsigned char xinb(unsigned short port) -{ - unsigned char val; - - val = inb(port); - pr_debug("%.2x=inb(%.4x)\n", val, port); - - return val; -} -#endif - -static inline unsigned char invert_revert(unsigned char ch) -{ - return bitrev8(~ch); -} - -static void str_invert_revert(unsigned char *b, int len) -{ - int i; - - for (i = 0; i < len; i++) - b[i] = invert_revert(b[i]); -} - -#define ATRLENCK(dev,pos) \ - if (pos>=dev->atr_len || pos>=MAX_ATR) \ - goto return_0; - -static unsigned int calc_baudv(unsigned char fidi) -{ - unsigned int wcrcf, wbrcf, fi_rfu, di_rfu; - - fi_rfu = 372; - di_rfu = 1; - - /* FI */ - switch ((fidi >> 4) & 0x0F) { - case 0x00: - wcrcf = 372; - break; - case 0x01: - wcrcf = 372; - break; - case 0x02: - wcrcf = 558; - break; - case 0x03: - wcrcf = 744; - break; - case 0x04: - wcrcf = 1116; - break; - case 0x05: - wcrcf = 1488; - break; - case 0x06: - wcrcf = 1860; - break; - case 0x07: - wcrcf = fi_rfu; - break; - case 0x08: - wcrcf = fi_rfu; - break; - case 0x09: - wcrcf = 512; - break; - case 0x0A: - wcrcf = 768; - break; - case 0x0B: - wcrcf = 1024; - break; - case 0x0C: - wcrcf = 1536; - break; - case 0x0D: - wcrcf = 2048; - break; - default: - wcrcf = fi_rfu; - break; - } - - /* DI */ - switch (fidi & 0x0F) { - case 0x00: - wbrcf = di_rfu; - break; - case 0x01: - wbrcf = 1; - break; - case 0x02: - wbrcf = 2; - break; - case 0x03: - wbrcf = 4; - break; - case 0x04: - wbrcf = 8; - break; - case 0x05: - wbrcf = 16; - break; - case 0x06: - wbrcf = 32; - break; - case 0x07: - wbrcf = di_rfu; - break; - case 0x08: - wbrcf = 12; - break; - case 0x09: - wbrcf = 20; - break; - default: - wbrcf = di_rfu; - break; - } - - return (wcrcf / wbrcf); -} - -static unsigned short io_read_num_rec_bytes(unsigned int iobase, - unsigned short *s) -{ - unsigned short tmp; - - tmp = *s = 0; - do { - *s = tmp; - tmp = inb(REG_NUM_BYTES(iobase)) | - (inb(REG_FLAGS0(iobase)) & 4 ? 0x100 : 0); - } while (tmp != *s); - - return *s; -} - -static int parse_atr(struct cm4000_dev *dev) -{ - unsigned char any_t1, any_t0; - unsigned char ch, ifno; - int ix, done; - - DEBUGP(3, dev, "-> parse_atr: dev->atr_len = %i\n", dev->atr_len); - - if (dev->atr_len < 3) { - DEBUGP(5, dev, "parse_atr: atr_len < 3\n"); - return 0; - } - - if (dev->atr[0] == 0x3f) - set_bit(IS_INVREV, &dev->flags); - else - clear_bit(IS_INVREV, &dev->flags); - ix = 1; - ifno = 1; - ch = dev->atr[1]; - dev->proto = 0; /* XXX PROTO */ - any_t1 = any_t0 = done = 0; - dev->ta1 = 0x11; /* defaults to 9600 baud */ - do { - if (ifno == 1 && (ch & 0x10)) { - /* read first interface byte and TA1 is present */ - dev->ta1 = dev->atr[2]; - DEBUGP(5, dev, "Card says FiDi is 0x%.2x\n", dev->ta1); - ifno++; - } else if ((ifno == 2) && (ch & 0x10)) { /* TA(2) */ - dev->ta1 = 0x11; - ifno++; - } - - DEBUGP(5, dev, "Yi=%.2x\n", ch & 0xf0); - ix += ((ch & 0x10) >> 4) /* no of int.face chars */ - +((ch & 0x20) >> 5) - + ((ch & 0x40) >> 6) - + ((ch & 0x80) >> 7); - /* ATRLENCK(dev,ix); */ - if (ch & 0x80) { /* TDi */ - ch = dev->atr[ix]; - if ((ch & 0x0f)) { - any_t1 = 1; - DEBUGP(5, dev, "card is capable of T=1\n"); - } else { - any_t0 = 1; - DEBUGP(5, dev, "card is capable of T=0\n"); - } - } else - done = 1; - } while (!done); - - DEBUGP(5, dev, "ix=%d noHist=%d any_t1=%d\n", - ix, dev->atr[1] & 15, any_t1); - if (ix + 1 + (dev->atr[1] & 0x0f) + any_t1 != dev->atr_len) { - DEBUGP(5, dev, "length error\n"); - return 0; - } - if (any_t0) - set_bit(IS_ANY_T0, &dev->flags); - - if (any_t1) { /* compute csum */ - dev->atr_csum = 0; -#ifdef ATR_CSUM - for (i = 1; i < dev->atr_len; i++) - dev->atr_csum ^= dev->atr[i]; - if (dev->atr_csum) { - set_bit(IS_BAD_CSUM, &dev->flags); - DEBUGP(5, dev, "bad checksum\n"); - goto return_0; - } -#endif - if (any_t0 == 0) - dev->proto = 1; /* XXX PROTO */ - set_bit(IS_ANY_T1, &dev->flags); - } - - return 1; -} - -struct card_fixup { - char atr[12]; - u_int8_t atr_len; - u_int8_t stopbits; -}; - -static struct card_fixup card_fixups[] = { - { /* ACOS */ - .atr = { 0x3b, 0xb3, 0x11, 0x00, 0x00, 0x41, 0x01 }, - .atr_len = 7, - .stopbits = 0x03, - }, - { /* Motorola */ - .atr = {0x3b, 0x76, 0x13, 0x00, 0x00, 0x80, 0x62, 0x07, - 0x41, 0x81, 0x81 }, - .atr_len = 11, - .stopbits = 0x04, - }, -}; - -static void set_cardparameter(struct cm4000_dev *dev) -{ - int i; - unsigned int iobase = dev->p_dev->resource[0]->start; - u_int8_t stopbits = 0x02; /* ISO default */ - - DEBUGP(3, dev, "-> set_cardparameter\n"); - - dev->flags1 = dev->flags1 | (((dev->baudv - 1) & 0x0100) >> 8); - xoutb(dev->flags1, REG_FLAGS1(iobase)); - DEBUGP(5, dev, "flags1 = 0x%02x\n", dev->flags1); - - /* set baudrate */ - xoutb((unsigned char)((dev->baudv - 1) & 0xFF), REG_BAUDRATE(iobase)); - - DEBUGP(5, dev, "baudv = %i -> write 0x%02x\n", dev->baudv, - ((dev->baudv - 1) & 0xFF)); - - /* set stopbits */ - for (i = 0; i < ARRAY_SIZE(card_fixups); i++) { - if (!memcmp(dev->atr, card_fixups[i].atr, - card_fixups[i].atr_len)) - stopbits = card_fixups[i].stopbits; - } - xoutb(stopbits, REG_STOPBITS(iobase)); - - DEBUGP(3, dev, "<- set_cardparameter\n"); -} - -static int set_protocol(struct cm4000_dev *dev, struct ptsreq *ptsreq) -{ - - unsigned long tmp, i; - unsigned short num_bytes_read; - unsigned char pts_reply[4]; - ssize_t rc; - unsigned int iobase = dev->p_dev->resource[0]->start; - - rc = 0; - - DEBUGP(3, dev, "-> set_protocol\n"); - DEBUGP(5, dev, "ptsreq->Protocol = 0x%.8x, ptsreq->Flags=0x%.8x, " - "ptsreq->pts1=0x%.2x, ptsreq->pts2=0x%.2x, " - "ptsreq->pts3=0x%.2x\n", (unsigned int)ptsreq->protocol, - (unsigned int)ptsreq->flags, ptsreq->pts1, ptsreq->pts2, - ptsreq->pts3); - - /* Fill PTS structure */ - dev->pts[0] = 0xff; - dev->pts[1] = 0x00; - tmp = ptsreq->protocol; - while ((tmp = (tmp >> 1)) > 0) - dev->pts[1]++; - dev->proto = dev->pts[1]; /* Set new protocol */ - dev->pts[1] = (0x01 << 4) | (dev->pts[1]); - - /* Correct Fi/Di according to CM4000 Fi/Di table */ - DEBUGP(5, dev, "Ta(1) from ATR is 0x%.2x\n", dev->ta1); - /* set Fi/Di according to ATR TA(1) */ - dev->pts[2] = fi_di_table[dev->ta1 & 0x0F][(dev->ta1 >> 4) & 0x0F]; - - /* Calculate PCK character */ - dev->pts[3] = dev->pts[0] ^ dev->pts[1] ^ dev->pts[2]; - - DEBUGP(5, dev, "pts0=%.2x, pts1=%.2x, pts2=%.2x, pts3=%.2x\n", - dev->pts[0], dev->pts[1], dev->pts[2], dev->pts[3]); - - /* check card convention */ - if (test_bit(IS_INVREV, &dev->flags)) - str_invert_revert(dev->pts, 4); - - /* reset SM */ - xoutb(0x80, REG_FLAGS0(iobase)); - - /* Enable access to the message buffer */ - DEBUGP(5, dev, "Enable access to the messages buffer\n"); - dev->flags1 = 0x20 /* T_Active */ - | (test_bit(IS_INVREV, &dev->flags) ? 0x02 : 0x00) /* inv parity */ - | ((dev->baudv >> 8) & 0x01); /* MSB-baud */ - xoutb(dev->flags1, REG_FLAGS1(iobase)); - - DEBUGP(5, dev, "Enable message buffer -> flags1 = 0x%.2x\n", - dev->flags1); - - /* write challenge to the buffer */ - DEBUGP(5, dev, "Write challenge to buffer: "); - for (i = 0; i < 4; i++) { - xoutb(i, REG_BUF_ADDR(iobase)); - xoutb(dev->pts[i], REG_BUF_DATA(iobase)); /* buf data */ -#ifdef CM4000_DEBUG - pr_debug("0x%.2x ", dev->pts[i]); - } - pr_debug("\n"); -#else - } -#endif - - /* set number of bytes to write */ - DEBUGP(5, dev, "Set number of bytes to write\n"); - xoutb(0x04, REG_NUM_SEND(iobase)); - - /* Trigger CARDMAN CONTROLLER */ - xoutb(0x50, REG_FLAGS0(iobase)); - - /* Monitor progress */ - /* wait for xmit done */ - DEBUGP(5, dev, "Waiting for NumRecBytes getting valid\n"); - - for (i = 0; i < 100; i++) { - if (inb(REG_FLAGS0(iobase)) & 0x08) { - DEBUGP(5, dev, "NumRecBytes is valid\n"); - break; - } - /* can not sleep as this is in atomic context */ - mdelay(10); - } - if (i == 100) { - DEBUGP(5, dev, "Timeout waiting for NumRecBytes getting " - "valid\n"); - rc = -EIO; - goto exit_setprotocol; - } - - DEBUGP(5, dev, "Reading NumRecBytes\n"); - for (i = 0; i < 100; i++) { - io_read_num_rec_bytes(iobase, &num_bytes_read); - if (num_bytes_read >= 4) { - DEBUGP(2, dev, "NumRecBytes = %i\n", num_bytes_read); - if (num_bytes_read > 4) { - rc = -EIO; - goto exit_setprotocol; - } - break; - } - /* can not sleep as this is in atomic context */ - mdelay(10); - } - - /* check whether it is a short PTS reply? */ - if (num_bytes_read == 3) - i = 0; - - if (i == 100) { - DEBUGP(5, dev, "Timeout reading num_bytes_read\n"); - rc = -EIO; - goto exit_setprotocol; - } - - DEBUGP(5, dev, "Reset the CARDMAN CONTROLLER\n"); - xoutb(0x80, REG_FLAGS0(iobase)); - - /* Read PPS reply */ - DEBUGP(5, dev, "Read PPS reply\n"); - for (i = 0; i < num_bytes_read; i++) { - xoutb(i, REG_BUF_ADDR(iobase)); - pts_reply[i] = inb(REG_BUF_DATA(iobase)); - } - -#ifdef CM4000_DEBUG - DEBUGP(2, dev, "PTSreply: "); - for (i = 0; i < num_bytes_read; i++) { - pr_debug("0x%.2x ", pts_reply[i]); - } - pr_debug("\n"); -#endif /* CM4000_DEBUG */ - - DEBUGP(5, dev, "Clear Tactive in Flags1\n"); - xoutb(0x20, REG_FLAGS1(iobase)); - - /* Compare ptsreq and ptsreply */ - if ((dev->pts[0] == pts_reply[0]) && - (dev->pts[1] == pts_reply[1]) && - (dev->pts[2] == pts_reply[2]) && (dev->pts[3] == pts_reply[3])) { - /* setcardparameter according to PPS */ - dev->baudv = calc_baudv(dev->pts[2]); - set_cardparameter(dev); - } else if ((dev->pts[0] == pts_reply[0]) && - ((dev->pts[1] & 0xef) == pts_reply[1]) && - ((pts_reply[0] ^ pts_reply[1]) == pts_reply[2])) { - /* short PTS reply, set card parameter to default values */ - dev->baudv = calc_baudv(0x11); - set_cardparameter(dev); - } else - rc = -EIO; - -exit_setprotocol: - DEBUGP(3, dev, "<- set_protocol\n"); - return rc; -} - -static int io_detect_cm4000(unsigned int iobase, struct cm4000_dev *dev) -{ - - /* note: statemachine is assumed to be reset */ - if (inb(REG_FLAGS0(iobase)) & 8) { - clear_bit(IS_ATR_VALID, &dev->flags); - set_bit(IS_CMM_ABSENT, &dev->flags); - return 0; /* detect CMM = 1 -> failure */ - } - /* xoutb(0x40, REG_FLAGS1(iobase)); detectCMM */ - xoutb(dev->flags1 | 0x40, REG_FLAGS1(iobase)); - if ((inb(REG_FLAGS0(iobase)) & 8) == 0) { - clear_bit(IS_ATR_VALID, &dev->flags); - set_bit(IS_CMM_ABSENT, &dev->flags); - return 0; /* detect CMM=0 -> failure */ - } - /* clear detectCMM again by restoring original flags1 */ - xoutb(dev->flags1, REG_FLAGS1(iobase)); - return 1; -} - -static void terminate_monitor(struct cm4000_dev *dev) -{ - - /* tell the monitor to stop and wait until - * it terminates. - */ - DEBUGP(3, dev, "-> terminate_monitor\n"); - wait_event_interruptible(dev->devq, - test_and_set_bit(LOCK_MONITOR, - (void *)&dev->flags)); - - /* now, LOCK_MONITOR has been set. - * allow a last cycle in the monitor. - * the monitor will indicate that it has - * finished by clearing this bit. - */ - DEBUGP(5, dev, "Now allow last cycle of monitor!\n"); - while (test_bit(LOCK_MONITOR, (void *)&dev->flags)) - msleep(25); - - DEBUGP(5, dev, "Delete timer\n"); - del_timer_sync(&dev->timer); -#ifdef CM4000_DEBUG - dev->monitor_running = 0; -#endif - - DEBUGP(3, dev, "<- terminate_monitor\n"); -} - -/* - * monitor the card every 50msec. as a side-effect, retrieve the - * atr once a card is inserted. another side-effect of retrieving the - * atr is that the card will be powered on, so there is no need to - * power on the card explicitly from the application: the driver - * is already doing that for you. - */ - -static void monitor_card(struct timer_list *t) -{ - struct cm4000_dev *dev = from_timer(dev, t, timer); - unsigned int iobase = dev->p_dev->resource[0]->start; - unsigned short s; - struct ptsreq ptsreq; - int i, atrc; - - DEBUGP(7, dev, "-> monitor_card\n"); - - /* if someone has set the lock for us: we're done! */ - if (test_and_set_bit(LOCK_MONITOR, &dev->flags)) { - DEBUGP(4, dev, "About to stop monitor\n"); - /* no */ - dev->rlen = - dev->rpos = - dev->atr_csum = dev->atr_len_retry = dev->cwarn = 0; - dev->mstate = M_FETCH_ATR; - clear_bit(LOCK_MONITOR, &dev->flags); - /* close et al. are sleeping on devq, so wake it */ - wake_up_interruptible(&dev->devq); - DEBUGP(2, dev, "<- monitor_card (we are done now)\n"); - return; - } - - /* try to lock io: if it is already locked, just add another timer */ - if (test_and_set_bit(LOCK_IO, (void *)&dev->flags)) { - DEBUGP(4, dev, "Couldn't get IO lock\n"); - goto return_with_timer; - } - - /* is a card/a reader inserted at all ? */ - dev->flags0 = xinb(REG_FLAGS0(iobase)); - DEBUGP(7, dev, "dev->flags0 = 0x%2x\n", dev->flags0); - DEBUGP(7, dev, "smartcard present: %s\n", - dev->flags0 & 1 ? "yes" : "no"); - DEBUGP(7, dev, "cardman present: %s\n", - dev->flags0 == 0xff ? "no" : "yes"); - - if ((dev->flags0 & 1) == 0 /* no smartcard inserted */ - || dev->flags0 == 0xff) { /* no cardman inserted */ - /* no */ - dev->rlen = - dev->rpos = - dev->atr_csum = dev->atr_len_retry = dev->cwarn = 0; - dev->mstate = M_FETCH_ATR; - - dev->flags &= 0x000000ff; /* only keep IO and MONITOR locks */ - - if (dev->flags0 == 0xff) { - DEBUGP(4, dev, "set IS_CMM_ABSENT bit\n"); - set_bit(IS_CMM_ABSENT, &dev->flags); - } else if (test_bit(IS_CMM_ABSENT, &dev->flags)) { - DEBUGP(4, dev, "clear IS_CMM_ABSENT bit " - "(card is removed)\n"); - clear_bit(IS_CMM_ABSENT, &dev->flags); - } - - goto release_io; - } else if ((dev->flags0 & 1) && test_bit(IS_CMM_ABSENT, &dev->flags)) { - /* cardman and card present but cardman was absent before - * (after suspend with inserted card) */ - DEBUGP(4, dev, "clear IS_CMM_ABSENT bit (card is inserted)\n"); - clear_bit(IS_CMM_ABSENT, &dev->flags); - } - - if (test_bit(IS_ATR_VALID, &dev->flags) == 1) { - DEBUGP(7, dev, "believe ATR is already valid (do nothing)\n"); - goto release_io; - } - - switch (dev->mstate) { - case M_CARDOFF: { - unsigned char flags0; - - DEBUGP(4, dev, "M_CARDOFF\n"); - flags0 = inb(REG_FLAGS0(iobase)); - if (flags0 & 0x02) { - /* wait until Flags0 indicate power is off */ - dev->mdelay = T_10MSEC; - } else { - /* Flags0 indicate power off and no card inserted now; - * Reset CARDMAN CONTROLLER */ - xoutb(0x80, REG_FLAGS0(iobase)); - - /* prepare for fetching ATR again: after card off ATR - * is read again automatically */ - dev->rlen = - dev->rpos = - dev->atr_csum = - dev->atr_len_retry = dev->cwarn = 0; - dev->mstate = M_FETCH_ATR; - - /* minimal gap between CARDOFF and read ATR is 50msec */ - dev->mdelay = T_50MSEC; - } - break; - } - case M_FETCH_ATR: - DEBUGP(4, dev, "M_FETCH_ATR\n"); - xoutb(0x80, REG_FLAGS0(iobase)); - DEBUGP(4, dev, "Reset BAUDV to 9600\n"); - dev->baudv = 0x173; /* 9600 */ - xoutb(0x02, REG_STOPBITS(iobase)); /* stopbits=2 */ - xoutb(0x73, REG_BAUDRATE(iobase)); /* baud value */ - xoutb(0x21, REG_FLAGS1(iobase)); /* T_Active=1, baud - value */ - /* warm start vs. power on: */ - xoutb(dev->flags0 & 2 ? 0x46 : 0x44, REG_FLAGS0(iobase)); - dev->mdelay = T_40MSEC; - dev->mstate = M_TIMEOUT_WAIT; - break; - case M_TIMEOUT_WAIT: - DEBUGP(4, dev, "M_TIMEOUT_WAIT\n"); - /* numRecBytes */ - io_read_num_rec_bytes(iobase, &dev->atr_len); - dev->mdelay = T_10MSEC; - dev->mstate = M_READ_ATR_LEN; - break; - case M_READ_ATR_LEN: - DEBUGP(4, dev, "M_READ_ATR_LEN\n"); - /* infinite loop possible, since there is no timeout */ - -#define MAX_ATR_LEN_RETRY 100 - - if (dev->atr_len == io_read_num_rec_bytes(iobase, &s)) { - if (dev->atr_len_retry++ >= MAX_ATR_LEN_RETRY) { /* + XX msec */ - dev->mdelay = T_10MSEC; - dev->mstate = M_READ_ATR; - } - } else { - dev->atr_len = s; - dev->atr_len_retry = 0; /* set new timeout */ - } - - DEBUGP(4, dev, "Current ATR_LEN = %i\n", dev->atr_len); - break; - case M_READ_ATR: - DEBUGP(4, dev, "M_READ_ATR\n"); - xoutb(0x80, REG_FLAGS0(iobase)); /* reset SM */ - for (i = 0; i < dev->atr_len; i++) { - xoutb(i, REG_BUF_ADDR(iobase)); - dev->atr[i] = inb(REG_BUF_DATA(iobase)); - } - /* Deactivate T_Active flags */ - DEBUGP(4, dev, "Deactivate T_Active flags\n"); - dev->flags1 = 0x01; - xoutb(dev->flags1, REG_FLAGS1(iobase)); - - /* atr is present (which doesn't mean it's valid) */ - set_bit(IS_ATR_PRESENT, &dev->flags); - if (dev->atr[0] == 0x03) - str_invert_revert(dev->atr, dev->atr_len); - atrc = parse_atr(dev); - if (atrc == 0) { /* atr invalid */ - dev->mdelay = 0; - dev->mstate = M_BAD_CARD; - } else { - dev->mdelay = T_50MSEC; - dev->mstate = M_ATR_PRESENT; - set_bit(IS_ATR_VALID, &dev->flags); - } - - if (test_bit(IS_ATR_VALID, &dev->flags) == 1) { - DEBUGP(4, dev, "monitor_card: ATR valid\n"); - /* if ta1 == 0x11, no PPS necessary (default values) */ - /* do not do PPS with multi protocol cards */ - if ((test_bit(IS_AUTOPPS_ACT, &dev->flags) == 0) && - (dev->ta1 != 0x11) && - !(test_bit(IS_ANY_T0, &dev->flags) && - test_bit(IS_ANY_T1, &dev->flags))) { - DEBUGP(4, dev, "Perform AUTOPPS\n"); - set_bit(IS_AUTOPPS_ACT, &dev->flags); - ptsreq.protocol = (0x01 << dev->proto); - ptsreq.flags = 0x01; - ptsreq.pts1 = 0x00; - ptsreq.pts2 = 0x00; - ptsreq.pts3 = 0x00; - if (set_protocol(dev, &ptsreq) == 0) { - DEBUGP(4, dev, "AUTOPPS ret SUCC\n"); - clear_bit(IS_AUTOPPS_ACT, &dev->flags); - wake_up_interruptible(&dev->atrq); - } else { - DEBUGP(4, dev, "AUTOPPS failed: " - "repower using defaults\n"); - /* prepare for repowering */ - clear_bit(IS_ATR_PRESENT, &dev->flags); - clear_bit(IS_ATR_VALID, &dev->flags); - dev->rlen = - dev->rpos = - dev->atr_csum = - dev->atr_len_retry = dev->cwarn = 0; - dev->mstate = M_FETCH_ATR; - - dev->mdelay = T_50MSEC; - } - } else { - /* for cards which use slightly different - * params (extra guard time) */ - set_cardparameter(dev); - if (test_bit(IS_AUTOPPS_ACT, &dev->flags) == 1) - DEBUGP(4, dev, "AUTOPPS already active " - "2nd try:use default values\n"); - if (dev->ta1 == 0x11) - DEBUGP(4, dev, "No AUTOPPS necessary " - "TA(1)==0x11\n"); - if (test_bit(IS_ANY_T0, &dev->flags) - && test_bit(IS_ANY_T1, &dev->flags)) - DEBUGP(4, dev, "Do NOT perform AUTOPPS " - "with multiprotocol cards\n"); - clear_bit(IS_AUTOPPS_ACT, &dev->flags); - wake_up_interruptible(&dev->atrq); - } - } else { - DEBUGP(4, dev, "ATR invalid\n"); - wake_up_interruptible(&dev->atrq); - } - break; - case M_BAD_CARD: - DEBUGP(4, dev, "M_BAD_CARD\n"); - /* slow down warning, but prompt immediately after insertion */ - if (dev->cwarn == 0 || dev->cwarn == 10) { - set_bit(IS_BAD_CARD, &dev->flags); - dev_warn(&dev->p_dev->dev, MODULE_NAME ": "); - if (test_bit(IS_BAD_CSUM, &dev->flags)) { - DEBUGP(4, dev, "ATR checksum (0x%.2x, should " - "be zero) failed\n", dev->atr_csum); - } -#ifdef CM4000_DEBUG - else if (test_bit(IS_BAD_LENGTH, &dev->flags)) { - DEBUGP(4, dev, "ATR length error\n"); - } else { - DEBUGP(4, dev, "card damaged or wrong way " - "inserted\n"); - } -#endif - dev->cwarn = 0; - wake_up_interruptible(&dev->atrq); /* wake open */ - } - dev->cwarn++; - dev->mdelay = T_100MSEC; - dev->mstate = M_FETCH_ATR; - break; - default: - DEBUGP(7, dev, "Unknown action\n"); - break; /* nothing */ - } - -release_io: - DEBUGP(7, dev, "release_io\n"); - clear_bit(LOCK_IO, &dev->flags); - wake_up_interruptible(&dev->ioq); /* whoever needs IO */ - -return_with_timer: - DEBUGP(7, dev, "<- monitor_card (returns with timer)\n"); - mod_timer(&dev->timer, jiffies + dev->mdelay); - clear_bit(LOCK_MONITOR, &dev->flags); -} - -/* Interface to userland (file_operations) */ - -static ssize_t cmm_read(struct file *filp, __user char *buf, size_t count, - loff_t *ppos) -{ - struct cm4000_dev *dev = filp->private_data; - unsigned int iobase = dev->p_dev->resource[0]->start; - ssize_t rc; - int i, j, k; - - DEBUGP(2, dev, "-> cmm_read(%s,%d)\n", current->comm, current->pid); - - if (count == 0) /* according to manpage */ - return 0; - - if (!pcmcia_dev_present(dev->p_dev) || /* device removed */ - test_bit(IS_CMM_ABSENT, &dev->flags)) - return -ENODEV; - - if (test_bit(IS_BAD_CSUM, &dev->flags)) - return -EIO; - - /* also see the note about this in cmm_write */ - if (wait_event_interruptible - (dev->atrq, - ((filp->f_flags & O_NONBLOCK) - || (test_bit(IS_ATR_PRESENT, (void *)&dev->flags) != 0)))) { - if (filp->f_flags & O_NONBLOCK) - return -EAGAIN; - return -ERESTARTSYS; - } - - if (test_bit(IS_ATR_VALID, &dev->flags) == 0) - return -EIO; - - /* this one implements blocking IO */ - if (wait_event_interruptible - (dev->readq, - ((filp->f_flags & O_NONBLOCK) || (dev->rpos < dev->rlen)))) { - if (filp->f_flags & O_NONBLOCK) - return -EAGAIN; - return -ERESTARTSYS; - } - - /* lock io */ - if (wait_event_interruptible - (dev->ioq, - ((filp->f_flags & O_NONBLOCK) - || (test_and_set_bit(LOCK_IO, (void *)&dev->flags) == 0)))) { - if (filp->f_flags & O_NONBLOCK) - return -EAGAIN; - return -ERESTARTSYS; - } - - rc = 0; - dev->flags0 = inb(REG_FLAGS0(iobase)); - if ((dev->flags0 & 1) == 0 /* no smartcard inserted */ - || dev->flags0 == 0xff) { /* no cardman inserted */ - clear_bit(IS_ATR_VALID, &dev->flags); - if (dev->flags0 & 1) { - set_bit(IS_CMM_ABSENT, &dev->flags); - rc = -ENODEV; - } else { - rc = -EIO; - } - goto release_io; - } - - DEBUGP(4, dev, "begin read answer\n"); - j = min(count, (size_t)(dev->rlen - dev->rpos)); - k = dev->rpos; - if (k + j > 255) - j = 256 - k; - DEBUGP(4, dev, "read1 j=%d\n", j); - for (i = 0; i < j; i++) { - xoutb(k++, REG_BUF_ADDR(iobase)); - dev->rbuf[i] = xinb(REG_BUF_DATA(iobase)); - } - j = min(count, (size_t)(dev->rlen - dev->rpos)); - if (k + j > 255) { - DEBUGP(4, dev, "read2 j=%d\n", j); - dev->flags1 |= 0x10; /* MSB buf addr set */ - xoutb(dev->flags1, REG_FLAGS1(iobase)); - for (; i < j; i++) { - xoutb(k++, REG_BUF_ADDR(iobase)); - dev->rbuf[i] = xinb(REG_BUF_DATA(iobase)); - } - } - - if (dev->proto == 0 && count > dev->rlen - dev->rpos && i) { - DEBUGP(4, dev, "T=0 and count > buffer\n"); - dev->rbuf[i] = dev->rbuf[i - 1]; - dev->rbuf[i - 1] = dev->procbyte; - j++; - } - count = j; - - dev->rpos = dev->rlen + 1; - - /* Clear T1Active */ - DEBUGP(4, dev, "Clear T1Active\n"); - dev->flags1 &= 0xdf; - xoutb(dev->flags1, REG_FLAGS1(iobase)); - - xoutb(0, REG_FLAGS1(iobase)); /* clear detectCMM */ - /* last check before exit */ - if (!io_detect_cm4000(iobase, dev)) { - rc = -ENODEV; - goto release_io; - } - - if (test_bit(IS_INVREV, &dev->flags) && count > 0) - str_invert_revert(dev->rbuf, count); - - if (copy_to_user(buf, dev->rbuf, count)) - rc = -EFAULT; - -release_io: - clear_bit(LOCK_IO, &dev->flags); - wake_up_interruptible(&dev->ioq); - - DEBUGP(2, dev, "<- cmm_read returns: rc = %zi\n", - (rc < 0 ? rc : count)); - return rc < 0 ? rc : count; -} - -static ssize_t cmm_write(struct file *filp, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct cm4000_dev *dev = filp->private_data; - unsigned int iobase = dev->p_dev->resource[0]->start; - unsigned short s; - unsigned char infolen; - unsigned char sendT0; - unsigned short nsend; - unsigned short nr; - ssize_t rc; - int i; - - DEBUGP(2, dev, "-> cmm_write(%s,%d)\n", current->comm, current->pid); - - if (count == 0) /* according to manpage */ - return 0; - - if (dev->proto == 0 && count < 4) { - /* T0 must have at least 4 bytes */ - DEBUGP(4, dev, "T0 short write\n"); - return -EIO; - } - - nr = count & 0x1ff; /* max bytes to write */ - - sendT0 = dev->proto ? 0 : nr > 5 ? 0x08 : 0; - - if (!pcmcia_dev_present(dev->p_dev) || /* device removed */ - test_bit(IS_CMM_ABSENT, &dev->flags)) - return -ENODEV; - - if (test_bit(IS_BAD_CSUM, &dev->flags)) { - DEBUGP(4, dev, "bad csum\n"); - return -EIO; - } - - /* - * wait for atr to become valid. - * note: it is important to lock this code. if we dont, the monitor - * could be run between test_bit and the call to sleep on the - * atr-queue. if *then* the monitor detects atr valid, it will wake up - * any process on the atr-queue, *but* since we have been interrupted, - * we do not yet sleep on this queue. this would result in a missed - * wake_up and the calling process would sleep forever (until - * interrupted). also, do *not* restore_flags before sleep_on, because - * this could result in the same situation! - */ - if (wait_event_interruptible - (dev->atrq, - ((filp->f_flags & O_NONBLOCK) - || (test_bit(IS_ATR_PRESENT, (void *)&dev->flags) != 0)))) { - if (filp->f_flags & O_NONBLOCK) - return -EAGAIN; - return -ERESTARTSYS; - } - - if (test_bit(IS_ATR_VALID, &dev->flags) == 0) { /* invalid atr */ - DEBUGP(4, dev, "invalid ATR\n"); - return -EIO; - } - - /* lock io */ - if (wait_event_interruptible - (dev->ioq, - ((filp->f_flags & O_NONBLOCK) - || (test_and_set_bit(LOCK_IO, (void *)&dev->flags) == 0)))) { - if (filp->f_flags & O_NONBLOCK) - return -EAGAIN; - return -ERESTARTSYS; - } - - if (copy_from_user(dev->sbuf, buf, ((count > 512) ? 512 : count))) - return -EFAULT; - - rc = 0; - dev->flags0 = inb(REG_FLAGS0(iobase)); - if ((dev->flags0 & 1) == 0 /* no smartcard inserted */ - || dev->flags0 == 0xff) { /* no cardman inserted */ - clear_bit(IS_ATR_VALID, &dev->flags); - if (dev->flags0 & 1) { - set_bit(IS_CMM_ABSENT, &dev->flags); - rc = -ENODEV; - } else { - DEBUGP(4, dev, "IO error\n"); - rc = -EIO; - } - goto release_io; - } - - xoutb(0x80, REG_FLAGS0(iobase)); /* reset SM */ - - if (!io_detect_cm4000(iobase, dev)) { - rc = -ENODEV; - goto release_io; - } - - /* reflect T=0 send/read mode in flags1 */ - dev->flags1 |= (sendT0); - - set_cardparameter(dev); - - /* dummy read, reset flag procedure received */ - inb(REG_FLAGS1(iobase)); - - dev->flags1 = 0x20 /* T_Active */ - | (sendT0) - | (test_bit(IS_INVREV, &dev->flags) ? 2 : 0)/* inverse parity */ - | (((dev->baudv - 1) & 0x0100) >> 8); /* MSB-Baud */ - DEBUGP(1, dev, "set dev->flags1 = 0x%.2x\n", dev->flags1); - xoutb(dev->flags1, REG_FLAGS1(iobase)); - - /* xmit data */ - DEBUGP(4, dev, "Xmit data\n"); - for (i = 0; i < nr; i++) { - if (i >= 256) { - dev->flags1 = 0x20 /* T_Active */ - | (sendT0) /* SendT0 */ - /* inverse parity: */ - | (test_bit(IS_INVREV, &dev->flags) ? 2 : 0) - | (((dev->baudv - 1) & 0x0100) >> 8) /* MSB-Baud */ - | 0x10; /* set address high */ - DEBUGP(4, dev, "dev->flags = 0x%.2x - set address " - "high\n", dev->flags1); - xoutb(dev->flags1, REG_FLAGS1(iobase)); - } - if (test_bit(IS_INVREV, &dev->flags)) { - DEBUGP(4, dev, "Apply inverse convention for 0x%.2x " - "-> 0x%.2x\n", (unsigned char)dev->sbuf[i], - invert_revert(dev->sbuf[i])); - xoutb(i, REG_BUF_ADDR(iobase)); - xoutb(invert_revert(dev->sbuf[i]), - REG_BUF_DATA(iobase)); - } else { - xoutb(i, REG_BUF_ADDR(iobase)); - xoutb(dev->sbuf[i], REG_BUF_DATA(iobase)); - } - } - DEBUGP(4, dev, "Xmit done\n"); - - if (dev->proto == 0) { - /* T=0 proto: 0 byte reply */ - if (nr == 4) { - DEBUGP(4, dev, "T=0 assumes 0 byte reply\n"); - xoutb(i, REG_BUF_ADDR(iobase)); - if (test_bit(IS_INVREV, &dev->flags)) - xoutb(0xff, REG_BUF_DATA(iobase)); - else - xoutb(0x00, REG_BUF_DATA(iobase)); - } - - /* numSendBytes */ - if (sendT0) - nsend = nr; - else { - if (nr == 4) - nsend = 5; - else { - nsend = 5 + (unsigned char)dev->sbuf[4]; - if (dev->sbuf[4] == 0) - nsend += 0x100; - } - } - } else - nsend = nr; - - /* T0: output procedure byte */ - if (test_bit(IS_INVREV, &dev->flags)) { - DEBUGP(4, dev, "T=0 set Procedure byte (inverse-reverse) " - "0x%.2x\n", invert_revert(dev->sbuf[1])); - xoutb(invert_revert(dev->sbuf[1]), REG_NUM_BYTES(iobase)); - } else { - DEBUGP(4, dev, "T=0 set Procedure byte 0x%.2x\n", dev->sbuf[1]); - xoutb(dev->sbuf[1], REG_NUM_BYTES(iobase)); - } - - DEBUGP(1, dev, "set NumSendBytes = 0x%.2x\n", - (unsigned char)(nsend & 0xff)); - xoutb((unsigned char)(nsend & 0xff), REG_NUM_SEND(iobase)); - - DEBUGP(1, dev, "Trigger CARDMAN CONTROLLER (0x%.2x)\n", - 0x40 /* SM_Active */ - | (dev->flags0 & 2 ? 0 : 4) /* power on if needed */ - |(dev->proto ? 0x10 : 0x08) /* T=1/T=0 */ - |(nsend & 0x100) >> 8 /* MSB numSendBytes */ ); - xoutb(0x40 /* SM_Active */ - | (dev->flags0 & 2 ? 0 : 4) /* power on if needed */ - |(dev->proto ? 0x10 : 0x08) /* T=1/T=0 */ - |(nsend & 0x100) >> 8, /* MSB numSendBytes */ - REG_FLAGS0(iobase)); - - /* wait for xmit done */ - if (dev->proto == 1) { - DEBUGP(4, dev, "Wait for xmit done\n"); - for (i = 0; i < 1000; i++) { - if (inb(REG_FLAGS0(iobase)) & 0x08) - break; - msleep_interruptible(10); - } - if (i == 1000) { - DEBUGP(4, dev, "timeout waiting for xmit done\n"); - rc = -EIO; - goto release_io; - } - } - - /* T=1: wait for infoLen */ - - infolen = 0; - if (dev->proto) { - /* wait until infoLen is valid */ - for (i = 0; i < 6000; i++) { /* max waiting time of 1 min */ - io_read_num_rec_bytes(iobase, &s); - if (s >= 3) { - infolen = inb(REG_FLAGS1(iobase)); - DEBUGP(4, dev, "infolen=%d\n", infolen); - break; - } - msleep_interruptible(10); - } - if (i == 6000) { - DEBUGP(4, dev, "timeout waiting for infoLen\n"); - rc = -EIO; - goto release_io; - } - } else - clear_bit(IS_PROCBYTE_PRESENT, &dev->flags); - - /* numRecBytes | bit9 of numRecytes */ - io_read_num_rec_bytes(iobase, &dev->rlen); - for (i = 0; i < 600; i++) { /* max waiting time of 2 sec */ - if (dev->proto) { - if (dev->rlen >= infolen + 4) - break; - } - msleep_interruptible(10); - /* numRecBytes | bit9 of numRecytes */ - io_read_num_rec_bytes(iobase, &s); - if (s > dev->rlen) { - DEBUGP(1, dev, "NumRecBytes inc (reset timeout)\n"); - i = 0; /* reset timeout */ - dev->rlen = s; - } - /* T=0: we are done when numRecBytes doesn't - * increment any more and NoProcedureByte - * is set and numRecBytes == bytes sent + 6 - * (header bytes + data + 1 for sw2) - * except when the card replies an error - * which means, no data will be sent back. - */ - else if (dev->proto == 0) { - if ((inb(REG_BUF_ADDR(iobase)) & 0x80)) { - /* no procedure byte received since last read */ - DEBUGP(1, dev, "NoProcedure byte set\n"); - /* i=0; */ - } else { - /* procedure byte received since last read */ - DEBUGP(1, dev, "NoProcedure byte unset " - "(reset timeout)\n"); - dev->procbyte = inb(REG_FLAGS1(iobase)); - DEBUGP(1, dev, "Read procedure byte 0x%.2x\n", - dev->procbyte); - i = 0; /* resettimeout */ - } - if (inb(REG_FLAGS0(iobase)) & 0x08) { - DEBUGP(1, dev, "T0Done flag (read reply)\n"); - break; - } - } - if (dev->proto) - infolen = inb(REG_FLAGS1(iobase)); - } - if (i == 600) { - DEBUGP(1, dev, "timeout waiting for numRecBytes\n"); - rc = -EIO; - goto release_io; - } else { - if (dev->proto == 0) { - DEBUGP(1, dev, "Wait for T0Done bit to be set\n"); - for (i = 0; i < 1000; i++) { - if (inb(REG_FLAGS0(iobase)) & 0x08) - break; - msleep_interruptible(10); - } - if (i == 1000) { - DEBUGP(1, dev, "timeout waiting for T0Done\n"); - rc = -EIO; - goto release_io; - } - - dev->procbyte = inb(REG_FLAGS1(iobase)); - DEBUGP(4, dev, "Read procedure byte 0x%.2x\n", - dev->procbyte); - - io_read_num_rec_bytes(iobase, &dev->rlen); - DEBUGP(4, dev, "Read NumRecBytes = %i\n", dev->rlen); - - } - } - /* T=1: read offset=zero, T=0: read offset=after challenge */ - dev->rpos = dev->proto ? 0 : nr == 4 ? 5 : nr > dev->rlen ? 5 : nr; - DEBUGP(4, dev, "dev->rlen = %i, dev->rpos = %i, nr = %i\n", - dev->rlen, dev->rpos, nr); - -release_io: - DEBUGP(4, dev, "Reset SM\n"); - xoutb(0x80, REG_FLAGS0(iobase)); /* reset SM */ - - if (rc < 0) { - DEBUGP(4, dev, "Write failed but clear T_Active\n"); - dev->flags1 &= 0xdf; - xoutb(dev->flags1, REG_FLAGS1(iobase)); - } - - clear_bit(LOCK_IO, &dev->flags); - wake_up_interruptible(&dev->ioq); - wake_up_interruptible(&dev->readq); /* tell read we have data */ - - /* ITSEC E2: clear write buffer */ - memset((char *)dev->sbuf, 0, 512); - - /* return error or actually written bytes */ - DEBUGP(2, dev, "<- cmm_write\n"); - return rc < 0 ? rc : nr; -} - -static void start_monitor(struct cm4000_dev *dev) -{ - DEBUGP(3, dev, "-> start_monitor\n"); - if (!dev->monitor_running) { - DEBUGP(5, dev, "create, init and add timer\n"); - timer_setup(&dev->timer, monitor_card, 0); - dev->monitor_running = 1; - mod_timer(&dev->timer, jiffies); - } else - DEBUGP(5, dev, "monitor already running\n"); - DEBUGP(3, dev, "<- start_monitor\n"); -} - -static void stop_monitor(struct cm4000_dev *dev) -{ - DEBUGP(3, dev, "-> stop_monitor\n"); - if (dev->monitor_running) { - DEBUGP(5, dev, "stopping monitor\n"); - terminate_monitor(dev); - /* reset monitor SM */ - clear_bit(IS_ATR_VALID, &dev->flags); - clear_bit(IS_ATR_PRESENT, &dev->flags); - } else - DEBUGP(5, dev, "monitor already stopped\n"); - DEBUGP(3, dev, "<- stop_monitor\n"); -} - -static long cmm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) -{ - struct cm4000_dev *dev = filp->private_data; - unsigned int iobase = dev->p_dev->resource[0]->start; - struct inode *inode = file_inode(filp); - struct pcmcia_device *link; - int rc; - void __user *argp = (void __user *)arg; -#ifdef CM4000_DEBUG - char *ioctl_names[CM_IOC_MAXNR + 1] = { - [_IOC_NR(CM_IOCGSTATUS)] "CM_IOCGSTATUS", - [_IOC_NR(CM_IOCGATR)] "CM_IOCGATR", - [_IOC_NR(CM_IOCARDOFF)] "CM_IOCARDOFF", - [_IOC_NR(CM_IOCSPTS)] "CM_IOCSPTS", - [_IOC_NR(CM_IOSDBGLVL)] "CM4000_DBGLVL", - }; - DEBUGP(3, dev, "cmm_ioctl(device=%d.%d) %s\n", imajor(inode), - iminor(inode), ioctl_names[_IOC_NR(cmd)]); -#endif - - mutex_lock(&cmm_mutex); - rc = -ENODEV; - link = dev_table[iminor(inode)]; - if (!pcmcia_dev_present(link)) { - DEBUGP(4, dev, "DEV_OK false\n"); - goto out; - } - - if (test_bit(IS_CMM_ABSENT, &dev->flags)) { - DEBUGP(4, dev, "CMM_ABSENT flag set\n"); - goto out; - } - rc = -EINVAL; - - if (_IOC_TYPE(cmd) != CM_IOC_MAGIC) { - DEBUGP(4, dev, "ioctype mismatch\n"); - goto out; - } - if (_IOC_NR(cmd) > CM_IOC_MAXNR) { - DEBUGP(4, dev, "iocnr mismatch\n"); - goto out; - } - rc = 0; - - switch (cmd) { - case CM_IOCGSTATUS: - DEBUGP(4, dev, " ... in CM_IOCGSTATUS\n"); - { - int status; - - /* clear other bits, but leave inserted & powered as - * they are */ - status = dev->flags0 & 3; - if (test_bit(IS_ATR_PRESENT, &dev->flags)) - status |= CM_ATR_PRESENT; - if (test_bit(IS_ATR_VALID, &dev->flags)) - status |= CM_ATR_VALID; - if (test_bit(IS_CMM_ABSENT, &dev->flags)) - status |= CM_NO_READER; - if (test_bit(IS_BAD_CARD, &dev->flags)) - status |= CM_BAD_CARD; - if (copy_to_user(argp, &status, sizeof(int))) - rc = -EFAULT; - } - break; - case CM_IOCGATR: - DEBUGP(4, dev, "... in CM_IOCGATR\n"); - { - struct atreq __user *atreq = argp; - int tmp; - /* allow nonblocking io and being interrupted */ - if (wait_event_interruptible - (dev->atrq, - ((filp->f_flags & O_NONBLOCK) - || (test_bit(IS_ATR_PRESENT, (void *)&dev->flags) - != 0)))) { - if (filp->f_flags & O_NONBLOCK) - rc = -EAGAIN; - else - rc = -ERESTARTSYS; - break; - } - - rc = -EFAULT; - if (test_bit(IS_ATR_VALID, &dev->flags) == 0) { - tmp = -1; - if (copy_to_user(&(atreq->atr_len), &tmp, - sizeof(int))) - break; - } else { - if (copy_to_user(atreq->atr, dev->atr, - dev->atr_len)) - break; - - tmp = dev->atr_len; - if (copy_to_user(&(atreq->atr_len), &tmp, sizeof(int))) - break; - } - rc = 0; - break; - } - case CM_IOCARDOFF: - -#ifdef CM4000_DEBUG - DEBUGP(4, dev, "... in CM_IOCARDOFF\n"); - if (dev->flags0 & 0x01) { - DEBUGP(4, dev, " Card inserted\n"); - } else { - DEBUGP(2, dev, " No card inserted\n"); - } - if (dev->flags0 & 0x02) { - DEBUGP(4, dev, " Card powered\n"); - } else { - DEBUGP(2, dev, " Card not powered\n"); - } -#endif - - /* is a card inserted and powered? */ - if ((dev->flags0 & 0x01) && (dev->flags0 & 0x02)) { - - /* get IO lock */ - if (wait_event_interruptible - (dev->ioq, - ((filp->f_flags & O_NONBLOCK) - || (test_and_set_bit(LOCK_IO, (void *)&dev->flags) - == 0)))) { - if (filp->f_flags & O_NONBLOCK) - rc = -EAGAIN; - else - rc = -ERESTARTSYS; - break; - } - /* Set Flags0 = 0x42 */ - DEBUGP(4, dev, "Set Flags0=0x42 \n"); - xoutb(0x42, REG_FLAGS0(iobase)); - clear_bit(IS_ATR_PRESENT, &dev->flags); - clear_bit(IS_ATR_VALID, &dev->flags); - dev->mstate = M_CARDOFF; - clear_bit(LOCK_IO, &dev->flags); - if (wait_event_interruptible - (dev->atrq, - ((filp->f_flags & O_NONBLOCK) - || (test_bit(IS_ATR_VALID, (void *)&dev->flags) != - 0)))) { - if (filp->f_flags & O_NONBLOCK) - rc = -EAGAIN; - else - rc = -ERESTARTSYS; - break; - } - } - /* release lock */ - clear_bit(LOCK_IO, &dev->flags); - wake_up_interruptible(&dev->ioq); - - rc = 0; - break; - case CM_IOCSPTS: - { - struct ptsreq krnptsreq; - - if (copy_from_user(&krnptsreq, argp, - sizeof(struct ptsreq))) { - rc = -EFAULT; - break; - } - - rc = 0; - DEBUGP(4, dev, "... in CM_IOCSPTS\n"); - /* wait for ATR to get valid */ - if (wait_event_interruptible - (dev->atrq, - ((filp->f_flags & O_NONBLOCK) - || (test_bit(IS_ATR_PRESENT, (void *)&dev->flags) - != 0)))) { - if (filp->f_flags & O_NONBLOCK) - rc = -EAGAIN; - else - rc = -ERESTARTSYS; - break; - } - /* get IO lock */ - if (wait_event_interruptible - (dev->ioq, - ((filp->f_flags & O_NONBLOCK) - || (test_and_set_bit(LOCK_IO, (void *)&dev->flags) - == 0)))) { - if (filp->f_flags & O_NONBLOCK) - rc = -EAGAIN; - else - rc = -ERESTARTSYS; - break; - } - - if ((rc = set_protocol(dev, &krnptsreq)) != 0) { - /* auto power_on again */ - dev->mstate = M_FETCH_ATR; - clear_bit(IS_ATR_VALID, &dev->flags); - } - /* release lock */ - clear_bit(LOCK_IO, &dev->flags); - wake_up_interruptible(&dev->ioq); - - } - break; -#ifdef CM4000_DEBUG - case CM_IOSDBGLVL: - rc = -ENOTTY; - break; -#endif - default: - DEBUGP(4, dev, "... in default (unknown IOCTL code)\n"); - rc = -ENOTTY; - } -out: - mutex_unlock(&cmm_mutex); - return rc; -} - -static int cmm_open(struct inode *inode, struct file *filp) -{ - struct cm4000_dev *dev; - struct pcmcia_device *link; - int minor = iminor(inode); - int ret; - - if (minor >= CM4000_MAX_DEV) - return -ENODEV; - - mutex_lock(&cmm_mutex); - link = dev_table[minor]; - if (link == NULL || !pcmcia_dev_present(link)) { - ret = -ENODEV; - goto out; - } - - if (link->open) { - ret = -EBUSY; - goto out; - } - - dev = link->priv; - filp->private_data = dev; - - DEBUGP(2, dev, "-> cmm_open(device=%d.%d process=%s,%d)\n", - imajor(inode), minor, current->comm, current->pid); - - /* init device variables, they may be "polluted" after close - * or, the device may never have been closed (i.e. open failed) - */ - - ZERO_DEV(dev); - - /* opening will always block since the - * monitor will be started by open, which - * means we have to wait for ATR becoming - * valid = block until valid (or card - * inserted) - */ - if (filp->f_flags & O_NONBLOCK) { - ret = -EAGAIN; - goto out; - } - - dev->mdelay = T_50MSEC; - - /* start monitoring the cardstatus */ - start_monitor(dev); - - link->open = 1; /* only one open per device */ - - DEBUGP(2, dev, "<- cmm_open\n"); - ret = stream_open(inode, filp); -out: - mutex_unlock(&cmm_mutex); - return ret; -} - -static int cmm_close(struct inode *inode, struct file *filp) -{ - struct cm4000_dev *dev; - struct pcmcia_device *link; - int minor = iminor(inode); - - if (minor >= CM4000_MAX_DEV) - return -ENODEV; - - link = dev_table[minor]; - if (link == NULL) - return -ENODEV; - - dev = link->priv; - - DEBUGP(2, dev, "-> cmm_close(maj/min=%d.%d)\n", - imajor(inode), minor); - - stop_monitor(dev); - - ZERO_DEV(dev); - - link->open = 0; /* only one open per device */ - wake_up(&dev->devq); /* socket removed? */ - - DEBUGP(2, dev, "cmm_close\n"); - return 0; -} - -static void cmm_cm4000_release(struct pcmcia_device * link) -{ - struct cm4000_dev *dev = link->priv; - - /* dont terminate the monitor, rather rely on - * close doing that for us. - */ - DEBUGP(3, dev, "-> cmm_cm4000_release\n"); - while (link->open) { - printk(KERN_INFO MODULE_NAME ": delaying release until " - "process has terminated\n"); - /* note: don't interrupt us: - * close the applications which own - * the devices _first_ ! - */ - wait_event(dev->devq, (link->open == 0)); - } - /* dev->devq=NULL; this cannot be zeroed earlier */ - DEBUGP(3, dev, "<- cmm_cm4000_release\n"); - return; -} - -/*==== Interface to PCMCIA Layer =======================================*/ - -static int cm4000_config_check(struct pcmcia_device *p_dev, void *priv_data) -{ - return pcmcia_request_io(p_dev); -} - -static int cm4000_config(struct pcmcia_device * link, int devno) -{ - link->config_flags |= CONF_AUTO_SET_IO; - - /* read the config-tuples */ - if (pcmcia_loop_config(link, cm4000_config_check, NULL)) - goto cs_release; - - if (pcmcia_enable_device(link)) - goto cs_release; - - return 0; - -cs_release: - cm4000_release(link); - return -ENODEV; -} - -static int cm4000_suspend(struct pcmcia_device *link) -{ - struct cm4000_dev *dev; - - dev = link->priv; - stop_monitor(dev); - - return 0; -} - -static int cm4000_resume(struct pcmcia_device *link) -{ - struct cm4000_dev *dev; - - dev = link->priv; - if (link->open) - start_monitor(dev); - - return 0; -} - -static void cm4000_release(struct pcmcia_device *link) -{ - cmm_cm4000_release(link); /* delay release until device closed */ - pcmcia_disable_device(link); -} - -static int cm4000_probe(struct pcmcia_device *link) -{ - struct cm4000_dev *dev; - int i, ret; - - for (i = 0; i < CM4000_MAX_DEV; i++) - if (dev_table[i] == NULL) - break; - - if (i == CM4000_MAX_DEV) { - printk(KERN_NOTICE MODULE_NAME ": all devices in use\n"); - return -ENODEV; - } - - /* create a new cm4000_cs device */ - dev = kzalloc(sizeof(struct cm4000_dev), GFP_KERNEL); - if (dev == NULL) - return -ENOMEM; - - dev->p_dev = link; - link->priv = dev; - dev_table[i] = link; - - init_waitqueue_head(&dev->devq); - init_waitqueue_head(&dev->ioq); - init_waitqueue_head(&dev->atrq); - init_waitqueue_head(&dev->readq); - - ret = cm4000_config(link, i); - if (ret) { - dev_table[i] = NULL; - kfree(dev); - return ret; - } - - device_create(cmm_class, NULL, MKDEV(major, i), NULL, "cmm%d", i); - - return 0; -} - -static void cm4000_detach(struct pcmcia_device *link) -{ - struct cm4000_dev *dev = link->priv; - int devno; - - /* find device */ - for (devno = 0; devno < CM4000_MAX_DEV; devno++) - if (dev_table[devno] == link) - break; - if (devno == CM4000_MAX_DEV) - return; - - stop_monitor(dev); - - cm4000_release(link); - - dev_table[devno] = NULL; - kfree(dev); - - device_destroy(cmm_class, MKDEV(major, devno)); - - return; -} - -static const struct file_operations cm4000_fops = { - .owner = THIS_MODULE, - .read = cmm_read, - .write = cmm_write, - .unlocked_ioctl = cmm_ioctl, - .open = cmm_open, - .release= cmm_close, - .llseek = no_llseek, -}; - -static const struct pcmcia_device_id cm4000_ids[] = { - PCMCIA_DEVICE_MANF_CARD(0x0223, 0x0002), - PCMCIA_DEVICE_PROD_ID12("CardMan", "4000", 0x2FB368CA, 0xA2BD8C39), - PCMCIA_DEVICE_NULL, -}; -MODULE_DEVICE_TABLE(pcmcia, cm4000_ids); - -static struct pcmcia_driver cm4000_driver = { - .owner = THIS_MODULE, - .name = "cm4000_cs", - .probe = cm4000_probe, - .remove = cm4000_detach, - .suspend = cm4000_suspend, - .resume = cm4000_resume, - .id_table = cm4000_ids, -}; - -static int __init cmm_init(void) -{ - int rc; - - cmm_class = class_create(THIS_MODULE, "cardman_4000"); - if (IS_ERR(cmm_class)) - return PTR_ERR(cmm_class); - - major = register_chrdev(0, DEVICE_NAME, &cm4000_fops); - if (major < 0) { - printk(KERN_WARNING MODULE_NAME - ": could not get major number\n"); - class_destroy(cmm_class); - return major; - } - - rc = pcmcia_register_driver(&cm4000_driver); - if (rc < 0) { - unregister_chrdev(major, DEVICE_NAME); - class_destroy(cmm_class); - return rc; - } - - return 0; -} - -static void __exit cmm_exit(void) -{ - pcmcia_unregister_driver(&cm4000_driver); - unregister_chrdev(major, DEVICE_NAME); - class_destroy(cmm_class); -}; - -module_init(cmm_init); -module_exit(cmm_exit); -MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/char/pcmcia/cm4040_cs.c b/drivers/char/pcmcia/cm4040_cs.c deleted file mode 100644 index 827711911da4..000000000000 --- a/drivers/char/pcmcia/cm4040_cs.c +++ /dev/null @@ -1,684 +0,0 @@ -/* - * A driver for the Omnikey PCMCIA smartcard reader CardMan 4040 - * - * (c) 2000-2004 Omnikey AG (http://www.omnikey.com/) - * - * (C) 2005-2006 Harald Welte - * - add support for poll() - * - driver cleanup - * - add waitqueues - * - adhere to linux kernel coding style and policies - * - support 2.6.13 "new style" pcmcia interface - * - add class interface for udev device creation - * - * The device basically is a USB CCID compliant device that has been - * attached to an I/O-Mapped FIFO. - * - * All rights reserved, Dual BSD/GPL Licensed. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "cm4040_cs.h" - - -#define reader_to_dev(x) (&x->p_dev->dev) - -/* n (debug level) is ignored */ -/* additional debug output may be enabled by re-compiling with - * CM4040_DEBUG set */ -/* #define CM4040_DEBUG */ -#define DEBUGP(n, rdr, x, args...) do { \ - dev_dbg(reader_to_dev(rdr), "%s:" x, \ - __func__ , ## args); \ - } while (0) - -static DEFINE_MUTEX(cm4040_mutex); - -#define CCID_DRIVER_BULK_DEFAULT_TIMEOUT (150*HZ) -#define CCID_DRIVER_ASYNC_POWERUP_TIMEOUT (35*HZ) -#define CCID_DRIVER_MINIMUM_TIMEOUT (3*HZ) -#define READ_WRITE_BUFFER_SIZE 512 -#define POLL_LOOP_COUNT 1000 - -/* how often to poll for fifo status change */ -#define POLL_PERIOD msecs_to_jiffies(10) - -static void reader_release(struct pcmcia_device *link); - -static int major; -static struct class *cmx_class; - -#define BS_READABLE 0x01 -#define BS_WRITABLE 0x02 - -struct reader_dev { - struct pcmcia_device *p_dev; - wait_queue_head_t devq; - wait_queue_head_t poll_wait; - wait_queue_head_t read_wait; - wait_queue_head_t write_wait; - unsigned long buffer_status; - unsigned long timeout; - unsigned char s_buf[READ_WRITE_BUFFER_SIZE]; - unsigned char r_buf[READ_WRITE_BUFFER_SIZE]; - struct timer_list poll_timer; -}; - -static struct pcmcia_device *dev_table[CM_MAX_DEV]; - -#ifndef CM4040_DEBUG -#define xoutb outb -#define xinb inb -#else -static inline void xoutb(unsigned char val, unsigned short port) -{ - pr_debug("outb(val=%.2x,port=%.4x)\n", val, port); - outb(val, port); -} - -static inline unsigned char xinb(unsigned short port) -{ - unsigned char val; - - val = inb(port); - pr_debug("%.2x=inb(%.4x)\n", val, port); - return val; -} -#endif - -/* poll the device fifo status register. not to be confused with - * the poll syscall. */ -static void cm4040_do_poll(struct timer_list *t) -{ - struct reader_dev *dev = from_timer(dev, t, poll_timer); - unsigned int obs = xinb(dev->p_dev->resource[0]->start - + REG_OFFSET_BUFFER_STATUS); - - if ((obs & BSR_BULK_IN_FULL)) { - set_bit(BS_READABLE, &dev->buffer_status); - DEBUGP(4, dev, "waking up read_wait\n"); - wake_up_interruptible(&dev->read_wait); - } else - clear_bit(BS_READABLE, &dev->buffer_status); - - if (!(obs & BSR_BULK_OUT_FULL)) { - set_bit(BS_WRITABLE, &dev->buffer_status); - DEBUGP(4, dev, "waking up write_wait\n"); - wake_up_interruptible(&dev->write_wait); - } else - clear_bit(BS_WRITABLE, &dev->buffer_status); - - if (dev->buffer_status) - wake_up_interruptible(&dev->poll_wait); - - mod_timer(&dev->poll_timer, jiffies + POLL_PERIOD); -} - -static void cm4040_stop_poll(struct reader_dev *dev) -{ - del_timer_sync(&dev->poll_timer); -} - -static int wait_for_bulk_out_ready(struct reader_dev *dev) -{ - int i, rc; - int iobase = dev->p_dev->resource[0]->start; - - for (i = 0; i < POLL_LOOP_COUNT; i++) { - if ((xinb(iobase + REG_OFFSET_BUFFER_STATUS) - & BSR_BULK_OUT_FULL) == 0) { - DEBUGP(4, dev, "BulkOut empty (i=%d)\n", i); - return 1; - } - } - - DEBUGP(4, dev, "wait_event_interruptible_timeout(timeout=%ld\n", - dev->timeout); - rc = wait_event_interruptible_timeout(dev->write_wait, - test_and_clear_bit(BS_WRITABLE, - &dev->buffer_status), - dev->timeout); - - if (rc > 0) - DEBUGP(4, dev, "woke up: BulkOut empty\n"); - else if (rc == 0) - DEBUGP(4, dev, "woke up: BulkOut full, returning 0 :(\n"); - else if (rc < 0) - DEBUGP(4, dev, "woke up: signal arrived\n"); - - return rc; -} - -/* Write to Sync Control Register */ -static int write_sync_reg(unsigned char val, struct reader_dev *dev) -{ - int iobase = dev->p_dev->resource[0]->start; - int rc; - - rc = wait_for_bulk_out_ready(dev); - if (rc <= 0) - return rc; - - xoutb(val, iobase + REG_OFFSET_SYNC_CONTROL); - rc = wait_for_bulk_out_ready(dev); - if (rc <= 0) - return rc; - - return 1; -} - -static int wait_for_bulk_in_ready(struct reader_dev *dev) -{ - int i, rc; - int iobase = dev->p_dev->resource[0]->start; - - for (i = 0; i < POLL_LOOP_COUNT; i++) { - if ((xinb(iobase + REG_OFFSET_BUFFER_STATUS) - & BSR_BULK_IN_FULL) == BSR_BULK_IN_FULL) { - DEBUGP(3, dev, "BulkIn full (i=%d)\n", i); - return 1; - } - } - - DEBUGP(4, dev, "wait_event_interruptible_timeout(timeout=%ld\n", - dev->timeout); - rc = wait_event_interruptible_timeout(dev->read_wait, - test_and_clear_bit(BS_READABLE, - &dev->buffer_status), - dev->timeout); - if (rc > 0) - DEBUGP(4, dev, "woke up: BulkIn full\n"); - else if (rc == 0) - DEBUGP(4, dev, "woke up: BulkIn not full, returning 0 :(\n"); - else if (rc < 0) - DEBUGP(4, dev, "woke up: signal arrived\n"); - - return rc; -} - -static ssize_t cm4040_read(struct file *filp, char __user *buf, - size_t count, loff_t *ppos) -{ - struct reader_dev *dev = filp->private_data; - int iobase = dev->p_dev->resource[0]->start; - size_t bytes_to_read; - unsigned long i; - size_t min_bytes_to_read; - int rc; - - DEBUGP(2, dev, "-> cm4040_read(%s,%d)\n", current->comm, current->pid); - - if (count == 0) - return 0; - - if (count < 10) - return -EFAULT; - - if (filp->f_flags & O_NONBLOCK) { - DEBUGP(4, dev, "filep->f_flags O_NONBLOCK set\n"); - DEBUGP(2, dev, "<- cm4040_read (failure)\n"); - return -EAGAIN; - } - - if (!pcmcia_dev_present(dev->p_dev)) - return -ENODEV; - - for (i = 0; i < 5; i++) { - rc = wait_for_bulk_in_ready(dev); - if (rc <= 0) { - DEBUGP(5, dev, "wait_for_bulk_in_ready rc=%.2x\n", rc); - DEBUGP(2, dev, "<- cm4040_read (failed)\n"); - if (rc == -ERESTARTSYS) - return rc; - return -EIO; - } - dev->r_buf[i] = xinb(iobase + REG_OFFSET_BULK_IN); -#ifdef CM4040_DEBUG - pr_debug("%lu:%2x ", i, dev->r_buf[i]); - } - pr_debug("\n"); -#else - } -#endif - - bytes_to_read = 5 + le32_to_cpu(*(__le32 *)&dev->r_buf[1]); - - DEBUGP(6, dev, "BytesToRead=%zu\n", bytes_to_read); - - min_bytes_to_read = min(count, bytes_to_read + 5); - min_bytes_to_read = min_t(size_t, min_bytes_to_read, READ_WRITE_BUFFER_SIZE); - - DEBUGP(6, dev, "Min=%zu\n", min_bytes_to_read); - - for (i = 0; i < (min_bytes_to_read-5); i++) { - rc = wait_for_bulk_in_ready(dev); - if (rc <= 0) { - DEBUGP(5, dev, "wait_for_bulk_in_ready rc=%.2x\n", rc); - DEBUGP(2, dev, "<- cm4040_read (failed)\n"); - if (rc == -ERESTARTSYS) - return rc; - return -EIO; - } - dev->r_buf[i+5] = xinb(iobase + REG_OFFSET_BULK_IN); -#ifdef CM4040_DEBUG - pr_debug("%lu:%2x ", i, dev->r_buf[i]); - } - pr_debug("\n"); -#else - } -#endif - - *ppos = min_bytes_to_read; - if (copy_to_user(buf, dev->r_buf, min_bytes_to_read)) - return -EFAULT; - - rc = wait_for_bulk_in_ready(dev); - if (rc <= 0) { - DEBUGP(5, dev, "wait_for_bulk_in_ready rc=%.2x\n", rc); - DEBUGP(2, dev, "<- cm4040_read (failed)\n"); - if (rc == -ERESTARTSYS) - return rc; - return -EIO; - } - - rc = write_sync_reg(SCR_READER_TO_HOST_DONE, dev); - if (rc <= 0) { - DEBUGP(5, dev, "write_sync_reg c=%.2x\n", rc); - DEBUGP(2, dev, "<- cm4040_read (failed)\n"); - if (rc == -ERESTARTSYS) - return rc; - else - return -EIO; - } - - xinb(iobase + REG_OFFSET_BULK_IN); - - DEBUGP(2, dev, "<- cm4040_read (successfully)\n"); - return min_bytes_to_read; -} - -static ssize_t cm4040_write(struct file *filp, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct reader_dev *dev = filp->private_data; - int iobase = dev->p_dev->resource[0]->start; - ssize_t rc; - int i; - unsigned int bytes_to_write; - - DEBUGP(2, dev, "-> cm4040_write(%s,%d)\n", current->comm, current->pid); - - if (count == 0) { - DEBUGP(2, dev, "<- cm4040_write empty read (successfully)\n"); - return 0; - } - - if ((count < 5) || (count > READ_WRITE_BUFFER_SIZE)) { - DEBUGP(2, dev, "<- cm4040_write buffersize=%zd < 5\n", count); - return -EIO; - } - - if (filp->f_flags & O_NONBLOCK) { - DEBUGP(4, dev, "filep->f_flags O_NONBLOCK set\n"); - DEBUGP(4, dev, "<- cm4040_write (failure)\n"); - return -EAGAIN; - } - - if (!pcmcia_dev_present(dev->p_dev)) - return -ENODEV; - - bytes_to_write = count; - if (copy_from_user(dev->s_buf, buf, bytes_to_write)) - return -EFAULT; - - switch (dev->s_buf[0]) { - case CMD_PC_TO_RDR_XFRBLOCK: - case CMD_PC_TO_RDR_SECURE: - case CMD_PC_TO_RDR_TEST_SECURE: - case CMD_PC_TO_RDR_OK_SECURE: - dev->timeout = CCID_DRIVER_BULK_DEFAULT_TIMEOUT; - break; - - case CMD_PC_TO_RDR_ICCPOWERON: - dev->timeout = CCID_DRIVER_ASYNC_POWERUP_TIMEOUT; - break; - - case CMD_PC_TO_RDR_GETSLOTSTATUS: - case CMD_PC_TO_RDR_ICCPOWEROFF: - case CMD_PC_TO_RDR_GETPARAMETERS: - case CMD_PC_TO_RDR_RESETPARAMETERS: - case CMD_PC_TO_RDR_SETPARAMETERS: - case CMD_PC_TO_RDR_ESCAPE: - case CMD_PC_TO_RDR_ICCCLOCK: - default: - dev->timeout = CCID_DRIVER_MINIMUM_TIMEOUT; - break; - } - - rc = write_sync_reg(SCR_HOST_TO_READER_START, dev); - if (rc <= 0) { - DEBUGP(5, dev, "write_sync_reg c=%.2zx\n", rc); - DEBUGP(2, dev, "<- cm4040_write (failed)\n"); - if (rc == -ERESTARTSYS) - return rc; - else - return -EIO; - } - - DEBUGP(4, dev, "start \n"); - - for (i = 0; i < bytes_to_write; i++) { - rc = wait_for_bulk_out_ready(dev); - if (rc <= 0) { - DEBUGP(5, dev, "wait_for_bulk_out_ready rc=%.2zx\n", - rc); - DEBUGP(2, dev, "<- cm4040_write (failed)\n"); - if (rc == -ERESTARTSYS) - return rc; - else - return -EIO; - } - - xoutb(dev->s_buf[i],iobase + REG_OFFSET_BULK_OUT); - } - DEBUGP(4, dev, "end\n"); - - rc = write_sync_reg(SCR_HOST_TO_READER_DONE, dev); - - if (rc <= 0) { - DEBUGP(5, dev, "write_sync_reg c=%.2zx\n", rc); - DEBUGP(2, dev, "<- cm4040_write (failed)\n"); - if (rc == -ERESTARTSYS) - return rc; - else - return -EIO; - } - - DEBUGP(2, dev, "<- cm4040_write (successfully)\n"); - return count; -} - -static __poll_t cm4040_poll(struct file *filp, poll_table *wait) -{ - struct reader_dev *dev = filp->private_data; - __poll_t mask = 0; - - poll_wait(filp, &dev->poll_wait, wait); - - if (test_and_clear_bit(BS_READABLE, &dev->buffer_status)) - mask |= EPOLLIN | EPOLLRDNORM; - if (test_and_clear_bit(BS_WRITABLE, &dev->buffer_status)) - mask |= EPOLLOUT | EPOLLWRNORM; - - DEBUGP(2, dev, "<- cm4040_poll(%u)\n", mask); - - return mask; -} - -static int cm4040_open(struct inode *inode, struct file *filp) -{ - struct reader_dev *dev; - struct pcmcia_device *link; - int minor = iminor(inode); - int ret; - - if (minor >= CM_MAX_DEV) - return -ENODEV; - - mutex_lock(&cm4040_mutex); - link = dev_table[minor]; - if (link == NULL || !pcmcia_dev_present(link)) { - ret = -ENODEV; - goto out; - } - - if (link->open) { - ret = -EBUSY; - goto out; - } - - dev = link->priv; - filp->private_data = dev; - - if (filp->f_flags & O_NONBLOCK) { - DEBUGP(4, dev, "filep->f_flags O_NONBLOCK set\n"); - ret = -EAGAIN; - goto out; - } - - link->open = 1; - - mod_timer(&dev->poll_timer, jiffies + POLL_PERIOD); - - DEBUGP(2, dev, "<- cm4040_open (successfully)\n"); - ret = nonseekable_open(inode, filp); -out: - mutex_unlock(&cm4040_mutex); - return ret; -} - -static int cm4040_close(struct inode *inode, struct file *filp) -{ - struct reader_dev *dev = filp->private_data; - struct pcmcia_device *link; - int minor = iminor(inode); - - DEBUGP(2, dev, "-> cm4040_close(maj/min=%d.%d)\n", imajor(inode), - iminor(inode)); - - if (minor >= CM_MAX_DEV) - return -ENODEV; - - link = dev_table[minor]; - if (link == NULL) - return -ENODEV; - - cm4040_stop_poll(dev); - - link->open = 0; - wake_up(&dev->devq); - - DEBUGP(2, dev, "<- cm4040_close\n"); - return 0; -} - -static void cm4040_reader_release(struct pcmcia_device *link) -{ - struct reader_dev *dev = link->priv; - - DEBUGP(3, dev, "-> cm4040_reader_release\n"); - while (link->open) { - DEBUGP(3, dev, MODULE_NAME ": delaying release " - "until process has terminated\n"); - wait_event(dev->devq, (link->open == 0)); - } - DEBUGP(3, dev, "<- cm4040_reader_release\n"); - return; -} - -static int cm4040_config_check(struct pcmcia_device *p_dev, void *priv_data) -{ - return pcmcia_request_io(p_dev); -} - - -static int reader_config(struct pcmcia_device *link, int devno) -{ - struct reader_dev *dev; - int fail_rc; - - link->config_flags |= CONF_AUTO_SET_IO; - - if (pcmcia_loop_config(link, cm4040_config_check, NULL)) - goto cs_release; - - fail_rc = pcmcia_enable_device(link); - if (fail_rc != 0) { - dev_info(&link->dev, "pcmcia_enable_device failed 0x%x\n", - fail_rc); - goto cs_release; - } - - dev = link->priv; - - DEBUGP(2, dev, "device " DEVICE_NAME "%d at %pR\n", devno, - link->resource[0]); - DEBUGP(2, dev, "<- reader_config (succ)\n"); - - return 0; - -cs_release: - reader_release(link); - return -ENODEV; -} - -static void reader_release(struct pcmcia_device *link) -{ - cm4040_reader_release(link); - pcmcia_disable_device(link); -} - -static int reader_probe(struct pcmcia_device *link) -{ - struct reader_dev *dev; - int i, ret; - - for (i = 0; i < CM_MAX_DEV; i++) { - if (dev_table[i] == NULL) - break; - } - - if (i == CM_MAX_DEV) - return -ENODEV; - - dev = kzalloc(sizeof(struct reader_dev), GFP_KERNEL); - if (dev == NULL) - return -ENOMEM; - - dev->timeout = CCID_DRIVER_MINIMUM_TIMEOUT; - dev->buffer_status = 0; - - link->priv = dev; - dev->p_dev = link; - - dev_table[i] = link; - - init_waitqueue_head(&dev->devq); - init_waitqueue_head(&dev->poll_wait); - init_waitqueue_head(&dev->read_wait); - init_waitqueue_head(&dev->write_wait); - timer_setup(&dev->poll_timer, cm4040_do_poll, 0); - - ret = reader_config(link, i); - if (ret) { - dev_table[i] = NULL; - kfree(dev); - return ret; - } - - device_create(cmx_class, NULL, MKDEV(major, i), NULL, "cmx%d", i); - - return 0; -} - -static void reader_detach(struct pcmcia_device *link) -{ - struct reader_dev *dev = link->priv; - int devno; - - /* find device */ - for (devno = 0; devno < CM_MAX_DEV; devno++) { - if (dev_table[devno] == link) - break; - } - if (devno == CM_MAX_DEV) - return; - - reader_release(link); - - dev_table[devno] = NULL; - kfree(dev); - - device_destroy(cmx_class, MKDEV(major, devno)); - - return; -} - -static const struct file_operations reader_fops = { - .owner = THIS_MODULE, - .read = cm4040_read, - .write = cm4040_write, - .open = cm4040_open, - .release = cm4040_close, - .poll = cm4040_poll, - .llseek = no_llseek, -}; - -static const struct pcmcia_device_id cm4040_ids[] = { - PCMCIA_DEVICE_MANF_CARD(0x0223, 0x0200), - PCMCIA_DEVICE_PROD_ID12("OMNIKEY", "CardMan 4040", - 0xE32CDD8C, 0x8F23318B), - PCMCIA_DEVICE_NULL, -}; -MODULE_DEVICE_TABLE(pcmcia, cm4040_ids); - -static struct pcmcia_driver reader_driver = { - .owner = THIS_MODULE, - .name = "cm4040_cs", - .probe = reader_probe, - .remove = reader_detach, - .id_table = cm4040_ids, -}; - -static int __init cm4040_init(void) -{ - int rc; - - cmx_class = class_create(THIS_MODULE, "cardman_4040"); - if (IS_ERR(cmx_class)) - return PTR_ERR(cmx_class); - - major = register_chrdev(0, DEVICE_NAME, &reader_fops); - if (major < 0) { - printk(KERN_WARNING MODULE_NAME - ": could not get major number\n"); - class_destroy(cmx_class); - return major; - } - - rc = pcmcia_register_driver(&reader_driver); - if (rc < 0) { - unregister_chrdev(major, DEVICE_NAME); - class_destroy(cmx_class); - return rc; - } - - return 0; -} - -static void __exit cm4040_exit(void) -{ - pcmcia_unregister_driver(&reader_driver); - unregister_chrdev(major, DEVICE_NAME); - class_destroy(cmx_class); -} - -module_init(cm4040_init); -module_exit(cm4040_exit); -MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/char/pcmcia/cm4040_cs.h b/drivers/char/pcmcia/cm4040_cs.h deleted file mode 100644 index e2ffff995d51..000000000000 --- a/drivers/char/pcmcia/cm4040_cs.h +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _CM4040_H_ -#define _CM4040_H_ - -#define CM_MAX_DEV 4 - -#define DEVICE_NAME "cmx" -#define MODULE_NAME "cm4040_cs" - -#define REG_OFFSET_BULK_OUT 0 -#define REG_OFFSET_BULK_IN 0 -#define REG_OFFSET_BUFFER_STATUS 1 -#define REG_OFFSET_SYNC_CONTROL 2 - -#define BSR_BULK_IN_FULL 0x02 -#define BSR_BULK_OUT_FULL 0x01 - -#define SCR_HOST_TO_READER_START 0x80 -#define SCR_ABORT 0x40 -#define SCR_EN_NOTIFY 0x20 -#define SCR_ACK_NOTIFY 0x10 -#define SCR_READER_TO_HOST_DONE 0x08 -#define SCR_HOST_TO_READER_DONE 0x04 -#define SCR_PULSE_INTERRUPT 0x02 -#define SCR_POWER_DOWN 0x01 - - -#define CMD_PC_TO_RDR_ICCPOWERON 0x62 -#define CMD_PC_TO_RDR_GETSLOTSTATUS 0x65 -#define CMD_PC_TO_RDR_ICCPOWEROFF 0x63 -#define CMD_PC_TO_RDR_SECURE 0x69 -#define CMD_PC_TO_RDR_GETPARAMETERS 0x6C -#define CMD_PC_TO_RDR_RESETPARAMETERS 0x6D -#define CMD_PC_TO_RDR_SETPARAMETERS 0x61 -#define CMD_PC_TO_RDR_XFRBLOCK 0x6F -#define CMD_PC_TO_RDR_ESCAPE 0x6B -#define CMD_PC_TO_RDR_ICCCLOCK 0x6E -#define CMD_PC_TO_RDR_TEST_SECURE 0x74 -#define CMD_PC_TO_RDR_OK_SECURE 0x89 - - -#define CMD_RDR_TO_PC_SLOTSTATUS 0x81 -#define CMD_RDR_TO_PC_DATABLOCK 0x80 -#define CMD_RDR_TO_PC_PARAMETERS 0x82 -#define CMD_RDR_TO_PC_ESCAPE 0x83 -#define CMD_RDR_TO_PC_OK_SECURE 0x89 - -#endif /* _CM4040_H_ */ diff --git a/drivers/char/pcmcia/scr24x_cs.c b/drivers/char/pcmcia/scr24x_cs.c deleted file mode 100644 index 1bdce08fae3d..000000000000 --- a/drivers/char/pcmcia/scr24x_cs.c +++ /dev/null @@ -1,359 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SCR24x PCMCIA Smart Card Reader Driver - * - * Copyright (C) 2005-2006 TL Sudheendran - * Copyright (C) 2016 Lubomir Rintel - * - * Derived from "scr24x_v4.2.6_Release.tar.gz" driver by TL Sudheendran. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#define CCID_HEADER_SIZE 10 -#define CCID_LENGTH_OFFSET 1 -#define CCID_MAX_LEN 271 - -#define SCR24X_DATA(n) (1 + n) -#define SCR24X_CMD_STATUS 7 -#define CMD_START 0x40 -#define CMD_WRITE_BYTE 0x41 -#define CMD_READ_BYTE 0x42 -#define STATUS_BUSY 0x80 - -struct scr24x_dev { - struct device *dev; - struct cdev c_dev; - unsigned char buf[CCID_MAX_LEN]; - int devno; - struct mutex lock; - struct kref refcnt; - u8 __iomem *regs; -}; - -#define SCR24X_DEVS 8 -static DECLARE_BITMAP(scr24x_minors, SCR24X_DEVS); - -static struct class *scr24x_class; -static dev_t scr24x_devt; - -static void scr24x_delete(struct kref *kref) -{ - struct scr24x_dev *dev = container_of(kref, struct scr24x_dev, - refcnt); - - kfree(dev); -} - -static int scr24x_wait_ready(struct scr24x_dev *dev) -{ - u_char status; - int timeout = 100; - - do { - status = ioread8(dev->regs + SCR24X_CMD_STATUS); - if (!(status & STATUS_BUSY)) - return 0; - - msleep(20); - } while (--timeout); - - return -EIO; -} - -static int scr24x_open(struct inode *inode, struct file *filp) -{ - struct scr24x_dev *dev = container_of(inode->i_cdev, - struct scr24x_dev, c_dev); - - kref_get(&dev->refcnt); - filp->private_data = dev; - - return stream_open(inode, filp); -} - -static int scr24x_release(struct inode *inode, struct file *filp) -{ - struct scr24x_dev *dev = filp->private_data; - - /* We must not take the dev->lock here as scr24x_delete() - * might be called to remove the dev structure altogether. - * We don't need the lock anyway, since after the reference - * acquired in probe() is released in remove() the chrdev - * is already unregistered and noone can possibly acquire - * a reference via open() anymore. */ - kref_put(&dev->refcnt, scr24x_delete); - return 0; -} - -static int read_chunk(struct scr24x_dev *dev, size_t offset, size_t limit) -{ - size_t i, y; - int ret; - - for (i = offset; i < limit; i += 5) { - iowrite8(CMD_READ_BYTE, dev->regs + SCR24X_CMD_STATUS); - ret = scr24x_wait_ready(dev); - if (ret < 0) - return ret; - - for (y = 0; y < 5 && i + y < limit; y++) - dev->buf[i + y] = ioread8(dev->regs + SCR24X_DATA(y)); - } - - return 0; -} - -static ssize_t scr24x_read(struct file *filp, char __user *buf, size_t count, - loff_t *ppos) -{ - struct scr24x_dev *dev = filp->private_data; - int ret; - int len; - - if (count < CCID_HEADER_SIZE) - return -EINVAL; - - if (mutex_lock_interruptible(&dev->lock)) - return -ERESTARTSYS; - - if (!dev->dev) { - ret = -ENODEV; - goto out; - } - - ret = scr24x_wait_ready(dev); - if (ret < 0) - goto out; - len = CCID_HEADER_SIZE; - ret = read_chunk(dev, 0, len); - if (ret < 0) - goto out; - - len += le32_to_cpu(*(__le32 *)(&dev->buf[CCID_LENGTH_OFFSET])); - if (len > sizeof(dev->buf)) { - ret = -EIO; - goto out; - } - ret = read_chunk(dev, CCID_HEADER_SIZE, len); - if (ret < 0) - goto out; - - if (len < count) - count = len; - - if (copy_to_user(buf, dev->buf, count)) { - ret = -EFAULT; - goto out; - } - - ret = count; -out: - mutex_unlock(&dev->lock); - return ret; -} - -static ssize_t scr24x_write(struct file *filp, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct scr24x_dev *dev = filp->private_data; - size_t i, y; - int ret; - - if (mutex_lock_interruptible(&dev->lock)) - return -ERESTARTSYS; - - if (!dev->dev) { - ret = -ENODEV; - goto out; - } - - if (count > sizeof(dev->buf)) { - ret = -EINVAL; - goto out; - } - - if (copy_from_user(dev->buf, buf, count)) { - ret = -EFAULT; - goto out; - } - - ret = scr24x_wait_ready(dev); - if (ret < 0) - goto out; - - iowrite8(CMD_START, dev->regs + SCR24X_CMD_STATUS); - ret = scr24x_wait_ready(dev); - if (ret < 0) - goto out; - - for (i = 0; i < count; i += 5) { - for (y = 0; y < 5 && i + y < count; y++) - iowrite8(dev->buf[i + y], dev->regs + SCR24X_DATA(y)); - - iowrite8(CMD_WRITE_BYTE, dev->regs + SCR24X_CMD_STATUS); - ret = scr24x_wait_ready(dev); - if (ret < 0) - goto out; - } - - ret = count; -out: - mutex_unlock(&dev->lock); - return ret; -} - -static const struct file_operations scr24x_fops = { - .owner = THIS_MODULE, - .read = scr24x_read, - .write = scr24x_write, - .open = scr24x_open, - .release = scr24x_release, - .llseek = no_llseek, -}; - -static int scr24x_config_check(struct pcmcia_device *link, void *priv_data) -{ - if (resource_size(link->resource[PCMCIA_IOPORT_0]) != 0x11) - return -ENODEV; - return pcmcia_request_io(link); -} - -static int scr24x_probe(struct pcmcia_device *link) -{ - struct scr24x_dev *dev; - int ret; - - dev = kzalloc(sizeof(*dev), GFP_KERNEL); - if (!dev) - return -ENOMEM; - - dev->devno = find_first_zero_bit(scr24x_minors, SCR24X_DEVS); - if (dev->devno >= SCR24X_DEVS) { - ret = -EBUSY; - goto err; - } - - mutex_init(&dev->lock); - kref_init(&dev->refcnt); - - link->priv = dev; - link->config_flags |= CONF_ENABLE_IRQ | CONF_AUTO_SET_IO; - - ret = pcmcia_loop_config(link, scr24x_config_check, NULL); - if (ret < 0) - goto err; - - dev->dev = &link->dev; - dev->regs = devm_ioport_map(&link->dev, - link->resource[PCMCIA_IOPORT_0]->start, - resource_size(link->resource[PCMCIA_IOPORT_0])); - if (!dev->regs) { - ret = -EIO; - goto err; - } - - cdev_init(&dev->c_dev, &scr24x_fops); - dev->c_dev.owner = THIS_MODULE; - ret = cdev_add(&dev->c_dev, MKDEV(MAJOR(scr24x_devt), dev->devno), 1); - if (ret < 0) - goto err; - - ret = pcmcia_enable_device(link); - if (ret < 0) { - pcmcia_disable_device(link); - goto err; - } - - device_create(scr24x_class, NULL, MKDEV(MAJOR(scr24x_devt), dev->devno), - NULL, "scr24x%d", dev->devno); - - dev_info(&link->dev, "SCR24x Chip Card Interface\n"); - return 0; - -err: - if (dev->devno < SCR24X_DEVS) - clear_bit(dev->devno, scr24x_minors); - kfree (dev); - return ret; -} - -static void scr24x_remove(struct pcmcia_device *link) -{ - struct scr24x_dev *dev = (struct scr24x_dev *)link->priv; - - device_destroy(scr24x_class, MKDEV(MAJOR(scr24x_devt), dev->devno)); - mutex_lock(&dev->lock); - pcmcia_disable_device(link); - cdev_del(&dev->c_dev); - clear_bit(dev->devno, scr24x_minors); - dev->dev = NULL; - mutex_unlock(&dev->lock); - - kref_put(&dev->refcnt, scr24x_delete); -} - -static const struct pcmcia_device_id scr24x_ids[] = { - PCMCIA_DEVICE_PROD_ID12("HP", "PC Card Smart Card Reader", - 0x53cb94f9, 0xbfdf89a5), - PCMCIA_DEVICE_PROD_ID1("SCR241 PCMCIA", 0x6271efa3), - PCMCIA_DEVICE_PROD_ID1("SCR243 PCMCIA", 0x2054e8de), - PCMCIA_DEVICE_PROD_ID1("SCR24x PCMCIA", 0x54a33665), - PCMCIA_DEVICE_NULL -}; -MODULE_DEVICE_TABLE(pcmcia, scr24x_ids); - -static struct pcmcia_driver scr24x_driver = { - .owner = THIS_MODULE, - .name = "scr24x_cs", - .probe = scr24x_probe, - .remove = scr24x_remove, - .id_table = scr24x_ids, -}; - -static int __init scr24x_init(void) -{ - int ret; - - scr24x_class = class_create(THIS_MODULE, "scr24x"); - if (IS_ERR(scr24x_class)) - return PTR_ERR(scr24x_class); - - ret = alloc_chrdev_region(&scr24x_devt, 0, SCR24X_DEVS, "scr24x"); - if (ret < 0) { - class_destroy(scr24x_class); - return ret; - } - - ret = pcmcia_register_driver(&scr24x_driver); - if (ret < 0) { - unregister_chrdev_region(scr24x_devt, SCR24X_DEVS); - class_destroy(scr24x_class); - } - - return ret; -} - -static void __exit scr24x_exit(void) -{ - pcmcia_unregister_driver(&scr24x_driver); - unregister_chrdev_region(scr24x_devt, SCR24X_DEVS); - class_destroy(scr24x_class); -} - -module_init(scr24x_init); -module_exit(scr24x_exit); - -MODULE_AUTHOR("Lubomir Rintel"); -MODULE_DESCRIPTION("SCR24x PCMCIA Smart Card Reader Driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/char/pcmcia/synclink_cs.c b/drivers/char/pcmcia/synclink_cs.c deleted file mode 100644 index 6ddfeb2fe98f..000000000000 --- a/drivers/char/pcmcia/synclink_cs.c +++ /dev/null @@ -1,4290 +0,0 @@ -/* - * linux/drivers/char/pcmcia/synclink_cs.c - * - * $Id: synclink_cs.c,v 4.34 2005/09/08 13:20:54 paulkf Exp $ - * - * Device driver for Microgate SyncLink PC Card - * multiprotocol serial adapter. - * - * written by Paul Fulghum for Microgate Corporation - * paulkf@microgate.com - * - * Microgate and SyncLink are trademarks of Microgate Corporation - * - * This code is released under the GNU General Public License (GPL) - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#define VERSION(ver,rel,seq) (((ver)<<16) | ((rel)<<8) | (seq)) -#if defined(__i386__) -# define BREAKPOINT() asm(" int $3"); -#else -# define BREAKPOINT() { } -#endif - -#define MAX_DEVICE_COUNT 4 - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#if defined(CONFIG_HDLC) || (defined(CONFIG_HDLC_MODULE) && defined(CONFIG_SYNCLINK_CS_MODULE)) -#define SYNCLINK_GENERIC_HDLC 1 -#else -#define SYNCLINK_GENERIC_HDLC 0 -#endif - -#define GET_USER(error,value,addr) error = get_user(value,addr) -#define COPY_FROM_USER(error,dest,src,size) error = copy_from_user(dest,src,size) ? -EFAULT : 0 -#define PUT_USER(error,value,addr) error = put_user(value,addr) -#define COPY_TO_USER(error,dest,src,size) error = copy_to_user(dest,src,size) ? -EFAULT : 0 - -#include - -static MGSL_PARAMS default_params = { - MGSL_MODE_HDLC, /* unsigned long mode */ - 0, /* unsigned char loopback; */ - HDLC_FLAG_UNDERRUN_ABORT15, /* unsigned short flags; */ - HDLC_ENCODING_NRZI_SPACE, /* unsigned char encoding; */ - 0, /* unsigned long clock_speed; */ - 0xff, /* unsigned char addr_filter; */ - HDLC_CRC_16_CCITT, /* unsigned short crc_type; */ - HDLC_PREAMBLE_LENGTH_8BITS, /* unsigned char preamble_length; */ - HDLC_PREAMBLE_PATTERN_NONE, /* unsigned char preamble; */ - 9600, /* unsigned long data_rate; */ - 8, /* unsigned char data_bits; */ - 1, /* unsigned char stop_bits; */ - ASYNC_PARITY_NONE /* unsigned char parity; */ -}; - -typedef struct { - int count; - unsigned char status; - char data[1]; -} RXBUF; - -/* The queue of BH actions to be performed */ - -#define BH_RECEIVE 1 -#define BH_TRANSMIT 2 -#define BH_STATUS 4 - -#define IO_PIN_SHUTDOWN_LIMIT 100 - -#define RELEVANT_IFLAG(iflag) (iflag & (IGNBRK|BRKINT|IGNPAR|PARMRK|INPCK)) - -struct _input_signal_events { - int ri_up; - int ri_down; - int dsr_up; - int dsr_down; - int dcd_up; - int dcd_down; - int cts_up; - int cts_down; -}; - - -/* - * Device instance data structure - */ - -typedef struct _mgslpc_info { - struct tty_port port; - void *if_ptr; /* General purpose pointer (used by SPPP) */ - int magic; - int line; - - struct mgsl_icount icount; - - int timeout; - int x_char; /* xon/xoff character */ - unsigned char read_status_mask; - unsigned char ignore_status_mask; - - unsigned char *tx_buf; - int tx_put; - int tx_get; - int tx_count; - - /* circular list of fixed length rx buffers */ - - unsigned char *rx_buf; /* memory allocated for all rx buffers */ - int rx_buf_total_size; /* size of memory allocated for rx buffers */ - int rx_put; /* index of next empty rx buffer */ - int rx_get; /* index of next full rx buffer */ - int rx_buf_size; /* size in bytes of single rx buffer */ - int rx_buf_count; /* total number of rx buffers */ - int rx_frame_count; /* number of full rx buffers */ - - wait_queue_head_t status_event_wait_q; - wait_queue_head_t event_wait_q; - struct timer_list tx_timer; /* HDLC transmit timeout timer */ - struct _mgslpc_info *next_device; /* device list link */ - - unsigned short imra_value; - unsigned short imrb_value; - unsigned char pim_value; - - spinlock_t lock; - struct work_struct task; /* task structure for scheduling bh */ - - u32 max_frame_size; - - u32 pending_bh; - - bool bh_running; - bool bh_requested; - - int dcd_chkcount; /* check counts to prevent */ - int cts_chkcount; /* too many IRQs if a signal */ - int dsr_chkcount; /* is floating */ - int ri_chkcount; - - bool rx_enabled; - bool rx_overflow; - - bool tx_enabled; - bool tx_active; - bool tx_aborting; - u32 idle_mode; - - int if_mode; /* serial interface selection (RS-232, v.35 etc) */ - - char device_name[25]; /* device instance name */ - - unsigned int io_base; /* base I/O address of adapter */ - unsigned int irq_level; - - MGSL_PARAMS params; /* communications parameters */ - - unsigned char serial_signals; /* current serial signal states */ - - bool irq_occurred; /* for diagnostics use */ - char testing_irq; - unsigned int init_error; /* startup error (DIAGS) */ - - char *flag_buf; - bool drop_rts_on_tx_done; - - struct _input_signal_events input_signal_events; - - /* PCMCIA support */ - struct pcmcia_device *p_dev; - int stop; - - /* SPPP/Cisco HDLC device parts */ - int netcount; - spinlock_t netlock; - -#if SYNCLINK_GENERIC_HDLC - struct net_device *netdev; -#endif - -} MGSLPC_INFO; - -#define MGSLPC_MAGIC 0x5402 - -/* - * The size of the serial xmit buffer is 1 page, or 4096 bytes - */ -#define TXBUFSIZE 4096 - - -#define CHA 0x00 /* channel A offset */ -#define CHB 0x40 /* channel B offset */ - -/* - * FIXME: PPC has PVR defined in asm/reg.h. For now we just undef it. - */ -#undef PVR - -#define RXFIFO 0 -#define TXFIFO 0 -#define STAR 0x20 -#define CMDR 0x20 -#define RSTA 0x21 -#define PRE 0x21 -#define MODE 0x22 -#define TIMR 0x23 -#define XAD1 0x24 -#define XAD2 0x25 -#define RAH1 0x26 -#define RAH2 0x27 -#define DAFO 0x27 -#define RAL1 0x28 -#define RFC 0x28 -#define RHCR 0x29 -#define RAL2 0x29 -#define RBCL 0x2a -#define XBCL 0x2a -#define RBCH 0x2b -#define XBCH 0x2b -#define CCR0 0x2c -#define CCR1 0x2d -#define CCR2 0x2e -#define CCR3 0x2f -#define VSTR 0x34 -#define BGR 0x34 -#define RLCR 0x35 -#define AML 0x36 -#define AMH 0x37 -#define GIS 0x38 -#define IVA 0x38 -#define IPC 0x39 -#define ISR 0x3a -#define IMR 0x3a -#define PVR 0x3c -#define PIS 0x3d -#define PIM 0x3d -#define PCR 0x3e -#define CCR4 0x3f - -// IMR/ISR - -#define IRQ_BREAK_ON BIT15 // rx break detected -#define IRQ_DATAOVERRUN BIT14 // receive data overflow -#define IRQ_ALLSENT BIT13 // all sent -#define IRQ_UNDERRUN BIT12 // transmit data underrun -#define IRQ_TIMER BIT11 // timer interrupt -#define IRQ_CTS BIT10 // CTS status change -#define IRQ_TXREPEAT BIT9 // tx message repeat -#define IRQ_TXFIFO BIT8 // transmit pool ready -#define IRQ_RXEOM BIT7 // receive message end -#define IRQ_EXITHUNT BIT6 // receive frame start -#define IRQ_RXTIME BIT6 // rx char timeout -#define IRQ_DCD BIT2 // carrier detect status change -#define IRQ_OVERRUN BIT1 // receive frame overflow -#define IRQ_RXFIFO BIT0 // receive pool full - -// STAR - -#define XFW BIT6 // transmit FIFO write enable -#define CEC BIT2 // command executing -#define CTS BIT1 // CTS state - -#define PVR_DTR BIT0 -#define PVR_DSR BIT1 -#define PVR_RI BIT2 -#define PVR_AUTOCTS BIT3 -#define PVR_RS232 0x20 /* 0010b */ -#define PVR_V35 0xe0 /* 1110b */ -#define PVR_RS422 0x40 /* 0100b */ - -/* Register access functions */ - -#define write_reg(info, reg, val) outb((val),(info)->io_base + (reg)) -#define read_reg(info, reg) inb((info)->io_base + (reg)) - -#define read_reg16(info, reg) inw((info)->io_base + (reg)) -#define write_reg16(info, reg, val) outw((val), (info)->io_base + (reg)) - -#define set_reg_bits(info, reg, mask) \ - write_reg(info, (reg), \ - (unsigned char) (read_reg(info, (reg)) | (mask))) -#define clear_reg_bits(info, reg, mask) \ - write_reg(info, (reg), \ - (unsigned char) (read_reg(info, (reg)) & ~(mask))) -/* - * interrupt enable/disable routines - */ -static void irq_disable(MGSLPC_INFO *info, unsigned char channel, unsigned short mask) -{ - if (channel == CHA) { - info->imra_value |= mask; - write_reg16(info, CHA + IMR, info->imra_value); - } else { - info->imrb_value |= mask; - write_reg16(info, CHB + IMR, info->imrb_value); - } -} -static void irq_enable(MGSLPC_INFO *info, unsigned char channel, unsigned short mask) -{ - if (channel == CHA) { - info->imra_value &= ~mask; - write_reg16(info, CHA + IMR, info->imra_value); - } else { - info->imrb_value &= ~mask; - write_reg16(info, CHB + IMR, info->imrb_value); - } -} - -#define port_irq_disable(info, mask) \ - { info->pim_value |= (mask); write_reg(info, PIM, info->pim_value); } - -#define port_irq_enable(info, mask) \ - { info->pim_value &= ~(mask); write_reg(info, PIM, info->pim_value); } - -static void rx_start(MGSLPC_INFO *info); -static void rx_stop(MGSLPC_INFO *info); - -static void tx_start(MGSLPC_INFO *info, struct tty_struct *tty); -static void tx_stop(MGSLPC_INFO *info); -static void tx_set_idle(MGSLPC_INFO *info); - -static void get_signals(MGSLPC_INFO *info); -static void set_signals(MGSLPC_INFO *info); - -static void reset_device(MGSLPC_INFO *info); - -static void hdlc_mode(MGSLPC_INFO *info); -static void async_mode(MGSLPC_INFO *info); - -static void tx_timeout(struct timer_list *t); - -static bool carrier_raised(struct tty_port *port); -static void dtr_rts(struct tty_port *port, bool active); - -#if SYNCLINK_GENERIC_HDLC -#define dev_to_port(D) (dev_to_hdlc(D)->priv) -static void hdlcdev_tx_done(MGSLPC_INFO *info); -static void hdlcdev_rx(MGSLPC_INFO *info, char *buf, int size); -static int hdlcdev_init(MGSLPC_INFO *info); -static void hdlcdev_exit(MGSLPC_INFO *info); -#endif - -static void trace_block(MGSLPC_INFO *info,const char* data, int count, int xmit); - -static bool register_test(MGSLPC_INFO *info); -static bool irq_test(MGSLPC_INFO *info); -static int adapter_test(MGSLPC_INFO *info); - -static int claim_resources(MGSLPC_INFO *info); -static void release_resources(MGSLPC_INFO *info); -static int mgslpc_add_device(MGSLPC_INFO *info); -static void mgslpc_remove_device(MGSLPC_INFO *info); - -static bool rx_get_frame(MGSLPC_INFO *info, struct tty_struct *tty); -static void rx_reset_buffers(MGSLPC_INFO *info); -static int rx_alloc_buffers(MGSLPC_INFO *info); -static void rx_free_buffers(MGSLPC_INFO *info); - -static irqreturn_t mgslpc_isr(int irq, void *dev_id); - -/* - * Bottom half interrupt handlers - */ -static void bh_handler(struct work_struct *work); -static void bh_transmit(MGSLPC_INFO *info, struct tty_struct *tty); -static void bh_status(MGSLPC_INFO *info); - -/* - * ioctl handlers - */ -static int tiocmget(struct tty_struct *tty); -static int tiocmset(struct tty_struct *tty, - unsigned int set, unsigned int clear); -static int get_stats(MGSLPC_INFO *info, struct mgsl_icount __user *user_icount); -static int get_params(MGSLPC_INFO *info, MGSL_PARAMS __user *user_params); -static int set_params(MGSLPC_INFO *info, MGSL_PARAMS __user *new_params, struct tty_struct *tty); -static int get_txidle(MGSLPC_INFO *info, int __user *idle_mode); -static int set_txidle(MGSLPC_INFO *info, int idle_mode); -static int set_txenable(MGSLPC_INFO *info, int enable, struct tty_struct *tty); -static int tx_abort(MGSLPC_INFO *info); -static int set_rxenable(MGSLPC_INFO *info, int enable); -static int wait_events(MGSLPC_INFO *info, int __user *mask); - -static MGSLPC_INFO *mgslpc_device_list = NULL; -static int mgslpc_device_count = 0; - -/* - * Set this param to non-zero to load eax with the - * .text section address and breakpoint on module load. - * This is useful for use with gdb and add-symbol-file command. - */ -static bool break_on_load; - -/* - * Driver major number, defaults to zero to get auto - * assigned major number. May be forced as module parameter. - */ -static int ttymajor=0; - -static int debug_level = 0; -static int maxframe[MAX_DEVICE_COUNT] = {0,}; - -module_param(break_on_load, bool, 0); -module_param(ttymajor, int, 0); -module_param(debug_level, int, 0); -module_param_array(maxframe, int, NULL, 0); - -MODULE_LICENSE("GPL"); - -static char *driver_name = "SyncLink PC Card driver"; -static char *driver_version = "$Revision: 4.34 $"; - -static struct tty_driver *serial_driver; - -/* number of characters left in xmit buffer before we ask for more */ -#define WAKEUP_CHARS 256 - -static void mgslpc_change_params(MGSLPC_INFO *info, struct tty_struct *tty); -static void mgslpc_wait_until_sent(struct tty_struct *tty, int timeout); - -/* PCMCIA prototypes */ - -static int mgslpc_config(struct pcmcia_device *link); -static void mgslpc_release(u_long arg); -static void mgslpc_detach(struct pcmcia_device *p_dev); - -/* - * 1st function defined in .text section. Calling this function in - * init_module() followed by a breakpoint allows a remote debugger - * (gdb) to get the .text address for the add-symbol-file command. - * This allows remote debugging of dynamically loadable modules. - */ -static void* mgslpc_get_text_ptr(void) -{ - return mgslpc_get_text_ptr; -} - -/* - * line discipline callback wrappers - * - * The wrappers maintain line discipline references - * while calling into the line discipline. - * - * ldisc_receive_buf - pass receive data to line discipline - */ - -static void ldisc_receive_buf(struct tty_struct *tty, - const __u8 *data, char *flags, int count) -{ - struct tty_ldisc *ld; - if (!tty) - return; - ld = tty_ldisc_ref(tty); - if (ld) { - if (ld->ops->receive_buf) - ld->ops->receive_buf(tty, data, flags, count); - tty_ldisc_deref(ld); - } -} - -static const struct tty_port_operations mgslpc_port_ops = { - .carrier_raised = carrier_raised, - .dtr_rts = dtr_rts -}; - -static int mgslpc_probe(struct pcmcia_device *link) -{ - MGSLPC_INFO *info; - int ret; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("mgslpc_attach\n"); - - info = kzalloc(sizeof(MGSLPC_INFO), GFP_KERNEL); - if (!info) { - printk("Error can't allocate device instance data\n"); - return -ENOMEM; - } - - info->magic = MGSLPC_MAGIC; - tty_port_init(&info->port); - info->port.ops = &mgslpc_port_ops; - INIT_WORK(&info->task, bh_handler); - info->max_frame_size = 4096; - init_waitqueue_head(&info->status_event_wait_q); - init_waitqueue_head(&info->event_wait_q); - spin_lock_init(&info->lock); - spin_lock_init(&info->netlock); - memcpy(&info->params,&default_params,sizeof(MGSL_PARAMS)); - info->idle_mode = HDLC_TXIDLE_FLAGS; - info->imra_value = 0xffff; - info->imrb_value = 0xffff; - info->pim_value = 0xff; - - info->p_dev = link; - link->priv = info; - - /* Initialize the struct pcmcia_device structure */ - - ret = mgslpc_config(link); - if (ret != 0) - goto failed; - - ret = mgslpc_add_device(info); - if (ret != 0) - goto failed_release; - - return 0; - -failed_release: - mgslpc_release((u_long)link); -failed: - tty_port_destroy(&info->port); - kfree(info); - return ret; -} - -/* Card has been inserted. - */ - -static int mgslpc_ioprobe(struct pcmcia_device *p_dev, void *priv_data) -{ - return pcmcia_request_io(p_dev); -} - -static int mgslpc_config(struct pcmcia_device *link) -{ - MGSLPC_INFO *info = link->priv; - int ret; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("mgslpc_config(0x%p)\n", link); - - link->config_flags |= CONF_ENABLE_IRQ | CONF_AUTO_SET_IO; - - ret = pcmcia_loop_config(link, mgslpc_ioprobe, NULL); - if (ret != 0) - goto failed; - - link->config_index = 8; - link->config_regs = PRESENT_OPTION; - - ret = pcmcia_request_irq(link, mgslpc_isr); - if (ret) - goto failed; - ret = pcmcia_enable_device(link); - if (ret) - goto failed; - - info->io_base = link->resource[0]->start; - info->irq_level = link->irq; - return 0; - -failed: - mgslpc_release((u_long)link); - return -ENODEV; -} - -/* Card has been removed. - * Unregister device and release PCMCIA configuration. - * If device is open, postpone until it is closed. - */ -static void mgslpc_release(u_long arg) -{ - struct pcmcia_device *link = (struct pcmcia_device *)arg; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("mgslpc_release(0x%p)\n", link); - - pcmcia_disable_device(link); -} - -static void mgslpc_detach(struct pcmcia_device *link) -{ - if (debug_level >= DEBUG_LEVEL_INFO) - printk("mgslpc_detach(0x%p)\n", link); - - ((MGSLPC_INFO *)link->priv)->stop = 1; - mgslpc_release((u_long)link); - - mgslpc_remove_device((MGSLPC_INFO *)link->priv); -} - -static int mgslpc_suspend(struct pcmcia_device *link) -{ - MGSLPC_INFO *info = link->priv; - - info->stop = 1; - - return 0; -} - -static int mgslpc_resume(struct pcmcia_device *link) -{ - MGSLPC_INFO *info = link->priv; - - info->stop = 0; - - return 0; -} - - -static inline bool mgslpc_paranoia_check(MGSLPC_INFO *info, - char *name, const char *routine) -{ -#ifdef MGSLPC_PARANOIA_CHECK - static const char *badmagic = - "Warning: bad magic number for mgsl struct (%s) in %s\n"; - static const char *badinfo = - "Warning: null mgslpc_info for (%s) in %s\n"; - - if (!info) { - printk(badinfo, name, routine); - return true; - } - if (info->magic != MGSLPC_MAGIC) { - printk(badmagic, name, routine); - return true; - } -#else - if (!info) - return true; -#endif - return false; -} - - -#define CMD_RXFIFO BIT7 // release current rx FIFO -#define CMD_RXRESET BIT6 // receiver reset -#define CMD_RXFIFO_READ BIT5 -#define CMD_START_TIMER BIT4 -#define CMD_TXFIFO BIT3 // release current tx FIFO -#define CMD_TXEOM BIT1 // transmit end message -#define CMD_TXRESET BIT0 // transmit reset - -static bool wait_command_complete(MGSLPC_INFO *info, unsigned char channel) -{ - int i = 0; - /* wait for command completion */ - while (read_reg(info, (unsigned char)(channel+STAR)) & BIT2) { - udelay(1); - if (i++ == 1000) - return false; - } - return true; -} - -static void issue_command(MGSLPC_INFO *info, unsigned char channel, unsigned char cmd) -{ - wait_command_complete(info, channel); - write_reg(info, (unsigned char) (channel + CMDR), cmd); -} - -static void tx_pause(struct tty_struct *tty) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (mgslpc_paranoia_check(info, tty->name, "tx_pause")) - return; - if (debug_level >= DEBUG_LEVEL_INFO) - printk("tx_pause(%s)\n", info->device_name); - - spin_lock_irqsave(&info->lock, flags); - if (info->tx_enabled) - tx_stop(info); - spin_unlock_irqrestore(&info->lock, flags); -} - -static void tx_release(struct tty_struct *tty) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (mgslpc_paranoia_check(info, tty->name, "tx_release")) - return; - if (debug_level >= DEBUG_LEVEL_INFO) - printk("tx_release(%s)\n", info->device_name); - - spin_lock_irqsave(&info->lock, flags); - if (!info->tx_enabled) - tx_start(info, tty); - spin_unlock_irqrestore(&info->lock, flags); -} - -/* Return next bottom half action to perform. - * or 0 if nothing to do. - */ -static int bh_action(MGSLPC_INFO *info) -{ - unsigned long flags; - int rc = 0; - - spin_lock_irqsave(&info->lock, flags); - - if (info->pending_bh & BH_RECEIVE) { - info->pending_bh &= ~BH_RECEIVE; - rc = BH_RECEIVE; - } else if (info->pending_bh & BH_TRANSMIT) { - info->pending_bh &= ~BH_TRANSMIT; - rc = BH_TRANSMIT; - } else if (info->pending_bh & BH_STATUS) { - info->pending_bh &= ~BH_STATUS; - rc = BH_STATUS; - } - - if (!rc) { - /* Mark BH routine as complete */ - info->bh_running = false; - info->bh_requested = false; - } - - spin_unlock_irqrestore(&info->lock, flags); - - return rc; -} - -static void bh_handler(struct work_struct *work) -{ - MGSLPC_INFO *info = container_of(work, MGSLPC_INFO, task); - struct tty_struct *tty; - int action; - - if (debug_level >= DEBUG_LEVEL_BH) - printk("%s(%d):bh_handler(%s) entry\n", - __FILE__,__LINE__,info->device_name); - - info->bh_running = true; - tty = tty_port_tty_get(&info->port); - - while((action = bh_action(info)) != 0) { - - /* Process work item */ - if (debug_level >= DEBUG_LEVEL_BH) - printk("%s(%d):bh_handler() work item action=%d\n", - __FILE__,__LINE__,action); - - switch (action) { - - case BH_RECEIVE: - while(rx_get_frame(info, tty)); - break; - case BH_TRANSMIT: - bh_transmit(info, tty); - break; - case BH_STATUS: - bh_status(info); - break; - default: - /* unknown work item ID */ - printk("Unknown work item ID=%08X!\n", action); - break; - } - } - - tty_kref_put(tty); - if (debug_level >= DEBUG_LEVEL_BH) - printk("%s(%d):bh_handler(%s) exit\n", - __FILE__,__LINE__,info->device_name); -} - -static void bh_transmit(MGSLPC_INFO *info, struct tty_struct *tty) -{ - if (debug_level >= DEBUG_LEVEL_BH) - printk("bh_transmit() entry on %s\n", info->device_name); - - if (tty) - tty_wakeup(tty); -} - -static void bh_status(MGSLPC_INFO *info) -{ - info->ri_chkcount = 0; - info->dsr_chkcount = 0; - info->dcd_chkcount = 0; - info->cts_chkcount = 0; -} - -/* eom: non-zero = end of frame */ -static void rx_ready_hdlc(MGSLPC_INFO *info, int eom) -{ - unsigned char data[2]; - unsigned char fifo_count, read_count, i; - RXBUF *buf = (RXBUF*)(info->rx_buf + (info->rx_put * info->rx_buf_size)); - - if (debug_level >= DEBUG_LEVEL_ISR) - printk("%s(%d):rx_ready_hdlc(eom=%d)\n", __FILE__, __LINE__, eom); - - if (!info->rx_enabled) - return; - - if (info->rx_frame_count >= info->rx_buf_count) { - /* no more free buffers */ - issue_command(info, CHA, CMD_RXRESET); - info->pending_bh |= BH_RECEIVE; - info->rx_overflow = true; - info->icount.buf_overrun++; - return; - } - - if (eom) { - /* end of frame, get FIFO count from RBCL register */ - fifo_count = (unsigned char)(read_reg(info, CHA+RBCL) & 0x1f); - if (fifo_count == 0) - fifo_count = 32; - } else - fifo_count = 32; - - do { - if (fifo_count == 1) { - read_count = 1; - data[0] = read_reg(info, CHA + RXFIFO); - } else { - read_count = 2; - *((unsigned short *) data) = read_reg16(info, CHA + RXFIFO); - } - fifo_count -= read_count; - if (!fifo_count && eom) - buf->status = data[--read_count]; - - for (i = 0; i < read_count; i++) { - if (buf->count >= info->max_frame_size) { - /* frame too large, reset receiver and reset current buffer */ - issue_command(info, CHA, CMD_RXRESET); - buf->count = 0; - return; - } - *(buf->data + buf->count) = data[i]; - buf->count++; - } - } while (fifo_count); - - if (eom) { - info->pending_bh |= BH_RECEIVE; - info->rx_frame_count++; - info->rx_put++; - if (info->rx_put >= info->rx_buf_count) - info->rx_put = 0; - } - issue_command(info, CHA, CMD_RXFIFO); -} - -static void rx_ready_async(MGSLPC_INFO *info, int tcd) -{ - struct tty_port *port = &info->port; - unsigned char data, status, flag; - int fifo_count; - int work = 0; - struct mgsl_icount *icount = &info->icount; - - if (tcd) { - /* early termination, get FIFO count from RBCL register */ - fifo_count = (unsigned char)(read_reg(info, CHA+RBCL) & 0x1f); - - /* Zero fifo count could mean 0 or 32 bytes available. - * If BIT5 of STAR is set then at least 1 byte is available. - */ - if (!fifo_count && (read_reg(info,CHA+STAR) & BIT5)) - fifo_count = 32; - } else - fifo_count = 32; - - tty_buffer_request_room(port, fifo_count); - /* Flush received async data to receive data buffer. */ - while (fifo_count) { - data = read_reg(info, CHA + RXFIFO); - status = read_reg(info, CHA + RXFIFO); - fifo_count -= 2; - - icount->rx++; - flag = TTY_NORMAL; - - // if no frameing/crc error then save data - // BIT7:parity error - // BIT6:framing error - - if (status & (BIT7 | BIT6)) { - if (status & BIT7) - icount->parity++; - else - icount->frame++; - - /* discard char if tty control flags say so */ - if (status & info->ignore_status_mask) - continue; - - status &= info->read_status_mask; - - if (status & BIT7) - flag = TTY_PARITY; - else if (status & BIT6) - flag = TTY_FRAME; - } - work += tty_insert_flip_char(port, data, flag); - } - issue_command(info, CHA, CMD_RXFIFO); - - if (debug_level >= DEBUG_LEVEL_ISR) { - printk("%s(%d):rx_ready_async", - __FILE__,__LINE__); - printk("%s(%d):rx=%d brk=%d parity=%d frame=%d overrun=%d\n", - __FILE__,__LINE__,icount->rx,icount->brk, - icount->parity,icount->frame,icount->overrun); - } - - if (work) - tty_flip_buffer_push(port); -} - - -static void tx_done(MGSLPC_INFO *info, struct tty_struct *tty) -{ - if (!info->tx_active) - return; - - info->tx_active = false; - info->tx_aborting = false; - - if (info->params.mode == MGSL_MODE_ASYNC) - return; - - info->tx_count = info->tx_put = info->tx_get = 0; - del_timer(&info->tx_timer); - - if (info->drop_rts_on_tx_done) { - get_signals(info); - if (info->serial_signals & SerialSignal_RTS) { - info->serial_signals &= ~SerialSignal_RTS; - set_signals(info); - } - info->drop_rts_on_tx_done = false; - } - -#if SYNCLINK_GENERIC_HDLC - if (info->netcount) - hdlcdev_tx_done(info); - else -#endif - { - if (tty && (tty->flow.stopped || tty->hw_stopped)) { - tx_stop(info); - return; - } - info->pending_bh |= BH_TRANSMIT; - } -} - -static void tx_ready(MGSLPC_INFO *info, struct tty_struct *tty) -{ - unsigned char fifo_count = 32; - int c; - - if (debug_level >= DEBUG_LEVEL_ISR) - printk("%s(%d):tx_ready(%s)\n", __FILE__, __LINE__, info->device_name); - - if (info->params.mode == MGSL_MODE_HDLC) { - if (!info->tx_active) - return; - } else { - if (tty && (tty->flow.stopped || tty->hw_stopped)) { - tx_stop(info); - return; - } - if (!info->tx_count) - info->tx_active = false; - } - - if (!info->tx_count) - return; - - while (info->tx_count && fifo_count) { - c = min(2, min_t(int, fifo_count, min(info->tx_count, TXBUFSIZE - info->tx_get))); - - if (c == 1) { - write_reg(info, CHA + TXFIFO, *(info->tx_buf + info->tx_get)); - } else { - write_reg16(info, CHA + TXFIFO, - *((unsigned short*)(info->tx_buf + info->tx_get))); - } - info->tx_count -= c; - info->tx_get = (info->tx_get + c) & (TXBUFSIZE - 1); - fifo_count -= c; - } - - if (info->params.mode == MGSL_MODE_ASYNC) { - if (info->tx_count < WAKEUP_CHARS) - info->pending_bh |= BH_TRANSMIT; - issue_command(info, CHA, CMD_TXFIFO); - } else { - if (info->tx_count) - issue_command(info, CHA, CMD_TXFIFO); - else - issue_command(info, CHA, CMD_TXFIFO + CMD_TXEOM); - } -} - -static void cts_change(MGSLPC_INFO *info, struct tty_struct *tty) -{ - get_signals(info); - if ((info->cts_chkcount)++ >= IO_PIN_SHUTDOWN_LIMIT) - irq_disable(info, CHB, IRQ_CTS); - info->icount.cts++; - if (info->serial_signals & SerialSignal_CTS) - info->input_signal_events.cts_up++; - else - info->input_signal_events.cts_down++; - wake_up_interruptible(&info->status_event_wait_q); - wake_up_interruptible(&info->event_wait_q); - - if (tty && tty_port_cts_enabled(&info->port)) { - if (tty->hw_stopped) { - if (info->serial_signals & SerialSignal_CTS) { - if (debug_level >= DEBUG_LEVEL_ISR) - printk("CTS tx start..."); - tty->hw_stopped = 0; - tx_start(info, tty); - info->pending_bh |= BH_TRANSMIT; - return; - } - } else { - if (!(info->serial_signals & SerialSignal_CTS)) { - if (debug_level >= DEBUG_LEVEL_ISR) - printk("CTS tx stop..."); - tty->hw_stopped = 1; - tx_stop(info); - } - } - } - info->pending_bh |= BH_STATUS; -} - -static void dcd_change(MGSLPC_INFO *info, struct tty_struct *tty) -{ - get_signals(info); - if ((info->dcd_chkcount)++ >= IO_PIN_SHUTDOWN_LIMIT) - irq_disable(info, CHB, IRQ_DCD); - info->icount.dcd++; - if (info->serial_signals & SerialSignal_DCD) { - info->input_signal_events.dcd_up++; - } - else - info->input_signal_events.dcd_down++; -#if SYNCLINK_GENERIC_HDLC - if (info->netcount) { - if (info->serial_signals & SerialSignal_DCD) - netif_carrier_on(info->netdev); - else - netif_carrier_off(info->netdev); - } -#endif - wake_up_interruptible(&info->status_event_wait_q); - wake_up_interruptible(&info->event_wait_q); - - if (tty_port_check_carrier(&info->port)) { - if (debug_level >= DEBUG_LEVEL_ISR) - printk("%s CD now %s...", info->device_name, - (info->serial_signals & SerialSignal_DCD) ? "on" : "off"); - if (info->serial_signals & SerialSignal_DCD) - wake_up_interruptible(&info->port.open_wait); - else { - if (debug_level >= DEBUG_LEVEL_ISR) - printk("doing serial hangup..."); - if (tty) - tty_hangup(tty); - } - } - info->pending_bh |= BH_STATUS; -} - -static void dsr_change(MGSLPC_INFO *info) -{ - get_signals(info); - if ((info->dsr_chkcount)++ >= IO_PIN_SHUTDOWN_LIMIT) - port_irq_disable(info, PVR_DSR); - info->icount.dsr++; - if (info->serial_signals & SerialSignal_DSR) - info->input_signal_events.dsr_up++; - else - info->input_signal_events.dsr_down++; - wake_up_interruptible(&info->status_event_wait_q); - wake_up_interruptible(&info->event_wait_q); - info->pending_bh |= BH_STATUS; -} - -static void ri_change(MGSLPC_INFO *info) -{ - get_signals(info); - if ((info->ri_chkcount)++ >= IO_PIN_SHUTDOWN_LIMIT) - port_irq_disable(info, PVR_RI); - info->icount.rng++; - if (info->serial_signals & SerialSignal_RI) - info->input_signal_events.ri_up++; - else - info->input_signal_events.ri_down++; - wake_up_interruptible(&info->status_event_wait_q); - wake_up_interruptible(&info->event_wait_q); - info->pending_bh |= BH_STATUS; -} - -/* Interrupt service routine entry point. - * - * Arguments: - * - * irq interrupt number that caused interrupt - * dev_id device ID supplied during interrupt registration - */ -static irqreturn_t mgslpc_isr(int dummy, void *dev_id) -{ - MGSLPC_INFO *info = dev_id; - struct tty_struct *tty; - unsigned short isr; - unsigned char gis, pis; - int count=0; - - if (debug_level >= DEBUG_LEVEL_ISR) - printk("mgslpc_isr(%d) entry.\n", info->irq_level); - - if (!(info->p_dev->_locked)) - return IRQ_HANDLED; - - tty = tty_port_tty_get(&info->port); - - spin_lock(&info->lock); - - while ((gis = read_reg(info, CHA + GIS))) { - if (debug_level >= DEBUG_LEVEL_ISR) - printk("mgslpc_isr %s gis=%04X\n", info->device_name,gis); - - if ((gis & 0x70) || count > 1000) { - printk("synclink_cs:hardware failed or ejected\n"); - break; - } - count++; - - if (gis & (BIT1 | BIT0)) { - isr = read_reg16(info, CHB + ISR); - if (isr & IRQ_DCD) - dcd_change(info, tty); - if (isr & IRQ_CTS) - cts_change(info, tty); - } - if (gis & (BIT3 | BIT2)) - { - isr = read_reg16(info, CHA + ISR); - if (isr & IRQ_TIMER) { - info->irq_occurred = true; - irq_disable(info, CHA, IRQ_TIMER); - } - - /* receive IRQs */ - if (isr & IRQ_EXITHUNT) { - info->icount.exithunt++; - wake_up_interruptible(&info->event_wait_q); - } - if (isr & IRQ_BREAK_ON) { - info->icount.brk++; - if (info->port.flags & ASYNC_SAK) - do_SAK(tty); - } - if (isr & IRQ_RXTIME) { - issue_command(info, CHA, CMD_RXFIFO_READ); - } - if (isr & (IRQ_RXEOM | IRQ_RXFIFO)) { - if (info->params.mode == MGSL_MODE_HDLC) - rx_ready_hdlc(info, isr & IRQ_RXEOM); - else - rx_ready_async(info, isr & IRQ_RXEOM); - } - - /* transmit IRQs */ - if (isr & IRQ_UNDERRUN) { - if (info->tx_aborting) - info->icount.txabort++; - else - info->icount.txunder++; - tx_done(info, tty); - } - else if (isr & IRQ_ALLSENT) { - info->icount.txok++; - tx_done(info, tty); - } - else if (isr & IRQ_TXFIFO) - tx_ready(info, tty); - } - if (gis & BIT7) { - pis = read_reg(info, CHA + PIS); - if (pis & BIT1) - dsr_change(info); - if (pis & BIT2) - ri_change(info); - } - } - - /* Request bottom half processing if there's something - * for it to do and the bh is not already running - */ - - if (info->pending_bh && !info->bh_running && !info->bh_requested) { - if (debug_level >= DEBUG_LEVEL_ISR) - printk("%s(%d):%s queueing bh task.\n", - __FILE__,__LINE__,info->device_name); - schedule_work(&info->task); - info->bh_requested = true; - } - - spin_unlock(&info->lock); - tty_kref_put(tty); - - if (debug_level >= DEBUG_LEVEL_ISR) - printk("%s(%d):mgslpc_isr(%d)exit.\n", - __FILE__, __LINE__, info->irq_level); - - return IRQ_HANDLED; -} - -/* Initialize and start device. - */ -static int startup(MGSLPC_INFO * info, struct tty_struct *tty) -{ - int retval = 0; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):startup(%s)\n", __FILE__, __LINE__, info->device_name); - - if (tty_port_initialized(&info->port)) - return 0; - - if (!info->tx_buf) { - /* allocate a page of memory for a transmit buffer */ - info->tx_buf = (unsigned char *)get_zeroed_page(GFP_KERNEL); - if (!info->tx_buf) { - printk(KERN_ERR"%s(%d):%s can't allocate transmit buffer\n", - __FILE__, __LINE__, info->device_name); - return -ENOMEM; - } - } - - info->pending_bh = 0; - - memset(&info->icount, 0, sizeof(info->icount)); - - timer_setup(&info->tx_timer, tx_timeout, 0); - - /* Allocate and claim adapter resources */ - retval = claim_resources(info); - - /* perform existence check and diagnostics */ - if (!retval) - retval = adapter_test(info); - - if (retval) { - if (capable(CAP_SYS_ADMIN) && tty) - set_bit(TTY_IO_ERROR, &tty->flags); - release_resources(info); - return retval; - } - - /* program hardware for current parameters */ - mgslpc_change_params(info, tty); - - if (tty) - clear_bit(TTY_IO_ERROR, &tty->flags); - - tty_port_set_initialized(&info->port, true); - - return 0; -} - -/* Called by mgslpc_close() and mgslpc_hangup() to shutdown hardware - */ -static void shutdown(MGSLPC_INFO * info, struct tty_struct *tty) -{ - unsigned long flags; - - if (!tty_port_initialized(&info->port)) - return; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_shutdown(%s)\n", - __FILE__, __LINE__, info->device_name); - - /* clear status wait queue because status changes */ - /* can't happen after shutting down the hardware */ - wake_up_interruptible(&info->status_event_wait_q); - wake_up_interruptible(&info->event_wait_q); - - del_timer_sync(&info->tx_timer); - - if (info->tx_buf) { - free_page((unsigned long) info->tx_buf); - info->tx_buf = NULL; - } - - spin_lock_irqsave(&info->lock, flags); - - rx_stop(info); - tx_stop(info); - - /* TODO:disable interrupts instead of reset to preserve signal states */ - reset_device(info); - - if (!tty || C_HUPCL(tty)) { - info->serial_signals &= ~(SerialSignal_RTS | SerialSignal_DTR); - set_signals(info); - } - - spin_unlock_irqrestore(&info->lock, flags); - - release_resources(info); - - if (tty) - set_bit(TTY_IO_ERROR, &tty->flags); - - tty_port_set_initialized(&info->port, false); -} - -static void mgslpc_program_hw(MGSLPC_INFO *info, struct tty_struct *tty) -{ - unsigned long flags; - - spin_lock_irqsave(&info->lock, flags); - - rx_stop(info); - tx_stop(info); - info->tx_count = info->tx_put = info->tx_get = 0; - - if (info->params.mode == MGSL_MODE_HDLC || info->netcount) - hdlc_mode(info); - else - async_mode(info); - - set_signals(info); - - info->dcd_chkcount = 0; - info->cts_chkcount = 0; - info->ri_chkcount = 0; - info->dsr_chkcount = 0; - - irq_enable(info, CHB, IRQ_DCD | IRQ_CTS); - port_irq_enable(info, (unsigned char) PVR_DSR | PVR_RI); - get_signals(info); - - if (info->netcount || (tty && C_CREAD(tty))) - rx_start(info); - - spin_unlock_irqrestore(&info->lock, flags); -} - -/* Reconfigure adapter based on new parameters - */ -static void mgslpc_change_params(MGSLPC_INFO *info, struct tty_struct *tty) -{ - unsigned cflag; - int bits_per_char; - - if (!tty) - return; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_change_params(%s)\n", - __FILE__, __LINE__, info->device_name); - - cflag = tty->termios.c_cflag; - - /* if B0 rate (hangup) specified then negate RTS and DTR */ - /* otherwise assert RTS and DTR */ - if (cflag & CBAUD) - info->serial_signals |= SerialSignal_RTS | SerialSignal_DTR; - else - info->serial_signals &= ~(SerialSignal_RTS | SerialSignal_DTR); - - /* byte size and parity */ - if ((cflag & CSIZE) != CS8) { - cflag &= ~CSIZE; - cflag |= CS7; - tty->termios.c_cflag = cflag; - } - info->params.data_bits = tty_get_char_size(cflag); - - if (cflag & CSTOPB) - info->params.stop_bits = 2; - else - info->params.stop_bits = 1; - - info->params.parity = ASYNC_PARITY_NONE; - if (cflag & PARENB) { - if (cflag & PARODD) - info->params.parity = ASYNC_PARITY_ODD; - else - info->params.parity = ASYNC_PARITY_EVEN; - if (cflag & CMSPAR) - info->params.parity = ASYNC_PARITY_SPACE; - } - - /* calculate number of jiffies to transmit a full - * FIFO (32 bytes) at specified data rate - */ - bits_per_char = info->params.data_bits + - info->params.stop_bits + 1; - - /* if port data rate is set to 460800 or less then - * allow tty settings to override, otherwise keep the - * current data rate. - */ - if (info->params.data_rate <= 460800) { - info->params.data_rate = tty_get_baud_rate(tty); - } - - if (info->params.data_rate) { - info->timeout = (32*HZ*bits_per_char) / - info->params.data_rate; - } - info->timeout += HZ/50; /* Add .02 seconds of slop */ - - tty_port_set_cts_flow(&info->port, cflag & CRTSCTS); - tty_port_set_check_carrier(&info->port, ~cflag & CLOCAL); - - /* process tty input control flags */ - - info->read_status_mask = 0; - if (I_INPCK(tty)) - info->read_status_mask |= BIT7 | BIT6; - if (I_IGNPAR(tty)) - info->ignore_status_mask |= BIT7 | BIT6; - - mgslpc_program_hw(info, tty); -} - -/* Add a character to the transmit buffer - */ -static int mgslpc_put_char(struct tty_struct *tty, unsigned char ch) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) { - printk("%s(%d):mgslpc_put_char(%d) on %s\n", - __FILE__, __LINE__, ch, info->device_name); - } - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_put_char")) - return 0; - - if (!info->tx_buf) - return 0; - - spin_lock_irqsave(&info->lock, flags); - - if (info->params.mode == MGSL_MODE_ASYNC || !info->tx_active) { - if (info->tx_count < TXBUFSIZE - 1) { - info->tx_buf[info->tx_put++] = ch; - info->tx_put &= TXBUFSIZE-1; - info->tx_count++; - } - } - - spin_unlock_irqrestore(&info->lock, flags); - return 1; -} - -/* Enable transmitter so remaining characters in the - * transmit buffer are sent. - */ -static void mgslpc_flush_chars(struct tty_struct *tty) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_flush_chars() entry on %s tx_count=%d\n", - __FILE__, __LINE__, info->device_name, info->tx_count); - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_flush_chars")) - return; - - if (info->tx_count <= 0 || tty->flow.stopped || - tty->hw_stopped || !info->tx_buf) - return; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_flush_chars() entry on %s starting transmitter\n", - __FILE__, __LINE__, info->device_name); - - spin_lock_irqsave(&info->lock, flags); - if (!info->tx_active) - tx_start(info, tty); - spin_unlock_irqrestore(&info->lock, flags); -} - -/* Send a block of data - * - * Arguments: - * - * tty pointer to tty information structure - * buf pointer to buffer containing send data - * count size of send data in bytes - * - * Returns: number of characters written - */ -static int mgslpc_write(struct tty_struct * tty, - const unsigned char *buf, int count) -{ - int c, ret = 0; - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_write(%s) count=%d\n", - __FILE__, __LINE__, info->device_name, count); - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_write") || - !info->tx_buf) - goto cleanup; - - if (info->params.mode == MGSL_MODE_HDLC) { - if (count > TXBUFSIZE) { - ret = -EIO; - goto cleanup; - } - if (info->tx_active) - goto cleanup; - else if (info->tx_count) - goto start; - } - - for (;;) { - c = min(count, - min(TXBUFSIZE - info->tx_count - 1, - TXBUFSIZE - info->tx_put)); - if (c <= 0) - break; - - memcpy(info->tx_buf + info->tx_put, buf, c); - - spin_lock_irqsave(&info->lock, flags); - info->tx_put = (info->tx_put + c) & (TXBUFSIZE-1); - info->tx_count += c; - spin_unlock_irqrestore(&info->lock, flags); - - buf += c; - count -= c; - ret += c; - } -start: - if (info->tx_count && !tty->flow.stopped && !tty->hw_stopped) { - spin_lock_irqsave(&info->lock, flags); - if (!info->tx_active) - tx_start(info, tty); - spin_unlock_irqrestore(&info->lock, flags); - } -cleanup: - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_write(%s) returning=%d\n", - __FILE__, __LINE__, info->device_name, ret); - return ret; -} - -/* Return the count of free bytes in transmit buffer - */ -static unsigned int mgslpc_write_room(struct tty_struct *tty) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - int ret; - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_write_room")) - return 0; - - if (info->params.mode == MGSL_MODE_HDLC) { - /* HDLC (frame oriented) mode */ - if (info->tx_active) - return 0; - else - return HDLC_MAX_FRAME_SIZE; - } else { - ret = TXBUFSIZE - info->tx_count - 1; - if (ret < 0) - ret = 0; - } - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_write_room(%s)=%d\n", - __FILE__, __LINE__, info->device_name, ret); - return ret; -} - -/* Return the count of bytes in transmit buffer - */ -static unsigned int mgslpc_chars_in_buffer(struct tty_struct *tty) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned int rc; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_chars_in_buffer(%s)\n", - __FILE__, __LINE__, info->device_name); - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_chars_in_buffer")) - return 0; - - if (info->params.mode == MGSL_MODE_HDLC) - rc = info->tx_active ? info->max_frame_size : 0; - else - rc = info->tx_count; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_chars_in_buffer(%s)=%u\n", - __FILE__, __LINE__, info->device_name, rc); - - return rc; -} - -/* Discard all data in the send buffer - */ -static void mgslpc_flush_buffer(struct tty_struct *tty) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_flush_buffer(%s) entry\n", - __FILE__, __LINE__, info->device_name); - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_flush_buffer")) - return; - - spin_lock_irqsave(&info->lock, flags); - info->tx_count = info->tx_put = info->tx_get = 0; - del_timer(&info->tx_timer); - spin_unlock_irqrestore(&info->lock, flags); - - wake_up_interruptible(&tty->write_wait); - tty_wakeup(tty); -} - -/* Send a high-priority XON/XOFF character - */ -static void mgslpc_send_xchar(struct tty_struct *tty, char ch) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_send_xchar(%s,%d)\n", - __FILE__, __LINE__, info->device_name, ch); - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_send_xchar")) - return; - - info->x_char = ch; - if (ch) { - spin_lock_irqsave(&info->lock, flags); - if (!info->tx_enabled) - tx_start(info, tty); - spin_unlock_irqrestore(&info->lock, flags); - } -} - -/* Signal remote device to throttle send data (our receive data) - */ -static void mgslpc_throttle(struct tty_struct * tty) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_throttle(%s) entry\n", - __FILE__, __LINE__, info->device_name); - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_throttle")) - return; - - if (I_IXOFF(tty)) - mgslpc_send_xchar(tty, STOP_CHAR(tty)); - - if (C_CRTSCTS(tty)) { - spin_lock_irqsave(&info->lock, flags); - info->serial_signals &= ~SerialSignal_RTS; - set_signals(info); - spin_unlock_irqrestore(&info->lock, flags); - } -} - -/* Signal remote device to stop throttling send data (our receive data) - */ -static void mgslpc_unthrottle(struct tty_struct * tty) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_unthrottle(%s) entry\n", - __FILE__, __LINE__, info->device_name); - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_unthrottle")) - return; - - if (I_IXOFF(tty)) { - if (info->x_char) - info->x_char = 0; - else - mgslpc_send_xchar(tty, START_CHAR(tty)); - } - - if (C_CRTSCTS(tty)) { - spin_lock_irqsave(&info->lock, flags); - info->serial_signals |= SerialSignal_RTS; - set_signals(info); - spin_unlock_irqrestore(&info->lock, flags); - } -} - -/* get the current serial statistics - */ -static int get_stats(MGSLPC_INFO * info, struct mgsl_icount __user *user_icount) -{ - int err; - if (debug_level >= DEBUG_LEVEL_INFO) - printk("get_params(%s)\n", info->device_name); - if (!user_icount) { - memset(&info->icount, 0, sizeof(info->icount)); - } else { - COPY_TO_USER(err, user_icount, &info->icount, sizeof(struct mgsl_icount)); - if (err) - return -EFAULT; - } - return 0; -} - -/* get the current serial parameters - */ -static int get_params(MGSLPC_INFO * info, MGSL_PARAMS __user *user_params) -{ - int err; - if (debug_level >= DEBUG_LEVEL_INFO) - printk("get_params(%s)\n", info->device_name); - COPY_TO_USER(err,user_params, &info->params, sizeof(MGSL_PARAMS)); - if (err) - return -EFAULT; - return 0; -} - -/* set the serial parameters - * - * Arguments: - * - * info pointer to device instance data - * new_params user buffer containing new serial params - * - * Returns: 0 if success, otherwise error code - */ -static int set_params(MGSLPC_INFO * info, MGSL_PARAMS __user *new_params, struct tty_struct *tty) -{ - unsigned long flags; - MGSL_PARAMS tmp_params; - int err; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):set_params %s\n", __FILE__,__LINE__, - info->device_name); - COPY_FROM_USER(err,&tmp_params, new_params, sizeof(MGSL_PARAMS)); - if (err) { - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):set_params(%s) user buffer copy failed\n", - __FILE__, __LINE__, info->device_name); - return -EFAULT; - } - - spin_lock_irqsave(&info->lock, flags); - memcpy(&info->params,&tmp_params,sizeof(MGSL_PARAMS)); - spin_unlock_irqrestore(&info->lock, flags); - - mgslpc_change_params(info, tty); - - return 0; -} - -static int get_txidle(MGSLPC_INFO * info, int __user *idle_mode) -{ - int err; - if (debug_level >= DEBUG_LEVEL_INFO) - printk("get_txidle(%s)=%d\n", info->device_name, info->idle_mode); - COPY_TO_USER(err,idle_mode, &info->idle_mode, sizeof(int)); - if (err) - return -EFAULT; - return 0; -} - -static int set_txidle(MGSLPC_INFO * info, int idle_mode) -{ - unsigned long flags; - if (debug_level >= DEBUG_LEVEL_INFO) - printk("set_txidle(%s,%d)\n", info->device_name, idle_mode); - spin_lock_irqsave(&info->lock, flags); - info->idle_mode = idle_mode; - tx_set_idle(info); - spin_unlock_irqrestore(&info->lock, flags); - return 0; -} - -static int get_interface(MGSLPC_INFO * info, int __user *if_mode) -{ - int err; - if (debug_level >= DEBUG_LEVEL_INFO) - printk("get_interface(%s)=%d\n", info->device_name, info->if_mode); - COPY_TO_USER(err,if_mode, &info->if_mode, sizeof(int)); - if (err) - return -EFAULT; - return 0; -} - -static int set_interface(MGSLPC_INFO * info, int if_mode) -{ - unsigned long flags; - unsigned char val; - if (debug_level >= DEBUG_LEVEL_INFO) - printk("set_interface(%s,%d)\n", info->device_name, if_mode); - spin_lock_irqsave(&info->lock, flags); - info->if_mode = if_mode; - - val = read_reg(info, PVR) & 0x0f; - switch (info->if_mode) - { - case MGSL_INTERFACE_RS232: val |= PVR_RS232; break; - case MGSL_INTERFACE_V35: val |= PVR_V35; break; - case MGSL_INTERFACE_RS422: val |= PVR_RS422; break; - } - write_reg(info, PVR, val); - - spin_unlock_irqrestore(&info->lock, flags); - return 0; -} - -static int set_txenable(MGSLPC_INFO * info, int enable, struct tty_struct *tty) -{ - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("set_txenable(%s,%d)\n", info->device_name, enable); - - spin_lock_irqsave(&info->lock, flags); - if (enable) { - if (!info->tx_enabled) - tx_start(info, tty); - } else { - if (info->tx_enabled) - tx_stop(info); - } - spin_unlock_irqrestore(&info->lock, flags); - return 0; -} - -static int tx_abort(MGSLPC_INFO * info) -{ - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("tx_abort(%s)\n", info->device_name); - - spin_lock_irqsave(&info->lock, flags); - if (info->tx_active && info->tx_count && - info->params.mode == MGSL_MODE_HDLC) { - /* clear data count so FIFO is not filled on next IRQ. - * This results in underrun and abort transmission. - */ - info->tx_count = info->tx_put = info->tx_get = 0; - info->tx_aborting = true; - } - spin_unlock_irqrestore(&info->lock, flags); - return 0; -} - -static int set_rxenable(MGSLPC_INFO * info, int enable) -{ - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("set_rxenable(%s,%d)\n", info->device_name, enable); - - spin_lock_irqsave(&info->lock, flags); - if (enable) { - if (!info->rx_enabled) - rx_start(info); - } else { - if (info->rx_enabled) - rx_stop(info); - } - spin_unlock_irqrestore(&info->lock, flags); - return 0; -} - -/* wait for specified event to occur - * - * Arguments: info pointer to device instance data - * mask pointer to bitmask of events to wait for - * Return Value: 0 if successful and bit mask updated with - * of events triggerred, - * otherwise error code - */ -static int wait_events(MGSLPC_INFO * info, int __user *mask_ptr) -{ - unsigned long flags; - int s; - int rc=0; - struct mgsl_icount cprev, cnow; - int events; - int mask; - struct _input_signal_events oldsigs, newsigs; - DECLARE_WAITQUEUE(wait, current); - - COPY_FROM_USER(rc,&mask, mask_ptr, sizeof(int)); - if (rc) - return -EFAULT; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("wait_events(%s,%d)\n", info->device_name, mask); - - spin_lock_irqsave(&info->lock, flags); - - /* return immediately if state matches requested events */ - get_signals(info); - s = info->serial_signals; - events = mask & - ( ((s & SerialSignal_DSR) ? MgslEvent_DsrActive:MgslEvent_DsrInactive) + - ((s & SerialSignal_DCD) ? MgslEvent_DcdActive:MgslEvent_DcdInactive) + - ((s & SerialSignal_CTS) ? MgslEvent_CtsActive:MgslEvent_CtsInactive) + - ((s & SerialSignal_RI) ? MgslEvent_RiActive :MgslEvent_RiInactive) ); - if (events) { - spin_unlock_irqrestore(&info->lock, flags); - goto exit; - } - - /* save current irq counts */ - cprev = info->icount; - oldsigs = info->input_signal_events; - - if ((info->params.mode == MGSL_MODE_HDLC) && - (mask & MgslEvent_ExitHuntMode)) - irq_enable(info, CHA, IRQ_EXITHUNT); - - set_current_state(TASK_INTERRUPTIBLE); - add_wait_queue(&info->event_wait_q, &wait); - - spin_unlock_irqrestore(&info->lock, flags); - - - for(;;) { - schedule(); - if (signal_pending(current)) { - rc = -ERESTARTSYS; - break; - } - - /* get current irq counts */ - spin_lock_irqsave(&info->lock, flags); - cnow = info->icount; - newsigs = info->input_signal_events; - set_current_state(TASK_INTERRUPTIBLE); - spin_unlock_irqrestore(&info->lock, flags); - - /* if no change, wait aborted for some reason */ - if (newsigs.dsr_up == oldsigs.dsr_up && - newsigs.dsr_down == oldsigs.dsr_down && - newsigs.dcd_up == oldsigs.dcd_up && - newsigs.dcd_down == oldsigs.dcd_down && - newsigs.cts_up == oldsigs.cts_up && - newsigs.cts_down == oldsigs.cts_down && - newsigs.ri_up == oldsigs.ri_up && - newsigs.ri_down == oldsigs.ri_down && - cnow.exithunt == cprev.exithunt && - cnow.rxidle == cprev.rxidle) { - rc = -EIO; - break; - } - - events = mask & - ( (newsigs.dsr_up != oldsigs.dsr_up ? MgslEvent_DsrActive:0) + - (newsigs.dsr_down != oldsigs.dsr_down ? MgslEvent_DsrInactive:0) + - (newsigs.dcd_up != oldsigs.dcd_up ? MgslEvent_DcdActive:0) + - (newsigs.dcd_down != oldsigs.dcd_down ? MgslEvent_DcdInactive:0) + - (newsigs.cts_up != oldsigs.cts_up ? MgslEvent_CtsActive:0) + - (newsigs.cts_down != oldsigs.cts_down ? MgslEvent_CtsInactive:0) + - (newsigs.ri_up != oldsigs.ri_up ? MgslEvent_RiActive:0) + - (newsigs.ri_down != oldsigs.ri_down ? MgslEvent_RiInactive:0) + - (cnow.exithunt != cprev.exithunt ? MgslEvent_ExitHuntMode:0) + - (cnow.rxidle != cprev.rxidle ? MgslEvent_IdleReceived:0) ); - if (events) - break; - - cprev = cnow; - oldsigs = newsigs; - } - - remove_wait_queue(&info->event_wait_q, &wait); - set_current_state(TASK_RUNNING); - - if (mask & MgslEvent_ExitHuntMode) { - spin_lock_irqsave(&info->lock, flags); - if (!waitqueue_active(&info->event_wait_q)) - irq_disable(info, CHA, IRQ_EXITHUNT); - spin_unlock_irqrestore(&info->lock, flags); - } -exit: - if (rc == 0) - PUT_USER(rc, events, mask_ptr); - return rc; -} - -static int modem_input_wait(MGSLPC_INFO *info,int arg) -{ - unsigned long flags; - int rc; - struct mgsl_icount cprev, cnow; - DECLARE_WAITQUEUE(wait, current); - - /* save current irq counts */ - spin_lock_irqsave(&info->lock, flags); - cprev = info->icount; - add_wait_queue(&info->status_event_wait_q, &wait); - set_current_state(TASK_INTERRUPTIBLE); - spin_unlock_irqrestore(&info->lock, flags); - - for(;;) { - schedule(); - if (signal_pending(current)) { - rc = -ERESTARTSYS; - break; - } - - /* get new irq counts */ - spin_lock_irqsave(&info->lock, flags); - cnow = info->icount; - set_current_state(TASK_INTERRUPTIBLE); - spin_unlock_irqrestore(&info->lock, flags); - - /* if no change, wait aborted for some reason */ - if (cnow.rng == cprev.rng && cnow.dsr == cprev.dsr && - cnow.dcd == cprev.dcd && cnow.cts == cprev.cts) { - rc = -EIO; - break; - } - - /* check for change in caller specified modem input */ - if ((arg & TIOCM_RNG && cnow.rng != cprev.rng) || - (arg & TIOCM_DSR && cnow.dsr != cprev.dsr) || - (arg & TIOCM_CD && cnow.dcd != cprev.dcd) || - (arg & TIOCM_CTS && cnow.cts != cprev.cts)) { - rc = 0; - break; - } - - cprev = cnow; - } - remove_wait_queue(&info->status_event_wait_q, &wait); - set_current_state(TASK_RUNNING); - return rc; -} - -/* return the state of the serial control and status signals - */ -static int tiocmget(struct tty_struct *tty) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned int result; - unsigned long flags; - - spin_lock_irqsave(&info->lock, flags); - get_signals(info); - spin_unlock_irqrestore(&info->lock, flags); - - result = ((info->serial_signals & SerialSignal_RTS) ? TIOCM_RTS:0) + - ((info->serial_signals & SerialSignal_DTR) ? TIOCM_DTR:0) + - ((info->serial_signals & SerialSignal_DCD) ? TIOCM_CAR:0) + - ((info->serial_signals & SerialSignal_RI) ? TIOCM_RNG:0) + - ((info->serial_signals & SerialSignal_DSR) ? TIOCM_DSR:0) + - ((info->serial_signals & SerialSignal_CTS) ? TIOCM_CTS:0); - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):%s tiocmget() value=%08X\n", - __FILE__, __LINE__, info->device_name, result); - return result; -} - -/* set modem control signals (DTR/RTS) - */ -static int tiocmset(struct tty_struct *tty, - unsigned int set, unsigned int clear) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):%s tiocmset(%x,%x)\n", - __FILE__, __LINE__, info->device_name, set, clear); - - if (set & TIOCM_RTS) - info->serial_signals |= SerialSignal_RTS; - if (set & TIOCM_DTR) - info->serial_signals |= SerialSignal_DTR; - if (clear & TIOCM_RTS) - info->serial_signals &= ~SerialSignal_RTS; - if (clear & TIOCM_DTR) - info->serial_signals &= ~SerialSignal_DTR; - - spin_lock_irqsave(&info->lock, flags); - set_signals(info); - spin_unlock_irqrestore(&info->lock, flags); - - return 0; -} - -/* Set or clear transmit break condition - * - * Arguments: tty pointer to tty instance data - * break_state -1=set break condition, 0=clear - */ -static int mgslpc_break(struct tty_struct *tty, int break_state) -{ - MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_break(%s,%d)\n", - __FILE__, __LINE__, info->device_name, break_state); - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_break")) - return -EINVAL; - - spin_lock_irqsave(&info->lock, flags); - if (break_state == -1) - set_reg_bits(info, CHA+DAFO, BIT6); - else - clear_reg_bits(info, CHA+DAFO, BIT6); - spin_unlock_irqrestore(&info->lock, flags); - return 0; -} - -static int mgslpc_get_icount(struct tty_struct *tty, - struct serial_icounter_struct *icount) -{ - MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data; - struct mgsl_icount cnow; /* kernel counter temps */ - unsigned long flags; - - spin_lock_irqsave(&info->lock, flags); - cnow = info->icount; - spin_unlock_irqrestore(&info->lock, flags); - - icount->cts = cnow.cts; - icount->dsr = cnow.dsr; - icount->rng = cnow.rng; - icount->dcd = cnow.dcd; - icount->rx = cnow.rx; - icount->tx = cnow.tx; - icount->frame = cnow.frame; - icount->overrun = cnow.overrun; - icount->parity = cnow.parity; - icount->brk = cnow.brk; - icount->buf_overrun = cnow.buf_overrun; - - return 0; -} - -/* Service an IOCTL request - * - * Arguments: - * - * tty pointer to tty instance data - * cmd IOCTL command code - * arg command argument/context - * - * Return Value: 0 if success, otherwise error code - */ -static int mgslpc_ioctl(struct tty_struct *tty, - unsigned int cmd, unsigned long arg) -{ - MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data; - void __user *argp = (void __user *)arg; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_ioctl %s cmd=%08X\n", __FILE__, __LINE__, - info->device_name, cmd); - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_ioctl")) - return -ENODEV; - - if (cmd != TIOCMIWAIT) { - if (tty_io_error(tty)) - return -EIO; - } - - switch (cmd) { - case MGSL_IOCGPARAMS: - return get_params(info, argp); - case MGSL_IOCSPARAMS: - return set_params(info, argp, tty); - case MGSL_IOCGTXIDLE: - return get_txidle(info, argp); - case MGSL_IOCSTXIDLE: - return set_txidle(info, (int)arg); - case MGSL_IOCGIF: - return get_interface(info, argp); - case MGSL_IOCSIF: - return set_interface(info,(int)arg); - case MGSL_IOCTXENABLE: - return set_txenable(info,(int)arg, tty); - case MGSL_IOCRXENABLE: - return set_rxenable(info,(int)arg); - case MGSL_IOCTXABORT: - return tx_abort(info); - case MGSL_IOCGSTATS: - return get_stats(info, argp); - case MGSL_IOCWAITEVENT: - return wait_events(info, argp); - case TIOCMIWAIT: - return modem_input_wait(info,(int)arg); - default: - return -ENOIOCTLCMD; - } - return 0; -} - -/* Set new termios settings - * - * Arguments: - * - * tty pointer to tty structure - * termios pointer to buffer to hold returned old termios - */ -static void mgslpc_set_termios(struct tty_struct *tty, - const struct ktermios *old_termios) -{ - MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_set_termios %s\n", __FILE__, __LINE__, - tty->driver->name); - - /* just return if nothing has changed */ - if ((tty->termios.c_cflag == old_termios->c_cflag) - && (RELEVANT_IFLAG(tty->termios.c_iflag) - == RELEVANT_IFLAG(old_termios->c_iflag))) - return; - - mgslpc_change_params(info, tty); - - /* Handle transition to B0 status */ - if ((old_termios->c_cflag & CBAUD) && !C_BAUD(tty)) { - info->serial_signals &= ~(SerialSignal_RTS | SerialSignal_DTR); - spin_lock_irqsave(&info->lock, flags); - set_signals(info); - spin_unlock_irqrestore(&info->lock, flags); - } - - /* Handle transition away from B0 status */ - if (!(old_termios->c_cflag & CBAUD) && C_BAUD(tty)) { - info->serial_signals |= SerialSignal_DTR; - if (!C_CRTSCTS(tty) || !tty_throttled(tty)) - info->serial_signals |= SerialSignal_RTS; - spin_lock_irqsave(&info->lock, flags); - set_signals(info); - spin_unlock_irqrestore(&info->lock, flags); - } - - /* Handle turning off CRTSCTS */ - if (old_termios->c_cflag & CRTSCTS && !C_CRTSCTS(tty)) { - tty->hw_stopped = 0; - tx_release(tty); - } -} - -static void mgslpc_close(struct tty_struct *tty, struct file * filp) -{ - MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data; - struct tty_port *port = &info->port; - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_close")) - return; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_close(%s) entry, count=%d\n", - __FILE__, __LINE__, info->device_name, port->count); - - if (tty_port_close_start(port, tty, filp) == 0) - goto cleanup; - - if (tty_port_initialized(port)) - mgslpc_wait_until_sent(tty, info->timeout); - - mgslpc_flush_buffer(tty); - - tty_ldisc_flush(tty); - shutdown(info, tty); - - tty_port_close_end(port, tty); - tty_port_tty_set(port, NULL); -cleanup: - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_close(%s) exit, count=%d\n", __FILE__, __LINE__, - tty->driver->name, port->count); -} - -/* Wait until the transmitter is empty. - */ -static void mgslpc_wait_until_sent(struct tty_struct *tty, int timeout) -{ - MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data; - unsigned long orig_jiffies, char_time; - - if (!info) - return; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_wait_until_sent(%s) entry\n", - __FILE__, __LINE__, info->device_name); - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_wait_until_sent")) - return; - - if (!tty_port_initialized(&info->port)) - goto exit; - - orig_jiffies = jiffies; - - /* Set check interval to 1/5 of estimated time to - * send a character, and make it at least 1. The check - * interval should also be less than the timeout. - * Note: use tight timings here to satisfy the NIST-PCTS. - */ - - if (info->params.data_rate) { - char_time = info->timeout/(32 * 5); - if (!char_time) - char_time++; - } else - char_time = 1; - - if (timeout) - char_time = min_t(unsigned long, char_time, timeout); - - if (info->params.mode == MGSL_MODE_HDLC) { - while (info->tx_active) { - msleep_interruptible(jiffies_to_msecs(char_time)); - if (signal_pending(current)) - break; - if (timeout && time_after(jiffies, orig_jiffies + timeout)) - break; - } - } else { - while ((info->tx_count || info->tx_active) && - info->tx_enabled) { - msleep_interruptible(jiffies_to_msecs(char_time)); - if (signal_pending(current)) - break; - if (timeout && time_after(jiffies, orig_jiffies + timeout)) - break; - } - } - -exit: - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_wait_until_sent(%s) exit\n", - __FILE__, __LINE__, info->device_name); -} - -/* Called by tty_hangup() when a hangup is signaled. - * This is the same as closing all open files for the port. - */ -static void mgslpc_hangup(struct tty_struct *tty) -{ - MGSLPC_INFO * info = (MGSLPC_INFO *)tty->driver_data; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_hangup(%s)\n", - __FILE__, __LINE__, info->device_name); - - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_hangup")) - return; - - mgslpc_flush_buffer(tty); - shutdown(info, tty); - tty_port_hangup(&info->port); -} - -static bool carrier_raised(struct tty_port *port) -{ - MGSLPC_INFO *info = container_of(port, MGSLPC_INFO, port); - unsigned long flags; - - spin_lock_irqsave(&info->lock, flags); - get_signals(info); - spin_unlock_irqrestore(&info->lock, flags); - - return info->serial_signals & SerialSignal_DCD; -} - -static void dtr_rts(struct tty_port *port, bool active) -{ - MGSLPC_INFO *info = container_of(port, MGSLPC_INFO, port); - unsigned long flags; - - spin_lock_irqsave(&info->lock, flags); - if (active) - info->serial_signals |= SerialSignal_RTS | SerialSignal_DTR; - else - info->serial_signals &= ~(SerialSignal_RTS | SerialSignal_DTR); - set_signals(info); - spin_unlock_irqrestore(&info->lock, flags); -} - - -static int mgslpc_open(struct tty_struct *tty, struct file * filp) -{ - MGSLPC_INFO *info; - struct tty_port *port; - int retval, line; - unsigned long flags; - - /* verify range of specified line number */ - line = tty->index; - if (line >= mgslpc_device_count) { - printk("%s(%d):mgslpc_open with invalid line #%d.\n", - __FILE__, __LINE__, line); - return -ENODEV; - } - - /* find the info structure for the specified line */ - info = mgslpc_device_list; - while(info && info->line != line) - info = info->next_device; - if (mgslpc_paranoia_check(info, tty->name, "mgslpc_open")) - return -ENODEV; - - port = &info->port; - tty->driver_data = info; - tty_port_tty_set(port, tty); - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_open(%s), old ref count = %d\n", - __FILE__, __LINE__, tty->driver->name, port->count); - - spin_lock_irqsave(&info->netlock, flags); - if (info->netcount) { - retval = -EBUSY; - spin_unlock_irqrestore(&info->netlock, flags); - goto cleanup; - } - spin_lock(&port->lock); - port->count++; - spin_unlock(&port->lock); - spin_unlock_irqrestore(&info->netlock, flags); - - if (port->count == 1) { - /* 1st open on this device, init hardware */ - retval = startup(info, tty); - if (retval < 0) - goto cleanup; - } - - retval = tty_port_block_til_ready(&info->port, tty, filp); - if (retval) { - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):block_til_ready(%s) returned %d\n", - __FILE__, __LINE__, info->device_name, retval); - goto cleanup; - } - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):mgslpc_open(%s) success\n", - __FILE__, __LINE__, info->device_name); - retval = 0; - -cleanup: - return retval; -} - -/* - * /proc fs routines.... - */ - -static inline void line_info(struct seq_file *m, MGSLPC_INFO *info) -{ - char stat_buf[30]; - unsigned long flags; - - seq_printf(m, "%s:io:%04X irq:%d", - info->device_name, info->io_base, info->irq_level); - - /* output current serial signal states */ - spin_lock_irqsave(&info->lock, flags); - get_signals(info); - spin_unlock_irqrestore(&info->lock, flags); - - stat_buf[0] = 0; - stat_buf[1] = 0; - if (info->serial_signals & SerialSignal_RTS) - strcat(stat_buf, "|RTS"); - if (info->serial_signals & SerialSignal_CTS) - strcat(stat_buf, "|CTS"); - if (info->serial_signals & SerialSignal_DTR) - strcat(stat_buf, "|DTR"); - if (info->serial_signals & SerialSignal_DSR) - strcat(stat_buf, "|DSR"); - if (info->serial_signals & SerialSignal_DCD) - strcat(stat_buf, "|CD"); - if (info->serial_signals & SerialSignal_RI) - strcat(stat_buf, "|RI"); - - if (info->params.mode == MGSL_MODE_HDLC) { - seq_printf(m, " HDLC txok:%d rxok:%d", - info->icount.txok, info->icount.rxok); - if (info->icount.txunder) - seq_printf(m, " txunder:%d", info->icount.txunder); - if (info->icount.txabort) - seq_printf(m, " txabort:%d", info->icount.txabort); - if (info->icount.rxshort) - seq_printf(m, " rxshort:%d", info->icount.rxshort); - if (info->icount.rxlong) - seq_printf(m, " rxlong:%d", info->icount.rxlong); - if (info->icount.rxover) - seq_printf(m, " rxover:%d", info->icount.rxover); - if (info->icount.rxcrc) - seq_printf(m, " rxcrc:%d", info->icount.rxcrc); - } else { - seq_printf(m, " ASYNC tx:%d rx:%d", - info->icount.tx, info->icount.rx); - if (info->icount.frame) - seq_printf(m, " fe:%d", info->icount.frame); - if (info->icount.parity) - seq_printf(m, " pe:%d", info->icount.parity); - if (info->icount.brk) - seq_printf(m, " brk:%d", info->icount.brk); - if (info->icount.overrun) - seq_printf(m, " oe:%d", info->icount.overrun); - } - - /* Append serial signal status to end */ - seq_printf(m, " %s\n", stat_buf+1); - - seq_printf(m, "txactive=%d bh_req=%d bh_run=%d pending_bh=%x\n", - info->tx_active,info->bh_requested,info->bh_running, - info->pending_bh); -} - -/* Called to print information about devices - */ -static int mgslpc_proc_show(struct seq_file *m, void *v) -{ - MGSLPC_INFO *info; - - seq_printf(m, "synclink driver:%s\n", driver_version); - - info = mgslpc_device_list; - while (info) { - line_info(m, info); - info = info->next_device; - } - return 0; -} - -static int rx_alloc_buffers(MGSLPC_INFO *info) -{ - /* each buffer has header and data */ - info->rx_buf_size = sizeof(RXBUF) + info->max_frame_size; - - /* calculate total allocation size for 8 buffers */ - info->rx_buf_total_size = info->rx_buf_size * 8; - - /* limit total allocated memory */ - if (info->rx_buf_total_size > 0x10000) - info->rx_buf_total_size = 0x10000; - - /* calculate number of buffers */ - info->rx_buf_count = info->rx_buf_total_size / info->rx_buf_size; - - info->rx_buf = kmalloc(info->rx_buf_total_size, GFP_KERNEL); - if (info->rx_buf == NULL) - return -ENOMEM; - - /* unused flag buffer to satisfy receive_buf calling interface */ - info->flag_buf = kzalloc(info->max_frame_size, GFP_KERNEL); - if (!info->flag_buf) { - kfree(info->rx_buf); - info->rx_buf = NULL; - return -ENOMEM; - } - - rx_reset_buffers(info); - return 0; -} - -static void rx_free_buffers(MGSLPC_INFO *info) -{ - kfree(info->rx_buf); - info->rx_buf = NULL; - kfree(info->flag_buf); - info->flag_buf = NULL; -} - -static int claim_resources(MGSLPC_INFO *info) -{ - if (rx_alloc_buffers(info) < 0) { - printk("Can't allocate rx buffer %s\n", info->device_name); - release_resources(info); - return -ENODEV; - } - return 0; -} - -static void release_resources(MGSLPC_INFO *info) -{ - if (debug_level >= DEBUG_LEVEL_INFO) - printk("release_resources(%s)\n", info->device_name); - rx_free_buffers(info); -} - -/* Add the specified device instance data structure to the - * global linked list of devices and increment the device count. - * - * Arguments: info pointer to device instance data - */ -static int mgslpc_add_device(MGSLPC_INFO *info) -{ - MGSLPC_INFO *current_dev = NULL; - struct device *tty_dev; - int ret; - - info->next_device = NULL; - info->line = mgslpc_device_count; - sprintf(info->device_name,"ttySLP%d",info->line); - - if (info->line < MAX_DEVICE_COUNT) { - if (maxframe[info->line]) - info->max_frame_size = maxframe[info->line]; - } - - mgslpc_device_count++; - - if (!mgslpc_device_list) - mgslpc_device_list = info; - else { - current_dev = mgslpc_device_list; - while (current_dev->next_device) - current_dev = current_dev->next_device; - current_dev->next_device = info; - } - - if (info->max_frame_size < 4096) - info->max_frame_size = 4096; - else if (info->max_frame_size > 65535) - info->max_frame_size = 65535; - - printk("SyncLink PC Card %s:IO=%04X IRQ=%d\n", - info->device_name, info->io_base, info->irq_level); - -#if SYNCLINK_GENERIC_HDLC - ret = hdlcdev_init(info); - if (ret != 0) - goto failed; -#endif - - tty_dev = tty_port_register_device(&info->port, serial_driver, info->line, - &info->p_dev->dev); - if (IS_ERR(tty_dev)) { - ret = PTR_ERR(tty_dev); -#if SYNCLINK_GENERIC_HDLC - hdlcdev_exit(info); -#endif - goto failed; - } - - return 0; - -failed: - if (current_dev) - current_dev->next_device = NULL; - else - mgslpc_device_list = NULL; - mgslpc_device_count--; - return ret; -} - -static void mgslpc_remove_device(MGSLPC_INFO *remove_info) -{ - MGSLPC_INFO *info = mgslpc_device_list; - MGSLPC_INFO *last = NULL; - - while(info) { - if (info == remove_info) { - if (last) - last->next_device = info->next_device; - else - mgslpc_device_list = info->next_device; - tty_unregister_device(serial_driver, info->line); -#if SYNCLINK_GENERIC_HDLC - hdlcdev_exit(info); -#endif - release_resources(info); - tty_port_destroy(&info->port); - kfree(info); - mgslpc_device_count--; - return; - } - last = info; - info = info->next_device; - } -} - -static const struct pcmcia_device_id mgslpc_ids[] = { - PCMCIA_DEVICE_MANF_CARD(0x02c5, 0x0050), - PCMCIA_DEVICE_NULL -}; -MODULE_DEVICE_TABLE(pcmcia, mgslpc_ids); - -static struct pcmcia_driver mgslpc_driver = { - .owner = THIS_MODULE, - .name = "synclink_cs", - .probe = mgslpc_probe, - .remove = mgslpc_detach, - .id_table = mgslpc_ids, - .suspend = mgslpc_suspend, - .resume = mgslpc_resume, -}; - -static const struct tty_operations mgslpc_ops = { - .open = mgslpc_open, - .close = mgslpc_close, - .write = mgslpc_write, - .put_char = mgslpc_put_char, - .flush_chars = mgslpc_flush_chars, - .write_room = mgslpc_write_room, - .chars_in_buffer = mgslpc_chars_in_buffer, - .flush_buffer = mgslpc_flush_buffer, - .ioctl = mgslpc_ioctl, - .throttle = mgslpc_throttle, - .unthrottle = mgslpc_unthrottle, - .send_xchar = mgslpc_send_xchar, - .break_ctl = mgslpc_break, - .wait_until_sent = mgslpc_wait_until_sent, - .set_termios = mgslpc_set_termios, - .stop = tx_pause, - .start = tx_release, - .hangup = mgslpc_hangup, - .tiocmget = tiocmget, - .tiocmset = tiocmset, - .get_icount = mgslpc_get_icount, - .proc_show = mgslpc_proc_show, -}; - -static int __init synclink_cs_init(void) -{ - int rc; - - if (break_on_load) { - mgslpc_get_text_ptr(); - BREAKPOINT(); - } - - serial_driver = tty_alloc_driver(MAX_DEVICE_COUNT, - TTY_DRIVER_REAL_RAW | - TTY_DRIVER_DYNAMIC_DEV); - if (IS_ERR(serial_driver)) { - rc = PTR_ERR(serial_driver); - goto err; - } - - /* Initialize the tty_driver structure */ - serial_driver->driver_name = "synclink_cs"; - serial_driver->name = "ttySLP"; - serial_driver->major = ttymajor; - serial_driver->minor_start = 64; - serial_driver->type = TTY_DRIVER_TYPE_SERIAL; - serial_driver->subtype = SERIAL_TYPE_NORMAL; - serial_driver->init_termios = tty_std_termios; - serial_driver->init_termios.c_cflag = - B9600 | CS8 | CREAD | HUPCL | CLOCAL; - tty_set_operations(serial_driver, &mgslpc_ops); - - rc = tty_register_driver(serial_driver); - if (rc < 0) { - printk(KERN_ERR "%s(%d):Couldn't register serial driver\n", - __FILE__, __LINE__); - goto err_put_tty; - } - - rc = pcmcia_register_driver(&mgslpc_driver); - if (rc < 0) - goto err_unreg_tty; - - printk(KERN_INFO "%s %s, tty major#%d\n", driver_name, driver_version, - serial_driver->major); - - return 0; -err_unreg_tty: - tty_unregister_driver(serial_driver); -err_put_tty: - tty_driver_kref_put(serial_driver); -err: - return rc; -} - -static void __exit synclink_cs_exit(void) -{ - pcmcia_unregister_driver(&mgslpc_driver); - tty_unregister_driver(serial_driver); - tty_driver_kref_put(serial_driver); -} - -module_init(synclink_cs_init); -module_exit(synclink_cs_exit); - -static void mgslpc_set_rate(MGSLPC_INFO *info, unsigned char channel, unsigned int rate) -{ - unsigned int M, N; - unsigned char val; - - /* note:standard BRG mode is broken in V3.2 chip - * so enhanced mode is always used - */ - - if (rate) { - N = 3686400 / rate; - if (!N) - N = 1; - N >>= 1; - for (M = 1; N > 64 && M < 16; M++) - N >>= 1; - N--; - - /* BGR[5..0] = N - * BGR[9..6] = M - * BGR[7..0] contained in BGR register - * BGR[9..8] contained in CCR2[7..6] - * divisor = (N+1)*2^M - * - * Note: M *must* not be zero (causes asymetric duty cycle) - */ - write_reg(info, (unsigned char) (channel + BGR), - (unsigned char) ((M << 6) + N)); - val = read_reg(info, (unsigned char) (channel + CCR2)) & 0x3f; - val |= ((M << 4) & 0xc0); - write_reg(info, (unsigned char) (channel + CCR2), val); - } -} - -/* Enabled the AUX clock output at the specified frequency. - */ -static void enable_auxclk(MGSLPC_INFO *info) -{ - unsigned char val; - - /* MODE - * - * 07..06 MDS[1..0] 10 = transparent HDLC mode - * 05 ADM Address Mode, 0 = no addr recognition - * 04 TMD Timer Mode, 0 = external - * 03 RAC Receiver Active, 0 = inactive - * 02 RTS 0=RTS active during xmit, 1=RTS always active - * 01 TRS Timer Resolution, 1=512 - * 00 TLP Test Loop, 0 = no loop - * - * 1000 0010 - */ - val = 0x82; - - /* channel B RTS is used to enable AUXCLK driver on SP505 */ - if (info->params.mode == MGSL_MODE_HDLC && info->params.clock_speed) - val |= BIT2; - write_reg(info, CHB + MODE, val); - - /* CCR0 - * - * 07 PU Power Up, 1=active, 0=power down - * 06 MCE Master Clock Enable, 1=enabled - * 05 Reserved, 0 - * 04..02 SC[2..0] Encoding - * 01..00 SM[1..0] Serial Mode, 00=HDLC - * - * 11000000 - */ - write_reg(info, CHB + CCR0, 0xc0); - - /* CCR1 - * - * 07 SFLG Shared Flag, 0 = disable shared flags - * 06 GALP Go Active On Loop, 0 = not used - * 05 GLP Go On Loop, 0 = not used - * 04 ODS Output Driver Select, 1=TxD is push-pull output - * 03 ITF Interframe Time Fill, 0=mark, 1=flag - * 02..00 CM[2..0] Clock Mode - * - * 0001 0111 - */ - write_reg(info, CHB + CCR1, 0x17); - - /* CCR2 (Channel B) - * - * 07..06 BGR[9..8] Baud rate bits 9..8 - * 05 BDF Baud rate divisor factor, 0=1, 1=BGR value - * 04 SSEL Clock source select, 1=submode b - * 03 TOE 0=TxCLK is input, 1=TxCLK is output - * 02 RWX Read/Write Exchange 0=disabled - * 01 C32, CRC select, 0=CRC-16, 1=CRC-32 - * 00 DIV, data inversion 0=disabled, 1=enabled - * - * 0011 1000 - */ - if (info->params.mode == MGSL_MODE_HDLC && info->params.clock_speed) - write_reg(info, CHB + CCR2, 0x38); - else - write_reg(info, CHB + CCR2, 0x30); - - /* CCR4 - * - * 07 MCK4 Master Clock Divide by 4, 1=enabled - * 06 EBRG Enhanced Baud Rate Generator Mode, 1=enabled - * 05 TST1 Test Pin, 0=normal operation - * 04 ICD Ivert Carrier Detect, 1=enabled (active low) - * 03..02 Reserved, must be 0 - * 01..00 RFT[1..0] RxFIFO Threshold 00=32 bytes - * - * 0101 0000 - */ - write_reg(info, CHB + CCR4, 0x50); - - /* if auxclk not enabled, set internal BRG so - * CTS transitions can be detected (requires TxC) - */ - if (info->params.mode == MGSL_MODE_HDLC && info->params.clock_speed) - mgslpc_set_rate(info, CHB, info->params.clock_speed); - else - mgslpc_set_rate(info, CHB, 921600); -} - -static void loopback_enable(MGSLPC_INFO *info) -{ - unsigned char val; - - /* CCR1:02..00 CM[2..0] Clock Mode = 111 (clock mode 7) */ - val = read_reg(info, CHA + CCR1) | (BIT2 | BIT1 | BIT0); - write_reg(info, CHA + CCR1, val); - - /* CCR2:04 SSEL Clock source select, 1=submode b */ - val = read_reg(info, CHA + CCR2) | (BIT4 | BIT5); - write_reg(info, CHA + CCR2, val); - - /* set LinkSpeed if available, otherwise default to 2Mbps */ - if (info->params.clock_speed) - mgslpc_set_rate(info, CHA, info->params.clock_speed); - else - mgslpc_set_rate(info, CHA, 1843200); - - /* MODE:00 TLP Test Loop, 1=loopback enabled */ - val = read_reg(info, CHA + MODE) | BIT0; - write_reg(info, CHA + MODE, val); -} - -static void hdlc_mode(MGSLPC_INFO *info) -{ - unsigned char val; - unsigned char clkmode, clksubmode; - - /* disable all interrupts */ - irq_disable(info, CHA, 0xffff); - irq_disable(info, CHB, 0xffff); - port_irq_disable(info, 0xff); - - /* assume clock mode 0a, rcv=RxC xmt=TxC */ - clkmode = clksubmode = 0; - if (info->params.flags & HDLC_FLAG_RXC_DPLL - && info->params.flags & HDLC_FLAG_TXC_DPLL) { - /* clock mode 7a, rcv = DPLL, xmt = DPLL */ - clkmode = 7; - } else if (info->params.flags & HDLC_FLAG_RXC_BRG - && info->params.flags & HDLC_FLAG_TXC_BRG) { - /* clock mode 7b, rcv = BRG, xmt = BRG */ - clkmode = 7; - clksubmode = 1; - } else if (info->params.flags & HDLC_FLAG_RXC_DPLL) { - if (info->params.flags & HDLC_FLAG_TXC_BRG) { - /* clock mode 6b, rcv = DPLL, xmt = BRG/16 */ - clkmode = 6; - clksubmode = 1; - } else { - /* clock mode 6a, rcv = DPLL, xmt = TxC */ - clkmode = 6; - } - } else if (info->params.flags & HDLC_FLAG_TXC_BRG) { - /* clock mode 0b, rcv = RxC, xmt = BRG */ - clksubmode = 1; - } - - /* MODE - * - * 07..06 MDS[1..0] 10 = transparent HDLC mode - * 05 ADM Address Mode, 0 = no addr recognition - * 04 TMD Timer Mode, 0 = external - * 03 RAC Receiver Active, 0 = inactive - * 02 RTS 0=RTS active during xmit, 1=RTS always active - * 01 TRS Timer Resolution, 1=512 - * 00 TLP Test Loop, 0 = no loop - * - * 1000 0010 - */ - val = 0x82; - if (info->params.loopback) - val |= BIT0; - - /* preserve RTS state */ - if (info->serial_signals & SerialSignal_RTS) - val |= BIT2; - write_reg(info, CHA + MODE, val); - - /* CCR0 - * - * 07 PU Power Up, 1=active, 0=power down - * 06 MCE Master Clock Enable, 1=enabled - * 05 Reserved, 0 - * 04..02 SC[2..0] Encoding - * 01..00 SM[1..0] Serial Mode, 00=HDLC - * - * 11000000 - */ - val = 0xc0; - switch (info->params.encoding) - { - case HDLC_ENCODING_NRZI: - val |= BIT3; - break; - case HDLC_ENCODING_BIPHASE_SPACE: - val |= BIT4; - break; // FM0 - case HDLC_ENCODING_BIPHASE_MARK: - val |= BIT4 | BIT2; - break; // FM1 - case HDLC_ENCODING_BIPHASE_LEVEL: - val |= BIT4 | BIT3; - break; // Manchester - } - write_reg(info, CHA + CCR0, val); - - /* CCR1 - * - * 07 SFLG Shared Flag, 0 = disable shared flags - * 06 GALP Go Active On Loop, 0 = not used - * 05 GLP Go On Loop, 0 = not used - * 04 ODS Output Driver Select, 1=TxD is push-pull output - * 03 ITF Interframe Time Fill, 0=mark, 1=flag - * 02..00 CM[2..0] Clock Mode - * - * 0001 0000 - */ - val = 0x10 + clkmode; - write_reg(info, CHA + CCR1, val); - - /* CCR2 - * - * 07..06 BGR[9..8] Baud rate bits 9..8 - * 05 BDF Baud rate divisor factor, 0=1, 1=BGR value - * 04 SSEL Clock source select, 1=submode b - * 03 TOE 0=TxCLK is input, 0=TxCLK is input - * 02 RWX Read/Write Exchange 0=disabled - * 01 C32, CRC select, 0=CRC-16, 1=CRC-32 - * 00 DIV, data inversion 0=disabled, 1=enabled - * - * 0000 0000 - */ - val = 0x00; - if (clkmode == 2 || clkmode == 3 || clkmode == 6 - || clkmode == 7 || (clkmode == 0 && clksubmode == 1)) - val |= BIT5; - if (clksubmode) - val |= BIT4; - if (info->params.crc_type == HDLC_CRC_32_CCITT) - val |= BIT1; - if (info->params.encoding == HDLC_ENCODING_NRZB) - val |= BIT0; - write_reg(info, CHA + CCR2, val); - - /* CCR3 - * - * 07..06 PRE[1..0] Preamble count 00=1, 01=2, 10=4, 11=8 - * 05 EPT Enable preamble transmission, 1=enabled - * 04 RADD Receive address pushed to FIFO, 0=disabled - * 03 CRL CRC Reset Level, 0=FFFF - * 02 RCRC Rx CRC 0=On 1=Off - * 01 TCRC Tx CRC 0=On 1=Off - * 00 PSD DPLL Phase Shift Disable - * - * 0000 0000 - */ - val = 0x00; - if (info->params.crc_type == HDLC_CRC_NONE) - val |= BIT2 | BIT1; - if (info->params.preamble != HDLC_PREAMBLE_PATTERN_NONE) - val |= BIT5; - switch (info->params.preamble_length) - { - case HDLC_PREAMBLE_LENGTH_16BITS: - val |= BIT6; - break; - case HDLC_PREAMBLE_LENGTH_32BITS: - val |= BIT6; - break; - case HDLC_PREAMBLE_LENGTH_64BITS: - val |= BIT7 | BIT6; - break; - } - write_reg(info, CHA + CCR3, val); - - /* PRE - Preamble pattern */ - val = 0; - switch (info->params.preamble) - { - case HDLC_PREAMBLE_PATTERN_FLAGS: val = 0x7e; break; - case HDLC_PREAMBLE_PATTERN_10: val = 0xaa; break; - case HDLC_PREAMBLE_PATTERN_01: val = 0x55; break; - case HDLC_PREAMBLE_PATTERN_ONES: val = 0xff; break; - } - write_reg(info, CHA + PRE, val); - - /* CCR4 - * - * 07 MCK4 Master Clock Divide by 4, 1=enabled - * 06 EBRG Enhanced Baud Rate Generator Mode, 1=enabled - * 05 TST1 Test Pin, 0=normal operation - * 04 ICD Ivert Carrier Detect, 1=enabled (active low) - * 03..02 Reserved, must be 0 - * 01..00 RFT[1..0] RxFIFO Threshold 00=32 bytes - * - * 0101 0000 - */ - val = 0x50; - write_reg(info, CHA + CCR4, val); - if (info->params.flags & HDLC_FLAG_RXC_DPLL) - mgslpc_set_rate(info, CHA, info->params.clock_speed * 16); - else - mgslpc_set_rate(info, CHA, info->params.clock_speed); - - /* RLCR Receive length check register - * - * 7 1=enable receive length check - * 6..0 Max frame length = (RL + 1) * 32 - */ - write_reg(info, CHA + RLCR, 0); - - /* XBCH Transmit Byte Count High - * - * 07 DMA mode, 0 = interrupt driven - * 06 NRM, 0=ABM (ignored) - * 05 CAS Carrier Auto Start - * 04 XC Transmit Continuously (ignored) - * 03..00 XBC[10..8] Transmit byte count bits 10..8 - * - * 0000 0000 - */ - val = 0x00; - if (info->params.flags & HDLC_FLAG_AUTO_DCD) - val |= BIT5; - write_reg(info, CHA + XBCH, val); - enable_auxclk(info); - if (info->params.loopback || info->testing_irq) - loopback_enable(info); - if (info->params.flags & HDLC_FLAG_AUTO_CTS) - { - irq_enable(info, CHB, IRQ_CTS); - /* PVR[3] 1=AUTO CTS active */ - set_reg_bits(info, CHA + PVR, BIT3); - } else - clear_reg_bits(info, CHA + PVR, BIT3); - - irq_enable(info, CHA, - IRQ_RXEOM | IRQ_RXFIFO | IRQ_ALLSENT | - IRQ_UNDERRUN | IRQ_TXFIFO); - issue_command(info, CHA, CMD_TXRESET + CMD_RXRESET); - wait_command_complete(info, CHA); - read_reg16(info, CHA + ISR); /* clear pending IRQs */ - - /* Master clock mode enabled above to allow reset commands - * to complete even if no data clocks are present. - * - * Disable master clock mode for normal communications because - * V3.2 of the ESCC2 has a bug that prevents the transmit all sent - * IRQ when in master clock mode. - * - * Leave master clock mode enabled for IRQ test because the - * timer IRQ used by the test can only happen in master clock mode. - */ - if (!info->testing_irq) - clear_reg_bits(info, CHA + CCR0, BIT6); - - tx_set_idle(info); - - tx_stop(info); - rx_stop(info); -} - -static void rx_stop(MGSLPC_INFO *info) -{ - if (debug_level >= DEBUG_LEVEL_ISR) - printk("%s(%d):rx_stop(%s)\n", - __FILE__, __LINE__, info->device_name); - - /* MODE:03 RAC Receiver Active, 0=inactive */ - clear_reg_bits(info, CHA + MODE, BIT3); - - info->rx_enabled = false; - info->rx_overflow = false; -} - -static void rx_start(MGSLPC_INFO *info) -{ - if (debug_level >= DEBUG_LEVEL_ISR) - printk("%s(%d):rx_start(%s)\n", - __FILE__, __LINE__, info->device_name); - - rx_reset_buffers(info); - info->rx_enabled = false; - info->rx_overflow = false; - - /* MODE:03 RAC Receiver Active, 1=active */ - set_reg_bits(info, CHA + MODE, BIT3); - - info->rx_enabled = true; -} - -static void tx_start(MGSLPC_INFO *info, struct tty_struct *tty) -{ - if (debug_level >= DEBUG_LEVEL_ISR) - printk("%s(%d):tx_start(%s)\n", - __FILE__, __LINE__, info->device_name); - - if (info->tx_count) { - /* If auto RTS enabled and RTS is inactive, then assert */ - /* RTS and set a flag indicating that the driver should */ - /* negate RTS when the transmission completes. */ - info->drop_rts_on_tx_done = false; - - if (info->params.flags & HDLC_FLAG_AUTO_RTS) { - get_signals(info); - if (!(info->serial_signals & SerialSignal_RTS)) { - info->serial_signals |= SerialSignal_RTS; - set_signals(info); - info->drop_rts_on_tx_done = true; - } - } - - if (info->params.mode == MGSL_MODE_ASYNC) { - if (!info->tx_active) { - info->tx_active = true; - tx_ready(info, tty); - } - } else { - info->tx_active = true; - tx_ready(info, tty); - mod_timer(&info->tx_timer, jiffies + - msecs_to_jiffies(5000)); - } - } - - if (!info->tx_enabled) - info->tx_enabled = true; -} - -static void tx_stop(MGSLPC_INFO *info) -{ - if (debug_level >= DEBUG_LEVEL_ISR) - printk("%s(%d):tx_stop(%s)\n", - __FILE__, __LINE__, info->device_name); - - del_timer(&info->tx_timer); - - info->tx_enabled = false; - info->tx_active = false; -} - -/* Reset the adapter to a known state and prepare it for further use. - */ -static void reset_device(MGSLPC_INFO *info) -{ - /* power up both channels (set BIT7) */ - write_reg(info, CHA + CCR0, 0x80); - write_reg(info, CHB + CCR0, 0x80); - write_reg(info, CHA + MODE, 0); - write_reg(info, CHB + MODE, 0); - - /* disable all interrupts */ - irq_disable(info, CHA, 0xffff); - irq_disable(info, CHB, 0xffff); - port_irq_disable(info, 0xff); - - /* PCR Port Configuration Register - * - * 07..04 DEC[3..0] Serial I/F select outputs - * 03 output, 1=AUTO CTS control enabled - * 02 RI Ring Indicator input 0=active - * 01 DSR input 0=active - * 00 DTR output 0=active - * - * 0000 0110 - */ - write_reg(info, PCR, 0x06); - - /* PVR Port Value Register - * - * 07..04 DEC[3..0] Serial I/F select (0000=disabled) - * 03 AUTO CTS output 1=enabled - * 02 RI Ring Indicator input - * 01 DSR input - * 00 DTR output (1=inactive) - * - * 0000 0001 - */ -// write_reg(info, PVR, PVR_DTR); - - /* IPC Interrupt Port Configuration - * - * 07 VIS 1=Masked interrupts visible - * 06..05 Reserved, 0 - * 04..03 SLA Slave address, 00 ignored - * 02 CASM Cascading Mode, 1=daisy chain - * 01..00 IC[1..0] Interrupt Config, 01=push-pull output, active low - * - * 0000 0101 - */ - write_reg(info, IPC, 0x05); -} - -static void async_mode(MGSLPC_INFO *info) -{ - unsigned char val; - - /* disable all interrupts */ - irq_disable(info, CHA, 0xffff); - irq_disable(info, CHB, 0xffff); - port_irq_disable(info, 0xff); - - /* MODE - * - * 07 Reserved, 0 - * 06 FRTS RTS State, 0=active - * 05 FCTS Flow Control on CTS - * 04 FLON Flow Control Enable - * 03 RAC Receiver Active, 0 = inactive - * 02 RTS 0=Auto RTS, 1=manual RTS - * 01 TRS Timer Resolution, 1=512 - * 00 TLP Test Loop, 0 = no loop - * - * 0000 0110 - */ - val = 0x06; - if (info->params.loopback) - val |= BIT0; - - /* preserve RTS state */ - if (!(info->serial_signals & SerialSignal_RTS)) - val |= BIT6; - write_reg(info, CHA + MODE, val); - - /* CCR0 - * - * 07 PU Power Up, 1=active, 0=power down - * 06 MCE Master Clock Enable, 1=enabled - * 05 Reserved, 0 - * 04..02 SC[2..0] Encoding, 000=NRZ - * 01..00 SM[1..0] Serial Mode, 11=Async - * - * 1000 0011 - */ - write_reg(info, CHA + CCR0, 0x83); - - /* CCR1 - * - * 07..05 Reserved, 0 - * 04 ODS Output Driver Select, 1=TxD is push-pull output - * 03 BCR Bit Clock Rate, 1=16x - * 02..00 CM[2..0] Clock Mode, 111=BRG - * - * 0001 1111 - */ - write_reg(info, CHA + CCR1, 0x1f); - - /* CCR2 (channel A) - * - * 07..06 BGR[9..8] Baud rate bits 9..8 - * 05 BDF Baud rate divisor factor, 0=1, 1=BGR value - * 04 SSEL Clock source select, 1=submode b - * 03 TOE 0=TxCLK is input, 0=TxCLK is input - * 02 RWX Read/Write Exchange 0=disabled - * 01 Reserved, 0 - * 00 DIV, data inversion 0=disabled, 1=enabled - * - * 0001 0000 - */ - write_reg(info, CHA + CCR2, 0x10); - - /* CCR3 - * - * 07..01 Reserved, 0 - * 00 PSD DPLL Phase Shift Disable - * - * 0000 0000 - */ - write_reg(info, CHA + CCR3, 0); - - /* CCR4 - * - * 07 MCK4 Master Clock Divide by 4, 1=enabled - * 06 EBRG Enhanced Baud Rate Generator Mode, 1=enabled - * 05 TST1 Test Pin, 0=normal operation - * 04 ICD Ivert Carrier Detect, 1=enabled (active low) - * 03..00 Reserved, must be 0 - * - * 0101 0000 - */ - write_reg(info, CHA + CCR4, 0x50); - mgslpc_set_rate(info, CHA, info->params.data_rate * 16); - - /* DAFO Data Format - * - * 07 Reserved, 0 - * 06 XBRK transmit break, 0=normal operation - * 05 Stop bits (0=1, 1=2) - * 04..03 PAR[1..0] Parity (01=odd, 10=even) - * 02 PAREN Parity Enable - * 01..00 CHL[1..0] Character Length (00=8, 01=7) - * - */ - val = 0x00; - if (info->params.data_bits != 8) - val |= BIT0; /* 7 bits */ - if (info->params.stop_bits != 1) - val |= BIT5; - if (info->params.parity != ASYNC_PARITY_NONE) - { - val |= BIT2; /* Parity enable */ - if (info->params.parity == ASYNC_PARITY_ODD) - val |= BIT3; - else - val |= BIT4; - } - write_reg(info, CHA + DAFO, val); - - /* RFC Rx FIFO Control - * - * 07 Reserved, 0 - * 06 DPS, 1=parity bit not stored in data byte - * 05 DXS, 0=all data stored in FIFO (including XON/XOFF) - * 04 RFDF Rx FIFO Data Format, 1=status byte stored in FIFO - * 03..02 RFTH[1..0], rx threshold, 11=16 status + 16 data byte - * 01 Reserved, 0 - * 00 TCDE Terminate Char Detect Enable, 0=disabled - * - * 0101 1100 - */ - write_reg(info, CHA + RFC, 0x5c); - - /* RLCR Receive length check register - * - * Max frame length = (RL + 1) * 32 - */ - write_reg(info, CHA + RLCR, 0); - - /* XBCH Transmit Byte Count High - * - * 07 DMA mode, 0 = interrupt driven - * 06 NRM, 0=ABM (ignored) - * 05 CAS Carrier Auto Start - * 04 XC Transmit Continuously (ignored) - * 03..00 XBC[10..8] Transmit byte count bits 10..8 - * - * 0000 0000 - */ - val = 0x00; - if (info->params.flags & HDLC_FLAG_AUTO_DCD) - val |= BIT5; - write_reg(info, CHA + XBCH, val); - if (info->params.flags & HDLC_FLAG_AUTO_CTS) - irq_enable(info, CHA, IRQ_CTS); - - /* MODE:03 RAC Receiver Active, 1=active */ - set_reg_bits(info, CHA + MODE, BIT3); - enable_auxclk(info); - if (info->params.flags & HDLC_FLAG_AUTO_CTS) { - irq_enable(info, CHB, IRQ_CTS); - /* PVR[3] 1=AUTO CTS active */ - set_reg_bits(info, CHA + PVR, BIT3); - } else - clear_reg_bits(info, CHA + PVR, BIT3); - irq_enable(info, CHA, - IRQ_RXEOM | IRQ_RXFIFO | IRQ_BREAK_ON | IRQ_RXTIME | - IRQ_ALLSENT | IRQ_TXFIFO); - issue_command(info, CHA, CMD_TXRESET + CMD_RXRESET); - wait_command_complete(info, CHA); - read_reg16(info, CHA + ISR); /* clear pending IRQs */ -} - -/* Set the HDLC idle mode for the transmitter. - */ -static void tx_set_idle(MGSLPC_INFO *info) -{ - /* Note: ESCC2 only supports flags and one idle modes */ - if (info->idle_mode == HDLC_TXIDLE_FLAGS) - set_reg_bits(info, CHA + CCR1, BIT3); - else - clear_reg_bits(info, CHA + CCR1, BIT3); -} - -/* get state of the V24 status (input) signals. - */ -static void get_signals(MGSLPC_INFO *info) -{ - unsigned char status = 0; - - /* preserve RTS and DTR */ - info->serial_signals &= SerialSignal_RTS | SerialSignal_DTR; - - if (read_reg(info, CHB + VSTR) & BIT7) - info->serial_signals |= SerialSignal_DCD; - if (read_reg(info, CHB + STAR) & BIT1) - info->serial_signals |= SerialSignal_CTS; - - status = read_reg(info, CHA + PVR); - if (!(status & PVR_RI)) - info->serial_signals |= SerialSignal_RI; - if (!(status & PVR_DSR)) - info->serial_signals |= SerialSignal_DSR; -} - -/* Set the state of RTS and DTR based on contents of - * serial_signals member of device extension. - */ -static void set_signals(MGSLPC_INFO *info) -{ - unsigned char val; - - val = read_reg(info, CHA + MODE); - if (info->params.mode == MGSL_MODE_ASYNC) { - if (info->serial_signals & SerialSignal_RTS) - val &= ~BIT6; - else - val |= BIT6; - } else { - if (info->serial_signals & SerialSignal_RTS) - val |= BIT2; - else - val &= ~BIT2; - } - write_reg(info, CHA + MODE, val); - - if (info->serial_signals & SerialSignal_DTR) - clear_reg_bits(info, CHA + PVR, PVR_DTR); - else - set_reg_bits(info, CHA + PVR, PVR_DTR); -} - -static void rx_reset_buffers(MGSLPC_INFO *info) -{ - RXBUF *buf; - int i; - - info->rx_put = 0; - info->rx_get = 0; - info->rx_frame_count = 0; - for (i=0 ; i < info->rx_buf_count ; i++) { - buf = (RXBUF*)(info->rx_buf + (i * info->rx_buf_size)); - buf->status = buf->count = 0; - } -} - -/* Attempt to return a received HDLC frame - * Only frames received without errors are returned. - * - * Returns true if frame returned, otherwise false - */ -static bool rx_get_frame(MGSLPC_INFO *info, struct tty_struct *tty) -{ - unsigned short status; - RXBUF *buf; - unsigned int framesize = 0; - unsigned long flags; - bool return_frame = false; - - if (info->rx_frame_count == 0) - return false; - - buf = (RXBUF*)(info->rx_buf + (info->rx_get * info->rx_buf_size)); - - status = buf->status; - - /* 07 VFR 1=valid frame - * 06 RDO 1=data overrun - * 05 CRC 1=OK, 0=error - * 04 RAB 1=frame aborted - */ - if ((status & 0xf0) != 0xA0) { - if (!(status & BIT7) || (status & BIT4)) - info->icount.rxabort++; - else if (status & BIT6) - info->icount.rxover++; - else if (!(status & BIT5)) { - info->icount.rxcrc++; - if (info->params.crc_type & HDLC_CRC_RETURN_EX) - return_frame = true; - } - framesize = 0; -#if SYNCLINK_GENERIC_HDLC - { - info->netdev->stats.rx_errors++; - info->netdev->stats.rx_frame_errors++; - } -#endif - } else - return_frame = true; - - if (return_frame) - framesize = buf->count; - - if (debug_level >= DEBUG_LEVEL_BH) - printk("%s(%d):rx_get_frame(%s) status=%04X size=%d\n", - __FILE__, __LINE__, info->device_name, status, framesize); - - if (debug_level >= DEBUG_LEVEL_DATA) - trace_block(info, buf->data, framesize, 0); - - if (framesize) { - if ((info->params.crc_type & HDLC_CRC_RETURN_EX && - framesize+1 > info->max_frame_size) || - framesize > info->max_frame_size) - info->icount.rxlong++; - else { - if (status & BIT5) - info->icount.rxok++; - - if (info->params.crc_type & HDLC_CRC_RETURN_EX) { - *(buf->data + framesize) = status & BIT5 ? RX_OK:RX_CRC_ERROR; - ++framesize; - } - -#if SYNCLINK_GENERIC_HDLC - if (info->netcount) - hdlcdev_rx(info, buf->data, framesize); - else -#endif - ldisc_receive_buf(tty, buf->data, info->flag_buf, framesize); - } - } - - spin_lock_irqsave(&info->lock, flags); - buf->status = buf->count = 0; - info->rx_frame_count--; - info->rx_get++; - if (info->rx_get >= info->rx_buf_count) - info->rx_get = 0; - spin_unlock_irqrestore(&info->lock, flags); - - return true; -} - -static bool register_test(MGSLPC_INFO *info) -{ - static unsigned char patterns[] = - { 0x00, 0xff, 0xaa, 0x55, 0x69, 0x96, 0x0f }; - static unsigned int count = ARRAY_SIZE(patterns); - unsigned int i; - bool rc = true; - unsigned long flags; - - spin_lock_irqsave(&info->lock, flags); - reset_device(info); - - for (i = 0; i < count; i++) { - write_reg(info, XAD1, patterns[i]); - write_reg(info, XAD2, patterns[(i + 1) % count]); - if ((read_reg(info, XAD1) != patterns[i]) || - (read_reg(info, XAD2) != patterns[(i + 1) % count])) { - rc = false; - break; - } - } - - spin_unlock_irqrestore(&info->lock, flags); - return rc; -} - -static bool irq_test(MGSLPC_INFO *info) -{ - unsigned long end_time; - unsigned long flags; - - spin_lock_irqsave(&info->lock, flags); - reset_device(info); - - info->testing_irq = true; - hdlc_mode(info); - - info->irq_occurred = false; - - /* init hdlc mode */ - - irq_enable(info, CHA, IRQ_TIMER); - write_reg(info, CHA + TIMR, 0); /* 512 cycles */ - issue_command(info, CHA, CMD_START_TIMER); - - spin_unlock_irqrestore(&info->lock, flags); - - end_time=100; - while(end_time-- && !info->irq_occurred) { - msleep_interruptible(10); - } - - info->testing_irq = false; - - spin_lock_irqsave(&info->lock, flags); - reset_device(info); - spin_unlock_irqrestore(&info->lock, flags); - - return info->irq_occurred; -} - -static int adapter_test(MGSLPC_INFO *info) -{ - if (!register_test(info)) { - info->init_error = DiagStatus_AddressFailure; - printk("%s(%d):Register test failure for device %s Addr=%04X\n", - __FILE__, __LINE__, info->device_name, (unsigned short)(info->io_base)); - return -ENODEV; - } - - if (!irq_test(info)) { - info->init_error = DiagStatus_IrqFailure; - printk("%s(%d):Interrupt test failure for device %s IRQ=%d\n", - __FILE__, __LINE__, info->device_name, (unsigned short)(info->irq_level)); - return -ENODEV; - } - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):device %s passed diagnostics\n", - __FILE__, __LINE__, info->device_name); - return 0; -} - -static void trace_block(MGSLPC_INFO *info,const char* data, int count, int xmit) -{ - int i; - int linecount; - if (xmit) - printk("%s tx data:\n", info->device_name); - else - printk("%s rx data:\n", info->device_name); - - while(count) { - if (count > 16) - linecount = 16; - else - linecount = count; - - for(i=0;i=040 && data[i]<=0176) - printk("%c", data[i]); - else - printk("."); - } - printk("\n"); - - data += linecount; - count -= linecount; - } -} - -/* HDLC frame time out - * update stats and do tx completion processing - */ -static void tx_timeout(struct timer_list *t) -{ - MGSLPC_INFO *info = from_timer(info, t, tx_timer); - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s(%d):tx_timeout(%s)\n", - __FILE__, __LINE__, info->device_name); - if (info->tx_active && - info->params.mode == MGSL_MODE_HDLC) { - info->icount.txtimeout++; - } - spin_lock_irqsave(&info->lock, flags); - info->tx_active = false; - info->tx_count = info->tx_put = info->tx_get = 0; - - spin_unlock_irqrestore(&info->lock, flags); - -#if SYNCLINK_GENERIC_HDLC - if (info->netcount) - hdlcdev_tx_done(info); - else -#endif - { - struct tty_struct *tty = tty_port_tty_get(&info->port); - bh_transmit(info, tty); - tty_kref_put(tty); - } -} - -#if SYNCLINK_GENERIC_HDLC - -/* - * called by generic HDLC layer when protocol selected (PPP, frame relay, etc.) - * set encoding and frame check sequence (FCS) options - * - * dev pointer to network device structure - * encoding serial encoding setting - * parity FCS setting - * - * returns 0 if success, otherwise error code - */ -static int hdlcdev_attach(struct net_device *dev, unsigned short encoding, - unsigned short parity) -{ - MGSLPC_INFO *info = dev_to_port(dev); - struct tty_struct *tty; - unsigned char new_encoding; - unsigned short new_crctype; - - /* return error if TTY interface open */ - if (info->port.count) - return -EBUSY; - - switch (encoding) - { - case ENCODING_NRZ: new_encoding = HDLC_ENCODING_NRZ; break; - case ENCODING_NRZI: new_encoding = HDLC_ENCODING_NRZI_SPACE; break; - case ENCODING_FM_MARK: new_encoding = HDLC_ENCODING_BIPHASE_MARK; break; - case ENCODING_FM_SPACE: new_encoding = HDLC_ENCODING_BIPHASE_SPACE; break; - case ENCODING_MANCHESTER: new_encoding = HDLC_ENCODING_BIPHASE_LEVEL; break; - default: return -EINVAL; - } - - switch (parity) - { - case PARITY_NONE: new_crctype = HDLC_CRC_NONE; break; - case PARITY_CRC16_PR1_CCITT: new_crctype = HDLC_CRC_16_CCITT; break; - case PARITY_CRC32_PR1_CCITT: new_crctype = HDLC_CRC_32_CCITT; break; - default: return -EINVAL; - } - - info->params.encoding = new_encoding; - info->params.crc_type = new_crctype; - - /* if network interface up, reprogram hardware */ - if (info->netcount) { - tty = tty_port_tty_get(&info->port); - mgslpc_program_hw(info, tty); - tty_kref_put(tty); - } - - return 0; -} - -/* - * called by generic HDLC layer to send frame - * - * skb socket buffer containing HDLC frame - * dev pointer to network device structure - */ -static netdev_tx_t hdlcdev_xmit(struct sk_buff *skb, - struct net_device *dev) -{ - MGSLPC_INFO *info = dev_to_port(dev); - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk(KERN_INFO "%s:hdlc_xmit(%s)\n", __FILE__, dev->name); - - /* stop sending until this frame completes */ - netif_stop_queue(dev); - - /* copy data to device buffers */ - skb_copy_from_linear_data(skb, info->tx_buf, skb->len); - info->tx_get = 0; - info->tx_put = info->tx_count = skb->len; - - /* update network statistics */ - dev->stats.tx_packets++; - dev->stats.tx_bytes += skb->len; - - /* done with socket buffer, so free it */ - dev_kfree_skb(skb); - - /* save start time for transmit timeout detection */ - netif_trans_update(dev); - - /* start hardware transmitter if necessary */ - spin_lock_irqsave(&info->lock, flags); - if (!info->tx_active) { - struct tty_struct *tty = tty_port_tty_get(&info->port); - tx_start(info, tty); - tty_kref_put(tty); - } - spin_unlock_irqrestore(&info->lock, flags); - - return NETDEV_TX_OK; -} - -/* - * called by network layer when interface enabled - * claim resources and initialize hardware - * - * dev pointer to network device structure - * - * returns 0 if success, otherwise error code - */ -static int hdlcdev_open(struct net_device *dev) -{ - MGSLPC_INFO *info = dev_to_port(dev); - struct tty_struct *tty; - int rc; - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s:hdlcdev_open(%s)\n", __FILE__, dev->name); - - /* generic HDLC layer open processing */ - rc = hdlc_open(dev); - if (rc != 0) - return rc; - - /* arbitrate between network and tty opens */ - spin_lock_irqsave(&info->netlock, flags); - if (info->port.count != 0 || info->netcount != 0) { - printk(KERN_WARNING "%s: hdlc_open returning busy\n", dev->name); - spin_unlock_irqrestore(&info->netlock, flags); - return -EBUSY; - } - info->netcount=1; - spin_unlock_irqrestore(&info->netlock, flags); - - tty = tty_port_tty_get(&info->port); - /* claim resources and init adapter */ - rc = startup(info, tty); - if (rc != 0) { - tty_kref_put(tty); - spin_lock_irqsave(&info->netlock, flags); - info->netcount=0; - spin_unlock_irqrestore(&info->netlock, flags); - return rc; - } - /* assert RTS and DTR, apply hardware settings */ - info->serial_signals |= SerialSignal_RTS | SerialSignal_DTR; - mgslpc_program_hw(info, tty); - tty_kref_put(tty); - - /* enable network layer transmit */ - netif_trans_update(dev); - netif_start_queue(dev); - - /* inform generic HDLC layer of current DCD status */ - spin_lock_irqsave(&info->lock, flags); - get_signals(info); - spin_unlock_irqrestore(&info->lock, flags); - if (info->serial_signals & SerialSignal_DCD) - netif_carrier_on(dev); - else - netif_carrier_off(dev); - return 0; -} - -/* - * called by network layer when interface is disabled - * shutdown hardware and release resources - * - * dev pointer to network device structure - * - * returns 0 if success, otherwise error code - */ -static int hdlcdev_close(struct net_device *dev) -{ - MGSLPC_INFO *info = dev_to_port(dev); - struct tty_struct *tty = tty_port_tty_get(&info->port); - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s:hdlcdev_close(%s)\n", __FILE__, dev->name); - - netif_stop_queue(dev); - - /* shutdown adapter and release resources */ - shutdown(info, tty); - tty_kref_put(tty); - hdlc_close(dev); - - spin_lock_irqsave(&info->netlock, flags); - info->netcount=0; - spin_unlock_irqrestore(&info->netlock, flags); - - return 0; -} - -/* - * called by network layer to process IOCTL call to network device - * - * dev pointer to network device structure - * ifs pointer to network interface settings structure - * - * returns 0 if success, otherwise error code - */ -static int hdlcdev_wan_ioctl(struct net_device *dev, struct if_settings *ifs) -{ - const size_t size = sizeof(sync_serial_settings); - sync_serial_settings new_line; - sync_serial_settings __user *line = ifs->ifs_ifsu.sync; - MGSLPC_INFO *info = dev_to_port(dev); - unsigned int flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("%s:hdlcdev_ioctl(%s)\n", __FILE__, dev->name); - - /* return error if TTY interface open */ - if (info->port.count) - return -EBUSY; - - memset(&new_line, 0, size); - - switch (ifs->type) { - case IF_GET_IFACE: /* return current sync_serial_settings */ - - ifs->type = IF_IFACE_SYNC_SERIAL; - if (ifs->size < size) { - ifs->size = size; /* data size wanted */ - return -ENOBUFS; - } - - flags = info->params.flags & (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_RXC_DPLL | - HDLC_FLAG_RXC_BRG | HDLC_FLAG_RXC_TXCPIN | - HDLC_FLAG_TXC_TXCPIN | HDLC_FLAG_TXC_DPLL | - HDLC_FLAG_TXC_BRG | HDLC_FLAG_TXC_RXCPIN); - - switch (flags){ - case (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_TXCPIN): new_line.clock_type = CLOCK_EXT; break; - case (HDLC_FLAG_RXC_BRG | HDLC_FLAG_TXC_BRG): new_line.clock_type = CLOCK_INT; break; - case (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_BRG): new_line.clock_type = CLOCK_TXINT; break; - case (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_RXCPIN): new_line.clock_type = CLOCK_TXFROMRX; break; - default: new_line.clock_type = CLOCK_DEFAULT; - } - - new_line.clock_rate = info->params.clock_speed; - new_line.loopback = info->params.loopback ? 1:0; - - if (copy_to_user(line, &new_line, size)) - return -EFAULT; - return 0; - - case IF_IFACE_SYNC_SERIAL: /* set sync_serial_settings */ - - if(!capable(CAP_NET_ADMIN)) - return -EPERM; - if (copy_from_user(&new_line, line, size)) - return -EFAULT; - - switch (new_line.clock_type) - { - case CLOCK_EXT: flags = HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_TXCPIN; break; - case CLOCK_TXFROMRX: flags = HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_RXCPIN; break; - case CLOCK_INT: flags = HDLC_FLAG_RXC_BRG | HDLC_FLAG_TXC_BRG; break; - case CLOCK_TXINT: flags = HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_BRG; break; - case CLOCK_DEFAULT: flags = info->params.flags & - (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_RXC_DPLL | - HDLC_FLAG_RXC_BRG | HDLC_FLAG_RXC_TXCPIN | - HDLC_FLAG_TXC_TXCPIN | HDLC_FLAG_TXC_DPLL | - HDLC_FLAG_TXC_BRG | HDLC_FLAG_TXC_RXCPIN); break; - default: return -EINVAL; - } - - if (new_line.loopback != 0 && new_line.loopback != 1) - return -EINVAL; - - info->params.flags &= ~(HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_RXC_DPLL | - HDLC_FLAG_RXC_BRG | HDLC_FLAG_RXC_TXCPIN | - HDLC_FLAG_TXC_TXCPIN | HDLC_FLAG_TXC_DPLL | - HDLC_FLAG_TXC_BRG | HDLC_FLAG_TXC_RXCPIN); - info->params.flags |= flags; - - info->params.loopback = new_line.loopback; - - if (flags & (HDLC_FLAG_RXC_BRG | HDLC_FLAG_TXC_BRG)) - info->params.clock_speed = new_line.clock_rate; - else - info->params.clock_speed = 0; - - /* if network interface up, reprogram hardware */ - if (info->netcount) { - struct tty_struct *tty = tty_port_tty_get(&info->port); - mgslpc_program_hw(info, tty); - tty_kref_put(tty); - } - return 0; - default: - return hdlc_ioctl(dev, ifs); - } -} - -/* - * called by network layer when transmit timeout is detected - * - * dev pointer to network device structure - */ -static void hdlcdev_tx_timeout(struct net_device *dev, unsigned int txqueue) -{ - MGSLPC_INFO *info = dev_to_port(dev); - unsigned long flags; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("hdlcdev_tx_timeout(%s)\n", dev->name); - - dev->stats.tx_errors++; - dev->stats.tx_aborted_errors++; - - spin_lock_irqsave(&info->lock, flags); - tx_stop(info); - spin_unlock_irqrestore(&info->lock, flags); - - netif_wake_queue(dev); -} - -/* - * called by device driver when transmit completes - * reenable network layer transmit if stopped - * - * info pointer to device instance information - */ -static void hdlcdev_tx_done(MGSLPC_INFO *info) -{ - if (netif_queue_stopped(info->netdev)) - netif_wake_queue(info->netdev); -} - -/* - * called by device driver when frame received - * pass frame to network layer - * - * info pointer to device instance information - * buf pointer to buffer contianing frame data - * size count of data bytes in buf - */ -static void hdlcdev_rx(MGSLPC_INFO *info, char *buf, int size) -{ - struct sk_buff *skb = dev_alloc_skb(size); - struct net_device *dev = info->netdev; - - if (debug_level >= DEBUG_LEVEL_INFO) - printk("hdlcdev_rx(%s)\n", dev->name); - - if (skb == NULL) { - printk(KERN_NOTICE "%s: can't alloc skb, dropping packet\n", dev->name); - dev->stats.rx_dropped++; - return; - } - - skb_put_data(skb, buf, size); - - skb->protocol = hdlc_type_trans(skb, dev); - - dev->stats.rx_packets++; - dev->stats.rx_bytes += size; - - netif_rx(skb); -} - -static const struct net_device_ops hdlcdev_ops = { - .ndo_open = hdlcdev_open, - .ndo_stop = hdlcdev_close, - .ndo_start_xmit = hdlc_start_xmit, - .ndo_siocwandev = hdlcdev_wan_ioctl, - .ndo_tx_timeout = hdlcdev_tx_timeout, -}; - -/* - * called by device driver when adding device instance - * do generic HDLC initialization - * - * info pointer to device instance information - * - * returns 0 if success, otherwise error code - */ -static int hdlcdev_init(MGSLPC_INFO *info) -{ - int rc; - struct net_device *dev; - hdlc_device *hdlc; - - /* allocate and initialize network and HDLC layer objects */ - - dev = alloc_hdlcdev(info); - if (dev == NULL) { - printk(KERN_ERR "%s:hdlc device allocation failure\n", __FILE__); - return -ENOMEM; - } - - /* for network layer reporting purposes only */ - dev->base_addr = info->io_base; - dev->irq = info->irq_level; - - /* network layer callbacks and settings */ - dev->netdev_ops = &hdlcdev_ops; - dev->watchdog_timeo = 10 * HZ; - dev->tx_queue_len = 50; - - /* generic HDLC layer callbacks and settings */ - hdlc = dev_to_hdlc(dev); - hdlc->attach = hdlcdev_attach; - hdlc->xmit = hdlcdev_xmit; - - /* register objects with HDLC layer */ - rc = register_hdlc_device(dev); - if (rc) { - printk(KERN_WARNING "%s:unable to register hdlc device\n", __FILE__); - free_netdev(dev); - return rc; - } - - info->netdev = dev; - return 0; -} - -/* - * called by device driver when removing device instance - * do generic HDLC cleanup - * - * info pointer to device instance information - */ -static void hdlcdev_exit(MGSLPC_INFO *info) -{ - unregister_hdlc_device(info->netdev); - free_netdev(info->netdev); - info->netdev = NULL; -} - -#endif /* CONFIG_HDLC */ - diff --git a/include/linux/cm4000_cs.h b/include/linux/cm4000_cs.h deleted file mode 100644 index ea4958e07a14..000000000000 --- a/include/linux/cm4000_cs.h +++ /dev/null @@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _CM4000_H_ -#define _CM4000_H_ - -#include - - -#define DEVICE_NAME "cmm" -#define MODULE_NAME "cm4000_cs" - -#endif /* _CM4000_H_ */ diff --git a/include/uapi/linux/cm4000_cs.h b/include/uapi/linux/cm4000_cs.h deleted file mode 100644 index c70a62ec8a49..000000000000 --- a/include/uapi/linux/cm4000_cs.h +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _UAPI_CM4000_H_ -#define _UAPI_CM4000_H_ - -#include -#include - -#define MAX_ATR 33 - -#define CM4000_MAX_DEV 4 - -/* those two structures are passed via ioctl() from/to userspace. They are - * used by existing userspace programs, so I kepth the awkward "bIFSD" naming - * not to break compilation of userspace apps. -HW */ - -typedef struct atreq { - __s32 atr_len; - unsigned char atr[64]; - __s32 power_act; - unsigned char bIFSD; - unsigned char bIFSC; -} atreq_t; - - -/* what is particularly stupid in the original driver is the arch-dependent - * member sizes. This leads to CONFIG_COMPAT breakage, since 32bit userspace - * will lay out the structure members differently than the 64bit kernel. - * - * I've changed "ptsreq.protocol" from "unsigned long" to "__u32". - * On 32bit this will make no difference. With 64bit kernels, it will make - * 32bit apps work, too. - */ - -typedef struct ptsreq { - __u32 protocol; /*T=0: 2^0, T=1: 2^1*/ - unsigned char flags; - unsigned char pts1; - unsigned char pts2; - unsigned char pts3; -} ptsreq_t; - -#define CM_IOC_MAGIC 'c' -#define CM_IOC_MAXNR 255 - -#define CM_IOCGSTATUS _IOR (CM_IOC_MAGIC, 0, unsigned char *) -#define CM_IOCGATR _IOWR(CM_IOC_MAGIC, 1, atreq_t *) -#define CM_IOCSPTS _IOW (CM_IOC_MAGIC, 2, ptsreq_t *) -#define CM_IOCSRDR _IO (CM_IOC_MAGIC, 3) -#define CM_IOCARDOFF _IO (CM_IOC_MAGIC, 4) - -#define CM_IOSDBGLVL _IOW(CM_IOC_MAGIC, 250, int*) - -/* card and device states */ -#define CM_CARD_INSERTED 0x01 -#define CM_CARD_POWERED 0x02 -#define CM_ATR_PRESENT 0x04 -#define CM_ATR_VALID 0x08 -#define CM_STATE_VALID 0x0f -/* extra info only from CM4000 */ -#define CM_NO_READER 0x10 -#define CM_BAD_CARD 0x20 - - -#endif /* _UAPI_CM4000_H_ */ -- cgit v1.2.3 From 8efc52743ecb5c69d2e4faba965a3e14418c2494 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 7 Mar 2023 15:38:16 -0600 Subject: misc: alcor_pci: Use PCI core to manage ASPM instead of open-coding "priv->ext_config_dev_aspm" was never set to a non-zero value. Therefore, alcor_pci_aspm_ctrl(priv, 1) did nothing, and alcor_pci_aspm_ctrl(priv, 0) always disabled ASPM in the device and the upstream bridge. The driver disabled ASPM in alcor_pci_probe() and alcor_resume(), so it's possible the device doesn't work well when ASPM is enabled. Remove all the ASPM-related code and replace the alcor_pci_aspm_ctrl(0) calls with pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1), which asks the PCI core to disable ASPM. Signed-off-by: Bjorn Helgaas Cc: Oleksij Rempel Reviewed-by: Oleksij Rempel Link: https://lore.kernel.org/r/20230307213816.886308-1-helgaas@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/misc/cardreader/alcor_pci.c | 144 +----------------------------------- include/linux/alcor_pci.h | 7 -- 2 files changed, 4 insertions(+), 147 deletions(-) (limited to 'include/linux') diff --git a/drivers/misc/cardreader/alcor_pci.c b/drivers/misc/cardreader/alcor_pci.c index 9080f9f150a2..5b637171c46c 100644 --- a/drivers/misc/cardreader/alcor_pci.c +++ b/drivers/misc/cardreader/alcor_pci.c @@ -95,137 +95,6 @@ u32 alcor_read32be(struct alcor_pci_priv *priv, unsigned int addr) } EXPORT_SYMBOL_GPL(alcor_read32be); -static int alcor_pci_find_cap_offset(struct alcor_pci_priv *priv, - struct pci_dev *pci) -{ - int where; - u8 val8; - u32 val32; - - where = ALCOR_CAP_START_OFFSET; - pci_read_config_byte(pci, where, &val8); - if (!val8) - return 0; - - where = (int)val8; - while (1) { - pci_read_config_dword(pci, where, &val32); - if (val32 == 0xffffffff) { - dev_dbg(priv->dev, "find_cap_offset invalid value %x.\n", - val32); - return 0; - } - - if ((val32 & 0xff) == 0x10) { - dev_dbg(priv->dev, "pcie cap offset: %x\n", where); - return where; - } - - if ((val32 & 0xff00) == 0x00) { - dev_dbg(priv->dev, "pci_find_cap_offset invalid value %x.\n", - val32); - break; - } - where = (int)((val32 >> 8) & 0xff); - } - - return 0; -} - -static void alcor_pci_init_check_aspm(struct alcor_pci_priv *priv) -{ - struct pci_dev *pci; - int where; - u32 val32; - - priv->pdev_cap_off = alcor_pci_find_cap_offset(priv, priv->pdev); - /* - * A device might be attached to root complex directly and - * priv->parent_pdev will be NULL. In this case we don't check its - * capability and disable ASPM completely. - */ - if (priv->parent_pdev) - priv->parent_cap_off = alcor_pci_find_cap_offset(priv, - priv->parent_pdev); - - if ((priv->pdev_cap_off == 0) || (priv->parent_cap_off == 0)) { - dev_dbg(priv->dev, "pci_cap_off: %x, parent_cap_off: %x\n", - priv->pdev_cap_off, priv->parent_cap_off); - return; - } - - /* link capability */ - pci = priv->pdev; - where = priv->pdev_cap_off + ALCOR_PCIE_LINK_CAP_OFFSET; - pci_read_config_dword(pci, where, &val32); - priv->pdev_aspm_cap = (u8)(val32 >> 10) & 0x03; - - pci = priv->parent_pdev; - where = priv->parent_cap_off + ALCOR_PCIE_LINK_CAP_OFFSET; - pci_read_config_dword(pci, where, &val32); - priv->parent_aspm_cap = (u8)(val32 >> 10) & 0x03; - - if (priv->pdev_aspm_cap != priv->parent_aspm_cap) { - u8 aspm_cap; - - dev_dbg(priv->dev, "pdev_aspm_cap: %x, parent_aspm_cap: %x\n", - priv->pdev_aspm_cap, priv->parent_aspm_cap); - aspm_cap = priv->pdev_aspm_cap & priv->parent_aspm_cap; - priv->pdev_aspm_cap = aspm_cap; - priv->parent_aspm_cap = aspm_cap; - } - - dev_dbg(priv->dev, "ext_config_dev_aspm: %x, pdev_aspm_cap: %x\n", - priv->ext_config_dev_aspm, priv->pdev_aspm_cap); - priv->ext_config_dev_aspm &= priv->pdev_aspm_cap; -} - -static void alcor_pci_aspm_ctrl(struct alcor_pci_priv *priv, u8 aspm_enable) -{ - struct pci_dev *pci; - u8 aspm_ctrl, i; - int where; - u32 val32; - - if ((!priv->pdev_cap_off) || (!priv->parent_cap_off)) { - dev_dbg(priv->dev, "pci_cap_off: %x, parent_cap_off: %x\n", - priv->pdev_cap_off, priv->parent_cap_off); - return; - } - - if (!priv->pdev_aspm_cap) - return; - - aspm_ctrl = 0; - if (aspm_enable) { - aspm_ctrl = priv->ext_config_dev_aspm; - - if (!aspm_ctrl) { - dev_dbg(priv->dev, "aspm_ctrl == 0\n"); - return; - } - } - - for (i = 0; i < 2; i++) { - - if (i) { - pci = priv->parent_pdev; - where = priv->parent_cap_off - + ALCOR_PCIE_LINK_CTRL_OFFSET; - } else { - pci = priv->pdev; - where = priv->pdev_cap_off - + ALCOR_PCIE_LINK_CTRL_OFFSET; - } - - pci_read_config_dword(pci, where, &val32); - val32 &= (~0x03); - val32 |= (aspm_ctrl & priv->pdev_aspm_cap); - pci_write_config_byte(pci, where, (u8)val32); - } - -} - static inline void alcor_mask_sd_irqs(struct alcor_pci_priv *priv) { alcor_write32(priv, 0, AU6601_REG_INT_ENABLE); @@ -308,7 +177,6 @@ static int alcor_pci_probe(struct pci_dev *pdev, pci_set_master(pdev); pci_set_drvdata(pdev, priv); - alcor_pci_init_check_aspm(priv); for (i = 0; i < ARRAY_SIZE(alcor_pci_cells); i++) { alcor_pci_cells[i].platform_data = priv; @@ -319,7 +187,7 @@ static int alcor_pci_probe(struct pci_dev *pdev, if (ret < 0) goto error_clear_drvdata; - alcor_pci_aspm_ctrl(priv, 0); + pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1); return 0; @@ -339,8 +207,6 @@ static void alcor_pci_remove(struct pci_dev *pdev) priv = pci_get_drvdata(pdev); - alcor_pci_aspm_ctrl(priv, 1); - mfd_remove_devices(&pdev->dev); ida_free(&alcor_pci_idr, priv->id); @@ -353,18 +219,16 @@ static void alcor_pci_remove(struct pci_dev *pdev) #ifdef CONFIG_PM_SLEEP static int alcor_suspend(struct device *dev) { - struct alcor_pci_priv *priv = dev_get_drvdata(dev); - - alcor_pci_aspm_ctrl(priv, 1); return 0; } static int alcor_resume(struct device *dev) { - struct alcor_pci_priv *priv = dev_get_drvdata(dev); - alcor_pci_aspm_ctrl(priv, 0); + pci_disable_link_state(priv->pdev, + PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1); + return 0; } #endif /* CONFIG_PM_SLEEP */ diff --git a/include/linux/alcor_pci.h b/include/linux/alcor_pci.h index 8274ed525e9f..c4a0b23846d8 100644 --- a/include/linux/alcor_pci.h +++ b/include/linux/alcor_pci.h @@ -268,13 +268,6 @@ struct alcor_pci_priv { unsigned long id; /* idr id */ struct alcor_dev_cfg *cfg; - - /* PCI ASPM related vars */ - int pdev_cap_off; - u8 pdev_aspm_cap; - int parent_cap_off; - u8 parent_aspm_cap; - u8 ext_config_dev_aspm; }; void alcor_write8(struct alcor_pci_priv *priv, u8 val, unsigned int addr); -- cgit v1.2.3 From 8f118f61540e23fb0e4ec84aec361f6758dbe93a Mon Sep 17 00:00:00 2001 From: Nava kishore Manne Date: Fri, 24 Feb 2023 17:37:37 +0530 Subject: firmware: xilinx: Add pm api function for PL config reg readback Adds PM API for performing Programmable Logic(PL) configuration register readback. It provides an interface to the firmware(pmufw) to readback the FPGA configuration register. Signed-off-by: Nava kishore Manne Acked-by: Xu Yilun Link: https://lore.kernel.org/r/20230224120738.329416-2-nava.kishore.manne@amd.com Signed-off-by: Greg Kroah-Hartman --- drivers/firmware/xilinx/zynqmp.c | 33 +++++++++++++++++++++++++++++++++ include/linux/firmware/xlnx-zynqmp.h | 11 +++++++++++ 2 files changed, 44 insertions(+) (limited to 'include/linux') diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c index acd83d29c866..3b3ffb223c4c 100644 --- a/drivers/firmware/xilinx/zynqmp.c +++ b/drivers/firmware/xilinx/zynqmp.c @@ -971,6 +971,39 @@ int zynqmp_pm_fpga_get_status(u32 *value) } EXPORT_SYMBOL_GPL(zynqmp_pm_fpga_get_status); +/** + * zynqmp_pm_fpga_get_config_status - Get the FPGA configuration status. + * @value: Buffer to store FPGA configuration status. + * + * This function provides access to the pmufw to get the FPGA configuration + * status + * + * Return: 0 on success, a negative value on error + */ +int zynqmp_pm_fpga_get_config_status(u32 *value) +{ + u32 ret_payload[PAYLOAD_ARG_CNT]; + u32 buf, lower_addr, upper_addr; + int ret; + + if (!value) + return -EINVAL; + + lower_addr = lower_32_bits((u64)&buf); + upper_addr = upper_32_bits((u64)&buf); + + ret = zynqmp_pm_invoke_fn(PM_FPGA_READ, + XILINX_ZYNQMP_PM_FPGA_CONFIG_STAT_OFFSET, + lower_addr, upper_addr, + XILINX_ZYNQMP_PM_FPGA_READ_CONFIG_REG, + ret_payload); + + *value = ret_payload[1]; + + return ret; +} +EXPORT_SYMBOL_GPL(zynqmp_pm_fpga_get_config_status); + /** * zynqmp_pm_pinctrl_request - Request Pin from firmware * @pin: Pin number to request diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index 0e4c70987e6a..f5da51677069 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -71,6 +71,10 @@ #define XILINX_ZYNQMP_PM_FPGA_FULL 0x0U #define XILINX_ZYNQMP_PM_FPGA_PARTIAL BIT(0) +/* FPGA Status Reg */ +#define XILINX_ZYNQMP_PM_FPGA_CONFIG_STAT_OFFSET 7U +#define XILINX_ZYNQMP_PM_FPGA_READ_CONFIG_REG 0U + /* * Node IDs for the Error Events. */ @@ -124,6 +128,7 @@ enum pm_api_id { PM_CLOCK_GETRATE = 42, PM_CLOCK_SETPARENT = 43, PM_CLOCK_GETPARENT = 44, + PM_FPGA_READ = 46, PM_SECURE_AES = 47, PM_FEATURE_CHECK = 63, }; @@ -519,6 +524,7 @@ int zynqmp_pm_aes_engine(const u64 address, u32 *out); int zynqmp_pm_sha_hash(const u64 address, const u32 size, const u32 flags); int zynqmp_pm_fpga_load(const u64 address, const u32 size, const u32 flags); int zynqmp_pm_fpga_get_status(u32 *value); +int zynqmp_pm_fpga_get_config_status(u32 *value); int zynqmp_pm_write_ggs(u32 index, u32 value); int zynqmp_pm_read_ggs(u32 index, u32 *value); int zynqmp_pm_write_pggs(u32 index, u32 value); @@ -725,6 +731,11 @@ static inline int zynqmp_pm_fpga_get_status(u32 *value) return -ENODEV; } +static inline int zynqmp_pm_fpga_get_config_status(u32 *value) +{ + return -ENODEV; +} + static inline int zynqmp_pm_write_ggs(u32 index, u32 value) { return -ENODEV; -- cgit v1.2.3 From ac3b43283923440900b4f36ca5f9f0b1ca43b70e Mon Sep 17 00:00:00 2001 From: Song Liu Date: Mon, 6 Feb 2023 16:28:02 -0800 Subject: module: replace module_layout with module_memory module_layout manages different types of memory (text, data, rodata, etc.) in one allocation, which is problematic for some reasons: 1. It is hard to enable CONFIG_STRICT_MODULE_RWX. 2. It is hard to use huge pages in modules (and not break strict rwx). 3. Many archs uses module_layout for arch-specific data, but it is not obvious how these data are used (are they RO, RX, or RW?) Improve the scenario by replacing 2 (or 3) module_layout per module with up to 7 module_memory per module: MOD_TEXT, MOD_DATA, MOD_RODATA, MOD_RO_AFTER_INIT, MOD_INIT_TEXT, MOD_INIT_DATA, MOD_INIT_RODATA, and allocating them separately. This adds slightly more entries to mod_tree (from up to 3 entries per module, to up to 7 entries per module). However, this at most adds a small constant overhead to __module_address(), which is expected to be fast. Various archs use module_layout for different data. These data are put into different module_memory based on their location in module_layout. IOW, data that used to go with text is allocated with MOD_MEM_TYPE_TEXT; data that used to go with data is allocated with MOD_MEM_TYPE_DATA, etc. module_memory simplifies quite some of the module code. For example, ARCH_WANTS_MODULES_DATA_IN_VMALLOC is a lot cleaner, as it just uses a different allocator for the data. kernel/module/strict_rwx.c is also much cleaner with module_memory. Signed-off-by: Song Liu Cc: Luis Chamberlain Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: Guenter Roeck Cc: Christophe Leroy Reviewed-by: Thomas Gleixner Reviewed-by: Christophe Leroy Reviewed-by: Luis Chamberlain Signed-off-by: Luis Chamberlain --- arch/arc/kernel/unwind.c | 12 +- arch/arm/kernel/module-plts.c | 9 +- arch/arm64/kernel/module-plts.c | 13 +- arch/ia64/kernel/module.c | 24 +-- arch/mips/kernel/vpe.c | 11 +- arch/parisc/kernel/module.c | 51 ++---- arch/powerpc/kernel/module_32.c | 7 +- arch/s390/kernel/module.c | 26 +-- arch/x86/kernel/callthunks.c | 4 +- arch/x86/kernel/module.c | 4 +- include/linux/module.h | 89 +++++++--- kernel/module/internal.h | 40 ++--- kernel/module/kallsyms.c | 58 ++++--- kernel/module/kdb.c | 17 +- kernel/module/main.c | 375 ++++++++++++++++++++-------------------- kernel/module/procfs.c | 16 +- kernel/module/strict_rwx.c | 99 ++--------- kernel/module/tree_lookup.c | 39 ++--- 18 files changed, 427 insertions(+), 467 deletions(-) (limited to 'include/linux') diff --git a/arch/arc/kernel/unwind.c b/arch/arc/kernel/unwind.c index 200270a94558..9270d0a713c3 100644 --- a/arch/arc/kernel/unwind.c +++ b/arch/arc/kernel/unwind.c @@ -369,6 +369,8 @@ void *unwind_add_table(struct module *module, const void *table_start, unsigned long table_size) { struct unwind_table *table; + struct module_memory *core_text; + struct module_memory *init_text; if (table_size <= 0) return NULL; @@ -377,11 +379,11 @@ void *unwind_add_table(struct module *module, const void *table_start, if (!table) return NULL; - init_unwind_table(table, module->name, - module->core_layout.base, module->core_layout.size, - module->init_layout.base, module->init_layout.size, - table_start, table_size, - NULL, 0); + core_text = &module->mem[MOD_TEXT]; + init_text = &module->mem[MOD_INIT_TEXT]; + + init_unwind_table(table, module->name, core_text->base, core_text->size, + init_text->base, init_text->size, table_start, table_size, NULL, 0); init_unwind_hdr(table, unw_hdr_alloc); diff --git a/arch/arm/kernel/module-plts.c b/arch/arm/kernel/module-plts.c index af7c322ebed6..f5a43fd8c163 100644 --- a/arch/arm/kernel/module-plts.c +++ b/arch/arm/kernel/module-plts.c @@ -28,11 +28,6 @@ static const u32 fixed_plts[] = { #endif }; -static bool in_init(const struct module *mod, unsigned long loc) -{ - return loc - (u32)mod->init_layout.base < mod->init_layout.size; -} - static void prealloc_fixed(struct mod_plt_sec *pltsec, struct plt_entries *plt) { int i; @@ -50,8 +45,8 @@ static void prealloc_fixed(struct mod_plt_sec *pltsec, struct plt_entries *plt) u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val) { - struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core : - &mod->arch.init; + struct mod_plt_sec *pltsec = !within_module_init(loc, mod) ? + &mod->arch.core : &mod->arch.init; struct plt_entries *plt; int idx; diff --git a/arch/arm64/kernel/module-plts.c b/arch/arm64/kernel/module-plts.c index 5a0a8f552a61..543493bf924d 100644 --- a/arch/arm64/kernel/module-plts.c +++ b/arch/arm64/kernel/module-plts.c @@ -65,17 +65,12 @@ static bool plt_entries_equal(const struct plt_entry *a, (q + aarch64_insn_adrp_get_offset(le32_to_cpu(b->adrp))); } -static bool in_init(const struct module *mod, void *loc) -{ - return (u64)loc - (u64)mod->init_layout.base < mod->init_layout.size; -} - u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs, void *loc, const Elf64_Rela *rela, Elf64_Sym *sym) { - struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core : - &mod->arch.init; + struct mod_plt_sec *pltsec = !within_module_init((unsigned long)loc, mod) ? + &mod->arch.core : &mod->arch.init; struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr; int i = pltsec->plt_num_entries; int j = i - 1; @@ -105,8 +100,8 @@ u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs, u64 module_emit_veneer_for_adrp(struct module *mod, Elf64_Shdr *sechdrs, void *loc, u64 val) { - struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core : - &mod->arch.init; + struct mod_plt_sec *pltsec = !within_module_init((unsigned long)loc, mod) ? + &mod->arch.core : &mod->arch.init; struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr; int i = pltsec->plt_num_entries++; u32 br; diff --git a/arch/ia64/kernel/module.c b/arch/ia64/kernel/module.c index 8f62cf97f691..3661135da9d9 100644 --- a/arch/ia64/kernel/module.c +++ b/arch/ia64/kernel/module.c @@ -485,19 +485,19 @@ module_frob_arch_sections (Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, char *secstrings, return 0; } -static inline int +static inline bool in_init (const struct module *mod, uint64_t addr) { - return addr - (uint64_t) mod->init_layout.base < mod->init_layout.size; + return within_module_init(addr, mod); } -static inline int +static inline bool in_core (const struct module *mod, uint64_t addr) { - return addr - (uint64_t) mod->core_layout.base < mod->core_layout.size; + return within_module_core(addr, mod); } -static inline int +static inline bool is_internal (const struct module *mod, uint64_t value) { return in_init(mod, value) || in_core(mod, value); @@ -677,7 +677,8 @@ do_reloc (struct module *mod, uint8_t r_type, Elf64_Sym *sym, uint64_t addend, break; case RV_BDREL: - val -= (uint64_t) (in_init(mod, val) ? mod->init_layout.base : mod->core_layout.base); + val -= (uint64_t) (in_init(mod, val) ? mod->mem[MOD_INIT_TEXT].base : + mod->mem[MOD_TEXT].base); break; case RV_LTV: @@ -812,15 +813,18 @@ apply_relocate_add (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symind * addresses have been selected... */ uint64_t gp; - if (mod->core_layout.size > MAX_LTOFF) + struct module_memory *mod_mem; + + mod_mem = &mod->mem[MOD_DATA]; + if (mod_mem->size > MAX_LTOFF) /* * This takes advantage of fact that SHF_ARCH_SMALL gets allocated * at the end of the module. */ - gp = mod->core_layout.size - MAX_LTOFF / 2; + gp = mod_mem->size - MAX_LTOFF / 2; else - gp = mod->core_layout.size / 2; - gp = (uint64_t) mod->core_layout.base + ((gp + 7) & -8); + gp = mod_mem->size / 2; + gp = (uint64_t) mod_mem->base + ((gp + 7) & -8); mod->arch.gp = gp; DEBUGP("%s: placing gp at 0x%lx\n", __func__, gp); } diff --git a/arch/mips/kernel/vpe.c b/arch/mips/kernel/vpe.c index 13294972707b..ab114c686f9d 100644 --- a/arch/mips/kernel/vpe.c +++ b/arch/mips/kernel/vpe.c @@ -199,18 +199,17 @@ static void layout_sections(struct module *mod, const Elf_Ehdr *hdr, for (m = 0; m < ARRAY_SIZE(masks); ++m) { for (i = 0; i < hdr->e_shnum; ++i) { Elf_Shdr *s = &sechdrs[i]; + struct module_memory *mod_mem; + + mod_mem = &mod->mem[MOD_TEXT]; if ((s->sh_flags & masks[m][0]) != masks[m][0] || (s->sh_flags & masks[m][1]) || s->sh_entsize != ~0UL) continue; s->sh_entsize = - get_offset((unsigned long *)&mod->core_layout.size, s); + get_offset((unsigned long *)&mod_mem->size, s); } - - if (m == 0) - mod->core_layout.text_size = mod->core_layout.size; - } } @@ -641,7 +640,7 @@ static int vpe_elfload(struct vpe *v) layout_sections(&mod, hdr, sechdrs, secstrings); } - v->load_addr = alloc_progmem(mod.core_layout.size); + v->load_addr = alloc_progmem(mod.mod_mem[MOD_TEXT].size); if (!v->load_addr) return -ENOMEM; diff --git a/arch/parisc/kernel/module.c b/arch/parisc/kernel/module.c index 7df140545b22..f6e38c4d3904 100644 --- a/arch/parisc/kernel/module.c +++ b/arch/parisc/kernel/module.c @@ -27,9 +27,9 @@ * We are not doing SEGREL32 handling correctly. According to the ABI, we * should do a value offset, like this: * if (in_init(me, (void *)val)) - * val -= (uint32_t)me->init_layout.base; + * val -= (uint32_t)me->mem[MOD_INIT_TEXT].base; * else - * val -= (uint32_t)me->core_layout.base; + * val -= (uint32_t)me->mem[MOD_TEXT].base; * However, SEGREL32 is used only for PARISC unwind entries, and we want * those entries to have an absolute address, and not just an offset. * @@ -76,25 +76,6 @@ * allows us to allocate up to 4095 GOT entries. */ #define MAX_GOTS 4095 -/* three functions to determine where in the module core - * or init pieces the location is */ -static inline int in_init(struct module *me, void *loc) -{ - return (loc >= me->init_layout.base && - loc <= (me->init_layout.base + me->init_layout.size)); -} - -static inline int in_core(struct module *me, void *loc) -{ - return (loc >= me->core_layout.base && - loc <= (me->core_layout.base + me->core_layout.size)); -} - -static inline int in_local(struct module *me, void *loc) -{ - return in_init(me, loc) || in_core(me, loc); -} - #ifndef CONFIG_64BIT struct got_entry { Elf32_Addr addr; @@ -302,6 +283,7 @@ int module_frob_arch_sections(CONST Elf_Ehdr *hdr, { unsigned long gots = 0, fdescs = 0, len; unsigned int i; + struct module_memory *mod_mem; len = hdr->e_shnum * sizeof(me->arch.section[0]); me->arch.section = kzalloc(len, GFP_KERNEL); @@ -346,14 +328,15 @@ int module_frob_arch_sections(CONST Elf_Ehdr *hdr, me->arch.section[s].stub_entries += count; } + mod_mem = &me->mem[MOD_TEXT]; /* align things a bit */ - me->core_layout.size = ALIGN(me->core_layout.size, 16); - me->arch.got_offset = me->core_layout.size; - me->core_layout.size += gots * sizeof(struct got_entry); + mod_mem->size = ALIGN(mod_mem->size, 16); + me->arch.got_offset = mod_mem->size; + mod_mem->size += gots * sizeof(struct got_entry); - me->core_layout.size = ALIGN(me->core_layout.size, 16); - me->arch.fdesc_offset = me->core_layout.size; - me->core_layout.size += fdescs * sizeof(Elf_Fdesc); + mod_mem->size = ALIGN(mod_mem->size, 16); + me->arch.fdesc_offset = mod_mem->size; + mod_mem->size += fdescs * sizeof(Elf_Fdesc); me->arch.got_max = gots; me->arch.fdesc_max = fdescs; @@ -371,7 +354,7 @@ static Elf64_Word get_got(struct module *me, unsigned long value, long addend) BUG_ON(value == 0); - got = me->core_layout.base + me->arch.got_offset; + got = me->mem[MOD_TEXT].base + me->arch.got_offset; for (i = 0; got[i].addr; i++) if (got[i].addr == value) goto out; @@ -389,7 +372,7 @@ static Elf64_Word get_got(struct module *me, unsigned long value, long addend) #ifdef CONFIG_64BIT static Elf_Addr get_fdesc(struct module *me, unsigned long value) { - Elf_Fdesc *fdesc = me->core_layout.base + me->arch.fdesc_offset; + Elf_Fdesc *fdesc = me->mem[MOD_TEXT].base + me->arch.fdesc_offset; if (!value) { printk(KERN_ERR "%s: zero OPD requested!\n", me->name); @@ -407,7 +390,7 @@ static Elf_Addr get_fdesc(struct module *me, unsigned long value) /* Create new one */ fdesc->addr = value; - fdesc->gp = (Elf_Addr)me->core_layout.base + me->arch.got_offset; + fdesc->gp = (Elf_Addr)me->mem[MOD_TEXT].base + me->arch.got_offset; return (Elf_Addr)fdesc; } #endif /* CONFIG_64BIT */ @@ -742,7 +725,7 @@ int apply_relocate_add(Elf_Shdr *sechdrs, loc, val); val += addend; /* can we reach it locally? */ - if (in_local(me, (void *)val)) { + if (within_module(val, me)) { /* this is the case where the symbol is local * to the module, but in a different section, * so stub the jump in case it's more than 22 @@ -801,7 +784,7 @@ int apply_relocate_add(Elf_Shdr *sechdrs, break; case R_PARISC_FPTR64: /* 64-bit function address */ - if(in_local(me, (void *)(val + addend))) { + if (within_module(val + addend, me)) { *loc64 = get_fdesc(me, val+addend); pr_debug("FDESC for %s at %llx points to %llx\n", strtab + sym->st_name, *loc64, @@ -839,7 +822,7 @@ register_unwind_table(struct module *me, table = (unsigned char *)sechdrs[me->arch.unwind_section].sh_addr; end = table + sechdrs[me->arch.unwind_section].sh_size; - gp = (Elf_Addr)me->core_layout.base + me->arch.got_offset; + gp = (Elf_Addr)me->mem[MOD_TEXT].base + me->arch.got_offset; pr_debug("register_unwind_table(), sect = %d at 0x%p - 0x%p (gp=0x%lx)\n", me->arch.unwind_section, table, end, gp); @@ -977,7 +960,7 @@ void module_arch_cleanup(struct module *mod) #ifdef CONFIG_64BIT void *dereference_module_function_descriptor(struct module *mod, void *ptr) { - unsigned long start_opd = (Elf64_Addr)mod->core_layout.base + + unsigned long start_opd = (Elf64_Addr)mod->mem[MOD_TEXT].base + mod->arch.fdesc_offset; unsigned long end_opd = start_opd + mod->arch.fdesc_count * sizeof(Elf64_Fdesc); diff --git a/arch/powerpc/kernel/module_32.c b/arch/powerpc/kernel/module_32.c index ea6536171778..816a63fd71fb 100644 --- a/arch/powerpc/kernel/module_32.c +++ b/arch/powerpc/kernel/module_32.c @@ -163,8 +163,7 @@ static uint32_t do_plt_call(void *location, pr_debug("Doing plt for call to 0x%x at 0x%x\n", val, (unsigned int)location); /* Init, or core PLT? */ - if (location >= mod->core_layout.base - && location < mod->core_layout.base + mod->core_layout.size) + if (within_module_core((unsigned long)location, mod)) entry = (void *)sechdrs[mod->arch.core_plt_section].sh_addr; else entry = (void *)sechdrs[mod->arch.init_plt_section].sh_addr; @@ -322,14 +321,14 @@ notrace int module_trampoline_target(struct module *mod, unsigned long addr, int module_finalize_ftrace(struct module *module, const Elf_Shdr *sechdrs) { - module->arch.tramp = do_plt_call(module->core_layout.base, + module->arch.tramp = do_plt_call(module->mem[MOD_TEXT].base, (unsigned long)ftrace_caller, sechdrs, module); if (!module->arch.tramp) return -ENOENT; #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS - module->arch.tramp_regs = do_plt_call(module->core_layout.base, + module->arch.tramp_regs = do_plt_call(module->mem[MOD_TEXT].base, (unsigned long)ftrace_regs_caller, sechdrs, module); if (!module->arch.tramp_regs) diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index 2d159b32885b..b9dbf0c483ed 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -126,6 +126,7 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, Elf_Rela *rela; char *strings; int nrela, i, j; + struct module_memory *mod_mem; /* Find symbol table and string table. */ symtab = NULL; @@ -173,14 +174,15 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, /* Increase core size by size of got & plt and set start offsets for got and plt. */ - me->core_layout.size = ALIGN(me->core_layout.size, 4); - me->arch.got_offset = me->core_layout.size; - me->core_layout.size += me->arch.got_size; - me->arch.plt_offset = me->core_layout.size; + mod_mem = &me->mem[MOD_TEXT]; + mod_mem->size = ALIGN(mod_mem->size, 4); + me->arch.got_offset = mod_mem->size; + mod_mem->size += me->arch.got_size; + me->arch.plt_offset = mod_mem->size; if (me->arch.plt_size) { if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_disable) me->arch.plt_size += PLT_ENTRY_SIZE; - me->core_layout.size += me->arch.plt_size; + mod_mem->size += me->arch.plt_size; } return 0; } @@ -304,7 +306,7 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, case R_390_GOTPLT64: /* 64 bit offset to jump slot. */ case R_390_GOTPLTENT: /* 32 bit rel. offset to jump slot >> 1. */ if (info->got_initialized == 0) { - Elf_Addr *gotent = me->core_layout.base + + Elf_Addr *gotent = me->mem[MOD_TEXT].base + me->arch.got_offset + info->got_offset; @@ -329,7 +331,7 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, rc = apply_rela_bits(loc, val, 0, 64, 0, write); else if (r_type == R_390_GOTENT || r_type == R_390_GOTPLTENT) { - val += (Elf_Addr) me->core_layout.base - loc; + val += (Elf_Addr) me->mem[MOD_TEXT].base - loc; rc = apply_rela_bits(loc, val, 1, 32, 1, write); } break; @@ -345,7 +347,7 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, char *plt_base; char *ip; - plt_base = me->core_layout.base + me->arch.plt_offset; + plt_base = me->mem[MOD_TEXT].base + me->arch.plt_offset; ip = plt_base + info->plt_offset; *(int *)insn = 0x0d10e310; /* basr 1,0 */ *(int *)&insn[4] = 0x100c0004; /* lg 1,12(1) */ @@ -375,7 +377,7 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, val - loc + 0xffffUL < 0x1ffffeUL) || (r_type == R_390_PLT32DBL && val - loc + 0xffffffffULL < 0x1fffffffeULL))) - val = (Elf_Addr) me->core_layout.base + + val = (Elf_Addr) me->mem[MOD_TEXT].base + me->arch.plt_offset + info->plt_offset; val += rela->r_addend - loc; @@ -397,7 +399,7 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, case R_390_GOTOFF32: /* 32 bit offset to GOT. */ case R_390_GOTOFF64: /* 64 bit offset to GOT. */ val = val + rela->r_addend - - ((Elf_Addr) me->core_layout.base + me->arch.got_offset); + ((Elf_Addr) me->mem[MOD_TEXT].base + me->arch.got_offset); if (r_type == R_390_GOTOFF16) rc = apply_rela_bits(loc, val, 0, 16, 0, write); else if (r_type == R_390_GOTOFF32) @@ -407,7 +409,7 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, break; case R_390_GOTPC: /* 32 bit PC relative offset to GOT. */ case R_390_GOTPCDBL: /* 32 bit PC rel. off. to GOT shifted by 1. */ - val = (Elf_Addr) me->core_layout.base + me->arch.got_offset + + val = (Elf_Addr) me->mem[MOD_TEXT].base + me->arch.got_offset + rela->r_addend - loc; if (r_type == R_390_GOTPC) rc = apply_rela_bits(loc, val, 1, 32, 0, write); @@ -515,7 +517,7 @@ int module_finalize(const Elf_Ehdr *hdr, !nospec_disable && me->arch.plt_size) { unsigned int *ij; - ij = me->core_layout.base + me->arch.plt_offset + + ij = me->mem[MOD_TEXT].base + me->arch.plt_offset + me->arch.plt_size - PLT_ENTRY_SIZE; ij[0] = 0xc6000000; /* exrl %r0,.+10 */ ij[1] = 0x0005a7f4; /* j . */ diff --git a/arch/x86/kernel/callthunks.c b/arch/x86/kernel/callthunks.c index ffea98f9064b..22ab13966427 100644 --- a/arch/x86/kernel/callthunks.c +++ b/arch/x86/kernel/callthunks.c @@ -330,8 +330,8 @@ void noinline callthunks_patch_module_calls(struct callthunk_sites *cs, struct module *mod) { struct core_text ct = { - .base = (unsigned long)mod->core_layout.base, - .end = (unsigned long)mod->core_layout.base + mod->core_layout.size, + .base = (unsigned long)mod->mem[MOD_TEXT].base, + .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size, .name = mod->name, }; diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index 84ad0e61ba6e..b05f62ee2344 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -362,8 +362,8 @@ int module_finalize(const Elf_Ehdr *hdr, } if (locks) { void *lseg = (void *)locks->sh_addr; - void *text = me->core_layout.base; - void *text_end = text + me->core_layout.text_size; + void *text = me->mem[MOD_TEXT].base; + void *text_end = text + me->mem[MOD_TEXT].size; alternatives_smp_module_add(me, me->name, lseg, lseg + locks->sh_size, text, text_end); diff --git a/include/linux/module.h b/include/linux/module.h index 4435ad9439ab..c0593a96e158 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -320,17 +320,47 @@ struct mod_tree_node { struct latch_tree_node node; }; -struct module_layout { - /* The actual code + data. */ +enum mod_mem_type { + MOD_TEXT = 0, + MOD_DATA, + MOD_RODATA, + MOD_RO_AFTER_INIT, + MOD_INIT_TEXT, + MOD_INIT_DATA, + MOD_INIT_RODATA, + + MOD_MEM_NUM_TYPES, + MOD_INVALID = -1, +}; + +#define mod_mem_type_is_init(type) \ + ((type) == MOD_INIT_TEXT || \ + (type) == MOD_INIT_DATA || \ + (type) == MOD_INIT_RODATA) + +#define mod_mem_type_is_core(type) (!mod_mem_type_is_init(type)) + +#define mod_mem_type_is_text(type) \ + ((type) == MOD_TEXT || \ + (type) == MOD_INIT_TEXT) + +#define mod_mem_type_is_data(type) (!mod_mem_type_is_text(type)) + +#define mod_mem_type_is_core_data(type) \ + (mod_mem_type_is_core(type) && \ + mod_mem_type_is_data(type)) + +#define for_each_mod_mem_type(type) \ + for (enum mod_mem_type (type) = 0; \ + (type) < MOD_MEM_NUM_TYPES; (type)++) + +#define for_class_mod_mem_type(type, class) \ + for_each_mod_mem_type(type) \ + if (mod_mem_type_is_##class(type)) + +struct module_memory { void *base; - /* Total size. */ unsigned int size; - /* The size of the executable code. */ - unsigned int text_size; - /* Size of RO section of the module (text+rodata) */ - unsigned int ro_size; - /* Size of RO after init section */ - unsigned int ro_after_init_size; #ifdef CONFIG_MODULES_TREE_LOOKUP struct mod_tree_node mtn; @@ -339,9 +369,9 @@ struct module_layout { #ifdef CONFIG_MODULES_TREE_LOOKUP /* Only touch one cacheline for common rbtree-for-core-layout case. */ -#define __module_layout_align ____cacheline_aligned +#define __module_memory_align ____cacheline_aligned #else -#define __module_layout_align +#define __module_memory_align #endif struct mod_kallsyms { @@ -426,12 +456,7 @@ struct module { /* Startup function. */ int (*init)(void); - /* Core layout: rbtree is accessed frequently, so keep together. */ - struct module_layout core_layout __module_layout_align; - struct module_layout init_layout; -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - struct module_layout data_layout; -#endif + struct module_memory mem[MOD_MEM_NUM_TYPES] __module_memory_align; /* Arch-specific module values */ struct mod_arch_specific arch; @@ -581,23 +606,35 @@ bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr); bool is_module_percpu_address(unsigned long addr); bool is_module_text_address(unsigned long addr); +static inline bool within_module_mem_type(unsigned long addr, + const struct module *mod, + enum mod_mem_type type) +{ + unsigned long base, size; + + base = (unsigned long)mod->mem[type].base; + size = mod->mem[type].size; + return addr - base < size; +} + static inline bool within_module_core(unsigned long addr, const struct module *mod) { -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - if ((unsigned long)mod->data_layout.base <= addr && - addr < (unsigned long)mod->data_layout.base + mod->data_layout.size) - return true; -#endif - return (unsigned long)mod->core_layout.base <= addr && - addr < (unsigned long)mod->core_layout.base + mod->core_layout.size; + for_class_mod_mem_type(type, core) { + if (within_module_mem_type(addr, mod, type)) + return true; + } + return false; } static inline bool within_module_init(unsigned long addr, const struct module *mod) { - return (unsigned long)mod->init_layout.base <= addr && - addr < (unsigned long)mod->init_layout.base + mod->init_layout.size; + for_class_mod_mem_type(type, init) { + if (within_module_mem_type(addr, mod, type)) + return true; + } + return false; } static inline bool within_module(unsigned long addr, const struct module *mod) diff --git a/kernel/module/internal.h b/kernel/module/internal.h index 2e2bf236f558..c6c44ceca07a 100644 --- a/kernel/module/internal.h +++ b/kernel/module/internal.h @@ -17,27 +17,19 @@ #define ARCH_SHF_SMALL 0 #endif -/* If this is set, the section belongs in the init part of the module */ -#define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG - 1)) -/* Maximum number of characters written by module_flags() */ -#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4) - -#ifndef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC -#define data_layout core_layout -#endif - /* - * Modules' sections will be aligned on page boundaries - * to ensure complete separation of code and data, but - * only when CONFIG_STRICT_MODULE_RWX=y + * Use highest 4 bits of sh_entsize to store the mod_mem_type of this + * section. This leaves 28 bits for offset on 32-bit systems, which is + * about 256 MiB (WARN_ON_ONCE if we exceed that). */ -static inline unsigned int strict_align(unsigned int size) -{ - if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) - return PAGE_ALIGN(size); - else - return size; -} + +#define SH_ENTSIZE_TYPE_BITS 4 +#define SH_ENTSIZE_TYPE_SHIFT (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS) +#define SH_ENTSIZE_TYPE_MASK ((1UL << SH_ENTSIZE_TYPE_BITS) - 1) +#define SH_ENTSIZE_OFFSET_MASK ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1) + +/* Maximum number of characters written by module_flags() */ +#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4) extern struct mutex module_mutex; extern struct list_head modules; @@ -101,8 +93,8 @@ int try_to_force_load(struct module *mod, const char *reason); bool find_symbol(struct find_symbol_arg *fsa); struct module *find_module_all(const char *name, size_t len, bool even_unformed); int cmp_name(const void *name, const void *sym); -long module_get_offset(struct module *mod, unsigned int *size, Elf_Shdr *sechdr, - unsigned int section); +long module_get_offset_and_type(struct module *mod, enum mod_mem_type type, + Elf_Shdr *sechdr, unsigned int section); char *module_flags(struct module *mod, char *buf, bool show_state); size_t module_flags_taint(unsigned long taints, char *buf); @@ -190,10 +182,13 @@ struct mod_tree_root { #endif unsigned long addr_min; unsigned long addr_max; +#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC + unsigned long data_addr_min; + unsigned long data_addr_max; +#endif }; extern struct mod_tree_root mod_tree; -extern struct mod_tree_root mod_data_tree; #ifdef CONFIG_MODULES_TREE_LOOKUP void mod_tree_insert(struct module *mod); @@ -224,7 +219,6 @@ void module_enable_nx(const struct module *mod); void module_enable_x(const struct module *mod); int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, char *secstrings, struct module *mod); -bool module_check_misalignment(const struct module *mod); #ifdef CONFIG_MODULE_SIG int module_sig_check(struct load_info *info, int flags); diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index ab2376a1be88..8e89f459ffdd 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -78,6 +78,7 @@ static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs, unsigned int shnum, unsigned int pcpundx) { const Elf_Shdr *sec; + enum mod_mem_type type; if (src->st_shndx == SHN_UNDEF || src->st_shndx >= shnum || @@ -90,11 +91,12 @@ static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs, #endif sec = sechdrs + src->st_shndx; + type = sec->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT; if (!(sec->sh_flags & SHF_ALLOC) #ifndef CONFIG_KALLSYMS_ALL || !(sec->sh_flags & SHF_EXECINSTR) #endif - || (sec->sh_entsize & INIT_OFFSET_MASK)) + || mod_mem_type_is_init(type)) return false; return true; @@ -113,11 +115,13 @@ void layout_symtab(struct module *mod, struct load_info *info) Elf_Shdr *strsect = info->sechdrs + info->index.str; const Elf_Sym *src; unsigned int i, nsrc, ndst, strtab_size = 0; + struct module_memory *mod_mem_data = &mod->mem[MOD_DATA]; + struct module_memory *mod_mem_init_data = &mod->mem[MOD_INIT_DATA]; /* Put symbol section at end of init part of module. */ symsect->sh_flags |= SHF_ALLOC; - symsect->sh_entsize = module_get_offset(mod, &mod->init_layout.size, symsect, - info->index.sym) | INIT_OFFSET_MASK; + symsect->sh_entsize = module_get_offset_and_type(mod, MOD_INIT_DATA, + symsect, info->index.sym); pr_debug("\t%s\n", info->secstrings + symsect->sh_name); src = (void *)info->hdr + symsect->sh_offset; @@ -134,28 +138,27 @@ void layout_symtab(struct module *mod, struct load_info *info) } /* Append room for core symbols at end of core part. */ - info->symoffs = ALIGN(mod->data_layout.size, symsect->sh_addralign ?: 1); - info->stroffs = mod->data_layout.size = info->symoffs + ndst * sizeof(Elf_Sym); - mod->data_layout.size += strtab_size; + info->symoffs = ALIGN(mod_mem_data->size, symsect->sh_addralign ?: 1); + info->stroffs = mod_mem_data->size = info->symoffs + ndst * sizeof(Elf_Sym); + mod_mem_data->size += strtab_size; /* Note add_kallsyms() computes strtab_size as core_typeoffs - stroffs */ - info->core_typeoffs = mod->data_layout.size; - mod->data_layout.size += ndst * sizeof(char); - mod->data_layout.size = strict_align(mod->data_layout.size); + info->core_typeoffs = mod_mem_data->size; + mod_mem_data->size += ndst * sizeof(char); /* Put string table section at end of init part of module. */ strsect->sh_flags |= SHF_ALLOC; - strsect->sh_entsize = module_get_offset(mod, &mod->init_layout.size, strsect, - info->index.str) | INIT_OFFSET_MASK; + strsect->sh_entsize = module_get_offset_and_type(mod, MOD_INIT_DATA, + strsect, info->index.str); pr_debug("\t%s\n", info->secstrings + strsect->sh_name); /* We'll tack temporary mod_kallsyms on the end. */ - mod->init_layout.size = ALIGN(mod->init_layout.size, - __alignof__(struct mod_kallsyms)); - info->mod_kallsyms_init_off = mod->init_layout.size; - mod->init_layout.size += sizeof(struct mod_kallsyms); - info->init_typeoffs = mod->init_layout.size; - mod->init_layout.size += nsrc * sizeof(char); - mod->init_layout.size = strict_align(mod->init_layout.size); + mod_mem_init_data->size = ALIGN(mod_mem_init_data->size, + __alignof__(struct mod_kallsyms)); + info->mod_kallsyms_init_off = mod_mem_init_data->size; + + mod_mem_init_data->size += sizeof(struct mod_kallsyms); + info->init_typeoffs = mod_mem_init_data->size; + mod_mem_init_data->size += nsrc * sizeof(char); } /* @@ -171,9 +174,11 @@ void add_kallsyms(struct module *mod, const struct load_info *info) char *s; Elf_Shdr *symsec = &info->sechdrs[info->index.sym]; unsigned long strtab_size; + void *data_base = mod->mem[MOD_DATA].base; + void *init_data_base = mod->mem[MOD_INIT_DATA].base; /* Set up to point into init section. */ - mod->kallsyms = (void __rcu *)mod->init_layout.base + + mod->kallsyms = (void __rcu *)init_data_base + info->mod_kallsyms_init_off; rcu_read_lock(); @@ -183,15 +188,15 @@ void add_kallsyms(struct module *mod, const struct load_info *info) /* Make sure we get permanent strtab: don't use info->strtab. */ rcu_dereference(mod->kallsyms)->strtab = (void *)info->sechdrs[info->index.str].sh_addr; - rcu_dereference(mod->kallsyms)->typetab = mod->init_layout.base + info->init_typeoffs; + rcu_dereference(mod->kallsyms)->typetab = init_data_base + info->init_typeoffs; /* * Now populate the cut down core kallsyms for after init * and set types up while we still have access to sections. */ - mod->core_kallsyms.symtab = dst = mod->data_layout.base + info->symoffs; - mod->core_kallsyms.strtab = s = mod->data_layout.base + info->stroffs; - mod->core_kallsyms.typetab = mod->data_layout.base + info->core_typeoffs; + mod->core_kallsyms.symtab = dst = data_base + info->symoffs; + mod->core_kallsyms.strtab = s = data_base + info->stroffs; + mod->core_kallsyms.typetab = data_base + info->core_typeoffs; strtab_size = info->core_typeoffs - info->stroffs; src = rcu_dereference(mod->kallsyms)->symtab; for (ndst = i = 0; i < rcu_dereference(mod->kallsyms)->num_symtab; i++) { @@ -267,12 +272,15 @@ static const char *find_kallsyms_symbol(struct module *mod, unsigned int i, best = 0; unsigned long nextval, bestval; struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms); + struct module_memory *mod_mem; /* At worse, next value is at end of module */ if (within_module_init(addr, mod)) - nextval = (unsigned long)mod->init_layout.base + mod->init_layout.text_size; + mod_mem = &mod->mem[MOD_INIT_TEXT]; else - nextval = (unsigned long)mod->core_layout.base + mod->core_layout.text_size; + mod_mem = &mod->mem[MOD_TEXT]; + + nextval = (unsigned long)mod_mem->base + mod_mem->size; bestval = kallsyms_symbol_value(&kallsyms->symtab[best]); diff --git a/kernel/module/kdb.c b/kernel/module/kdb.c index f4317f92e189..995c32d3698f 100644 --- a/kernel/module/kdb.c +++ b/kernel/module/kdb.c @@ -26,10 +26,11 @@ int kdb_lsmod(int argc, const char **argv) if (mod->state == MODULE_STATE_UNFORMED) continue; - kdb_printf("%-20s%8u", mod->name, mod->core_layout.size); -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - kdb_printf("/%8u", mod->data_layout.size); -#endif + kdb_printf("%-20s%8u", mod->name, mod->mem[MOD_TEXT].size); + kdb_printf("/%8u", mod->mem[MOD_RODATA].size); + kdb_printf("/%8u", mod->mem[MOD_RO_AFTER_INIT].size); + kdb_printf("/%8u", mod->mem[MOD_DATA].size); + kdb_printf(" 0x%px ", (void *)mod); #ifdef CONFIG_MODULE_UNLOAD kdb_printf("%4d ", module_refcount(mod)); @@ -40,10 +41,10 @@ int kdb_lsmod(int argc, const char **argv) kdb_printf(" (Loading)"); else kdb_printf(" (Live)"); - kdb_printf(" 0x%px", mod->core_layout.base); -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - kdb_printf("/0x%px", mod->data_layout.base); -#endif + kdb_printf(" 0x%px", mod->mem[MOD_TEXT].base); + kdb_printf("/0x%px", mod->mem[MOD_RODATA].base); + kdb_printf("/0x%px", mod->mem[MOD_RO_AFTER_INIT].base); + kdb_printf("/0x%px", mod->mem[MOD_DATA].base); #ifdef CONFIG_MODULE_UNLOAD { diff --git a/kernel/module/main.c b/kernel/module/main.c index d3be89de706d..2724bc1b9a90 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -80,12 +80,6 @@ struct mod_tree_root mod_tree __cacheline_aligned = { .addr_min = -1UL, }; -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC -struct mod_tree_root mod_data_tree __cacheline_aligned = { - .addr_min = -1UL, -}; -#endif - struct symsearch { const struct kernel_symbol *start, *stop; const s32 *crcs; @@ -93,14 +87,24 @@ struct symsearch { }; /* - * Bounds of module text, for speeding up __module_address. + * Bounds of module memory, for speeding up __module_address. * Protected by module_mutex. */ -static void __mod_update_bounds(void *base, unsigned int size, struct mod_tree_root *tree) +static void __mod_update_bounds(enum mod_mem_type type __maybe_unused, void *base, + unsigned int size, struct mod_tree_root *tree) { unsigned long min = (unsigned long)base; unsigned long max = min + size; +#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC + if (mod_mem_type_is_core_data(type)) { + if (min < tree->data_addr_min) + tree->data_addr_min = min; + if (max > tree->data_addr_max) + tree->data_addr_max = max; + return; + } +#endif if (min < tree->addr_min) tree->addr_min = min; if (max > tree->addr_max) @@ -109,12 +113,12 @@ static void __mod_update_bounds(void *base, unsigned int size, struct mod_tree_r static void mod_update_bounds(struct module *mod) { - __mod_update_bounds(mod->core_layout.base, mod->core_layout.size, &mod_tree); - if (mod->init_layout.size) - __mod_update_bounds(mod->init_layout.base, mod->init_layout.size, &mod_tree); -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - __mod_update_bounds(mod->data_layout.base, mod->data_layout.size, &mod_data_tree); -#endif + for_each_mod_mem_type(type) { + struct module_memory *mod_mem = &mod->mem[type]; + + if (mod_mem->size) + __mod_update_bounds(type, mod_mem->base, mod_mem->size, &mod_tree); + } } /* Block module loading/unloading? */ @@ -926,7 +930,13 @@ struct module_attribute module_uevent = static ssize_t show_coresize(struct module_attribute *mattr, struct module_kobject *mk, char *buffer) { - return sprintf(buffer, "%u\n", mk->mod->core_layout.size); + unsigned int size = mk->mod->mem[MOD_TEXT].size; + + if (!IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC)) { + for_class_mod_mem_type(type, core_data) + size += mk->mod->mem[type].size; + } + return sprintf(buffer, "%u\n", size); } static struct module_attribute modinfo_coresize = @@ -936,7 +946,11 @@ static struct module_attribute modinfo_coresize = static ssize_t show_datasize(struct module_attribute *mattr, struct module_kobject *mk, char *buffer) { - return sprintf(buffer, "%u\n", mk->mod->data_layout.size); + unsigned int size = 0; + + for_class_mod_mem_type(type, core_data) + size += mk->mod->mem[type].size; + return sprintf(buffer, "%u\n", size); } static struct module_attribute modinfo_datasize = @@ -946,7 +960,11 @@ static struct module_attribute modinfo_datasize = static ssize_t show_initsize(struct module_attribute *mattr, struct module_kobject *mk, char *buffer) { - return sprintf(buffer, "%u\n", mk->mod->init_layout.size); + unsigned int size = 0; + + for_class_mod_mem_type(type, init) + size += mk->mod->mem[type].size; + return sprintf(buffer, "%u\n", size); } static struct module_attribute modinfo_initsize = @@ -1143,6 +1161,46 @@ void __weak module_arch_freeing_init(struct module *mod) { } +static bool mod_mem_use_vmalloc(enum mod_mem_type type) +{ + return IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC) && + mod_mem_type_is_core_data(type); +} + +static void *module_memory_alloc(unsigned int size, enum mod_mem_type type) +{ + if (mod_mem_use_vmalloc(type)) + return vzalloc(size); + return module_alloc(size); +} + +static void module_memory_free(void *ptr, enum mod_mem_type type) +{ + if (mod_mem_use_vmalloc(type)) + vfree(ptr); + else + module_memfree(ptr); +} + +static void free_mod_mem(struct module *mod) +{ + for_each_mod_mem_type(type) { + struct module_memory *mod_mem = &mod->mem[type]; + + if (type == MOD_DATA) + continue; + + /* Free lock-classes; relies on the preceding sync_rcu(). */ + lockdep_free_key_range(mod_mem->base, mod_mem->size); + if (mod_mem->size) + module_memory_free(mod_mem->base, type); + } + + /* MOD_DATA hosts mod, so free it at last */ + lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size); + module_memory_free(mod->mem[MOD_DATA].base, MOD_DATA); +} + /* Free a module, remove from lists, etc. */ static void free_module(struct module *mod) { @@ -1189,18 +1247,10 @@ static void free_module(struct module *mod) /* This may be empty, but that's OK */ module_arch_freeing_init(mod); - module_memfree(mod->init_layout.base); kfree(mod->args); percpu_modfree(mod); - /* Free lock-classes; relies on the preceding sync_rcu(). */ - lockdep_free_key_range(mod->data_layout.base, mod->data_layout.size); - - /* Finally, free the core (containing the module structure) */ - module_memfree(mod->core_layout.base); -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - vfree(mod->data_layout.base); -#endif + free_mod_mem(mod); } void *__symbol_get(const char *symbol) @@ -1387,16 +1437,18 @@ unsigned int __weak arch_mod_section_prepend(struct module *mod, return 0; } -/* Update size with this section: return offset. */ -long module_get_offset(struct module *mod, unsigned int *size, - Elf_Shdr *sechdr, unsigned int section) +long module_get_offset_and_type(struct module *mod, enum mod_mem_type type, + Elf_Shdr *sechdr, unsigned int section) { - long ret; + long offset; + long mask = ((unsigned long)(type) & SH_ENTSIZE_TYPE_MASK) << SH_ENTSIZE_TYPE_SHIFT; - *size += arch_mod_section_prepend(mod, section); - ret = ALIGN(*size, sechdr->sh_addralign ?: 1); - *size = ret + sechdr->sh_size; - return ret; + mod->mem[type].size += arch_mod_section_prepend(mod, section); + offset = ALIGN(mod->mem[type].size, sechdr->sh_addralign ?: 1); + mod->mem[type].size = offset + sechdr->sh_size; + + WARN_ON_ONCE(offset & mask); + return offset | mask; } static bool module_init_layout_section(const char *sname) @@ -1408,15 +1460,11 @@ static bool module_init_layout_section(const char *sname) return module_init_section(sname); } -/* - * Lay out the SHF_ALLOC sections in a way not dissimilar to how ld - * might -- code, read-only data, read-write data, small data. Tally - * sizes, and place the offsets into sh_entsize fields: high bit means it - * belongs in init. - */ -static void layout_sections(struct module *mod, struct load_info *info) +static void __layout_sections(struct module *mod, struct load_info *info, bool is_init) { - static unsigned long const masks[][2] = { + unsigned int m, i; + + static const unsigned long masks[][2] = { /* * NOTE: all executable code must be the first section * in this array; otherwise modify the text_size @@ -1428,84 +1476,63 @@ static void layout_sections(struct module *mod, struct load_info *info) { SHF_WRITE | SHF_ALLOC, ARCH_SHF_SMALL }, { ARCH_SHF_SMALL | SHF_ALLOC, 0 } }; - unsigned int m, i; - - for (i = 0; i < info->hdr->e_shnum; i++) - info->sechdrs[i].sh_entsize = ~0UL; + static const int core_m_to_mem_type[] = { + MOD_TEXT, + MOD_RODATA, + MOD_RO_AFTER_INIT, + MOD_DATA, + MOD_INVALID, /* This is needed to match the masks array */ + }; + static const int init_m_to_mem_type[] = { + MOD_INIT_TEXT, + MOD_INIT_RODATA, + MOD_INVALID, + MOD_INIT_DATA, + MOD_INVALID, /* This is needed to match the masks array */ + }; - pr_debug("Core section allocation order:\n"); for (m = 0; m < ARRAY_SIZE(masks); ++m) { + enum mod_mem_type type = is_init ? init_m_to_mem_type[m] : core_m_to_mem_type[m]; + for (i = 0; i < info->hdr->e_shnum; ++i) { Elf_Shdr *s = &info->sechdrs[i]; const char *sname = info->secstrings + s->sh_name; - unsigned int *sizep; if ((s->sh_flags & masks[m][0]) != masks[m][0] || (s->sh_flags & masks[m][1]) || s->sh_entsize != ~0UL - || module_init_layout_section(sname)) + || is_init != module_init_layout_section(sname)) continue; - sizep = m ? &mod->data_layout.size : &mod->core_layout.size; - s->sh_entsize = module_get_offset(mod, sizep, s, i); - pr_debug("\t%s\n", sname); - } - switch (m) { - case 0: /* executable */ - mod->core_layout.size = strict_align(mod->core_layout.size); - mod->core_layout.text_size = mod->core_layout.size; - break; - case 1: /* RO: text and ro-data */ - mod->data_layout.size = strict_align(mod->data_layout.size); - mod->data_layout.ro_size = mod->data_layout.size; - break; - case 2: /* RO after init */ - mod->data_layout.size = strict_align(mod->data_layout.size); - mod->data_layout.ro_after_init_size = mod->data_layout.size; - break; - case 4: /* whole core */ - mod->data_layout.size = strict_align(mod->data_layout.size); - break; - } - } - - pr_debug("Init section allocation order:\n"); - for (m = 0; m < ARRAY_SIZE(masks); ++m) { - for (i = 0; i < info->hdr->e_shnum; ++i) { - Elf_Shdr *s = &info->sechdrs[i]; - const char *sname = info->secstrings + s->sh_name; - if ((s->sh_flags & masks[m][0]) != masks[m][0] - || (s->sh_flags & masks[m][1]) - || s->sh_entsize != ~0UL - || !module_init_layout_section(sname)) + if (WARN_ON_ONCE(type == MOD_INVALID)) continue; - s->sh_entsize = (module_get_offset(mod, &mod->init_layout.size, s, i) - | INIT_OFFSET_MASK); + + s->sh_entsize = module_get_offset_and_type(mod, type, s, i); pr_debug("\t%s\n", sname); } - switch (m) { - case 0: /* executable */ - mod->init_layout.size = strict_align(mod->init_layout.size); - mod->init_layout.text_size = mod->init_layout.size; - break; - case 1: /* RO: text and ro-data */ - mod->init_layout.size = strict_align(mod->init_layout.size); - mod->init_layout.ro_size = mod->init_layout.size; - break; - case 2: - /* - * RO after init doesn't apply to init_layout (only - * core_layout), so it just takes the value of ro_size. - */ - mod->init_layout.ro_after_init_size = mod->init_layout.ro_size; - break; - case 4: /* whole init */ - mod->init_layout.size = strict_align(mod->init_layout.size); - break; - } } } +/* + * Lay out the SHF_ALLOC sections in a way not dissimilar to how ld + * might -- code, read-only data, read-write data, small data. Tally + * sizes, and place the offsets into sh_entsize fields: high bit means it + * belongs in init. + */ +static void layout_sections(struct module *mod, struct load_info *info) +{ + unsigned int i; + + for (i = 0; i < info->hdr->e_shnum; i++) + info->sechdrs[i].sh_entsize = ~0UL; + + pr_debug("Core section allocation order:\n"); + __layout_sections(mod, info, false); + + pr_debug("Init section allocation order:\n"); + __layout_sections(mod, info, true); +} + static void set_license(struct module *mod, const char *license) { if (!license) @@ -2122,72 +2149,41 @@ static int move_module(struct module *mod, struct load_info *info) { int i; void *ptr; + enum mod_mem_type t; - /* Do the allocs. */ - ptr = module_alloc(mod->core_layout.size); - /* - * The pointer to this block is stored in the module structure - * which is inside the block. Just mark it as not being a - * leak. - */ - kmemleak_not_leak(ptr); - if (!ptr) - return -ENOMEM; - - memset(ptr, 0, mod->core_layout.size); - mod->core_layout.base = ptr; + for_each_mod_mem_type(type) { + if (!mod->mem[type].size) { + mod->mem[type].base = NULL; + continue; + } + mod->mem[type].size = PAGE_ALIGN(mod->mem[type].size); + ptr = module_memory_alloc(mod->mem[type].size, type); - if (mod->init_layout.size) { - ptr = module_alloc(mod->init_layout.size); /* * The pointer to this block is stored in the module structure - * which is inside the block. This block doesn't need to be - * scanned as it contains data and code that will be freed - * after the module is initialized. + * which is inside the block. Just mark it as not being a + * leak. */ kmemleak_ignore(ptr); if (!ptr) { - module_memfree(mod->core_layout.base); - return -ENOMEM; + t = type; + goto out_enomem; } - memset(ptr, 0, mod->init_layout.size); - mod->init_layout.base = ptr; - } else - mod->init_layout.base = NULL; - -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - /* Do the allocs. */ - ptr = vzalloc(mod->data_layout.size); - /* - * The pointer to this block is stored in the module structure - * which is inside the block. Just mark it as not being a - * leak. - */ - kmemleak_not_leak(ptr); - if (!ptr) { - module_memfree(mod->core_layout.base); - module_memfree(mod->init_layout.base); - return -ENOMEM; + memset(ptr, 0, mod->mem[type].size); + mod->mem[type].base = ptr; } - mod->data_layout.base = ptr; -#endif /* Transfer each section which specifies SHF_ALLOC */ pr_debug("final section addresses:\n"); for (i = 0; i < info->hdr->e_shnum; i++) { void *dest; Elf_Shdr *shdr = &info->sechdrs[i]; + enum mod_mem_type type = shdr->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT; if (!(shdr->sh_flags & SHF_ALLOC)) continue; - if (shdr->sh_entsize & INIT_OFFSET_MASK) - dest = mod->init_layout.base - + (shdr->sh_entsize & ~INIT_OFFSET_MASK); - else if (!(shdr->sh_flags & SHF_EXECINSTR)) - dest = mod->data_layout.base + shdr->sh_entsize; - else - dest = mod->core_layout.base + shdr->sh_entsize; + dest = mod->mem[type].base + (shdr->sh_entsize & SH_ENTSIZE_OFFSET_MASK); if (shdr->sh_type != SHT_NOBITS) memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size); @@ -2198,6 +2194,10 @@ static int move_module(struct module *mod, struct load_info *info) } return 0; +out_enomem: + for (t--; t >= 0; t--) + module_memory_free(mod->mem[t].base, t); + return -ENOMEM; } static int check_module_license_and_versions(struct module *mod) @@ -2242,12 +2242,14 @@ static void flush_module_icache(const struct module *mod) * Do it before processing of module parameters, so the module * can provide parameter accessor functions of its own. */ - if (mod->init_layout.base) - flush_icache_range((unsigned long)mod->init_layout.base, - (unsigned long)mod->init_layout.base - + mod->init_layout.size); - flush_icache_range((unsigned long)mod->core_layout.base, - (unsigned long)mod->core_layout.base + mod->core_layout.size); + for_each_mod_mem_type(type) { + const struct module_memory *mod_mem = &mod->mem[type]; + + if (mod_mem->size) { + flush_icache_range((unsigned long)mod_mem->base, + (unsigned long)mod_mem->base + mod_mem->size); + } + } } bool __weak module_elf_check_arch(Elf_Ehdr *hdr) @@ -2350,11 +2352,8 @@ static void module_deallocate(struct module *mod, struct load_info *info) { percpu_modfree(mod); module_arch_freeing_init(mod); - module_memfree(mod->init_layout.base); - module_memfree(mod->core_layout.base); -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - vfree(mod->data_layout.base); -#endif + + free_mod_mem(mod); } int __weak module_finalize(const Elf_Ehdr *hdr, @@ -2415,7 +2414,9 @@ static void do_mod_ctors(struct module *mod) /* For freeing module_init on success, in case kallsyms traversing */ struct mod_initfree { struct llist_node node; - void *module_init; + void *init_text; + void *init_data; + void *init_rodata; }; static void do_free_init(struct work_struct *w) @@ -2429,7 +2430,9 @@ static void do_free_init(struct work_struct *w) llist_for_each_safe(pos, n, list) { initfree = container_of(pos, struct mod_initfree, node); - module_memfree(initfree->module_init); + module_memfree(initfree->init_text); + module_memfree(initfree->init_data); + module_memfree(initfree->init_rodata); kfree(initfree); } } @@ -2456,7 +2459,9 @@ static noinline int do_init_module(struct module *mod) ret = -ENOMEM; goto fail; } - freeinit->module_init = mod->init_layout.base; + freeinit->init_text = mod->mem[MOD_INIT_TEXT].base; + freeinit->init_data = mod->mem[MOD_INIT_DATA].base; + freeinit->init_rodata = mod->mem[MOD_INIT_RODATA].base; do_mod_ctors(mod); /* Start the module */ @@ -2492,8 +2497,8 @@ static noinline int do_init_module(struct module *mod) if (!mod->async_probe_requested) async_synchronize_full(); - ftrace_free_mem(mod, mod->init_layout.base, mod->init_layout.base + - mod->init_layout.size); + ftrace_free_mem(mod, mod->mem[MOD_INIT_TEXT].base, + mod->mem[MOD_INIT_TEXT].base + mod->mem[MOD_INIT_TEXT].size); mutex_lock(&module_mutex); /* Drop initial reference. */ module_put(mod); @@ -2505,11 +2510,10 @@ static noinline int do_init_module(struct module *mod) module_enable_ro(mod, true); mod_tree_remove_init(mod); module_arch_freeing_init(mod); - mod->init_layout.base = NULL; - mod->init_layout.size = 0; - mod->init_layout.ro_size = 0; - mod->init_layout.ro_after_init_size = 0; - mod->init_layout.text_size = 0; + for_class_mod_mem_type(type, init) { + mod->mem[type].base = NULL; + mod->mem[type].size = 0; + } #ifdef CONFIG_DEBUG_INFO_BTF_MODULES /* .BTF is not SHF_ALLOC and will get removed, so sanitize pointer */ mod->btf_data = NULL; @@ -2628,9 +2632,6 @@ static int complete_formation(struct module *mod, struct load_info *info) module_bug_finalize(info->hdr, info->sechdrs, mod); module_cfi_finalize(info->hdr, info->sechdrs, mod); - if (module_check_misalignment(mod)) - goto out_misaligned; - module_enable_ro(mod, false); module_enable_nx(mod); module_enable_x(mod); @@ -2644,8 +2645,6 @@ static int complete_formation(struct module *mod, struct load_info *info) return 0; -out_misaligned: - err = -EINVAL; out: mutex_unlock(&module_mutex); return err; @@ -2909,7 +2908,10 @@ static int load_module(struct load_info *info, const char __user *uargs, mutex_unlock(&module_mutex); free_module: /* Free lock-classes; relies on the preceding sync_rcu() */ - lockdep_free_key_range(mod->data_layout.base, mod->data_layout.size); + for_class_mod_mem_type(type, core_data) { + lockdep_free_key_range(mod->mem[type].base, + mod->mem[type].size); + } module_deallocate(mod, info); free_copy: @@ -3060,20 +3062,21 @@ bool is_module_address(unsigned long addr) struct module *__module_address(unsigned long addr) { struct module *mod; - struct mod_tree_root *tree; if (addr >= mod_tree.addr_min && addr <= mod_tree.addr_max) - tree = &mod_tree; + goto lookup; + #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - else if (addr >= mod_data_tree.addr_min && addr <= mod_data_tree.addr_max) - tree = &mod_data_tree; + if (addr >= mod_tree.data_addr_min && addr <= mod_tree.data_addr_max) + goto lookup; #endif - else - return NULL; + return NULL; + +lookup: module_assert_mutex_or_preempt(); - mod = mod_find(addr, tree); + mod = mod_find(addr, &mod_tree); if (mod) { BUG_ON(!within_module(addr, mod)); if (mod->state == MODULE_STATE_UNFORMED) @@ -3113,8 +3116,8 @@ struct module *__module_text_address(unsigned long addr) struct module *mod = __module_address(addr); if (mod) { /* Make sure it's within the text section. */ - if (!within(addr, mod->init_layout.base, mod->init_layout.text_size) - && !within(addr, mod->core_layout.base, mod->core_layout.text_size)) + if (!within_module_mem_type(addr, mod, MOD_TEXT) && + !within_module_mem_type(addr, mod, MOD_INIT_TEXT)) mod = NULL; } return mod; diff --git a/kernel/module/procfs.c b/kernel/module/procfs.c index cf5b9f1e6ec4..0a4841e88adb 100644 --- a/kernel/module/procfs.c +++ b/kernel/module/procfs.c @@ -62,6 +62,15 @@ static void m_stop(struct seq_file *m, void *p) mutex_unlock(&module_mutex); } +static unsigned int module_total_size(struct module *mod) +{ + int size = 0; + + for_each_mod_mem_type(type) + size += mod->mem[type].size; + return size; +} + static int m_show(struct seq_file *m, void *p) { struct module *mod = list_entry(p, struct module, list); @@ -73,10 +82,7 @@ static int m_show(struct seq_file *m, void *p) if (mod->state == MODULE_STATE_UNFORMED) return 0; - size = mod->init_layout.size + mod->core_layout.size; -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - size += mod->data_layout.size; -#endif + size = module_total_size(mod); seq_printf(m, "%s %u", mod->name, size); print_unload_info(m, mod); @@ -86,7 +92,7 @@ static int m_show(struct seq_file *m, void *p) mod->state == MODULE_STATE_COMING ? "Loading" : "Live"); /* Used by oprofile and other similar tools. */ - value = m->private ? NULL : mod->core_layout.base; + value = m->private ? NULL : mod->mem[MOD_TEXT].base; seq_printf(m, " 0x%px", value); /* Taints info */ diff --git a/kernel/module/strict_rwx.c b/kernel/module/strict_rwx.c index 14fbea66f12f..a2b656b4e3d2 100644 --- a/kernel/module/strict_rwx.c +++ b/kernel/module/strict_rwx.c @@ -11,82 +11,25 @@ #include #include "internal.h" -/* - * LKM RO/NX protection: protect module's text/ro-data - * from modification and any data from execution. - * - * General layout of module is: - * [text] [read-only-data] [ro-after-init] [writable data] - * text_size -----^ ^ ^ ^ - * ro_size ------------------------| | | - * ro_after_init_size -----------------------------| | - * size -----------------------------------------------------------| - * - * These values are always page-aligned (as is base) when - * CONFIG_STRICT_MODULE_RWX is set. - */ +static void module_set_memory(const struct module *mod, enum mod_mem_type type, + int (*set_memory)(unsigned long start, int num_pages)) +{ + const struct module_memory *mod_mem = &mod->mem[type]; + + set_vm_flush_reset_perms(mod_mem->base); + set_memory((unsigned long)mod_mem->base, mod_mem->size >> PAGE_SHIFT); +} /* * Since some arches are moving towards PAGE_KERNEL module allocations instead - * of PAGE_KERNEL_EXEC, keep frob_text() and module_enable_x() independent of + * of PAGE_KERNEL_EXEC, keep module_enable_x() independent of * CONFIG_STRICT_MODULE_RWX because they are needed regardless of whether we * are strict. */ -static void frob_text(const struct module_layout *layout, - int (*set_memory)(unsigned long start, int num_pages)) -{ - set_memory((unsigned long)layout->base, - PAGE_ALIGN(layout->text_size) >> PAGE_SHIFT); -} - -static void frob_rodata(const struct module_layout *layout, - int (*set_memory)(unsigned long start, int num_pages)) -{ - set_memory((unsigned long)layout->base + layout->text_size, - (layout->ro_size - layout->text_size) >> PAGE_SHIFT); -} - -static void frob_ro_after_init(const struct module_layout *layout, - int (*set_memory)(unsigned long start, int num_pages)) -{ - set_memory((unsigned long)layout->base + layout->ro_size, - (layout->ro_after_init_size - layout->ro_size) >> PAGE_SHIFT); -} - -static void frob_writable_data(const struct module_layout *layout, - int (*set_memory)(unsigned long start, int num_pages)) -{ - set_memory((unsigned long)layout->base + layout->ro_after_init_size, - (layout->size - layout->ro_after_init_size) >> PAGE_SHIFT); -} - -static bool layout_check_misalignment(const struct module_layout *layout) -{ - return WARN_ON(!PAGE_ALIGNED(layout->base)) || - WARN_ON(!PAGE_ALIGNED(layout->text_size)) || - WARN_ON(!PAGE_ALIGNED(layout->ro_size)) || - WARN_ON(!PAGE_ALIGNED(layout->ro_after_init_size)) || - WARN_ON(!PAGE_ALIGNED(layout->size)); -} - -bool module_check_misalignment(const struct module *mod) -{ - if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) - return false; - - return layout_check_misalignment(&mod->core_layout) || - layout_check_misalignment(&mod->data_layout) || - layout_check_misalignment(&mod->init_layout); -} - void module_enable_x(const struct module *mod) { - if (!PAGE_ALIGNED(mod->core_layout.base) || - !PAGE_ALIGNED(mod->init_layout.base)) - return; - - frob_text(&mod->core_layout, set_memory_x); - frob_text(&mod->init_layout, set_memory_x); + for_class_mod_mem_type(type, text) + module_set_memory(mod, type, set_memory_x); } void module_enable_ro(const struct module *mod, bool after_init) @@ -98,16 +41,13 @@ void module_enable_ro(const struct module *mod, bool after_init) return; #endif - set_vm_flush_reset_perms(mod->core_layout.base); - set_vm_flush_reset_perms(mod->init_layout.base); - frob_text(&mod->core_layout, set_memory_ro); - - frob_rodata(&mod->data_layout, set_memory_ro); - frob_text(&mod->init_layout, set_memory_ro); - frob_rodata(&mod->init_layout, set_memory_ro); + module_set_memory(mod, MOD_TEXT, set_memory_ro); + module_set_memory(mod, MOD_INIT_TEXT, set_memory_ro); + module_set_memory(mod, MOD_RODATA, set_memory_ro); + module_set_memory(mod, MOD_INIT_RODATA, set_memory_ro); if (after_init) - frob_ro_after_init(&mod->data_layout, set_memory_ro); + module_set_memory(mod, MOD_RO_AFTER_INIT, set_memory_ro); } void module_enable_nx(const struct module *mod) @@ -115,11 +55,8 @@ void module_enable_nx(const struct module *mod) if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) return; - frob_rodata(&mod->data_layout, set_memory_nx); - frob_ro_after_init(&mod->data_layout, set_memory_nx); - frob_writable_data(&mod->data_layout, set_memory_nx); - frob_rodata(&mod->init_layout, set_memory_nx); - frob_writable_data(&mod->init_layout, set_memory_nx); + for_class_mod_mem_type(type, data) + module_set_memory(mod, type, set_memory_nx); } int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, diff --git a/kernel/module/tree_lookup.c b/kernel/module/tree_lookup.c index 8ec5cfd60496..277197977d43 100644 --- a/kernel/module/tree_lookup.c +++ b/kernel/module/tree_lookup.c @@ -21,16 +21,16 @@ static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n) { - struct module_layout *layout = container_of(n, struct module_layout, mtn.node); + struct module_memory *mod_mem = container_of(n, struct module_memory, mtn.node); - return (unsigned long)layout->base; + return (unsigned long)mod_mem->base; } static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n) { - struct module_layout *layout = container_of(n, struct module_layout, mtn.node); + struct module_memory *mod_mem = container_of(n, struct module_memory, mtn.node); - return (unsigned long)layout->size; + return (unsigned long)mod_mem->size; } static __always_inline bool @@ -77,32 +77,27 @@ static void __mod_tree_remove(struct mod_tree_node *node, struct mod_tree_root * */ void mod_tree_insert(struct module *mod) { - mod->core_layout.mtn.mod = mod; - mod->init_layout.mtn.mod = mod; - - __mod_tree_insert(&mod->core_layout.mtn, &mod_tree); - if (mod->init_layout.size) - __mod_tree_insert(&mod->init_layout.mtn, &mod_tree); - -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - mod->data_layout.mtn.mod = mod; - __mod_tree_insert(&mod->data_layout.mtn, &mod_data_tree); -#endif + for_each_mod_mem_type(type) { + mod->mem[type].mtn.mod = mod; + if (mod->mem[type].size) + __mod_tree_insert(&mod->mem[type].mtn, &mod_tree); + } } void mod_tree_remove_init(struct module *mod) { - if (mod->init_layout.size) - __mod_tree_remove(&mod->init_layout.mtn, &mod_tree); + for_class_mod_mem_type(type, init) { + if (mod->mem[type].size) + __mod_tree_remove(&mod->mem[type].mtn, &mod_tree); + } } void mod_tree_remove(struct module *mod) { - __mod_tree_remove(&mod->core_layout.mtn, &mod_tree); - mod_tree_remove_init(mod); -#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC - __mod_tree_remove(&mod->data_layout.mtn, &mod_data_tree); -#endif + for_each_mod_mem_type(type) { + if (mod->mem[type].size) + __mod_tree_remove(&mod->mem[type].mtn, &mod_tree); + } } struct module *mod_find(unsigned long addr, struct mod_tree_root *tree) -- cgit v1.2.3 From 042edf1ebb49a63f299c4cdfb9c1e2ba299c0b91 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Sat, 4 Feb 2023 05:44:46 +0000 Subject: module: make module_ktype structure constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit ee6d3dd4ed48 ("driver core: make kobj_type constant.") the driver core allows the usage of const struct kobj_type. Take advantage of this to constify the structure definition to prevent modification at runtime. Signed-off-by: Thomas Weißschuh Signed-off-by: Luis Chamberlain --- include/linux/module.h | 2 +- kernel/params.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/module.h b/include/linux/module.h index c0593a96e158..dbc9124b71bc 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -871,7 +871,7 @@ void *dereference_module_function_descriptor(struct module *mod, void *ptr) #ifdef CONFIG_SYSFS extern struct kset *module_kset; -extern struct kobj_type module_ktype; +extern const struct kobj_type module_ktype; #endif /* CONFIG_SYSFS */ #define symbol_request(x) try_then_request_module(symbol_get(x), "symbol:" #x) diff --git a/kernel/params.c b/kernel/params.c index 6e34ca89ebae..6a7548979aa9 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -948,7 +948,7 @@ static void module_kobj_release(struct kobject *kobj) complete(mk->kobj_completion); } -struct kobj_type module_ktype = { +const struct kobj_type module_ktype = { .release = module_kobj_release, .sysfs_ops = &module_sysfs_ops, }; -- cgit v1.2.3 From 7deabd67498869640c937c9bd83472574b7dea0b Mon Sep 17 00:00:00 2001 From: Jason Baron Date: Fri, 3 Mar 2023 11:50:56 -0500 Subject: dyndbg: use the module notifier callbacks Bring dynamic debug in line with other subsystems by using the module notifier callbacks. This results in a net decrease in core module code. Additionally, Jim Cromie has a new dynamic debug classmap feature, which requires that jump labels be initialized prior to dynamic debug. Specifically, the new feature toggles a jump label from the existing dynamic_debug_setup() function. However, this does not currently work properly, because jump labels are initialized via the 'module_notify_list' notifier chain, which is invoked after the current call to dynamic_debug_setup(). Thus, this patch ensures that jump labels are initialized prior to dynamic debug by setting the dynamic debug notifier priority to 0, while jump labels have the higher priority of 1. Tested by Jim using his new test case, and I've verfied the correct printing via: # modprobe test_dynamic_debug dyndbg. Link: https://lore.kernel.org/lkml/20230113193016.749791-21-jim.cromie@gmail.com/ Reported-by: kernel test robot Link: https://lore.kernel.org/oe-kbuild-all/202302190427.9iIK2NfJ-lkp@intel.com/ Tested-by: Jim Cromie Reviewed-by: Vincenzo Palazzo Cc: Peter Zijlstra CC: Jim Cromie Cc: Luis Chamberlain Cc: Greg Kroah-Hartman Signed-off-by: Jason Baron Signed-off-by: Luis Chamberlain --- include/linux/dynamic_debug.h | 13 ----------- include/linux/module.h | 4 ++++ kernel/module/internal.h | 2 -- kernel/module/main.c | 30 +++++++------------------- lib/dynamic_debug.c | 50 +++++++++++++++++++++++++++++++++++-------- 5 files changed, 53 insertions(+), 46 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h index 41682278d2e8..f401414341fb 100644 --- a/include/linux/dynamic_debug.h +++ b/include/linux/dynamic_debug.h @@ -130,9 +130,6 @@ struct ddebug_class_param { #if defined(CONFIG_DYNAMIC_DEBUG_CORE) -int ddebug_add_module(struct _ddebug_info *dyndbg, const char *modname); - -extern int ddebug_remove_module(const char *mod_name); extern __printf(2, 3) void __dynamic_pr_debug(struct _ddebug *descriptor, const char *fmt, ...); @@ -304,16 +301,6 @@ int param_get_dyndbg_classes(char *buffer, const struct kernel_param *kp); #include #include -static inline int ddebug_add_module(struct _ddebug_info *dinfo, const char *modname) -{ - return 0; -} - -static inline int ddebug_remove_module(const char *mod) -{ - return 0; -} - static inline int ddebug_dyndbg_module_param_cb(char *param, char *val, const char *modname) { diff --git a/include/linux/module.h b/include/linux/module.h index dbc9124b71bc..91726444d55f 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -579,6 +580,9 @@ struct module { struct error_injection_entry *ei_funcs; unsigned int num_ei_funcs; #endif +#ifdef CONFIG_DYNAMIC_DEBUG_CORE + struct _ddebug_info dyndbg_info; +#endif } ____cacheline_aligned __randomize_layout; #ifndef MODULE_ARCH_INIT #define MODULE_ARCH_INIT {} diff --git a/kernel/module/internal.h b/kernel/module/internal.h index c6c44ceca07a..e3883b7d4840 100644 --- a/kernel/module/internal.h +++ b/kernel/module/internal.h @@ -45,7 +45,6 @@ extern const struct kernel_symbol __stop___ksymtab_gpl[]; extern const s32 __start___kcrctab[]; extern const s32 __start___kcrctab_gpl[]; -#include struct load_info { const char *name; /* pointer to module in temporary copy, freed at end of load_module() */ @@ -55,7 +54,6 @@ struct load_info { Elf_Shdr *sechdrs; char *secstrings, *strtab; unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs; - struct _ddebug_info dyndbg; bool sig_ok; #ifdef CONFIG_KALLSYMS unsigned long mod_kallsyms_init_off; diff --git a/kernel/module/main.c b/kernel/module/main.c index 1fa529668a45..b4759f1695b7 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -1216,9 +1216,6 @@ static void free_module(struct module *mod) mod->state = MODULE_STATE_UNFORMED; mutex_unlock(&module_mutex); - /* Remove dynamic debug info */ - ddebug_remove_module(mod->name); - /* Arch-specific cleanup. */ module_arch_cleanup(mod); @@ -1619,19 +1616,6 @@ static void free_modinfo(struct module *mod) } } -static void dynamic_debug_setup(struct module *mod, struct _ddebug_info *dyndbg) -{ - if (!dyndbg->num_descs) - return; - ddebug_add_module(dyndbg, mod->name); -} - -static void dynamic_debug_remove(struct module *mod, struct _ddebug_info *dyndbg) -{ - if (dyndbg->num_descs) - ddebug_remove_module(mod->name); -} - void * __weak module_alloc(unsigned long size) { return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, @@ -2137,10 +2121,14 @@ static int find_module_sections(struct module *mod, struct load_info *info) if (section_addr(info, "__obsparm")) pr_warn("%s: Ignoring obsolete parameters\n", mod->name); - info->dyndbg.descs = section_objs(info, "__dyndbg", - sizeof(*info->dyndbg.descs), &info->dyndbg.num_descs); - info->dyndbg.classes = section_objs(info, "__dyndbg_classes", - sizeof(*info->dyndbg.classes), &info->dyndbg.num_classes); +#ifdef CONFIG_DYNAMIC_DEBUG_CORE + mod->dyndbg_info.descs = section_objs(info, "__dyndbg", + sizeof(*mod->dyndbg_info.descs), + &mod->dyndbg_info.num_descs); + mod->dyndbg_info.classes = section_objs(info, "__dyndbg_classes", + sizeof(*mod->dyndbg_info.classes), + &mod->dyndbg_info.num_classes); +#endif return 0; } @@ -2824,7 +2812,6 @@ static int load_module(struct load_info *info, const char __user *uargs, } init_build_id(mod, info); - dynamic_debug_setup(mod, &info->dyndbg); /* Ftrace init must be called in the MODULE_STATE_UNFORMED state */ ftrace_module_init(mod); @@ -2888,7 +2875,6 @@ static int load_module(struct load_info *info, const char __user *uargs, ddebug_cleanup: ftrace_release_mod(mod); - dynamic_debug_remove(mod, &info->dyndbg); synchronize_rcu(); kfree(mod->args); free_arch_cleanup: diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index 8136e5236b7b..fdd6d9800a70 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -1223,7 +1223,7 @@ static void ddebug_attach_module_classes(struct ddebug_table *dt, * Allocate a new ddebug_table for the given module * and add it to the global list. */ -static int __ddebug_add_module(struct _ddebug_info *di, const char *modname) +static int ddebug_add_module(struct _ddebug_info *di, const char *modname) { struct ddebug_table *dt; @@ -1262,11 +1262,6 @@ static int __ddebug_add_module(struct _ddebug_info *di, const char *modname) return 0; } -int ddebug_add_module(struct _ddebug_info *di, const char *modname) -{ - return __ddebug_add_module(di, modname); -} - /* helper for ddebug_dyndbg_(boot|module)_param_cb */ static int ddebug_dyndbg_param_cb(char *param, char *val, const char *modname, int on_err) @@ -1313,11 +1308,13 @@ static void ddebug_table_free(struct ddebug_table *dt) kfree(dt); } +#ifdef CONFIG_MODULES + /* * Called in response to a module being unloaded. Removes * any ddebug_table's which point at the module. */ -int ddebug_remove_module(const char *mod_name) +static int ddebug_remove_module(const char *mod_name) { struct ddebug_table *dt, *nextdt; int ret = -ENOENT; @@ -1336,6 +1333,33 @@ int ddebug_remove_module(const char *mod_name) return ret; } +static int ddebug_module_notify(struct notifier_block *self, unsigned long val, + void *data) +{ + struct module *mod = data; + int ret = 0; + + switch (val) { + case MODULE_STATE_COMING: + ret = ddebug_add_module(&mod->dyndbg_info, mod->name); + if (ret) + WARN(1, "Failed to allocate memory: dyndbg may not work properly.\n"); + break; + case MODULE_STATE_GOING: + ddebug_remove_module(mod->name); + break; + } + + return notifier_from_errno(ret); +} + +static struct notifier_block ddebug_module_nb = { + .notifier_call = ddebug_module_notify, + .priority = 0, /* dynamic debug depends on jump label */ +}; + +#endif /* CONFIG_MODULES */ + static void ddebug_remove_all_tables(void) { mutex_lock(&ddebug_lock); @@ -1387,6 +1411,14 @@ static int __init dynamic_debug_init(void) .num_classes = __stop___dyndbg_classes - __start___dyndbg_classes, }; +#ifdef CONFIG_MODULES + ret = register_module_notifier(&ddebug_module_nb); + if (ret) { + pr_warn("Failed to register dynamic debug module notifier\n"); + return ret; + } +#endif /* CONFIG_MODULES */ + if (&__start___dyndbg == &__stop___dyndbg) { if (IS_ENABLED(CONFIG_DYNAMIC_DEBUG)) { pr_warn("_ddebug table is empty in a CONFIG_DYNAMIC_DEBUG build\n"); @@ -1407,7 +1439,7 @@ static int __init dynamic_debug_init(void) mod_ct++; di.num_descs = mod_sites; di.descs = iter_mod_start; - ret = __ddebug_add_module(&di, modname); + ret = ddebug_add_module(&di, modname); if (ret) goto out_err; @@ -1418,7 +1450,7 @@ static int __init dynamic_debug_init(void) } di.num_descs = mod_sites; di.descs = iter_mod_start; - ret = __ddebug_add_module(&di, modname); + ret = ddebug_add_module(&di, modname); if (ret) goto out_err; -- cgit v1.2.3 From 0433686c6092f65b552eadad651f620e51b6aad1 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 24 Feb 2023 22:07:45 +0200 Subject: devres: Pass unique name of the resource to devm_add_action() Pass the unique name of the resource to devm_add_action(), so it will be easier to debug managed resources. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20230224200745.17324-1-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/devres.c | 11 ++++++----- include/linux/device.h | 5 ++++- 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/devres.c b/drivers/base/devres.c index c0e100074aa3..5c998cfac335 100644 --- a/drivers/base/devres.c +++ b/drivers/base/devres.c @@ -722,20 +722,21 @@ static void devm_action_release(struct device *dev, void *res) } /** - * devm_add_action() - add a custom action to list of managed resources + * __devm_add_action() - add a custom action to list of managed resources * @dev: Device that owns the action * @action: Function that should be called * @data: Pointer to data passed to @action implementation + * @name: Name of the resource (for debugging purposes) * * This adds a custom action to the list of managed resources so that * it gets executed as part of standard resource unwinding. */ -int devm_add_action(struct device *dev, void (*action)(void *), void *data) +int __devm_add_action(struct device *dev, void (*action)(void *), void *data, const char *name) { struct action_devres *devres; - devres = devres_alloc(devm_action_release, - sizeof(struct action_devres), GFP_KERNEL); + devres = __devres_alloc_node(devm_action_release, sizeof(struct action_devres), + GFP_KERNEL, NUMA_NO_NODE, name); if (!devres) return -ENOMEM; @@ -745,7 +746,7 @@ int devm_add_action(struct device *dev, void (*action)(void *), void *data) devres_add(dev, devres); return 0; } -EXPORT_SYMBOL_GPL(devm_add_action); +EXPORT_SYMBOL_GPL(__devm_add_action); /** * devm_remove_action() - removes previously added custom action diff --git a/include/linux/device.h b/include/linux/device.h index 19b6ba478fbf..0f128520f6e5 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -243,10 +243,13 @@ void __iomem *devm_of_iomap(struct device *dev, resource_size_t *size); /* allows to add/remove a custom action to devres stack */ -int devm_add_action(struct device *dev, void (*action)(void *), void *data); void devm_remove_action(struct device *dev, void (*action)(void *), void *data); void devm_release_action(struct device *dev, void (*action)(void *), void *data); +int __devm_add_action(struct device *dev, void (*action)(void *), void *data, const char *name); +#define devm_add_action(release, action, data) \ + __devm_add_action(release, action, data, #action) + static inline int devm_add_action_or_reset(struct device *dev, void (*action)(void *), void *data) { -- cgit v1.2.3 From 557aafac11530a283bebf2dea4cec62765d8df0f Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 10 Mar 2023 10:55:59 -0800 Subject: kernel/module: add documentation for try_module_get() There is quite a bit of tribal knowledge around proper use of try_module_get() and requiring *somehow* the module to still exist to use this call in a way that is safe. Document this bit of tribal knowledge. To be clear, you should only use try_module_get() *iff* you are 100% sure the module already does exist and is not on its way out. You can be sure the module still exists and is alive through: 1) Direct protection with its refcount: you know some earlier caller called __module_get() safely 2) Implied protection: there is an implied protection against module removal Having an idea of when you are sure __module_get() might be called earlier is easy to understand however the implied protection requires an example. We use sysfs an an example for implied protection without a direct module reference count bump. kernfs / sysfs uses its own internal reference counting for files being actively used, when such file are active they completely prevent the module from being removed. kernfs protects this with its kernfs_active(). Effort has been put into verifying the kernfs implied protection works by using a currently out-of-tree test_sysfs selftest test #32 [0]: ./tools/testing/selftests/sysfs/sysfs.sh -t 0032 Without kernfs / sysfs preventing module removal through its active reference count (kernfs_active()) the write would fail or worse, a crash would happen in this test and it does not. Similar safeguards are required for other users of try_module_get() *iff* they are not ensuring the above rule 1) is followed. [0] https://lore.kernel.org/lkml/20211029184500.2821444-4-mcgrof@kernel.org/ Signed-off-by: Luis Chamberlain --- include/linux/module.h | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/module.h b/include/linux/module.h index 91726444d55f..c3b357196470 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -671,10 +671,46 @@ void symbol_put_addr(void *addr); to handle the error case (which only happens with rmmod --wait). */ extern void __module_get(struct module *module); -/* This is the Right Way to get a module: if it fails, it's being removed, - * so pretend it's not there. */ +/** + * try_module_get() - take module refcount unless module is being removed + * @module: the module we should check for + * + * Only try to get a module reference count if the module is not being removed. + * This call will fail if the module is already being removed. + * + * Care must also be taken to ensure the module exists and is alive prior to + * usage of this call. This can be gauranteed through two means: + * + * 1) Direct protection: you know an earlier caller must have increased the + * module reference through __module_get(). This can typically be achieved + * by having another entity other than the module itself increment the + * module reference count. + * + * 2) Implied protection: there is an implied protection against module + * removal. An example of this is the implied protection used by kernfs / + * sysfs. The sysfs store / read file operations are guaranteed to exist + * through the use of kernfs's active reference (see kernfs_active()) and a + * sysfs / kernfs file removal cannot happen unless the same file is not + * active. Therefore, if a sysfs file is being read or written to the module + * which created it must still exist. It is therefore safe to use + * try_module_get() on module sysfs store / read ops. + * + * One of the real values to try_module_get() is the module_is_live() check + * which ensures that the caller of try_module_get() can yield to userspace + * module removal requests and gracefully fail if the module is on its way out. + * + * Returns true if the reference count was successfully incremented. + */ extern bool try_module_get(struct module *module); +/** + * module_put() - release a reference count to a module + * @module: the module we should release a reference count for + * + * If you successfully bump a reference count to a module with try_module_get(), + * when you are finished you must call module_put() to release that reference + * count. + */ extern void module_put(struct module *module); #else /*!CONFIG_MODULE_UNLOAD*/ -- cgit v1.2.3 From 4ad682e04c1694aba95627f60e89f83178c277c2 Mon Sep 17 00:00:00 2001 From: Mehdi Djait Date: Thu, 2 Mar 2023 14:04:35 +0100 Subject: iio: Improve the kernel-doc of iio_trigger_poll Move the kernel-doc of the function to industrialio-trigger.c Add a note on the context where the function is expected to be called. Signed-off-by: Mehdi Djait Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/bd84fc17e9d22eab998bf48720297f9a77689f45.1677761379.git.mehdi.djait.k@gmail.com Signed-off-by: Jonathan Cameron --- drivers/iio/industrialio-trigger.c | 6 ++++++ include/linux/iio/trigger.h | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c index a2f3cc2f65ef..270ec0a9f641 100644 --- a/drivers/iio/industrialio-trigger.c +++ b/drivers/iio/industrialio-trigger.c @@ -192,6 +192,12 @@ static void iio_trigger_notify_done_atomic(struct iio_trigger *trig) schedule_work(&trig->reenable_work); } +/** + * iio_trigger_poll() - Call the IRQ trigger handler of the consumers + * @trig: trigger which occurred + * + * This function should only be called from a hard IRQ context. + */ void iio_trigger_poll(struct iio_trigger *trig) { int i; diff --git a/include/linux/iio/trigger.h b/include/linux/iio/trigger.h index f6360d9a492d..42da55dc3aa7 100644 --- a/include/linux/iio/trigger.h +++ b/include/linux/iio/trigger.h @@ -151,12 +151,6 @@ void iio_trigger_unregister(struct iio_trigger *trig_info); **/ int iio_trigger_set_immutable(struct iio_dev *indio_dev, struct iio_trigger *trig); -/** - * iio_trigger_poll() - called on a trigger occurring - * @trig: trigger which occurred - * - * Typically called in relevant hardware interrupt handler. - **/ void iio_trigger_poll(struct iio_trigger *trig); void iio_trigger_poll_chained(struct iio_trigger *trig); -- cgit v1.2.3 From f700e55ef6ef9ec723164659ed4385900981c872 Mon Sep 17 00:00:00 2001 From: Mehdi Djait Date: Thu, 2 Mar 2023 14:04:36 +0100 Subject: iio: Rename iio_trigger_poll_chained and add kernel-doc Rename the function to iio_trigger_poll_nested. Add kernel-doc with a note on the context where the function is expected to be called. Signed-off-by: Mehdi Djait Link: https://lore.kernel.org/r/841b533cba28ca25a8e87280c44e45979166e8e2.1677761379.git.mehdi.djait.k@gmail.com Signed-off-by: Jonathan Cameron --- drivers/iio/accel/bma400_core.c | 2 +- drivers/iio/accel/kionix-kx022a.c | 2 +- drivers/iio/accel/mma8452.c | 2 +- drivers/iio/accel/msa311.c | 2 +- drivers/iio/adc/ad7606.c | 2 +- drivers/iio/adc/at91-sama5d2_adc.c | 2 +- drivers/iio/adc/max11410.c | 2 +- drivers/iio/common/st_sensors/st_sensors_trigger.c | 4 ++-- drivers/iio/gyro/fxas21002c_core.c | 2 +- drivers/iio/gyro/mpu3050-core.c | 2 +- drivers/iio/humidity/hts221_buffer.c | 2 +- drivers/iio/industrialio-trigger.c | 11 +++++++++-- drivers/iio/light/acpi-als.c | 2 +- drivers/iio/light/rpr0521.c | 2 +- drivers/iio/light/st_uvis25_core.c | 2 +- drivers/iio/light/vcnl4000.c | 2 +- drivers/iio/light/vcnl4035.c | 2 +- drivers/iio/potentiostat/lmp91000.c | 2 +- drivers/iio/pressure/zpa2326.c | 2 +- drivers/iio/proximity/as3935.c | 2 +- drivers/iio/trigger/iio-trig-loop.c | 2 +- include/linux/iio/trigger.h | 2 +- 22 files changed, 31 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/accel/bma400_core.c b/drivers/iio/accel/bma400_core.c index 623f37cbaf50..a68b845f5b4f 100644 --- a/drivers/iio/accel/bma400_core.c +++ b/drivers/iio/accel/bma400_core.c @@ -1688,7 +1688,7 @@ static irqreturn_t bma400_interrupt(int irq, void *private) if (FIELD_GET(BMA400_INT_DRDY_MSK, le16_to_cpu(data->status))) { mutex_unlock(&data->mutex); - iio_trigger_poll_chained(data->trig); + iio_trigger_poll_nested(data->trig); return IRQ_HANDLED; } diff --git a/drivers/iio/accel/kionix-kx022a.c b/drivers/iio/accel/kionix-kx022a.c index 9f8c7631f098..8dac0337c249 100644 --- a/drivers/iio/accel/kionix-kx022a.c +++ b/drivers/iio/accel/kionix-kx022a.c @@ -899,7 +899,7 @@ static irqreturn_t kx022a_irq_thread_handler(int irq, void *private) mutex_lock(&data->mutex); if (data->trigger_enabled) { - iio_trigger_poll_chained(data->trig); + iio_trigger_poll_nested(data->trig); ret = IRQ_HANDLED; } diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c index f97fb68e3a71..ea14e3aaa30a 100644 --- a/drivers/iio/accel/mma8452.c +++ b/drivers/iio/accel/mma8452.c @@ -1067,7 +1067,7 @@ static irqreturn_t mma8452_interrupt(int irq, void *p) return IRQ_NONE; if (src & MMA8452_INT_DRDY) { - iio_trigger_poll_chained(indio_dev->trig); + iio_trigger_poll_nested(indio_dev->trig); ret = IRQ_HANDLED; } diff --git a/drivers/iio/accel/msa311.c b/drivers/iio/accel/msa311.c index af94d3adf6d8..6690fa37da8f 100644 --- a/drivers/iio/accel/msa311.c +++ b/drivers/iio/accel/msa311.c @@ -951,7 +951,7 @@ static irqreturn_t msa311_irq_thread(int irq, void *p) } if (new_data_int_enabled) - iio_trigger_poll_chained(msa311->new_data_trig); + iio_trigger_poll_nested(msa311->new_data_trig); return IRQ_HANDLED; } diff --git a/drivers/iio/adc/ad7606.c b/drivers/iio/adc/ad7606.c index dd6b603f65ea..1928d9ae5bcf 100644 --- a/drivers/iio/adc/ad7606.c +++ b/drivers/iio/adc/ad7606.c @@ -477,7 +477,7 @@ static irqreturn_t ad7606_interrupt(int irq, void *dev_id) if (iio_buffer_enabled(indio_dev)) { gpiod_set_value(st->gpio_convst, 0); - iio_trigger_poll_chained(st->trig); + iio_trigger_poll_nested(st->trig); } else { complete(&st->completion); } diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c index 50d02e5fc6fc..b5d0c9ee284c 100644 --- a/drivers/iio/adc/at91-sama5d2_adc.c +++ b/drivers/iio/adc/at91-sama5d2_adc.c @@ -1194,7 +1194,7 @@ static void at91_dma_buffer_done(void *data) { struct iio_dev *indio_dev = data; - iio_trigger_poll_chained(indio_dev->trig); + iio_trigger_poll_nested(indio_dev->trig); } static int at91_adc_dma_start(struct iio_dev *indio_dev) diff --git a/drivers/iio/adc/max11410.c b/drivers/iio/adc/max11410.c index b74b689ee7de..237b2ce3f264 100644 --- a/drivers/iio/adc/max11410.c +++ b/drivers/iio/adc/max11410.c @@ -678,7 +678,7 @@ static irqreturn_t max11410_interrupt(int irq, void *dev_id) struct max11410_state *st = iio_priv(indio_dev); if (iio_buffer_enabled(indio_dev)) - iio_trigger_poll_chained(st->trig); + iio_trigger_poll_nested(st->trig); else complete(&st->completion); diff --git a/drivers/iio/common/st_sensors/st_sensors_trigger.c b/drivers/iio/common/st_sensors/st_sensors_trigger.c index 899b640c0a70..a0df9250a69f 100644 --- a/drivers/iio/common/st_sensors/st_sensors_trigger.c +++ b/drivers/iio/common/st_sensors/st_sensors_trigger.c @@ -85,7 +85,7 @@ static irqreturn_t st_sensors_irq_thread(int irq, void *p) */ if (sdata->hw_irq_trigger && st_sensors_new_samples_available(indio_dev, sdata)) { - iio_trigger_poll_chained(p); + iio_trigger_poll_nested(p); } else { dev_dbg(indio_dev->dev.parent, "spurious IRQ\n"); return IRQ_NONE; @@ -110,7 +110,7 @@ static irqreturn_t st_sensors_irq_thread(int irq, void *p) dev_dbg(indio_dev->dev.parent, "more samples came in during polling\n"); sdata->hw_timestamp = iio_get_time_ns(indio_dev); - iio_trigger_poll_chained(p); + iio_trigger_poll_nested(p); } return IRQ_HANDLED; diff --git a/drivers/iio/gyro/fxas21002c_core.c b/drivers/iio/gyro/fxas21002c_core.c index 3ea1d4613080..c28d17ca6f5e 100644 --- a/drivers/iio/gyro/fxas21002c_core.c +++ b/drivers/iio/gyro/fxas21002c_core.c @@ -813,7 +813,7 @@ static irqreturn_t fxas21002c_data_rdy_thread(int irq, void *private) if (!data_ready) return IRQ_NONE; - iio_trigger_poll_chained(data->dready_trig); + iio_trigger_poll_nested(data->dready_trig); return IRQ_HANDLED; } diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c index 6a6d84a3deda..a791ba3a693a 100644 --- a/drivers/iio/gyro/mpu3050-core.c +++ b/drivers/iio/gyro/mpu3050-core.c @@ -939,7 +939,7 @@ static irqreturn_t mpu3050_irq_thread(int irq, void *p) if (!(val & MPU3050_INT_STATUS_RAW_RDY)) return IRQ_NONE; - iio_trigger_poll_chained(p); + iio_trigger_poll_nested(p); return IRQ_HANDLED; } diff --git a/drivers/iio/humidity/hts221_buffer.c b/drivers/iio/humidity/hts221_buffer.c index 2a4107a79662..11ef38994a95 100644 --- a/drivers/iio/humidity/hts221_buffer.c +++ b/drivers/iio/humidity/hts221_buffer.c @@ -68,7 +68,7 @@ static irqreturn_t hts221_trigger_handler_thread(int irq, void *private) if (!(status & HTS221_RH_DRDY_MASK)) return IRQ_NONE; - iio_trigger_poll_chained(hw->trig); + iio_trigger_poll_nested(hw->trig); return IRQ_HANDLED; } diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c index 270ec0a9f641..784dc1e00310 100644 --- a/drivers/iio/industrialio-trigger.c +++ b/drivers/iio/industrialio-trigger.c @@ -222,7 +222,14 @@ irqreturn_t iio_trigger_generic_data_rdy_poll(int irq, void *private) } EXPORT_SYMBOL(iio_trigger_generic_data_rdy_poll); -void iio_trigger_poll_chained(struct iio_trigger *trig) +/** + * iio_trigger_poll_nested() - Call the threaded trigger handler of the + * consumers + * @trig: trigger which occurred + * + * This function should only be called from a kernel thread context. + */ +void iio_trigger_poll_nested(struct iio_trigger *trig) { int i; @@ -237,7 +244,7 @@ void iio_trigger_poll_chained(struct iio_trigger *trig) } } } -EXPORT_SYMBOL(iio_trigger_poll_chained); +EXPORT_SYMBOL(iio_trigger_poll_nested); void iio_trigger_notify_done(struct iio_trigger *trig) { diff --git a/drivers/iio/light/acpi-als.c b/drivers/iio/light/acpi-als.c index e1ff6f524f4b..2d91caf24dd0 100644 --- a/drivers/iio/light/acpi-als.c +++ b/drivers/iio/light/acpi-als.c @@ -108,7 +108,7 @@ static void acpi_als_notify(struct acpi_device *device, u32 event) if (iio_buffer_enabled(indio_dev) && iio_trigger_using_own(indio_dev)) { switch (event) { case ACPI_ALS_NOTIFY_ILLUMINANCE: - iio_trigger_poll_chained(als->trig); + iio_trigger_poll_nested(als->trig); break; default: /* Unhandled event */ diff --git a/drivers/iio/light/rpr0521.c b/drivers/iio/light/rpr0521.c index 668e444f6049..9d0218b7426e 100644 --- a/drivers/iio/light/rpr0521.c +++ b/drivers/iio/light/rpr0521.c @@ -431,7 +431,7 @@ static irqreturn_t rpr0521_drdy_irq_thread(int irq, void *private) struct rpr0521_data *data = iio_priv(indio_dev); if (rpr0521_is_triggered(data)) { - iio_trigger_poll_chained(data->drdy_trigger0); + iio_trigger_poll_nested(data->drdy_trigger0); return IRQ_HANDLED; } diff --git a/drivers/iio/light/st_uvis25_core.c b/drivers/iio/light/st_uvis25_core.c index c737d3e193ae..50f95c5d2060 100644 --- a/drivers/iio/light/st_uvis25_core.c +++ b/drivers/iio/light/st_uvis25_core.c @@ -161,7 +161,7 @@ static irqreturn_t st_uvis25_trigger_handler_thread(int irq, void *private) if (!(status & ST_UVIS25_REG_UV_DA_MASK)) return IRQ_NONE; - iio_trigger_poll_chained(hw->trig); + iio_trigger_poll_nested(hw->trig); return IRQ_HANDLED; } diff --git a/drivers/iio/light/vcnl4000.c b/drivers/iio/light/vcnl4000.c index 6bdfce9747f9..b9460c165d37 100644 --- a/drivers/iio/light/vcnl4000.c +++ b/drivers/iio/light/vcnl4000.c @@ -1078,7 +1078,7 @@ static irqreturn_t vcnl4010_irq_thread(int irq, void *p) } if (isr & VCNL4010_INT_DRDY && iio_buffer_enabled(indio_dev)) - iio_trigger_poll_chained(indio_dev->trig); + iio_trigger_poll_nested(indio_dev->trig); end: return IRQ_HANDLED; diff --git a/drivers/iio/light/vcnl4035.c b/drivers/iio/light/vcnl4035.c index 84148b944000..14e29330e972 100644 --- a/drivers/iio/light/vcnl4035.c +++ b/drivers/iio/light/vcnl4035.c @@ -89,7 +89,7 @@ static irqreturn_t vcnl4035_drdy_irq_thread(int irq, void *private) IIO_EV_TYPE_THRESH, IIO_EV_DIR_EITHER), iio_get_time_ns(indio_dev)); - iio_trigger_poll_chained(data->drdy_trigger0); + iio_trigger_poll_nested(data->drdy_trigger0); return IRQ_HANDLED; } diff --git a/drivers/iio/potentiostat/lmp91000.c b/drivers/iio/potentiostat/lmp91000.c index b82f093f1e6a..0083e858c21e 100644 --- a/drivers/iio/potentiostat/lmp91000.c +++ b/drivers/iio/potentiostat/lmp91000.c @@ -118,7 +118,7 @@ static int lmp91000_read(struct lmp91000_data *data, int channel, int *val) data->chan_select = channel != LMP91000_REG_MODECN_3LEAD; - iio_trigger_poll_chained(data->trig); + iio_trigger_poll_nested(data->trig); ret = wait_for_completion_timeout(&data->completion, HZ); reinit_completion(&data->completion); diff --git a/drivers/iio/pressure/zpa2326.c b/drivers/iio/pressure/zpa2326.c index 67119a9b95fc..421e059d1f19 100644 --- a/drivers/iio/pressure/zpa2326.c +++ b/drivers/iio/pressure/zpa2326.c @@ -829,7 +829,7 @@ static irqreturn_t zpa2326_handle_threaded_irq(int irq, void *data) } /* New sample available: dispatch internal trigger consumers. */ - iio_trigger_poll_chained(priv->trigger); + iio_trigger_poll_nested(priv->trigger); if (cont) /* diff --git a/drivers/iio/proximity/as3935.c b/drivers/iio/proximity/as3935.c index ebc95cf8f5f4..96fa97451cbf 100644 --- a/drivers/iio/proximity/as3935.c +++ b/drivers/iio/proximity/as3935.c @@ -257,7 +257,7 @@ static void as3935_event_work(struct work_struct *work) switch (val) { case AS3935_EVENT_INT: - iio_trigger_poll_chained(st->trig); + iio_trigger_poll_nested(st->trig); break; case AS3935_DISTURB_INT: case AS3935_NOISE_INT: diff --git a/drivers/iio/trigger/iio-trig-loop.c b/drivers/iio/trigger/iio-trig-loop.c index 96ec06bbe546..7aaed0611899 100644 --- a/drivers/iio/trigger/iio-trig-loop.c +++ b/drivers/iio/trigger/iio-trig-loop.c @@ -46,7 +46,7 @@ static int iio_loop_thread(void *data) set_freezable(); do { - iio_trigger_poll_chained(trig); + iio_trigger_poll_nested(trig); } while (likely(!kthread_freezable_should_stop(NULL))); return 0; diff --git a/include/linux/iio/trigger.h b/include/linux/iio/trigger.h index 42da55dc3aa7..51f52c5c6092 100644 --- a/include/linux/iio/trigger.h +++ b/include/linux/iio/trigger.h @@ -152,7 +152,7 @@ void iio_trigger_unregister(struct iio_trigger *trig_info); int iio_trigger_set_immutable(struct iio_dev *indio_dev, struct iio_trigger *trig); void iio_trigger_poll(struct iio_trigger *trig); -void iio_trigger_poll_chained(struct iio_trigger *trig); +void iio_trigger_poll_nested(struct iio_trigger *trig); irqreturn_t iio_trigger_generic_data_rdy_poll(int irq, void *private); -- cgit v1.2.3 From ce48f955860db841eeb4bcf9bb973ea3ef177b3a Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 2 Mar 2023 18:12:06 +0200 Subject: i3c: Correct reference to the I²C device data type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I²C peripheral devices that are connected to the controller are represented in the Linux kernel as objects of the struct i2c_client. Fix this in the header file. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20230302161206.38106-1-andriy.shevchenko@linux.intel.com Signed-off-by: Alexandre Belloni --- include/linux/i3c/master.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h index 604a126b78c8..a12cda9dc715 100644 --- a/include/linux/i3c/master.h +++ b/include/linux/i3c/master.h @@ -22,9 +22,10 @@ #define I3C_BROADCAST_ADDR 0x7e #define I3C_MAX_ADDR GENMASK(6, 0) +struct i2c_client; + struct i3c_master_controller; struct i3c_bus; -struct i2c_device; struct i3c_device; /** -- cgit v1.2.3 From 84706e9a75ffc3c950424bdfea06cabb4101a6b4 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Tue, 14 Mar 2023 09:54:00 +0800 Subject: soundwire: intel: add sync_arm/sync_go to ops The bus start/stop sequences can be reused between platforms if we add a couple of new callbacks. In following patches the code will be moved to a shared file. Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20230314015410.487311-7-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/intel.c | 16 ++++++++++------ drivers/soundwire/intel.h | 20 ++++++++++++++++++++ include/linux/soundwire/sdw_intel.h | 8 ++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c index 6fdb10117e59..902934cbb27b 100644 --- a/drivers/soundwire/intel.c +++ b/drivers/soundwire/intel.c @@ -686,7 +686,7 @@ static int intel_pre_bank_switch(struct sdw_intel *sdw) if (!bus->multi_link) return 0; - intel_shim_sync_arm(sdw); + sdw_intel_sync_arm(sdw); return 0; } @@ -720,7 +720,7 @@ static int intel_post_bank_switch(struct sdw_intel *sdw) goto unlock; } - ret = intel_shim_sync_go_unlocked(sdw); + ret = sdw_intel_sync_go_unlocked(sdw); unlock: mutex_unlock(sdw->link_res->shim_lock); @@ -1140,7 +1140,7 @@ static int intel_start_bus(struct sdw_intel *sdw) * gsync is enabled */ if (bus->multi_link) - intel_shim_sync_arm(sdw); + sdw_intel_sync_arm(sdw); ret = sdw_cdns_init(cdns); if (ret < 0) { @@ -1155,7 +1155,7 @@ static int intel_start_bus(struct sdw_intel *sdw) } if (bus->multi_link) { - ret = intel_shim_sync_go(sdw); + ret = sdw_intel_sync_go(sdw); if (ret < 0) { dev_err(dev, "%s: sync go failed: %d\n", __func__, ret); goto err_interrupt; @@ -1210,7 +1210,7 @@ static int intel_start_bus_after_reset(struct sdw_intel *sdw) * timeouts when gsync is enabled */ if (bus->multi_link) - intel_shim_sync_arm(sdw); + sdw_intel_sync_arm(sdw); /* * Re-initialize the IP since it was powered-off @@ -1239,7 +1239,7 @@ static int intel_start_bus_after_reset(struct sdw_intel *sdw) } if (bus->multi_link) { - ret = intel_shim_sync_go(sdw); + ret = sdw_intel_sync_go(sdw); if (ret < 0) { dev_err(sdw->cdns.dev, "sync go failed during resume\n"); goto err_interrupt; @@ -1342,6 +1342,10 @@ const struct sdw_intel_hw_ops sdw_intel_cnl_hw_ops = { .pre_bank_switch = intel_pre_bank_switch, .post_bank_switch = intel_post_bank_switch, + + .sync_arm = intel_shim_sync_arm, + .sync_go_unlocked = intel_shim_sync_go_unlocked, + .sync_go = intel_shim_sync_go, }; EXPORT_SYMBOL_NS(sdw_intel_cnl_hw_ops, SOUNDWIRE_INTEL); diff --git a/drivers/soundwire/intel.h b/drivers/soundwire/intel.h index 089c41babfc1..28b21a92e28b 100644 --- a/drivers/soundwire/intel.h +++ b/drivers/soundwire/intel.h @@ -167,4 +167,24 @@ static inline void sdw_intel_shim_wake(struct sdw_intel *sdw, bool wake_enable) SDW_INTEL_OPS(sdw, shim_wake)(sdw, wake_enable); } +static inline void sdw_intel_sync_arm(struct sdw_intel *sdw) +{ + if (SDW_INTEL_CHECK_OPS(sdw, sync_arm)) + SDW_INTEL_OPS(sdw, sync_arm)(sdw); +} + +static inline int sdw_intel_sync_go_unlocked(struct sdw_intel *sdw) +{ + if (SDW_INTEL_CHECK_OPS(sdw, sync_go_unlocked)) + return SDW_INTEL_OPS(sdw, sync_go_unlocked)(sdw); + return -ENOTSUPP; +} + +static inline int sdw_intel_sync_go(struct sdw_intel *sdw) +{ + if (SDW_INTEL_CHECK_OPS(sdw, sync_go)) + return SDW_INTEL_OPS(sdw, sync_go)(sdw); + return -ENOTSUPP; +} + #endif /* __SDW_INTEL_LOCAL_H */ diff --git a/include/linux/soundwire/sdw_intel.h b/include/linux/soundwire/sdw_intel.h index 91f0dc564fe5..06fa30929ebd 100644 --- a/include/linux/soundwire/sdw_intel.h +++ b/include/linux/soundwire/sdw_intel.h @@ -309,6 +309,10 @@ struct sdw_intel; * @shim_wake: enable/disable in-band wake management * @pre_bank_switch: helper for bus management * @post_bank_switch: helper for bus management + * @sync_arm: helper for multi-link synchronization + * @sync_go_unlocked: helper for multi-link synchronization - + * shim_lock is assumed to be locked at higher level + * @sync_go: helper for multi-link synchronization */ struct sdw_intel_hw_ops { void (*debugfs_init)(struct sdw_intel *sdw); @@ -330,6 +334,10 @@ struct sdw_intel_hw_ops { int (*pre_bank_switch)(struct sdw_intel *sdw); int (*post_bank_switch)(struct sdw_intel *sdw); + + void (*sync_arm)(struct sdw_intel *sdw); + int (*sync_go_unlocked)(struct sdw_intel *sdw); + int (*sync_go)(struct sdw_intel *sdw); }; extern const struct sdw_intel_hw_ops sdw_intel_cnl_hw_ops; -- cgit v1.2.3 From 1e76de2e5dfeff17f13afe8146449fec3f5b69f7 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Tue, 14 Mar 2023 09:54:03 +0800 Subject: soundwire: intel: add abstraction for cmdsync check If we add one more callback, we can have common bank switch sequences between old and new hardware: the only difference is where the CMDSYNC register is located. Signed-off-by: Pierre-Louis Bossart Reviewed-by: Rander Wang Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20230314015410.487311-10-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/intel.c | 24 +++++++++++++----------- drivers/soundwire/intel.h | 7 +++++++ include/linux/soundwire/sdw_intel.h | 3 +++ 3 files changed, 23 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c index 77d698908595..1131ecb4b5e7 100644 --- a/drivers/soundwire/intel.c +++ b/drivers/soundwire/intel.c @@ -325,6 +325,15 @@ static void intel_shim_wake(struct sdw_intel *sdw, bool wake_enable) mutex_unlock(sdw->link_res->shim_lock); } +static bool intel_check_cmdsync_unlocked(struct sdw_intel *sdw) +{ + void __iomem *shim = sdw->link_res->shim; + int sync_reg; + + sync_reg = intel_readl(shim, SDW_SHIM_SYNC); + return !!(sync_reg & SDW_SHIM_SYNC_CMDSYNC_MASK); +} + static int intel_link_power_up(struct sdw_intel *sdw) { unsigned int link_id = sdw->instance; @@ -695,8 +704,7 @@ static int intel_post_bank_switch(struct sdw_intel *sdw) { struct sdw_cdns *cdns = &sdw->cdns; struct sdw_bus *bus = &cdns->bus; - void __iomem *shim = sdw->link_res->shim; - int sync_reg, ret; + int ret = 0; /* Write to register only for multi-link */ if (!bus->multi_link) @@ -704,9 +712,6 @@ static int intel_post_bank_switch(struct sdw_intel *sdw) mutex_lock(sdw->link_res->shim_lock); - /* Read SYNC register */ - sync_reg = intel_readl(shim, SDW_SHIM_SYNC); - /* * post_bank_switch() ops is called from the bus in loop for * all the Masters in the steam with the expectation that @@ -715,13 +720,9 @@ static int intel_post_bank_switch(struct sdw_intel *sdw) * * So, set the SYNCGO bit only if CMDSYNC bit is set for any Master. */ - if (!(sync_reg & SDW_SHIM_SYNC_CMDSYNC_MASK)) { - ret = 0; - goto unlock; - } + if (sdw_intel_sync_check_cmdsync_unlocked(sdw)) + ret = sdw_intel_sync_go_unlocked(sdw); - ret = sdw_intel_sync_go_unlocked(sdw); -unlock: mutex_unlock(sdw->link_res->shim_lock); if (ret < 0) @@ -1147,6 +1148,7 @@ const struct sdw_intel_hw_ops sdw_intel_cnl_hw_ops = { .sync_arm = intel_shim_sync_arm, .sync_go_unlocked = intel_shim_sync_go_unlocked, .sync_go = intel_shim_sync_go, + .sync_check_cmdsync_unlocked = intel_check_cmdsync_unlocked, }; EXPORT_SYMBOL_NS(sdw_intel_cnl_hw_ops, SOUNDWIRE_INTEL); diff --git a/drivers/soundwire/intel.h b/drivers/soundwire/intel.h index abd1a500defa..7a69cf755954 100644 --- a/drivers/soundwire/intel.h +++ b/drivers/soundwire/intel.h @@ -187,6 +187,13 @@ static inline int sdw_intel_sync_go(struct sdw_intel *sdw) return -ENOTSUPP; } +static inline bool sdw_intel_sync_check_cmdsync_unlocked(struct sdw_intel *sdw) +{ + if (SDW_INTEL_CHECK_OPS(sdw, sync_check_cmdsync_unlocked)) + return SDW_INTEL_OPS(sdw, sync_check_cmdsync_unlocked)(sdw); + return false; +} + /* common bus management */ int intel_start_bus(struct sdw_intel *sdw); int intel_start_bus_after_reset(struct sdw_intel *sdw); diff --git a/include/linux/soundwire/sdw_intel.h b/include/linux/soundwire/sdw_intel.h index 06fa30929ebd..207701aeeb47 100644 --- a/include/linux/soundwire/sdw_intel.h +++ b/include/linux/soundwire/sdw_intel.h @@ -313,6 +313,8 @@ struct sdw_intel; * @sync_go_unlocked: helper for multi-link synchronization - * shim_lock is assumed to be locked at higher level * @sync_go: helper for multi-link synchronization + * @sync_check_cmdsync_unlocked: helper for multi-link synchronization + * and bank switch - shim_lock is assumed to be locked at higher level */ struct sdw_intel_hw_ops { void (*debugfs_init)(struct sdw_intel *sdw); @@ -338,6 +340,7 @@ struct sdw_intel_hw_ops { void (*sync_arm)(struct sdw_intel *sdw); int (*sync_go_unlocked)(struct sdw_intel *sdw); int (*sync_go)(struct sdw_intel *sdw); + bool (*sync_check_cmdsync_unlocked)(struct sdw_intel *sdw); }; extern const struct sdw_intel_hw_ops sdw_intel_cnl_hw_ops; -- cgit v1.2.3 From b91e6107119f62be8b51b9955294be52c0b4fc80 Mon Sep 17 00:00:00 2001 From: Emanuele Ghidoli Date: Mon, 13 Mar 2023 17:50:39 +0100 Subject: usb: misc: usb3503: support usb3803 and bypass mode Add support for USB3803 and bypass mode, with this change is also possible to move the component out of bypass mode. In bypass mode the downstream port 3 is connected to the upstream port with low switch resistance R_on. Controlling mode of operations: | RESET_N | BYPASS_N | Mode | -------------------------------- | 0 | 0 | standby | | 1 | 0 | bypass | | 1 | 1 | hub | Datasheet: https://ww1.microchip.com/downloads/aemDocuments/documents/UNG/ProductDocuments/DataSheets/00001691D.pdf Signed-off-by: Emanuele Ghidoli Signed-off-by: Francesco Dolcini Link: https://lore.kernel.org/r/20230313165039.255579-4-francesco@dolcini.it Signed-off-by: Greg Kroah-Hartman --- drivers/usb/misc/usb3503.c | 22 +++++++++++++++++++++- include/linux/platform_data/usb3503.h | 1 + 2 files changed, 22 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/usb/misc/usb3503.c b/drivers/usb/misc/usb3503.c index 3044db9fd8aa..c6cfd1edaf76 100644 --- a/drivers/usb/misc/usb3503.c +++ b/drivers/usb/misc/usb3503.c @@ -46,6 +46,7 @@ struct usb3503 { struct device *dev; struct clk *clk; u8 port_off_mask; + struct gpio_desc *bypass; struct gpio_desc *intn; struct gpio_desc *reset; struct gpio_desc *connect; @@ -109,18 +110,25 @@ static int usb3503_connect(struct usb3503 *hub) static int usb3503_switch_mode(struct usb3503 *hub, enum usb3503_mode mode) { struct device *dev = hub->dev; - int rst, conn; + int rst, bypass, conn; switch (mode) { case USB3503_MODE_HUB: conn = 1; rst = 0; + bypass = 0; break; case USB3503_MODE_STANDBY: conn = 0; rst = 1; + bypass = 1; dev_info(dev, "switched to STANDBY mode\n"); break; + case USB3503_MODE_BYPASS: + conn = 0; + rst = 0; + bypass = 1; + break; default: dev_err(dev, "unknown mode is requested\n"); return -EINVAL; @@ -132,6 +140,9 @@ static int usb3503_switch_mode(struct usb3503 *hub, enum usb3503_mode mode) if (hub->reset) gpiod_set_value_cansleep(hub->reset, rst); + if (hub->bypass) + gpiod_set_value_cansleep(hub->bypass, bypass); + if (conn) { /* Wait T_HUBINIT == 4ms for hub logic to stabilize */ usleep_range(4000, 10000); @@ -247,6 +258,14 @@ static int usb3503_probe(struct usb3503 *hub) if (hub->connect) gpiod_set_consumer_name(hub->connect, "usb3503 connect"); + hub->bypass = devm_gpiod_get_optional(dev, "bypass", GPIOD_OUT_HIGH); + if (IS_ERR(hub->bypass)) { + err = PTR_ERR(hub->bypass); + goto err_clk; + } + if (hub->bypass) + gpiod_set_consumer_name(hub->bypass, "usb3503 bypass"); + hub->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(hub->reset)) { err = PTR_ERR(hub->reset); @@ -382,6 +401,7 @@ MODULE_DEVICE_TABLE(i2c, usb3503_id); static const struct of_device_id usb3503_of_match[] = { { .compatible = "smsc,usb3503", }, { .compatible = "smsc,usb3503a", }, + { .compatible = "smsc,usb3803", }, {}, }; MODULE_DEVICE_TABLE(of, usb3503_of_match); diff --git a/include/linux/platform_data/usb3503.h b/include/linux/platform_data/usb3503.h index d01ef97ddf36..f3c942f396f8 100644 --- a/include/linux/platform_data/usb3503.h +++ b/include/linux/platform_data/usb3503.h @@ -12,6 +12,7 @@ enum usb3503_mode { USB3503_MODE_UNKNOWN, USB3503_MODE_HUB, USB3503_MODE_STANDBY, + USB3503_MODE_BYPASS, }; struct usb3503_platform_data { -- cgit v1.2.3 From d8708b80fa0e6e21bc0c9e7276ad0bccef73b6e7 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Wed, 8 Feb 2023 15:01:05 +0100 Subject: KVM: Change return type of kvm_arch_vm_ioctl() to "int" All kvm_arch_vm_ioctl() implementations now only deal with "int" types as return values, so we can change the return type of these functions to use "int" instead of "long". Signed-off-by: Thomas Huth Acked-by: Anup Patel Message-Id: <20230208140105.655814-7-thuth@redhat.com> Signed-off-by: Paolo Bonzini --- arch/arm64/kvm/arm.c | 3 +-- arch/mips/kvm/mips.c | 4 ++-- arch/powerpc/kvm/powerpc.c | 5 ++--- arch/riscv/kvm/vm.c | 3 +-- arch/s390/kvm/kvm-s390.c | 3 +-- arch/x86/kvm/x86.c | 3 +-- include/linux/kvm_host.h | 3 +-- 7 files changed, 9 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 3bd732eaf087..a43e1cb3b7e9 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1439,8 +1439,7 @@ static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm, } } -long kvm_arch_vm_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) +int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { struct kvm *kvm = filp->private_data; void __user *argp = (void __user *)arg; diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 36c8991b5d39..884be4ef99dc 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -993,9 +993,9 @@ void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, kvm_flush_remote_tlbs(kvm); } -long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { - long r; + int r; switch (ioctl) { default: diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 4c5405fc5538..c0bac9cf2d87 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -2371,12 +2371,11 @@ static int kvmppc_get_cpu_char(struct kvm_ppc_cpu_char *cp) } #endif -long kvm_arch_vm_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) +int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { struct kvm *kvm __maybe_unused = filp->private_data; void __user *argp = (void __user *)arg; - long r; + int r; switch (ioctl) { case KVM_PPC_GET_PVINFO: { diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c index 65a964d7e70d..c13130ab459a 100644 --- a/arch/riscv/kvm/vm.c +++ b/arch/riscv/kvm/vm.c @@ -87,8 +87,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) return r; } -long kvm_arch_vm_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) +int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { return -EINVAL; } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 7039abda6d49..4c3edccb9911 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -2898,8 +2898,7 @@ static int kvm_s390_vm_mem_op(struct kvm *kvm, struct kvm_s390_mem_op *mop) } } -long kvm_arch_vm_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) +int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { struct kvm *kvm = filp->private_data; void __user *argp = (void __user *)arg; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 4282daeaee84..237c483b1230 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -6667,8 +6667,7 @@ static int kvm_vm_ioctl_set_clock(struct kvm *kvm, void __user *argp) return 0; } -long kvm_arch_vm_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) +int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { struct kvm *kvm = filp->private_data; void __user *argp = (void __user *)arg; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 8ada23756b0e..90edc16d37e5 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1397,8 +1397,7 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, bool line_status); int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap); -long kvm_arch_vm_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg); +int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); long kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); -- cgit v1.2.3 From 8af70e202ac4ea5dd2d5561cc2675bc09b30412e Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Sat, 18 Feb 2023 17:21:21 +0000 Subject: leds: Fix reference to led_set_brightness() in doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The referenced function led_classdev_brightness_set() never existed. Fixes: 5ada28bf7675 ("led-class: always implement blinking") Signed-off-by: Thomas Weißschuh Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20230218-typo-led-set-v1-1-3c35362a2f2d@weissschuh.net --- include/linux/leds.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/leds.h b/include/linux/leds.h index d71201a968b6..aedf34165354 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -256,7 +256,7 @@ struct led_classdev *__must_check devm_of_led_get(struct device *dev, * * Note that if software blinking is active, simply calling * led_cdev->brightness_set() will not stop the blinking, - * use led_classdev_brightness_set() instead. + * use led_set_brightness() instead. */ void led_blink_set(struct led_classdev *led_cdev, unsigned long *delay_on, unsigned long *delay_off); -- cgit v1.2.3 From 428e106ae1ad4e45d3fd6978a753db475d0d0ec9 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Sun, 12 Mar 2023 14:26:00 +0300 Subject: mm: Introduce untagged_addr_remote() untagged_addr() removes tags/metadata from the address and brings it to the canonical form. The helper is implemented on arm64 and sparc. Both of them do untagging based on global rules. However, Linear Address Masking (LAM) on x86 introduces per-process settings for untagging. As a result, untagged_addr() is now only suitable for untagging addresses for the current proccess. The new helper untagged_addr_remote() has to be used when the address targets remote process. It requires the mmap lock for target mm to be taken. Signed-off-by: Kirill A. Shutemov Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Alexander Potapenko Link: https://lore.kernel.org/all/20230312112612.31869-6-kirill.shutemov%40linux.intel.com --- arch/sparc/include/asm/uaccess_64.h | 2 ++ drivers/vfio/vfio_iommu_type1.c | 2 +- fs/proc/task_mmu.c | 9 +++++++-- include/linux/mm.h | 11 ----------- include/linux/uaccess.h | 22 ++++++++++++++++++++++ mm/gup.c | 4 ++-- mm/madvise.c | 5 +++-- mm/migrate.c | 11 ++++++----- 8 files changed, 43 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index 94266a5c5b04..b825a5dd0210 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -8,8 +8,10 @@ #include #include +#include #include #include +#include #include #include diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 493c31de0edb..3d4dd9420c30 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -580,7 +580,7 @@ static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr, goto done; } - vaddr = untagged_addr(vaddr); + vaddr = untagged_addr_remote(mm, vaddr); retry: vma = vma_lookup(mm, vaddr); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 6a96e1713fd5..29fd6b1f4058 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1689,8 +1689,13 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* watch out for wraparound */ start_vaddr = end_vaddr; - if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) - start_vaddr = untagged_addr(svpfn << PAGE_SHIFT); + if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { + ret = mmap_read_lock_killable(mm); + if (ret) + goto out_free; + start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); + mmap_read_unlock(mm); + } /* Ensure the address is inside the task */ if (start_vaddr > mm->task_size) diff --git a/include/linux/mm.h b/include/linux/mm.h index 1f79667824eb..289ae4caf878 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -96,17 +96,6 @@ extern int mmap_rnd_compat_bits __read_mostly; #include #include -/* - * Architectures that support memory tagging (assigning tags to memory regions, - * embedding these tags into addresses that point to these memory regions, and - * checking that the memory and the pointer tags match on memory accesses) - * redefine this macro to strip tags from pointers. - * It's defined as noop for architectures that don't support memory tagging. - */ -#ifndef untagged_addr -#define untagged_addr(addr) (addr) -#endif - #ifndef __pa_symbol #define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0)) #endif diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index ab9728138ad6..3064314f4832 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -10,6 +10,28 @@ #include +/* + * Architectures that support memory tagging (assigning tags to memory regions, + * embedding these tags into addresses that point to these memory regions, and + * checking that the memory and the pointer tags match on memory accesses) + * redefine this macro to strip tags from pointers. + * + * Passing down mm_struct allows to define untagging rules on per-process + * basis. + * + * It's defined as noop for architectures that don't support memory tagging. + */ +#ifndef untagged_addr +#define untagged_addr(addr) (addr) +#endif + +#ifndef untagged_addr_remote +#define untagged_addr_remote(mm, addr) ({ \ + mmap_assert_locked(mm); \ + untagged_addr(addr); \ +}) +#endif + /* * Architectures should provide two primitives (raw_copy_{to,from}_user()) * and get rid of their private instances of copy_{to,from}_user() and diff --git a/mm/gup.c b/mm/gup.c index eab18ba045db..5ee8b682a0fe 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1085,7 +1085,7 @@ static long __get_user_pages(struct mm_struct *mm, if (!nr_pages) return 0; - start = untagged_addr(start); + start = untagged_addr_remote(mm, start); VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); @@ -1259,7 +1259,7 @@ int fixup_user_fault(struct mm_struct *mm, struct vm_area_struct *vma; vm_fault_t ret; - address = untagged_addr(address); + address = untagged_addr_remote(mm, address); if (unlocked) fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; diff --git a/mm/madvise.c b/mm/madvise.c index 340125d08c03..d4b67f36f70f 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -1402,8 +1402,6 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh size_t len; struct blk_plug plug; - start = untagged_addr(start); - if (!madvise_behavior_valid(behavior)) return -EINVAL; @@ -1435,6 +1433,9 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh mmap_read_lock(mm); } + start = untagged_addr_remote(mm, start); + end = start + len; + blk_start_plug(&plug); error = madvise_walk_vmas(mm, start, end, behavior, madvise_vma_behavior); diff --git a/mm/migrate.c b/mm/migrate.c index 98f1c11197a8..8cd11bc9208f 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2097,15 +2097,18 @@ static int do_move_pages_to_node(struct mm_struct *mm, * target node * 1 - when it has been queued */ -static int add_page_for_migration(struct mm_struct *mm, unsigned long addr, +static int add_page_for_migration(struct mm_struct *mm, const void __user *p, int node, struct list_head *pagelist, bool migrate_all) { struct vm_area_struct *vma; + unsigned long addr; struct page *page; int err; bool isolated; mmap_read_lock(mm); + addr = (unsigned long)untagged_addr_remote(mm, p); + err = -EFAULT; vma = vma_lookup(mm, addr); if (!vma || !vma_migratable(vma)) @@ -2211,7 +2214,6 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, for (i = start = 0; i < nr_pages; i++) { const void __user *p; - unsigned long addr; int node; err = -EFAULT; @@ -2219,7 +2221,6 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, goto out_flush; if (get_user(node, nodes + i)) goto out_flush; - addr = (unsigned long)untagged_addr(p); err = -ENODEV; if (node < 0 || node >= MAX_NUMNODES) @@ -2247,8 +2248,8 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, * Errors in the page lookup or isolation are not fatal and we simply * report them via status */ - err = add_page_for_migration(mm, addr, current_node, - &pagelist, flags & MPOL_MF_MOVE_ALL); + err = add_page_for_migration(mm, p, current_node, &pagelist, + flags & MPOL_MF_MOVE_ALL); if (err > 0) { /* The page is successfully queued for migration */ -- cgit v1.2.3 From f7d304343b9d2456ffba23b99d2345408251ea45 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Sun, 12 Mar 2023 14:26:04 +0300 Subject: mm: Expose untagging mask in /proc/$PID/status Add a line in /proc/$PID/status to report untag_mask. It can be used to find out LAM status of the process from the outside. It is useful for debuggers. Signed-off-by: Kirill A. Shutemov Signed-off-by: Dave Hansen Acked-by: Catalin Marinas Acked-by: Peter Zijlstra (Intel) Tested-by: Alexander Potapenko Link: https://lore.kernel.org/all/20230312112612.31869-10-kirill.shutemov%40linux.intel.com --- arch/arm64/include/asm/mmu_context.h | 6 ++++++ arch/sparc/include/asm/mmu_context_64.h | 6 ++++++ arch/x86/include/asm/mmu_context.h | 6 ++++++ fs/proc/array.c | 7 +++++++ include/linux/mmu_context.h | 7 +++++++ 5 files changed, 32 insertions(+) (limited to 'include/linux') diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 72dbd6400549..56911691bef0 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -288,6 +288,12 @@ void post_ttbr_update_workaround(void); unsigned long arm64_mm_context_get(struct mm_struct *mm); void arm64_mm_context_put(struct mm_struct *mm); +#define mm_untag_mask mm_untag_mask +static inline unsigned long mm_untag_mask(struct mm_struct *mm) +{ + return -1UL >> 8; +} + #include #endif /* !__ASSEMBLY__ */ diff --git a/arch/sparc/include/asm/mmu_context_64.h b/arch/sparc/include/asm/mmu_context_64.h index 7a8380c63aab..799e797c5cdd 100644 --- a/arch/sparc/include/asm/mmu_context_64.h +++ b/arch/sparc/include/asm/mmu_context_64.h @@ -185,6 +185,12 @@ static inline void finish_arch_post_lock_switch(void) } } +#define mm_untag_mask mm_untag_mask +static inline unsigned long mm_untag_mask(struct mm_struct *mm) +{ + return -1UL >> adi_nbits(); +} + #include #endif /* !(__ASSEMBLY__) */ diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index eb1387ac40fa..06eaaf75d572 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -104,6 +104,12 @@ static inline void dup_lam(struct mm_struct *oldmm, struct mm_struct *mm) mm->context.untag_mask = oldmm->context.untag_mask; } +#define mm_untag_mask mm_untag_mask +static inline unsigned long mm_untag_mask(struct mm_struct *mm) +{ + return mm->context.untag_mask; +} + static inline void mm_reset_untag_mask(struct mm_struct *mm) { mm->context.untag_mask = -1UL; diff --git a/fs/proc/array.c b/fs/proc/array.c index 9b0315d34c58..6daea628bc76 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -91,6 +91,7 @@ #include #include #include +#include #include #include "internal.h" @@ -423,6 +424,11 @@ static inline void task_thp_status(struct seq_file *m, struct mm_struct *mm) seq_printf(m, "THP_enabled:\t%d\n", thp_enabled); } +static inline void task_untag_mask(struct seq_file *m, struct mm_struct *mm) +{ + seq_printf(m, "untag_mask:\t%#lx\n", mm_untag_mask(mm)); +} + int proc_pid_status(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *task) { @@ -438,6 +444,7 @@ int proc_pid_status(struct seq_file *m, struct pid_namespace *ns, task_mem(m, mm); task_core_dumping(m, task); task_thp_status(m, mm); + task_untag_mask(m, mm); mmput(mm); } task_sig(m, task); diff --git a/include/linux/mmu_context.h b/include/linux/mmu_context.h index b9b970f7ab45..14b9c1fa05c4 100644 --- a/include/linux/mmu_context.h +++ b/include/linux/mmu_context.h @@ -28,4 +28,11 @@ static inline void leave_mm(int cpu) { } # define task_cpu_possible(cpu, p) cpumask_test_cpu((cpu), task_cpu_possible_mask(p)) #endif +#ifndef mm_untag_mask +static inline unsigned long mm_untag_mask(struct mm_struct *mm) +{ + return -1UL; +} +#endif + #endif -- cgit v1.2.3 From 400b9b93441cd4e2fe824a70140f3d5a2a9c802b Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Sun, 12 Mar 2023 14:26:05 +0300 Subject: iommu/sva: Replace pasid_valid() helper with mm_valid_pasid() Kernel has few users of pasid_valid() and all but one checks if the process has PASID allocated. The helper takes ioasid_t as the input. Replace the helper with mm_valid_pasid() that takes mm_struct as the argument. The only call that checks PASID that is not tied to mm_struct is open-codded now. This is preparatory patch. It helps avoid ifdeffery: no need to dereference mm->pasid in generic code to check if the process has PASID. Signed-off-by: Kirill A. Shutemov Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/all/20230312112612.31869-11-kirill.shutemov%40linux.intel.com --- arch/x86/kernel/traps.c | 6 +++--- drivers/iommu/iommu-sva.c | 4 ++-- include/linux/ioasid.h | 9 --------- include/linux/sched/mm.h | 8 +++++++- 4 files changed, 12 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index d317dc3d06a3..8b83d8fbce71 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -671,15 +671,15 @@ static bool try_fixup_enqcmd_gp(void) if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) return false; - pasid = current->mm->pasid; - /* * If the mm has not been allocated a * PASID, the #GP can not be fixed up. */ - if (!pasid_valid(pasid)) + if (!mm_valid_pasid(current->mm)) return false; + pasid = current->mm->pasid; + /* * Did this thread already have its PASID activated? * If so, the #GP must be from something else. diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 24bf9b2b58aa..4ee2929f0d7a 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -34,14 +34,14 @@ int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max) mutex_lock(&iommu_sva_lock); /* Is a PASID already associated with this mm? */ - if (pasid_valid(mm->pasid)) { + if (mm_valid_pasid(mm)) { if (mm->pasid < min || mm->pasid >= max) ret = -EOVERFLOW; goto out; } pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm); - if (!pasid_valid(pasid)) + if (pasid == INVALID_IOASID) ret = -ENOMEM; else mm_pasid_set(mm, pasid); diff --git a/include/linux/ioasid.h b/include/linux/ioasid.h index af1c9d62e642..836ae09e92c2 100644 --- a/include/linux/ioasid.h +++ b/include/linux/ioasid.h @@ -40,10 +40,6 @@ void *ioasid_find(struct ioasid_set *set, ioasid_t ioasid, int ioasid_register_allocator(struct ioasid_allocator_ops *allocator); void ioasid_unregister_allocator(struct ioasid_allocator_ops *allocator); int ioasid_set_data(ioasid_t ioasid, void *data); -static inline bool pasid_valid(ioasid_t ioasid) -{ - return ioasid != INVALID_IOASID; -} #else /* !CONFIG_IOASID */ static inline ioasid_t ioasid_alloc(struct ioasid_set *set, ioasid_t min, @@ -74,10 +70,5 @@ static inline int ioasid_set_data(ioasid_t ioasid, void *data) return -ENOTSUPP; } -static inline bool pasid_valid(ioasid_t ioasid) -{ - return false; -} - #endif /* CONFIG_IOASID */ #endif /* __LINUX_IOASID_H */ diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 2a243616f222..b69fe7e8c0ac 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -457,6 +457,11 @@ static inline void mm_pasid_init(struct mm_struct *mm) mm->pasid = INVALID_IOASID; } +static inline bool mm_valid_pasid(struct mm_struct *mm) +{ + return mm->pasid != INVALID_IOASID; +} + /* Associate a PASID with an mm_struct: */ static inline void mm_pasid_set(struct mm_struct *mm, u32 pasid) { @@ -465,13 +470,14 @@ static inline void mm_pasid_set(struct mm_struct *mm, u32 pasid) static inline void mm_pasid_drop(struct mm_struct *mm) { - if (pasid_valid(mm->pasid)) { + if (mm_valid_pasid(mm)) { ioasid_free(mm->pasid); mm->pasid = INVALID_IOASID; } } #else static inline void mm_pasid_init(struct mm_struct *mm) {} +static inline bool mm_valid_pasid(struct mm_struct *mm) { return false; } static inline void mm_pasid_set(struct mm_struct *mm, u32 pasid) {} static inline void mm_pasid_drop(struct mm_struct *mm) {} #endif -- cgit v1.2.3 From 23e5d9ec2bab53c4e5fbac675304e699726c1ac5 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Sun, 12 Mar 2023 14:26:06 +0300 Subject: x86/mm/iommu/sva: Make LAM and SVA mutually exclusive IOMMU and SVA-capable devices know nothing about LAM and only expect canonical addresses. An attempt to pass down tagged pointer will lead to address translation failure. By default do not allow to enable both LAM and use SVA in the same process. The new ARCH_FORCE_TAGGED_SVA arch_prctl() overrides the limitation. By using the arch_prctl() userspace takes responsibility to never pass tagged address to the device. Signed-off-by: Kirill A. Shutemov Signed-off-by: Dave Hansen Reviewed-by: Ashok Raj Reviewed-by: Jacob Pan Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/all/20230312112612.31869-12-kirill.shutemov%40linux.intel.com --- arch/x86/include/asm/mmu.h | 2 ++ arch/x86/include/asm/mmu_context.h | 6 ++++++ arch/x86/include/uapi/asm/prctl.h | 1 + arch/x86/kernel/process_64.c | 7 +++++++ drivers/iommu/iommu-sva.c | 4 ++++ include/linux/mmu_context.h | 7 +++++++ 6 files changed, 27 insertions(+) (limited to 'include/linux') diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h index e80762e998ce..0da5c227f490 100644 --- a/arch/x86/include/asm/mmu.h +++ b/arch/x86/include/asm/mmu.h @@ -14,6 +14,8 @@ #define MM_CONTEXT_HAS_VSYSCALL 1 /* Do not allow changing LAM mode */ #define MM_CONTEXT_LOCK_LAM 2 +/* Allow LAM and SVA coexisting */ +#define MM_CONTEXT_FORCE_TAGGED_SVA 3 /* * x86 has arch-specific MMU state beyond what lives in mm_struct. diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index 06eaaf75d572..4c396e9a384f 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -115,6 +115,12 @@ static inline void mm_reset_untag_mask(struct mm_struct *mm) mm->context.untag_mask = -1UL; } +#define arch_pgtable_dma_compat arch_pgtable_dma_compat +static inline bool arch_pgtable_dma_compat(struct mm_struct *mm) +{ + return !mm_lam_cr3_mask(mm) || + test_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &mm->context.flags); +} #else static inline unsigned long mm_lam_cr3_mask(struct mm_struct *mm) diff --git a/arch/x86/include/uapi/asm/prctl.h b/arch/x86/include/uapi/asm/prctl.h index a31e27b95b19..eb290d89cb32 100644 --- a/arch/x86/include/uapi/asm/prctl.h +++ b/arch/x86/include/uapi/asm/prctl.h @@ -23,5 +23,6 @@ #define ARCH_GET_UNTAG_MASK 0x4001 #define ARCH_ENABLE_TAGGED_ADDR 0x4002 #define ARCH_GET_MAX_TAG_BITS 0x4003 +#define ARCH_FORCE_TAGGED_SVA 0x4004 #endif /* _ASM_X86_PRCTL_H */ diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 186f34add865..b46924c9e46d 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -756,6 +756,10 @@ static int prctl_enable_tagged_addr(struct mm_struct *mm, unsigned long nr_bits) if (current->mm != mm) return -EINVAL; + if (mm_valid_pasid(mm) && + !test_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &mm->context.flags)) + return -EINTR; + if (mmap_write_lock_killable(mm)) return -EINTR; @@ -878,6 +882,9 @@ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2) (unsigned long __user *)arg2); case ARCH_ENABLE_TAGGED_ADDR: return prctl_enable_tagged_addr(task->mm, arg2); + case ARCH_FORCE_TAGGED_SVA: + set_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &task->mm->context.flags); + return 0; case ARCH_GET_MAX_TAG_BITS: if (!cpu_feature_enabled(X86_FEATURE_LAM)) return put_user(0, (unsigned long __user *)arg2); diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 4ee2929f0d7a..dd76a1a09cf7 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -2,6 +2,7 @@ /* * Helpers for IOMMU drivers implementing SVA */ +#include #include #include #include @@ -32,6 +33,9 @@ int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max) min == 0 || max < min) return -EINVAL; + if (!arch_pgtable_dma_compat(mm)) + return -EBUSY; + mutex_lock(&iommu_sva_lock); /* Is a PASID already associated with this mm? */ if (mm_valid_pasid(mm)) { diff --git a/include/linux/mmu_context.h b/include/linux/mmu_context.h index 14b9c1fa05c4..f2b7a3f04099 100644 --- a/include/linux/mmu_context.h +++ b/include/linux/mmu_context.h @@ -35,4 +35,11 @@ static inline unsigned long mm_untag_mask(struct mm_struct *mm) } #endif +#ifndef arch_pgtable_dma_compat +static inline bool arch_pgtable_dma_compat(struct mm_struct *mm) +{ + return true; +} +#endif + #endif -- cgit v1.2.3 From 6229ad9913ac794233fdb3ed20f1e899add0af6b Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Thu, 9 Mar 2023 10:09:18 +0200 Subject: serial: Remove extern from func prototypes in headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove unnecessary externs from function prototypes in serial_8250.h and serial_core.h. Suggested-by: Jiri Slaby Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20230309080923.11778-4-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/serial_8250.h | 41 ++++++++++++++++++----------------------- include/linux/serial_core.h | 13 ++++++------- 2 files changed, 24 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h index 741ed4807a9c..6f78f302d272 100644 --- a/include/linux/serial_8250.h +++ b/include/linux/serial_8250.h @@ -151,26 +151,22 @@ void serial8250_unregister_port(int line); void serial8250_suspend_port(int line); void serial8250_resume_port(int line); -extern int early_serial_setup(struct uart_port *port); - -extern int early_serial8250_setup(struct earlycon_device *device, - const char *options); -extern void serial8250_update_uartclk(struct uart_port *port, - unsigned int uartclk); -extern void serial8250_do_set_termios(struct uart_port *port, - struct ktermios *termios, const struct ktermios *old); -extern void serial8250_do_set_ldisc(struct uart_port *port, - struct ktermios *termios); -extern unsigned int serial8250_do_get_mctrl(struct uart_port *port); -extern int serial8250_do_startup(struct uart_port *port); -extern void serial8250_do_shutdown(struct uart_port *port); -extern void serial8250_do_pm(struct uart_port *port, unsigned int state, - unsigned int oldstate); -extern void serial8250_do_set_mctrl(struct uart_port *port, unsigned int mctrl); -extern void serial8250_do_set_divisor(struct uart_port *port, unsigned int baud, - unsigned int quot, - unsigned int quot_frac); -extern int fsl8250_handle_irq(struct uart_port *port); +int early_serial_setup(struct uart_port *port); +int early_serial8250_setup(struct earlycon_device *device, const char *options); + +void serial8250_update_uartclk(struct uart_port *port, unsigned int uartclk); +void serial8250_do_set_termios(struct uart_port *port, struct ktermios *termios, + const struct ktermios *old); +void serial8250_do_set_ldisc(struct uart_port *port, struct ktermios *termios); +unsigned int serial8250_do_get_mctrl(struct uart_port *port); +int serial8250_do_startup(struct uart_port *port); +void serial8250_do_shutdown(struct uart_port *port); +void serial8250_do_pm(struct uart_port *port, unsigned int state, + unsigned int oldstate); +void serial8250_do_set_mctrl(struct uart_port *port, unsigned int mctrl); +void serial8250_do_set_divisor(struct uart_port *port, unsigned int baud, + unsigned int quot, unsigned int quot_frac); +int fsl8250_handle_irq(struct uart_port *port); int serial8250_handle_irq(struct uart_port *port, unsigned int iir); u16 serial8250_rx_chars(struct uart_8250_port *up, u16 lsr); void serial8250_read_char(struct uart_8250_port *up, u16 lsr); @@ -183,9 +179,8 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, int serial8250_console_setup(struct uart_port *port, char *options, bool probe); int serial8250_console_exit(struct uart_port *port); -extern void serial8250_set_isa_configurator(void (*v) - (int port, struct uart_port *up, - u32 *capabilities)); +void serial8250_set_isa_configurator(void (*v)(int port, struct uart_port *up, + u32 *capabilities)); #ifdef CONFIG_SERIAL_8250_RT288X unsigned int au_serial_in(struct uart_port *p, int offset); diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 9e3e5e0d11b2..05d18a145b3a 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -812,9 +812,8 @@ extern const struct earlycon_id __earlycon_table_end[]; #define EARLYCON_DECLARE(_name, fn) OF_EARLYCON_DECLARE(_name, "", fn) -extern int of_setup_earlycon(const struct earlycon_id *match, - unsigned long node, - const char *options); +int of_setup_earlycon(const struct earlycon_id *match, unsigned long node, + const char *options); #ifdef CONFIG_SERIAL_EARLYCON extern bool earlycon_acpi_spcr_enable __initdata; @@ -897,11 +896,11 @@ static inline bool uart_softcts_mode(struct uart_port *uport) * The following are helper functions for the low level drivers. */ -extern void uart_handle_dcd_change(struct uart_port *uport, bool active); -extern void uart_handle_cts_change(struct uart_port *uport, bool active); +void uart_handle_dcd_change(struct uart_port *uport, bool active); +void uart_handle_cts_change(struct uart_port *uport, bool active); -extern void uart_insert_char(struct uart_port *port, unsigned int status, - unsigned int overrun, unsigned int ch, unsigned int flag); +void uart_insert_char(struct uart_port *port, unsigned int status, + unsigned int overrun, unsigned int ch, unsigned int flag); void uart_xchar_out(struct uart_port *uport, int offset); -- cgit v1.2.3 From b5def43a7b3e7e68d6b49a23166f8b886a26bb54 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Thu, 9 Mar 2023 10:09:21 +0200 Subject: serial: Make hw_stopped bool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert hw_stopped in uart_port to bool because its more appropriate type for how it is used. Also convert the local variable in uart_change_line_settings() caching old hw_stopped to bool. Signed-off-by: Ilpo Järvinen Link: https://lore.kernel.org/r/20230309080923.11778-7-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/serial_core.c | 6 +++--- include/linux/serial_core.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index ecdc5d9cdb53..31b69e61e71d 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -182,7 +182,7 @@ static void uart_change_line_settings(struct tty_struct *tty, struct uart_state { struct uart_port *uport = uart_port_check(state); struct ktermios *termios; - int hw_stopped; + bool hw_stopped; /* * If we have no tty, termios, or the port does not exist, @@ -3304,13 +3304,13 @@ void uart_handle_cts_change(struct uart_port *uport, bool active) if (uart_softcts_mode(uport)) { if (uport->hw_stopped) { if (active) { - uport->hw_stopped = 0; + uport->hw_stopped = false; uport->ops->start_tx(uport); uart_write_wakeup(uport); } } else { if (!active) { - uport->hw_stopped = 1; + uport->hw_stopped = true; uport->ops->stop_tx(uport); } } diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 05d18a145b3a..66ecec15a1bf 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -553,7 +553,7 @@ struct uart_port { #define UPSTAT_AUTOXOFF ((__force upstat_t) (1 << 4)) #define UPSTAT_SYNC_FIFO ((__force upstat_t) (1 << 5)) - int hw_stopped; /* sw-assisted CTS flow state */ + bool hw_stopped; /* sw-assisted CTS flow state */ unsigned int mctrl; /* current modem ctrl settings */ unsigned int frame_time; /* frame timing in ns */ unsigned int type; /* port type */ -- cgit v1.2.3 From 035173c91c6b53144295a5b546c6bad3f40fb8a9 Mon Sep 17 00:00:00 2001 From: Ilpo Järvinen Date: Thu, 9 Mar 2023 10:20:35 +0200 Subject: tty: Convert hw_stopped in tty_struct to bool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hw_stopped in tty_struct is used like bool, convert the variable type to bool. Signed-off-by: Ilpo Järvinen Acked-by: Ulf Hansson # For MMC Link: https://lore.kernel.org/r/20230309082035.14880-9-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/char/pcmcia/synclink_cs.c | 6 +++--- drivers/mmc/core/sdio_uart.c | 10 +++++----- drivers/tty/amiserial.c | 6 +++--- drivers/tty/mxser.c | 6 +++--- drivers/tty/synclink_gt.c | 6 +++--- include/linux/tty.h | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/drivers/char/pcmcia/synclink_cs.c b/drivers/char/pcmcia/synclink_cs.c index 6ddfeb2fe98f..97c5bfb9d58a 100644 --- a/drivers/char/pcmcia/synclink_cs.c +++ b/drivers/char/pcmcia/synclink_cs.c @@ -1060,7 +1060,7 @@ static void cts_change(MGSLPC_INFO *info, struct tty_struct *tty) if (info->serial_signals & SerialSignal_CTS) { if (debug_level >= DEBUG_LEVEL_ISR) printk("CTS tx start..."); - tty->hw_stopped = 0; + tty->hw_stopped = false; tx_start(info, tty); info->pending_bh |= BH_TRANSMIT; return; @@ -1069,7 +1069,7 @@ static void cts_change(MGSLPC_INFO *info, struct tty_struct *tty) if (!(info->serial_signals & SerialSignal_CTS)) { if (debug_level >= DEBUG_LEVEL_ISR) printk("CTS tx stop..."); - tty->hw_stopped = 1; + tty->hw_stopped = true; tx_stop(info); } } @@ -2312,7 +2312,7 @@ static void mgslpc_set_termios(struct tty_struct *tty, /* Handle turning off CRTSCTS */ if (old_termios->c_cflag & CRTSCTS && !C_CRTSCTS(tty)) { - tty->hw_stopped = 0; + tty->hw_stopped = false; tx_release(tty); } } diff --git a/drivers/mmc/core/sdio_uart.c b/drivers/mmc/core/sdio_uart.c index 50536fe59f1a..aa659758563f 100644 --- a/drivers/mmc/core/sdio_uart.c +++ b/drivers/mmc/core/sdio_uart.c @@ -478,13 +478,13 @@ static void sdio_uart_check_modem_status(struct sdio_uart_port *port) int cts = (status & UART_MSR_CTS); if (tty->hw_stopped) { if (cts) { - tty->hw_stopped = 0; + tty->hw_stopped = false; sdio_uart_start_tx(port); tty_wakeup(tty); } } else { if (!cts) { - tty->hw_stopped = 1; + tty->hw_stopped = true; sdio_uart_stop_tx(port); } } @@ -633,7 +633,7 @@ static int sdio_uart_activate(struct tty_port *tport, struct tty_struct *tty) if (C_CRTSCTS(tty)) if (!(sdio_uart_get_mctrl(port) & TIOCM_CTS)) - tty->hw_stopped = 1; + tty->hw_stopped = true; clear_bit(TTY_IO_ERROR, &tty->flags); @@ -882,14 +882,14 @@ static void sdio_uart_set_termios(struct tty_struct *tty, /* Handle turning off CRTSCTS */ if ((old_termios->c_cflag & CRTSCTS) && !(cflag & CRTSCTS)) { - tty->hw_stopped = 0; + tty->hw_stopped = false; sdio_uart_start_tx(port); } /* Handle turning on CRTSCTS */ if (!(old_termios->c_cflag & CRTSCTS) && (cflag & CRTSCTS)) { if (!(sdio_uart_get_mctrl(port) & TIOCM_CTS)) { - tty->hw_stopped = 1; + tty->hw_stopped = true; sdio_uart_stop_tx(port); } } diff --git a/drivers/tty/amiserial.c b/drivers/tty/amiserial.c index d7515d61659e..c06ad0a0744b 100644 --- a/drivers/tty/amiserial.c +++ b/drivers/tty/amiserial.c @@ -347,7 +347,7 @@ static void check_modem_status(struct serial_state *info) #if (defined(SERIAL_DEBUG_INTR) || defined(SERIAL_DEBUG_FLOW)) printk("CTS tx start..."); #endif - port->tty->hw_stopped = 0; + port->tty->hw_stopped = false; info->IER |= UART_IER_THRI; amiga_custom.intena = IF_SETCLR | IF_TBE; mb(); @@ -362,7 +362,7 @@ static void check_modem_status(struct serial_state *info) #if (defined(SERIAL_DEBUG_INTR) || defined(SERIAL_DEBUG_FLOW)) printk("CTS tx stop..."); #endif - port->tty->hw_stopped = 1; + port->tty->hw_stopped = true; info->IER &= ~UART_IER_THRI; /* disable Tx interrupt and remove any pending interrupts */ amiga_custom.intena = IF_TBE; @@ -1197,7 +1197,7 @@ static void rs_set_termios(struct tty_struct *tty, const struct ktermios *old_te /* Handle turning off CRTSCTS */ if ((old_termios->c_cflag & CRTSCTS) && !C_CRTSCTS(tty)) { - tty->hw_stopped = 0; + tty->hw_stopped = false; rs_start(tty); } diff --git a/drivers/tty/mxser.c b/drivers/tty/mxser.c index ef3116e87975..10855e66fda1 100644 --- a/drivers/tty/mxser.c +++ b/drivers/tty/mxser.c @@ -553,7 +553,7 @@ static void mxser_handle_cts(struct tty_struct *tty, struct mxser_port *info, if (tty->hw_stopped) { if (cts) { - tty->hw_stopped = 0; + tty->hw_stopped = false; if (!mxser_16550A_or_MUST(info)) __mxser_start_tx(info); @@ -563,7 +563,7 @@ static void mxser_handle_cts(struct tty_struct *tty, struct mxser_port *info, } else if (cts) return; - tty->hw_stopped = 1; + tty->hw_stopped = true; if (!mxser_16550A_or_MUST(info)) __mxser_stop_tx(info); } @@ -1361,7 +1361,7 @@ static void mxser_set_termios(struct tty_struct *tty, spin_unlock_irqrestore(&info->slock, flags); if ((old_termios->c_cflag & CRTSCTS) && !C_CRTSCTS(tty)) { - tty->hw_stopped = 0; + tty->hw_stopped = false; mxser_start(tty); } diff --git a/drivers/tty/synclink_gt.c b/drivers/tty/synclink_gt.c index 33f258d6fef9..543b3224dce9 100644 --- a/drivers/tty/synclink_gt.c +++ b/drivers/tty/synclink_gt.c @@ -730,7 +730,7 @@ static void set_termios(struct tty_struct *tty, /* Handle turning off CRTSCTS */ if ((old_termios->c_cflag & CRTSCTS) && !C_CRTSCTS(tty)) { - tty->hw_stopped = 0; + tty->hw_stopped = false; tx_release(tty); } } @@ -1953,13 +1953,13 @@ static void cts_change(struct slgt_info *info, unsigned short status) if (info->port.tty) { if (info->port.tty->hw_stopped) { if (info->signals & SerialSignal_CTS) { - info->port.tty->hw_stopped = 0; + info->port.tty->hw_stopped = false; info->pending_bh |= BH_TRANSMIT; return; } } else { if (!(info->signals & SerialSignal_CTS)) - info->port.tty->hw_stopped = 1; + info->port.tty->hw_stopped = true; } } } diff --git a/include/linux/tty.h b/include/linux/tty.h index 093935e97f42..60871a9d3212 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -227,7 +227,7 @@ struct tty_struct { unsigned long unused[0]; } __aligned(sizeof(unsigned long)) ctrl; - int hw_stopped; + bool hw_stopped; unsigned int receive_room; int flow_change; -- cgit v1.2.3 From 410e7088e971ad656170fe9768b072b267a95310 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 13 Mar 2023 13:31:00 +0200 Subject: devres: Pass unique name of the resource to devm_add_action_or_reset() All the same as it's done in the commit e32c80bbd2f9 ("devres: Pass unique name of the resource to devm_add_action()") applies to the devm_add_action_or_reset(), which this change makes real. This helps with debug resource management. Reported-and-tested-by: Mirsad Goran Todorovac Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20230313113100.59643-1-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/device.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/device.h b/include/linux/device.h index 0f128520f6e5..12dc08aa5c0f 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -250,17 +250,19 @@ int __devm_add_action(struct device *dev, void (*action)(void *), void *data, co #define devm_add_action(release, action, data) \ __devm_add_action(release, action, data, #action) -static inline int devm_add_action_or_reset(struct device *dev, - void (*action)(void *), void *data) +static inline int __devm_add_action_or_reset(struct device *dev, void (*action)(void *), + void *data, const char *name) { int ret; - ret = devm_add_action(dev, action, data); + ret = __devm_add_action(dev, action, data, name); if (ret) action(data); return ret; } +#define devm_add_action_or_reset(release, action, data) \ + __devm_add_action_or_reset(release, action, data, #action) /** * devm_alloc_percpu - Resource-managed alloc_percpu -- cgit v1.2.3 From 4a46ac9d64030d9f6f18baaf115a3558880c1ae2 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:18:32 +0100 Subject: driver core: class: specify the module owner in __class_register() There's no need to manually have to set the module owner of a class, the compiler should do it automatically for you, so add a module * to the __class_register() function and allow it to set the module owner automatically. This will let us move the module pointer out of struct class eventually, as it should not be embedded in there if we wish for it to be a read-only structure eventually. And, funny story, this module pointer isn't even being used for anything, so while we will keep it around for now, it's not like it even matters. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313181843.1207845-1-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 6 +++--- include/linux/device/class.h | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index 5983eead8391..90dc5788957a 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -154,7 +154,7 @@ static void class_remove_groups(struct class *cls, return sysfs_remove_groups(&cls->p->subsys.kobj, groups); } -int __class_register(struct class *cls, struct lock_class_key *key) +int __class_register(struct class *cls, struct module *owner, struct lock_class_key *key) { struct subsys_private *cp; int error; @@ -187,6 +187,7 @@ int __class_register(struct class *cls, struct lock_class_key *key) if (error) goto err_out; + cls->owner = owner; error = class_add_groups(class_get(cls), cls->class_groups); class_put(cls); if (error) { @@ -244,10 +245,9 @@ struct class *__class_create(struct module *owner, const char *name, } cls->name = name; - cls->owner = owner; cls->class_release = class_create_release; - retval = __class_register(cls, key); + retval = __class_register(cls, owner, key); if (retval) goto error; diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 42cc3fb44a84..d1ba4ee235dc 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -85,15 +85,16 @@ struct class_dev_iter { extern struct kobject *sysfs_dev_block_kobj; extern struct kobject *sysfs_dev_char_kobj; extern int __must_check __class_register(struct class *class, + struct module *owner, struct lock_class_key *key); extern void class_unregister(struct class *class); /* This is a #define to keep the compiler from merging different * instances of the __key variable */ -#define class_register(class) \ -({ \ - static struct lock_class_key __key; \ - __class_register(class, &__key); \ +#define class_register(class) \ +({ \ + static struct lock_class_key __key; \ + __class_register(class, THIS_MODULE, &__key); \ }) struct class_compat; -- cgit v1.2.3 From 6e30a66433afee90e902ced95d7136e8f7edcc7e Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:18:34 +0100 Subject: driver core: class: remove struct module owner out of struct class The module owner field for a struct class was never actually used, so remove it as it is not doing anything at all. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313181843.1207845-3-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 9 +++------ include/linux/device/class.h | 18 +++++++----------- 2 files changed, 10 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index 90dc5788957a..9439c6c7466f 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -154,7 +154,7 @@ static void class_remove_groups(struct class *cls, return sysfs_remove_groups(&cls->p->subsys.kobj, groups); } -int __class_register(struct class *cls, struct module *owner, struct lock_class_key *key) +int __class_register(struct class *cls, struct lock_class_key *key) { struct subsys_private *cp; int error; @@ -187,7 +187,6 @@ int __class_register(struct class *cls, struct module *owner, struct lock_class_ if (error) goto err_out; - cls->owner = owner; error = class_add_groups(class_get(cls), cls->class_groups); class_put(cls); if (error) { @@ -220,7 +219,6 @@ static void class_create_release(struct class *cls) /** * __class_create - create a struct class structure - * @owner: pointer to the module that is to "own" this struct class * @name: pointer to a string for the name of this class. * @key: the lock_class_key for this class; used by mutex lock debugging * @@ -232,8 +230,7 @@ static void class_create_release(struct class *cls) * Note, the pointer created here is to be destroyed when finished by * making a call to class_destroy(). */ -struct class *__class_create(struct module *owner, const char *name, - struct lock_class_key *key) +struct class *__class_create(const char *name, struct lock_class_key *key) { struct class *cls; int retval; @@ -247,7 +244,7 @@ struct class *__class_create(struct module *owner, const char *name, cls->name = name; cls->class_release = class_create_release; - retval = __class_register(cls, owner, key); + retval = __class_register(cls, key); if (retval) goto error; diff --git a/include/linux/device/class.h b/include/linux/device/class.h index d1ba4ee235dc..bf736f14f0c1 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -25,7 +25,6 @@ struct fwnode_handle; /** * struct class - device classes * @name: Name of the class. - * @owner: The module owner. * @class_groups: Default attributes of this class. * @dev_groups: Default attributes of the devices that belong to the class. * @dev_kobj: The kobject that represents this class and links it into the hierarchy. @@ -53,7 +52,6 @@ struct fwnode_handle; */ struct class { const char *name; - struct module *owner; const struct attribute_group **class_groups; const struct attribute_group **dev_groups; @@ -85,16 +83,15 @@ struct class_dev_iter { extern struct kobject *sysfs_dev_block_kobj; extern struct kobject *sysfs_dev_char_kobj; extern int __must_check __class_register(struct class *class, - struct module *owner, struct lock_class_key *key); extern void class_unregister(struct class *class); /* This is a #define to keep the compiler from merging different * instances of the __key variable */ -#define class_register(class) \ -({ \ - static struct lock_class_key __key; \ - __class_register(class, THIS_MODULE, &__key); \ +#define class_register(class) \ +({ \ + static struct lock_class_key __key; \ + __class_register(class, &__key); \ }) struct class_compat; @@ -250,8 +247,7 @@ struct class_interface { extern int __must_check class_interface_register(struct class_interface *); extern void class_interface_unregister(struct class_interface *); -extern struct class * __must_check __class_create(struct module *owner, - const char *name, +extern struct class * __must_check __class_create(const char *name, struct lock_class_key *key); extern void class_destroy(struct class *cls); @@ -260,7 +256,7 @@ extern void class_destroy(struct class *cls); /** * class_create - create a struct class structure - * @owner: pointer to the module that is to "own" this struct class + * @owner: dummy pointer, does nothing, will be removed soon. * @name: pointer to a string for the name of this class. * * This is used to create a struct class pointer that can then be used @@ -274,7 +270,7 @@ extern void class_destroy(struct class *cls); #define class_create(owner, name) \ ({ \ static struct lock_class_key __key; \ - __class_create(owner, name, &__key); \ + __class_create(name, &__key); \ }) -- cgit v1.2.3 From 1aaba11da9aa7d7d6b52a74d45b31cac118295a1 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:18:35 +0100 Subject: driver core: class: remove module * from class_create() The module pointer in class_create() never actually did anything, and it shouldn't have been requred to be set as a parameter even if it did something. So just remove it and fix up all callers of the function in the kernel tree at the same time. Cc: "Rafael J. Wysocki" Acked-by: Benjamin Tissoires Link: https://lore.kernel.org/r/20230313181843.1207845-4-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- arch/mips/kernel/mips-mt.c | 2 +- arch/mips/sibyte/common/sb_tbprof.c | 2 +- arch/powerpc/platforms/book3s/vas-api.c | 2 +- arch/x86/kernel/cpu/resctrl/pseudo_lock.c | 2 +- arch/x86/kernel/cpuid.c | 2 +- arch/x86/kernel/msr.c | 2 +- block/bsg.c | 2 +- drivers/accel/drm_accel.c | 2 +- drivers/accel/habanalabs/common/habanalabs_drv.c | 2 +- drivers/base/power/wakeup_stats.c | 2 +- drivers/block/aoe/aoechr.c | 2 +- drivers/block/rnbd/rnbd-clt-sysfs.c | 2 +- drivers/block/rnbd/rnbd-srv-sysfs.c | 2 +- drivers/block/ublk_drv.c | 2 +- drivers/char/bsr.c | 2 +- drivers/char/dsp56k.c | 2 +- drivers/char/ipmi/ipmi_devintf.c | 2 +- drivers/char/lp.c | 2 +- drivers/char/mem.c | 2 +- drivers/char/misc.c | 2 +- drivers/char/pcmcia/cm4000_cs.c | 2 +- drivers/char/pcmcia/cm4040_cs.c | 2 +- drivers/char/pcmcia/scr24x_cs.c | 2 +- drivers/char/ppdev.c | 2 +- drivers/char/tpm/tpm-interface.c | 4 ++-- drivers/char/virtio_console.c | 2 +- drivers/char/xilinx_hwicap/xilinx_hwicap.c | 2 +- drivers/char/xillybus/xillybus_class.c | 2 +- drivers/comedi/comedi_fops.c | 2 +- drivers/comedi/drivers/comedi_test.c | 2 +- drivers/crypto/qat/qat_common/adf_ctl_drv.c | 2 +- drivers/dca/dca-sysfs.c | 2 +- drivers/devfreq/devfreq-event.c | 2 +- drivers/devfreq/devfreq.c | 2 +- drivers/dma-buf/dma-heap.c | 2 +- drivers/extcon/extcon.c | 2 +- drivers/fpga/fpga-bridge.c | 2 +- drivers/fpga/fpga-mgr.c | 2 +- drivers/fpga/fpga-region.c | 2 +- drivers/gnss/core.c | 2 +- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 2 +- drivers/gpu/drm/display/drm_dp_aux_dev.c | 2 +- drivers/gpu/drm/drm_sysfs.c | 2 +- drivers/hid/hid-roccat-arvo.c | 2 +- drivers/hid/hid-roccat-isku.c | 2 +- drivers/hid/hid-roccat-kone.c | 2 +- drivers/hid/hid-roccat-koneplus.c | 2 +- drivers/hid/hid-roccat-konepure.c | 2 +- drivers/hid/hid-roccat-kovaplus.c | 2 +- drivers/hid/hid-roccat-pyra.c | 2 +- drivers/hid/hid-roccat-ryos.c | 2 +- drivers/hid/hid-roccat-savu.c | 2 +- drivers/hid/hidraw.c | 2 +- drivers/i2c/i2c-dev.c | 2 +- drivers/infiniband/core/uverbs_main.c | 2 +- drivers/infiniband/hw/hfi1/device.c | 4 ++-- drivers/infiniband/hw/qib/qib_file_ops.c | 2 +- drivers/infiniband/ulp/rtrs/rtrs-clt.c | 2 +- drivers/infiniband/ulp/rtrs/rtrs-srv.c | 2 +- drivers/isdn/capi/capi.c | 2 +- drivers/isdn/mISDN/dsp_pipeline.c | 2 +- drivers/leds/led-class.c | 2 +- drivers/macintosh/adb.c | 2 +- drivers/media/dvb-core/dvbdev.c | 2 +- drivers/media/rc/lirc_dev.c | 2 +- drivers/misc/c2port/core.c | 2 +- drivers/misc/cxl/file.c | 2 +- drivers/misc/genwqe/card_base.c | 2 +- drivers/misc/hpilo.c | 2 +- drivers/misc/mei/main.c | 2 +- drivers/misc/ocxl/file.c | 2 +- drivers/misc/phantom.c | 2 +- drivers/misc/uacce/uacce.c | 2 +- drivers/most/most_cdev.c | 2 +- drivers/net/ethernet/hisilicon/hns/hnae.c | 2 +- drivers/net/ppp/ppp_generic.c | 2 +- drivers/net/wireless/mac80211_hwsim.c | 2 +- drivers/net/wwan/wwan_core.c | 2 +- drivers/net/wwan/wwan_hwsim.c | 2 +- drivers/nvdimm/bus.c | 2 +- drivers/nvme/host/core.c | 6 +++--- drivers/nvme/host/fabrics.c | 2 +- drivers/nvme/target/fcloop.c | 2 +- drivers/pci/endpoint/pci-epc-core.c | 2 +- drivers/pci/switch/switchtec.c | 2 +- drivers/phy/phy-core.c | 2 +- drivers/power/supply/power_supply_core.c | 2 +- drivers/pps/pps.c | 2 +- drivers/ptp/ptp_clock.c | 2 +- drivers/rapidio/devices/rio_mport_cdev.c | 2 +- drivers/rapidio/rio_cm.c | 2 +- drivers/rpmsg/rpmsg_core.c | 2 +- drivers/rtc/class.c | 2 +- drivers/s390/char/hmcdrv_dev.c | 2 +- drivers/s390/char/raw3270.c | 2 +- drivers/s390/char/tape_class.c | 2 +- drivers/s390/char/vmlogrdr.c | 2 +- drivers/s390/char/vmur.c | 2 +- drivers/s390/crypto/zcrypt_api.c | 2 +- drivers/sbus/char/oradax.c | 2 +- drivers/scsi/ch.c | 2 +- drivers/scsi/cxlflash/main.c | 2 +- drivers/scsi/pmcraid.c | 2 +- drivers/scsi/sg.c | 2 +- drivers/spi/spidev.c | 2 +- drivers/staging/fieldbus/anybuss/arcx-anybus.c | 2 +- drivers/staging/greybus/authentication.c | 2 +- drivers/staging/greybus/fw-management.c | 2 +- drivers/staging/greybus/raw.c | 2 +- drivers/staging/pi433/pi433_if.c | 2 +- drivers/staging/vme_user/vme_user.c | 2 +- drivers/tee/tee_core.c | 2 +- drivers/tty/tty_io.c | 2 +- drivers/tty/vt/vc_screen.c | 2 +- drivers/tty/vt/vt.c | 2 +- drivers/usb/core/file.c | 2 +- drivers/usb/gadget/function/f_hid.c | 2 +- drivers/usb/gadget/function/f_printer.c | 2 +- drivers/usb/gadget/udc/core.c | 2 +- drivers/usb/mon/mon_bin.c | 2 +- drivers/usb/roles/class.c | 2 +- drivers/vdpa/vdpa_user/vduse_dev.c | 2 +- drivers/vfio/group.c | 2 +- drivers/vfio/vfio_main.c | 2 +- drivers/video/backlight/backlight.c | 2 +- drivers/video/backlight/lcd.c | 2 +- drivers/video/fbdev/core/fbmem.c | 2 +- fs/coda/psdev.c | 2 +- fs/fuse/cuse.c | 2 +- fs/pstore/pmsg.c | 2 +- include/linux/device/class.h | 3 +-- mm/backing-dev.c | 2 +- net/bluetooth/hci_sysfs.c | 2 +- net/netfilter/xt_IDLETIMER.c | 2 +- samples/vfio-mdev/mbochs.c | 2 +- samples/vfio-mdev/mdpy.c | 2 +- samples/vfio-mdev/mtty.c | 2 +- sound/sound_core.c | 2 +- tools/testing/nvdimm/test/ndtest.c | 2 +- tools/testing/nvdimm/test/nfit.c | 2 +- 140 files changed, 144 insertions(+), 145 deletions(-) (limited to 'include/linux') diff --git a/arch/mips/kernel/mips-mt.c b/arch/mips/kernel/mips-mt.c index dc023a979803..f88b7919f11f 100644 --- a/arch/mips/kernel/mips-mt.c +++ b/arch/mips/kernel/mips-mt.c @@ -234,7 +234,7 @@ static int __init mips_mt_init(void) { struct class *mtc; - mtc = class_create(THIS_MODULE, "mt"); + mtc = class_create("mt"); if (IS_ERR(mtc)) return PTR_ERR(mtc); diff --git a/arch/mips/sibyte/common/sb_tbprof.c b/arch/mips/sibyte/common/sb_tbprof.c index bc47681e825a..ac376dfb4e7c 100644 --- a/arch/mips/sibyte/common/sb_tbprof.c +++ b/arch/mips/sibyte/common/sb_tbprof.c @@ -550,7 +550,7 @@ static int __init sbprof_tb_init(void) return -EIO; } - tbc = class_create(THIS_MODULE, "sb_tracebuffer"); + tbc = class_create("sb_tracebuffer"); if (IS_ERR(tbc)) { err = PTR_ERR(tbc); goto out_chrdev; diff --git a/arch/powerpc/platforms/book3s/vas-api.c b/arch/powerpc/platforms/book3s/vas-api.c index 36c21648d19a..77ea9335fd04 100644 --- a/arch/powerpc/platforms/book3s/vas-api.c +++ b/arch/powerpc/platforms/book3s/vas-api.c @@ -581,7 +581,7 @@ int vas_register_coproc_api(struct module *mod, enum vas_cop_type cop_type, pr_devel("%s device allocated, dev [%i,%i]\n", name, MAJOR(coproc_device.devt), MINOR(coproc_device.devt)); - coproc_device.class = class_create(mod, name); + coproc_device.class = class_create(name); if (IS_ERR(coproc_device.class)) { rc = PTR_ERR(coproc_device.class); pr_err("Unable to create %s class %d\n", name, rc); diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index 524f8ff3e69c..458cb7419502 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -1580,7 +1580,7 @@ int rdt_pseudo_lock_init(void) pseudo_lock_major = ret; - pseudo_lock_class = class_create(THIS_MODULE, "pseudo_lock"); + pseudo_lock_class = class_create("pseudo_lock"); if (IS_ERR(pseudo_lock_class)) { ret = PTR_ERR(pseudo_lock_class); unregister_chrdev(pseudo_lock_major, "pseudo_lock"); diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c index 621ba9c0f17a..bdc0d5539b57 100644 --- a/arch/x86/kernel/cpuid.c +++ b/arch/x86/kernel/cpuid.c @@ -154,7 +154,7 @@ static int __init cpuid_init(void) CPUID_MAJOR); return -EBUSY; } - cpuid_class = class_create(THIS_MODULE, "cpuid"); + cpuid_class = class_create("cpuid"); if (IS_ERR(cpuid_class)) { err = PTR_ERR(cpuid_class); goto out_chrdev; diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index 708751311786..7bb17d37db01 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -263,7 +263,7 @@ static int __init msr_init(void) pr_err("unable to get major %d for msr\n", MSR_MAJOR); return -EBUSY; } - msr_class = class_create(THIS_MODULE, "msr"); + msr_class = class_create("msr"); if (IS_ERR(msr_class)) { err = PTR_ERR(msr_class); goto out_chrdev; diff --git a/block/bsg.c b/block/bsg.c index 30fcc865ef4f..7eca43f33d7f 100644 --- a/block/bsg.c +++ b/block/bsg.c @@ -245,7 +245,7 @@ static int __init bsg_init(void) dev_t devid; int ret; - bsg_class = class_create(THIS_MODULE, "bsg"); + bsg_class = class_create("bsg"); if (IS_ERR(bsg_class)) return PTR_ERR(bsg_class); bsg_class->devnode = bsg_devnode; diff --git a/drivers/accel/drm_accel.c b/drivers/accel/drm_accel.c index 1b69824286fd..4a9baf02439e 100644 --- a/drivers/accel/drm_accel.c +++ b/drivers/accel/drm_accel.c @@ -34,7 +34,7 @@ static char *accel_devnode(const struct device *dev, umode_t *mode) static int accel_sysfs_init(void) { - accel_class = class_create(THIS_MODULE, "accel"); + accel_class = class_create("accel"); if (IS_ERR(accel_class)) return PTR_ERR(accel_class); diff --git a/drivers/accel/habanalabs/common/habanalabs_drv.c b/drivers/accel/habanalabs/common/habanalabs_drv.c index 03dae57dc838..3538e89e4e5e 100644 --- a/drivers/accel/habanalabs/common/habanalabs_drv.c +++ b/drivers/accel/habanalabs/common/habanalabs_drv.c @@ -702,7 +702,7 @@ static int __init hl_init(void) hl_major = MAJOR(dev); - hl_class = class_create(THIS_MODULE, HL_NAME); + hl_class = class_create(HL_NAME); if (IS_ERR(hl_class)) { pr_err("failed to allocate class\n"); rc = PTR_ERR(hl_class); diff --git a/drivers/base/power/wakeup_stats.c b/drivers/base/power/wakeup_stats.c index 924fac493c4f..6732ed2869f9 100644 --- a/drivers/base/power/wakeup_stats.c +++ b/drivers/base/power/wakeup_stats.c @@ -210,7 +210,7 @@ void wakeup_source_sysfs_remove(struct wakeup_source *ws) static int __init wakeup_sources_sysfs_init(void) { - wakeup_class = class_create(THIS_MODULE, "wakeup"); + wakeup_class = class_create("wakeup"); return PTR_ERR_OR_ZERO(wakeup_class); } diff --git a/drivers/block/aoe/aoechr.c b/drivers/block/aoe/aoechr.c index 7a368c90467d..4c666f72203f 100644 --- a/drivers/block/aoe/aoechr.c +++ b/drivers/block/aoe/aoechr.c @@ -290,7 +290,7 @@ aoechr_init(void) } init_completion(&emsgs_comp); spin_lock_init(&emsgs_lock); - aoe_class = class_create(THIS_MODULE, "aoe"); + aoe_class = class_create("aoe"); if (IS_ERR(aoe_class)) { unregister_chrdev(AOE_MAJOR, "aoechr"); return PTR_ERR(aoe_class); diff --git a/drivers/block/rnbd/rnbd-clt-sysfs.c b/drivers/block/rnbd/rnbd-clt-sysfs.c index e7c7d9a68168..8c6087949794 100644 --- a/drivers/block/rnbd/rnbd-clt-sysfs.c +++ b/drivers/block/rnbd/rnbd-clt-sysfs.c @@ -646,7 +646,7 @@ int rnbd_clt_create_sysfs_files(void) { int err; - rnbd_dev_class = class_create(THIS_MODULE, "rnbd-client"); + rnbd_dev_class = class_create("rnbd-client"); if (IS_ERR(rnbd_dev_class)) return PTR_ERR(rnbd_dev_class); diff --git a/drivers/block/rnbd/rnbd-srv-sysfs.c b/drivers/block/rnbd/rnbd-srv-sysfs.c index 297a6924ff4e..d5d9267e1fa5 100644 --- a/drivers/block/rnbd/rnbd-srv-sysfs.c +++ b/drivers/block/rnbd/rnbd-srv-sysfs.c @@ -215,7 +215,7 @@ int rnbd_srv_create_sysfs_files(void) { int err; - rnbd_dev_class = class_create(THIS_MODULE, "rnbd-server"); + rnbd_dev_class = class_create("rnbd-server"); if (IS_ERR(rnbd_dev_class)) return PTR_ERR(rnbd_dev_class); diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index d1d1c8d606c8..2eb46419562b 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2266,7 +2266,7 @@ static int __init ublk_init(void) if (ret) goto unregister_mis; - ublk_chr_class = class_create(THIS_MODULE, "ublk-char"); + ublk_chr_class = class_create("ublk-char"); if (IS_ERR(ublk_chr_class)) { ret = PTR_ERR(ublk_chr_class); goto free_chrdev_region; diff --git a/drivers/char/bsr.c b/drivers/char/bsr.c index d5f943938427..ff429ba02fa4 100644 --- a/drivers/char/bsr.c +++ b/drivers/char/bsr.c @@ -293,7 +293,7 @@ static int __init bsr_init(void) if (!np) goto out_err; - bsr_class = class_create(THIS_MODULE, "bsr"); + bsr_class = class_create("bsr"); if (IS_ERR(bsr_class)) { printk(KERN_ERR "class_create() failed for bsr_class\n"); ret = PTR_ERR(bsr_class); diff --git a/drivers/char/dsp56k.c b/drivers/char/dsp56k.c index 06749e295ada..b3eaf3e5ef2e 100644 --- a/drivers/char/dsp56k.c +++ b/drivers/char/dsp56k.c @@ -504,7 +504,7 @@ static int __init dsp56k_init_driver(void) printk("DSP56k driver: Unable to register driver\n"); return -ENODEV; } - dsp56k_class = class_create(THIS_MODULE, "dsp56k"); + dsp56k_class = class_create("dsp56k"); if (IS_ERR(dsp56k_class)) { err = PTR_ERR(dsp56k_class); goto out_chrdev; diff --git a/drivers/char/ipmi/ipmi_devintf.c b/drivers/char/ipmi/ipmi_devintf.c index d160fa4c73fe..73e5a9e28f85 100644 --- a/drivers/char/ipmi/ipmi_devintf.c +++ b/drivers/char/ipmi/ipmi_devintf.c @@ -860,7 +860,7 @@ static int __init init_ipmi_devintf(void) pr_info("ipmi device interface\n"); - ipmi_class = class_create(THIS_MODULE, "ipmi"); + ipmi_class = class_create("ipmi"); if (IS_ERR(ipmi_class)) { pr_err("ipmi: can't register device class\n"); return PTR_ERR(ipmi_class); diff --git a/drivers/char/lp.c b/drivers/char/lp.c index 38aad99ebb61..70cfc5140c2c 100644 --- a/drivers/char/lp.c +++ b/drivers/char/lp.c @@ -1049,7 +1049,7 @@ static int __init lp_init(void) return -EIO; } - lp_class = class_create(THIS_MODULE, "printer"); + lp_class = class_create("printer"); if (IS_ERR(lp_class)) { err = PTR_ERR(lp_class); goto out_reg; diff --git a/drivers/char/mem.c b/drivers/char/mem.c index ffb101d349f0..f494d31f2b98 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -762,7 +762,7 @@ static int __init chr_dev_init(void) if (register_chrdev(MEM_MAJOR, "mem", &memory_fops)) printk("unable to get major %d for memory devs\n", MEM_MAJOR); - mem_class = class_create(THIS_MODULE, "mem"); + mem_class = class_create("mem"); if (IS_ERR(mem_class)) return PTR_ERR(mem_class); diff --git a/drivers/char/misc.c b/drivers/char/misc.c index 7a1388b0572b..1c44c29a666e 100644 --- a/drivers/char/misc.c +++ b/drivers/char/misc.c @@ -286,7 +286,7 @@ static int __init misc_init(void) struct proc_dir_entry *ret; ret = proc_create_seq("misc", 0, NULL, &misc_seq_ops); - misc_class = class_create(THIS_MODULE, "misc"); + misc_class = class_create("misc"); err = PTR_ERR(misc_class); if (IS_ERR(misc_class)) goto fail_remove; diff --git a/drivers/char/pcmcia/cm4000_cs.c b/drivers/char/pcmcia/cm4000_cs.c index e656f42a28ac..7f96d8571a53 100644 --- a/drivers/char/pcmcia/cm4000_cs.c +++ b/drivers/char/pcmcia/cm4000_cs.c @@ -1878,7 +1878,7 @@ static int __init cmm_init(void) { int rc; - cmm_class = class_create(THIS_MODULE, "cardman_4000"); + cmm_class = class_create("cardman_4000"); if (IS_ERR(cmm_class)) return PTR_ERR(cmm_class); diff --git a/drivers/char/pcmcia/cm4040_cs.c b/drivers/char/pcmcia/cm4040_cs.c index 827711911da4..11ff59e2b963 100644 --- a/drivers/char/pcmcia/cm4040_cs.c +++ b/drivers/char/pcmcia/cm4040_cs.c @@ -650,7 +650,7 @@ static int __init cm4040_init(void) { int rc; - cmx_class = class_create(THIS_MODULE, "cardman_4040"); + cmx_class = class_create("cardman_4040"); if (IS_ERR(cmx_class)) return PTR_ERR(cmx_class); diff --git a/drivers/char/pcmcia/scr24x_cs.c b/drivers/char/pcmcia/scr24x_cs.c index 1bdce08fae3d..870781f5a08c 100644 --- a/drivers/char/pcmcia/scr24x_cs.c +++ b/drivers/char/pcmcia/scr24x_cs.c @@ -325,7 +325,7 @@ static int __init scr24x_init(void) { int ret; - scr24x_class = class_create(THIS_MODULE, "scr24x"); + scr24x_class = class_create("scr24x"); if (IS_ERR(scr24x_class)) return PTR_ERR(scr24x_class); diff --git a/drivers/char/ppdev.c b/drivers/char/ppdev.c index 38b46c7d1737..81ed58157b15 100644 --- a/drivers/char/ppdev.c +++ b/drivers/char/ppdev.c @@ -841,7 +841,7 @@ static int __init ppdev_init(void) pr_warn(CHRDEV ": unable to get major %d\n", PP_MAJOR); return -EIO; } - ppdev_class = class_create(THIS_MODULE, CHRDEV); + ppdev_class = class_create(CHRDEV); if (IS_ERR(ppdev_class)) { err = PTR_ERR(ppdev_class); goto out_chrdev; diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c index 7e513b771832..8763c820d1f8 100644 --- a/drivers/char/tpm/tpm-interface.c +++ b/drivers/char/tpm/tpm-interface.c @@ -466,13 +466,13 @@ static int __init tpm_init(void) { int rc; - tpm_class = class_create(THIS_MODULE, "tpm"); + tpm_class = class_create("tpm"); if (IS_ERR(tpm_class)) { pr_err("couldn't create tpm class\n"); return PTR_ERR(tpm_class); } - tpmrm_class = class_create(THIS_MODULE, "tpmrm"); + tpmrm_class = class_create("tpmrm"); if (IS_ERR(tpmrm_class)) { pr_err("couldn't create tpmrm class\n"); rc = PTR_ERR(tpmrm_class); diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c index d5ac4d955bc8..b65c809a4e97 100644 --- a/drivers/char/virtio_console.c +++ b/drivers/char/virtio_console.c @@ -2244,7 +2244,7 @@ static int __init virtio_console_init(void) { int err; - pdrvdata.class = class_create(THIS_MODULE, "virtio-ports"); + pdrvdata.class = class_create("virtio-ports"); if (IS_ERR(pdrvdata.class)) { err = PTR_ERR(pdrvdata.class); pr_err("Error %d creating virtio-ports class\n", err); diff --git a/drivers/char/xilinx_hwicap/xilinx_hwicap.c b/drivers/char/xilinx_hwicap/xilinx_hwicap.c index 74a4928aea1d..a46f637da959 100644 --- a/drivers/char/xilinx_hwicap/xilinx_hwicap.c +++ b/drivers/char/xilinx_hwicap/xilinx_hwicap.c @@ -856,7 +856,7 @@ static int __init hwicap_module_init(void) dev_t devt; int retval; - icap_class = class_create(THIS_MODULE, "xilinx_config"); + icap_class = class_create("xilinx_config"); mutex_init(&icap_sem); devt = MKDEV(XHWICAP_MAJOR, XHWICAP_MINOR); diff --git a/drivers/char/xillybus/xillybus_class.c b/drivers/char/xillybus/xillybus_class.c index e9a288e61c15..89926fe9d813 100644 --- a/drivers/char/xillybus/xillybus_class.c +++ b/drivers/char/xillybus/xillybus_class.c @@ -242,7 +242,7 @@ EXPORT_SYMBOL(xillybus_find_inode); static int __init xillybus_class_init(void) { - xillybus_class = class_create(THIS_MODULE, "xillybus"); + xillybus_class = class_create("xillybus"); if (IS_ERR(xillybus_class)) { pr_warn("Failed to register xillybus class\n"); diff --git a/drivers/comedi/comedi_fops.c b/drivers/comedi/comedi_fops.c index b982903aaa46..8e43918d38c4 100644 --- a/drivers/comedi/comedi_fops.c +++ b/drivers/comedi/comedi_fops.c @@ -3383,7 +3383,7 @@ static int __init comedi_init(void) if (retval) goto out_unregister_chrdev_region; - comedi_class = class_create(THIS_MODULE, "comedi"); + comedi_class = class_create("comedi"); if (IS_ERR(comedi_class)) { retval = PTR_ERR(comedi_class); pr_err("failed to create class\n"); diff --git a/drivers/comedi/drivers/comedi_test.c b/drivers/comedi/drivers/comedi_test.c index 0b5c0af1cebf..c02dc19a679b 100644 --- a/drivers/comedi/drivers/comedi_test.c +++ b/drivers/comedi/drivers/comedi_test.c @@ -795,7 +795,7 @@ static int __init comedi_test_init(void) } if (!config_mode) { - ctcls = class_create(THIS_MODULE, CLASS_NAME); + ctcls = class_create(CLASS_NAME); if (IS_ERR(ctcls)) { pr_warn("comedi_test: unable to create class\n"); goto clean3; diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index 9190532b27eb..75a50fde5644 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -56,7 +56,7 @@ static int adf_chr_drv_create(void) return -EFAULT; } - adf_ctl_drv.drv_class = class_create(THIS_MODULE, DEVICE_NAME); + adf_ctl_drv.drv_class = class_create(DEVICE_NAME); if (IS_ERR(adf_ctl_drv.drv_class)) { pr_err("QAT: class_create failed for adf_ctl\n"); goto err_chrdev_unreg; diff --git a/drivers/dca/dca-sysfs.c b/drivers/dca/dca-sysfs.c index 21ebd0af268b..fcc83ede0909 100644 --- a/drivers/dca/dca-sysfs.c +++ b/drivers/dca/dca-sysfs.c @@ -74,7 +74,7 @@ int __init dca_sysfs_init(void) idr_init(&dca_idr); spin_lock_init(&dca_idr_lock); - dca_class = class_create(THIS_MODULE, "dca"); + dca_class = class_create("dca"); if (IS_ERR(dca_class)) { idr_destroy(&dca_idr); return PTR_ERR(dca_class); diff --git a/drivers/devfreq/devfreq-event.c b/drivers/devfreq/devfreq-event.c index f041edccd107..3ebac2496679 100644 --- a/drivers/devfreq/devfreq-event.c +++ b/drivers/devfreq/devfreq-event.c @@ -469,7 +469,7 @@ ATTRIBUTE_GROUPS(devfreq_event); static int __init devfreq_event_init(void) { - devfreq_event_class = class_create(THIS_MODULE, "devfreq-event"); + devfreq_event_class = class_create("devfreq-event"); if (IS_ERR(devfreq_event_class)) { pr_err("%s: couldn't create class\n", __FILE__); return PTR_ERR(devfreq_event_class); diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index 817c71da391a..e36cbb920ec8 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -1988,7 +1988,7 @@ DEFINE_SHOW_ATTRIBUTE(devfreq_summary); static int __init devfreq_init(void) { - devfreq_class = class_create(THIS_MODULE, "devfreq"); + devfreq_class = class_create("devfreq"); if (IS_ERR(devfreq_class)) { pr_err("%s: couldn't create class\n", __FILE__); return PTR_ERR(devfreq_class); diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c index c9e41e8a9e27..84ae708fafe7 100644 --- a/drivers/dma-buf/dma-heap.c +++ b/drivers/dma-buf/dma-heap.c @@ -314,7 +314,7 @@ static int dma_heap_init(void) if (ret) return ret; - dma_heap_class = class_create(THIS_MODULE, DEVNAME); + dma_heap_class = class_create(DEVNAME); if (IS_ERR(dma_heap_class)) { unregister_chrdev_region(dma_heap_devt, NUM_HEAP_MINORS); return PTR_ERR(dma_heap_class); diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c index e1c71359b605..d43ba8e7260d 100644 --- a/drivers/extcon/extcon.c +++ b/drivers/extcon/extcon.c @@ -1013,7 +1013,7 @@ ATTRIBUTE_GROUPS(extcon); static int create_extcon_class(void) { if (!extcon_class) { - extcon_class = class_create(THIS_MODULE, "extcon"); + extcon_class = class_create("extcon"); if (IS_ERR(extcon_class)) return PTR_ERR(extcon_class); extcon_class->dev_groups = extcon_groups; diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c index 5cd40acab5bf..6a521c83a1ed 100644 --- a/drivers/fpga/fpga-bridge.c +++ b/drivers/fpga/fpga-bridge.c @@ -416,7 +416,7 @@ static void fpga_bridge_dev_release(struct device *dev) static int __init fpga_bridge_dev_init(void) { - fpga_bridge_class = class_create(THIS_MODULE, "fpga_bridge"); + fpga_bridge_class = class_create("fpga_bridge"); if (IS_ERR(fpga_bridge_class)) return PTR_ERR(fpga_bridge_class); diff --git a/drivers/fpga/fpga-mgr.c b/drivers/fpga/fpga-mgr.c index 8efa67620e21..eb583f86a0b9 100644 --- a/drivers/fpga/fpga-mgr.c +++ b/drivers/fpga/fpga-mgr.c @@ -971,7 +971,7 @@ static int __init fpga_mgr_class_init(void) { pr_info("FPGA manager framework\n"); - fpga_mgr_class = class_create(THIS_MODULE, "fpga_manager"); + fpga_mgr_class = class_create("fpga_manager"); if (IS_ERR(fpga_mgr_class)) return PTR_ERR(fpga_mgr_class); diff --git a/drivers/fpga/fpga-region.c b/drivers/fpga/fpga-region.c index 27ff9dea04ae..ccf6fdab1360 100644 --- a/drivers/fpga/fpga-region.c +++ b/drivers/fpga/fpga-region.c @@ -293,7 +293,7 @@ static void fpga_region_dev_release(struct device *dev) */ static int __init fpga_region_init(void) { - fpga_region_class = class_create(THIS_MODULE, "fpga_region"); + fpga_region_class = class_create("fpga_region"); if (IS_ERR(fpga_region_class)) return PTR_ERR(fpga_region_class); diff --git a/drivers/gnss/core.c b/drivers/gnss/core.c index 77a4b280c552..48f2ee0f78c4 100644 --- a/drivers/gnss/core.c +++ b/drivers/gnss/core.c @@ -387,7 +387,7 @@ static int __init gnss_module_init(void) return ret; } - gnss_class = class_create(THIS_MODULE, "gnss"); + gnss_class = class_create("gnss"); if (IS_ERR(gnss_class)) { ret = PTR_ERR(gnss_class); pr_err("failed to create class: %d\n", ret); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index a0e30f21e12e..20e75bd74bed 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -93,7 +93,7 @@ int kfd_chardev_init(void) if (err < 0) goto err_register_chrdev; - kfd_class = class_create(THIS_MODULE, kfd_dev_name); + kfd_class = class_create(kfd_dev_name); err = PTR_ERR(kfd_class); if (IS_ERR(kfd_class)) goto err_class_create; diff --git a/drivers/gpu/drm/display/drm_dp_aux_dev.c b/drivers/gpu/drm/display/drm_dp_aux_dev.c index 098e482e65a2..29555b9f03c8 100644 --- a/drivers/gpu/drm/display/drm_dp_aux_dev.c +++ b/drivers/gpu/drm/display/drm_dp_aux_dev.c @@ -330,7 +330,7 @@ int drm_dp_aux_dev_init(void) { int res; - drm_dp_aux_dev_class = class_create(THIS_MODULE, "drm_dp_aux_dev"); + drm_dp_aux_dev_class = class_create("drm_dp_aux_dev"); if (IS_ERR(drm_dp_aux_dev_class)) { return PTR_ERR(drm_dp_aux_dev_class); } diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c index 183130355997..3c22a803201d 100644 --- a/drivers/gpu/drm/drm_sysfs.c +++ b/drivers/gpu/drm/drm_sysfs.c @@ -112,7 +112,7 @@ int drm_sysfs_init(void) { int err; - drm_class = class_create(THIS_MODULE, "drm"); + drm_class = class_create("drm"); if (IS_ERR(drm_class)) return PTR_ERR(drm_class); diff --git a/drivers/hid/hid-roccat-arvo.c b/drivers/hid/hid-roccat-arvo.c index d94ee0539421..ea6b79b3aeeb 100644 --- a/drivers/hid/hid-roccat-arvo.c +++ b/drivers/hid/hid-roccat-arvo.c @@ -433,7 +433,7 @@ static int __init arvo_init(void) { int retval; - arvo_class = class_create(THIS_MODULE, "arvo"); + arvo_class = class_create("arvo"); if (IS_ERR(arvo_class)) return PTR_ERR(arvo_class); arvo_class->dev_groups = arvo_groups; diff --git a/drivers/hid/hid-roccat-isku.c b/drivers/hid/hid-roccat-isku.c index e95d59cd8d07..3903a2cea00c 100644 --- a/drivers/hid/hid-roccat-isku.c +++ b/drivers/hid/hid-roccat-isku.c @@ -435,7 +435,7 @@ static struct hid_driver isku_driver = { static int __init isku_init(void) { int retval; - isku_class = class_create(THIS_MODULE, "isku"); + isku_class = class_create("isku"); if (IS_ERR(isku_class)) return PTR_ERR(isku_class); isku_class->dev_groups = isku_groups; diff --git a/drivers/hid/hid-roccat-kone.c b/drivers/hid/hid-roccat-kone.c index 76da04801ca9..945ae236fb45 100644 --- a/drivers/hid/hid-roccat-kone.c +++ b/drivers/hid/hid-roccat-kone.c @@ -890,7 +890,7 @@ static int __init kone_init(void) int retval; /* class name has to be same as driver name */ - kone_class = class_create(THIS_MODULE, "kone"); + kone_class = class_create("kone"); if (IS_ERR(kone_class)) return PTR_ERR(kone_class); kone_class->dev_groups = kone_groups; diff --git a/drivers/hid/hid-roccat-koneplus.c b/drivers/hid/hid-roccat-koneplus.c index 1896c69ea512..97b83b6f53dd 100644 --- a/drivers/hid/hid-roccat-koneplus.c +++ b/drivers/hid/hid-roccat-koneplus.c @@ -549,7 +549,7 @@ static int __init koneplus_init(void) int retval; /* class name has to be same as driver name */ - koneplus_class = class_create(THIS_MODULE, "koneplus"); + koneplus_class = class_create("koneplus"); if (IS_ERR(koneplus_class)) return PTR_ERR(koneplus_class); koneplus_class->dev_groups = koneplus_groups; diff --git a/drivers/hid/hid-roccat-konepure.c b/drivers/hid/hid-roccat-konepure.c index cf8eeb33a125..a297756f2410 100644 --- a/drivers/hid/hid-roccat-konepure.c +++ b/drivers/hid/hid-roccat-konepure.c @@ -207,7 +207,7 @@ static int __init konepure_init(void) { int retval; - konepure_class = class_create(THIS_MODULE, "konepure"); + konepure_class = class_create("konepure"); if (IS_ERR(konepure_class)) return PTR_ERR(konepure_class); konepure_class->dev_groups = konepure_groups; diff --git a/drivers/hid/hid-roccat-kovaplus.c b/drivers/hid/hid-roccat-kovaplus.c index 6fb9b9563769..1a1d96e11683 100644 --- a/drivers/hid/hid-roccat-kovaplus.c +++ b/drivers/hid/hid-roccat-kovaplus.c @@ -638,7 +638,7 @@ static int __init kovaplus_init(void) { int retval; - kovaplus_class = class_create(THIS_MODULE, "kovaplus"); + kovaplus_class = class_create("kovaplus"); if (IS_ERR(kovaplus_class)) return PTR_ERR(kovaplus_class); kovaplus_class->dev_groups = kovaplus_groups; diff --git a/drivers/hid/hid-roccat-pyra.c b/drivers/hid/hid-roccat-pyra.c index 4fcc8e7d276f..15528c3b013c 100644 --- a/drivers/hid/hid-roccat-pyra.c +++ b/drivers/hid/hid-roccat-pyra.c @@ -585,7 +585,7 @@ static int __init pyra_init(void) int retval; /* class name has to be same as driver name */ - pyra_class = class_create(THIS_MODULE, "pyra"); + pyra_class = class_create("pyra"); if (IS_ERR(pyra_class)) return PTR_ERR(pyra_class); pyra_class->dev_groups = pyra_groups; diff --git a/drivers/hid/hid-roccat-ryos.c b/drivers/hid/hid-roccat-ryos.c index 5bf1971a2b14..0eb17a3b925d 100644 --- a/drivers/hid/hid-roccat-ryos.c +++ b/drivers/hid/hid-roccat-ryos.c @@ -216,7 +216,7 @@ static int __init ryos_init(void) { int retval; - ryos_class = class_create(THIS_MODULE, "ryos"); + ryos_class = class_create("ryos"); if (IS_ERR(ryos_class)) return PTR_ERR(ryos_class); ryos_class->dev_groups = ryos_groups; diff --git a/drivers/hid/hid-roccat-savu.c b/drivers/hid/hid-roccat-savu.c index a784bb4ee651..93be7acef673 100644 --- a/drivers/hid/hid-roccat-savu.c +++ b/drivers/hid/hid-roccat-savu.c @@ -204,7 +204,7 @@ static int __init savu_init(void) { int retval; - savu_class = class_create(THIS_MODULE, "savu"); + savu_class = class_create("savu"); if (IS_ERR(savu_class)) return PTR_ERR(savu_class); savu_class->dev_groups = savu_groups; diff --git a/drivers/hid/hidraw.c b/drivers/hid/hidraw.c index 197b1e7bf029..93e62b161501 100644 --- a/drivers/hid/hidraw.c +++ b/drivers/hid/hidraw.c @@ -618,7 +618,7 @@ int __init hidraw_init(void) hidraw_major = MAJOR(dev_id); - hidraw_class = class_create(THIS_MODULE, "hidraw"); + hidraw_class = class_create("hidraw"); if (IS_ERR(hidraw_class)) { result = PTR_ERR(hidraw_class); goto error_cdev; diff --git a/drivers/i2c/i2c-dev.c b/drivers/i2c/i2c-dev.c index 107623c4cc14..ef93064cfdb3 100644 --- a/drivers/i2c/i2c-dev.c +++ b/drivers/i2c/i2c-dev.c @@ -739,7 +739,7 @@ static int __init i2c_dev_init(void) if (res) goto out; - i2c_dev_class = class_create(THIS_MODULE, "i2c-dev"); + i2c_dev_class = class_create("i2c-dev"); if (IS_ERR(i2c_dev_class)) { res = PTR_ERR(i2c_dev_class); goto out_unreg_chrdev; diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index bdb179a09d77..fbace69672ca 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -1264,7 +1264,7 @@ static int __init ib_uverbs_init(void) goto out_alloc; } - uverbs_class = class_create(THIS_MODULE, "infiniband_verbs"); + uverbs_class = class_create("infiniband_verbs"); if (IS_ERR(uverbs_class)) { ret = PTR_ERR(uverbs_class); pr_err("user_verbs: couldn't create class infiniband_verbs\n"); diff --git a/drivers/infiniband/hw/hfi1/device.c b/drivers/infiniband/hw/hfi1/device.c index 1f4496032170..05be0d119f79 100644 --- a/drivers/infiniband/hw/hfi1/device.c +++ b/drivers/infiniband/hw/hfi1/device.c @@ -102,7 +102,7 @@ int __init dev_init(void) goto done; } - class = class_create(THIS_MODULE, class_name()); + class = class_create(class_name()); if (IS_ERR(class)) { ret = PTR_ERR(class); pr_err("Could not create device class (err %d)\n", -ret); @@ -111,7 +111,7 @@ int __init dev_init(void) } class->devnode = hfi1_devnode; - user_class = class_create(THIS_MODULE, class_name_user()); + user_class = class_create(class_name_user()); if (IS_ERR(user_class)) { ret = PTR_ERR(user_class); pr_err("Could not create device class for user accessible files (err %d)\n", diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c index 80fe92a21f96..c07c95fd3cfa 100644 --- a/drivers/infiniband/hw/qib/qib_file_ops.c +++ b/drivers/infiniband/hw/qib/qib_file_ops.c @@ -2326,7 +2326,7 @@ int __init qib_dev_init(void) goto done; } - qib_class = class_create(THIS_MODULE, "ipath"); + qib_class = class_create("ipath"); if (IS_ERR(qib_class)) { ret = PTR_ERR(qib_class); pr_err("Could not create device class (err %d)\n", -ret); diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index 80abf45a197a..edb2e3a25880 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -3163,7 +3163,7 @@ static int __init rtrs_client_init(void) { rtrs_rdma_dev_pd_init(0, &dev_pd); - rtrs_clt_dev_class = class_create(THIS_MODULE, "rtrs-client"); + rtrs_clt_dev_class = class_create("rtrs-client"); if (IS_ERR(rtrs_clt_dev_class)) { pr_err("Failed to create rtrs-client dev class\n"); return PTR_ERR(rtrs_clt_dev_class); diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index d1703e2c0b82..c38901e2c8f4 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -2253,7 +2253,7 @@ static int __init rtrs_server_init(void) err); return err; } - rtrs_dev_class = class_create(THIS_MODULE, "rtrs-server"); + rtrs_dev_class = class_create("rtrs-server"); if (IS_ERR(rtrs_dev_class)) { err = PTR_ERR(rtrs_dev_class); goto out_err; diff --git a/drivers/isdn/capi/capi.c b/drivers/isdn/capi/capi.c index 0f00be62438d..45a4043c5042 100644 --- a/drivers/isdn/capi/capi.c +++ b/drivers/isdn/capi/capi.c @@ -1393,7 +1393,7 @@ static int __init capi_init(void) kcapi_exit(); return major_ret; } - capi_class = class_create(THIS_MODULE, "capi"); + capi_class = class_create("capi"); if (IS_ERR(capi_class)) { unregister_chrdev(capi_major, "capi20"); kcapi_exit(); diff --git a/drivers/isdn/mISDN/dsp_pipeline.c b/drivers/isdn/mISDN/dsp_pipeline.c index cfbcd9e973c2..09b72f14d4b7 100644 --- a/drivers/isdn/mISDN/dsp_pipeline.c +++ b/drivers/isdn/mISDN/dsp_pipeline.c @@ -131,7 +131,7 @@ EXPORT_SYMBOL(mISDN_dsp_element_unregister); int dsp_pipeline_module_init(void) { - elements_class = class_create(THIS_MODULE, "dsp_pipeline"); + elements_class = class_create("dsp_pipeline"); if (IS_ERR(elements_class)) return PTR_ERR(elements_class); diff --git a/drivers/leds/led-class.c b/drivers/leds/led-class.c index a6b3adcd044a..9255bc11f99d 100644 --- a/drivers/leds/led-class.c +++ b/drivers/leds/led-class.c @@ -626,7 +626,7 @@ EXPORT_SYMBOL_GPL(devm_led_classdev_unregister); static int __init leds_init(void) { - leds_class = class_create(THIS_MODULE, "leds"); + leds_class = class_create("leds"); if (IS_ERR(leds_class)) return PTR_ERR(leds_class); leds_class->pm = &leds_class_dev_pm_ops; diff --git a/drivers/macintosh/adb.c b/drivers/macintosh/adb.c index 23bd0c77ac1a..57e987cf84b2 100644 --- a/drivers/macintosh/adb.c +++ b/drivers/macintosh/adb.c @@ -888,7 +888,7 @@ adbdev_init(void) return; } - adb_dev_class = class_create(THIS_MODULE, "adb"); + adb_dev_class = class_create("adb"); if (IS_ERR(adb_dev_class)) return; device_create(adb_dev_class, NULL, MKDEV(ADB_MAJOR, 0), NULL, "adb"); diff --git a/drivers/media/dvb-core/dvbdev.c b/drivers/media/dvb-core/dvbdev.c index 0ed087caf7f3..e9b3ce09e534 100644 --- a/drivers/media/dvb-core/dvbdev.c +++ b/drivers/media/dvb-core/dvbdev.c @@ -1063,7 +1063,7 @@ static int __init init_dvbdev(void) goto error; } - dvb_class = class_create(THIS_MODULE, "dvb"); + dvb_class = class_create("dvb"); if (IS_ERR(dvb_class)) { retval = PTR_ERR(dvb_class); goto error; diff --git a/drivers/media/rc/lirc_dev.c b/drivers/media/rc/lirc_dev.c index 25ab61dae126..043d23aaa3cb 100644 --- a/drivers/media/rc/lirc_dev.c +++ b/drivers/media/rc/lirc_dev.c @@ -785,7 +785,7 @@ int __init lirc_dev_init(void) { int retval; - lirc_class = class_create(THIS_MODULE, "lirc"); + lirc_class = class_create("lirc"); if (IS_ERR(lirc_class)) { pr_err("class_create failed\n"); return PTR_ERR(lirc_class); diff --git a/drivers/misc/c2port/core.c b/drivers/misc/c2port/core.c index fb9a1b49ff6d..f574c83b82cf 100644 --- a/drivers/misc/c2port/core.c +++ b/drivers/misc/c2port/core.c @@ -977,7 +977,7 @@ static int __init c2port_init(void) printk(KERN_INFO "Silicon Labs C2 port support v. " DRIVER_VERSION " - (C) 2007 Rodolfo Giometti\n"); - c2port_class = class_create(THIS_MODULE, "c2port"); + c2port_class = class_create("c2port"); if (IS_ERR(c2port_class)) { printk(KERN_ERR "c2port: failed to allocate class\n"); return PTR_ERR(c2port_class); diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c index 5878329b011a..144d1f2d78ce 100644 --- a/drivers/misc/cxl/file.c +++ b/drivers/misc/cxl/file.c @@ -678,7 +678,7 @@ int __init cxl_file_init(void) pr_devel("CXL device allocated, MAJOR %i\n", MAJOR(cxl_dev)); - cxl_class = class_create(THIS_MODULE, "cxl"); + cxl_class = class_create("cxl"); if (IS_ERR(cxl_class)) { pr_err("Unable to create CXL class\n"); rc = PTR_ERR(cxl_class); diff --git a/drivers/misc/genwqe/card_base.c b/drivers/misc/genwqe/card_base.c index 5b63d179b24e..02628288cd0f 100644 --- a/drivers/misc/genwqe/card_base.c +++ b/drivers/misc/genwqe/card_base.c @@ -1363,7 +1363,7 @@ static int __init genwqe_init_module(void) { int rc; - class_genwqe = class_create(THIS_MODULE, GENWQE_DEVNAME); + class_genwqe = class_create(GENWQE_DEVNAME); if (IS_ERR(class_genwqe)) { pr_err("[%s] create class failed\n", __func__); return -ENOMEM; diff --git a/drivers/misc/hpilo.c b/drivers/misc/hpilo.c index 8d00df9243c4..2c3a991d6e88 100644 --- a/drivers/misc/hpilo.c +++ b/drivers/misc/hpilo.c @@ -888,7 +888,7 @@ static int __init ilo_init(void) int error; dev_t dev; - ilo_class = class_create(THIS_MODULE, "iLO"); + ilo_class = class_create("iLO"); if (IS_ERR(ilo_class)) { error = PTR_ERR(ilo_class); goto out; diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c index 632d4ae21e46..76c771a424f7 100644 --- a/drivers/misc/mei/main.c +++ b/drivers/misc/mei/main.c @@ -1275,7 +1275,7 @@ static int __init mei_init(void) { int ret; - mei_class = class_create(THIS_MODULE, "mei"); + mei_class = class_create("mei"); if (IS_ERR(mei_class)) { pr_err("couldn't create class\n"); ret = PTR_ERR(mei_class); diff --git a/drivers/misc/ocxl/file.c b/drivers/misc/ocxl/file.c index 3b058654b45b..6e63f060e4cc 100644 --- a/drivers/misc/ocxl/file.c +++ b/drivers/misc/ocxl/file.c @@ -601,7 +601,7 @@ int ocxl_file_init(void) return rc; } - ocxl_class = class_create(THIS_MODULE, "ocxl"); + ocxl_class = class_create("ocxl"); if (IS_ERR(ocxl_class)) { pr_err("Unable to create ocxl class\n"); unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS); diff --git a/drivers/misc/phantom.c b/drivers/misc/phantom.c index ce72e46a2e73..7966a6b8b5b3 100644 --- a/drivers/misc/phantom.c +++ b/drivers/misc/phantom.c @@ -503,7 +503,7 @@ static int __init phantom_init(void) int retval; dev_t dev; - phantom_class = class_create(THIS_MODULE, "phantom"); + phantom_class = class_create("phantom"); if (IS_ERR(phantom_class)) { retval = PTR_ERR(phantom_class); printk(KERN_ERR "phantom: can't register phantom class\n"); diff --git a/drivers/misc/uacce/uacce.c b/drivers/misc/uacce/uacce.c index 07023397afc7..346bd7cf2e94 100644 --- a/drivers/misc/uacce/uacce.c +++ b/drivers/misc/uacce/uacce.c @@ -620,7 +620,7 @@ static int __init uacce_init(void) { int ret; - uacce_class = class_create(THIS_MODULE, UACCE_NAME); + uacce_class = class_create(UACCE_NAME); if (IS_ERR(uacce_class)) return PTR_ERR(uacce_class); diff --git a/drivers/most/most_cdev.c b/drivers/most/most_cdev.c index 4ee536980f71..3ed8f461e01e 100644 --- a/drivers/most/most_cdev.c +++ b/drivers/most/most_cdev.c @@ -491,7 +491,7 @@ static int __init most_cdev_init(void) { int err; - comp.class = class_create(THIS_MODULE, "most_cdev"); + comp.class = class_create("most_cdev"); if (IS_ERR(comp.class)) return PTR_ERR(comp.class); diff --git a/drivers/net/ethernet/hisilicon/hns/hnae.c b/drivers/net/ethernet/hisilicon/hns/hnae.c index 9b26f0f2c748..8a1027ad340d 100644 --- a/drivers/net/ethernet/hisilicon/hns/hnae.c +++ b/drivers/net/ethernet/hisilicon/hns/hnae.c @@ -448,7 +448,7 @@ EXPORT_SYMBOL(hnae_ae_unregister); static int __init hnae_init(void) { - hnae_class = class_create(THIS_MODULE, "hnae"); + hnae_class = class_create("hnae"); return PTR_ERR_OR_ZERO(hnae_class); } diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c index 1d71f5276241..a9beacd552cf 100644 --- a/drivers/net/ppp/ppp_generic.c +++ b/drivers/net/ppp/ppp_generic.c @@ -1394,7 +1394,7 @@ static int __init ppp_init(void) goto out_net; } - ppp_class = class_create(THIS_MODULE, "ppp"); + ppp_class = class_create("ppp"); if (IS_ERR(ppp_class)) { err = PTR_ERR(ppp_class); goto out_chrdev; diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c index 4cc4eaf80b14..68a7e09a195e 100644 --- a/drivers/net/wireless/mac80211_hwsim.c +++ b/drivers/net/wireless/mac80211_hwsim.c @@ -5748,7 +5748,7 @@ static int __init init_mac80211_hwsim(void) if (err) goto out_exit_netlink; - hwsim_class = class_create(THIS_MODULE, "mac80211_hwsim"); + hwsim_class = class_create("mac80211_hwsim"); if (IS_ERR(hwsim_class)) { err = PTR_ERR(hwsim_class); goto out_exit_virtio; diff --git a/drivers/net/wwan/wwan_core.c b/drivers/net/wwan/wwan_core.c index 966d0ccd2276..51bbd6bb74b5 100644 --- a/drivers/net/wwan/wwan_core.c +++ b/drivers/net/wwan/wwan_core.c @@ -1174,7 +1174,7 @@ static int __init wwan_init(void) if (err) return err; - wwan_class = class_create(THIS_MODULE, "wwan"); + wwan_class = class_create("wwan"); if (IS_ERR(wwan_class)) { err = PTR_ERR(wwan_class); goto unregister; diff --git a/drivers/net/wwan/wwan_hwsim.c b/drivers/net/wwan/wwan_hwsim.c index 2397a903d8f5..edb5fc803211 100644 --- a/drivers/net/wwan/wwan_hwsim.c +++ b/drivers/net/wwan/wwan_hwsim.c @@ -511,7 +511,7 @@ static int __init wwan_hwsim_init(void) if (!wwan_wq) return -ENOMEM; - wwan_hwsim_class = class_create(THIS_MODULE, "wwan_hwsim"); + wwan_hwsim_class = class_create("wwan_hwsim"); if (IS_ERR(wwan_hwsim_class)) { err = PTR_ERR(wwan_hwsim_class); goto err_wq_destroy; diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index 4976a0069e9c..954dbc105fc8 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -1320,7 +1320,7 @@ int __init nvdimm_bus_init(void) goto err_dimm_chrdev; nvdimm_major = rc; - nd_class = class_create(THIS_MODULE, "nd"); + nd_class = class_create("nd"); if (IS_ERR(nd_class)) { rc = PTR_ERR(nd_class); goto err_class; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index c2730b116dc6..877a61fbd78b 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -5381,14 +5381,14 @@ static int __init nvme_core_init(void) if (result < 0) goto destroy_delete_wq; - nvme_class = class_create(THIS_MODULE, "nvme"); + nvme_class = class_create("nvme"); if (IS_ERR(nvme_class)) { result = PTR_ERR(nvme_class); goto unregister_chrdev; } nvme_class->dev_uevent = nvme_class_uevent; - nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem"); + nvme_subsys_class = class_create("nvme-subsystem"); if (IS_ERR(nvme_subsys_class)) { result = PTR_ERR(nvme_subsys_class); goto destroy_class; @@ -5399,7 +5399,7 @@ static int __init nvme_core_init(void) if (result < 0) goto destroy_subsys_class; - nvme_ns_chr_class = class_create(THIS_MODULE, "nvme-generic"); + nvme_ns_chr_class = class_create("nvme-generic"); if (IS_ERR(nvme_ns_chr_class)) { result = PTR_ERR(nvme_ns_chr_class); goto unregister_generic_ns; diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index bbaa04a0c502..0069ebff85df 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -1254,7 +1254,7 @@ static int __init nvmf_init(void) if (!nvmf_default_host) return -ENOMEM; - nvmf_class = class_create(THIS_MODULE, "nvme-fabrics"); + nvmf_class = class_create("nvme-fabrics"); if (IS_ERR(nvmf_class)) { pr_err("couldn't register class nvme-fabrics\n"); ret = PTR_ERR(nvmf_class); diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c index 5c16372f3b53..af0d01797e5e 100644 --- a/drivers/nvme/target/fcloop.c +++ b/drivers/nvme/target/fcloop.c @@ -1562,7 +1562,7 @@ static int __init fcloop_init(void) { int ret; - fcloop_class = class_create(THIS_MODULE, "fcloop"); + fcloop_class = class_create("fcloop"); if (IS_ERR(fcloop_class)) { pr_err("couldn't register class fcloop\n"); ret = PTR_ERR(fcloop_class); diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c index 9440d9811eea..46c9a5c3ca14 100644 --- a/drivers/pci/endpoint/pci-epc-core.c +++ b/drivers/pci/endpoint/pci-epc-core.c @@ -860,7 +860,7 @@ EXPORT_SYMBOL_GPL(__devm_pci_epc_create); static int __init pci_epc_init(void) { - pci_epc_class = class_create(THIS_MODULE, "pci_epc"); + pci_epc_class = class_create("pci_epc"); if (IS_ERR(pci_epc_class)) { pr_err("failed to create pci epc class --> %ld\n", PTR_ERR(pci_epc_class)); diff --git a/drivers/pci/switch/switchtec.c b/drivers/pci/switch/switchtec.c index 3d6f17ff2429..d837da055921 100644 --- a/drivers/pci/switch/switchtec.c +++ b/drivers/pci/switch/switchtec.c @@ -1804,7 +1804,7 @@ static int __init switchtec_init(void) if (rc) return rc; - switchtec_class = class_create(THIS_MODULE, "switchtec"); + switchtec_class = class_create("switchtec"); if (IS_ERR(switchtec_class)) { rc = PTR_ERR(switchtec_class); goto err_create_class; diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c index 9951efc03eaa..6464dcb56d56 100644 --- a/drivers/phy/phy-core.c +++ b/drivers/phy/phy-core.c @@ -1233,7 +1233,7 @@ static void phy_release(struct device *dev) static int __init phy_core_init(void) { - phy_class = class_create(THIS_MODULE, "phy"); + phy_class = class_create("phy"); if (IS_ERR(phy_class)) { pr_err("failed to create phy class --> %ld\n", PTR_ERR(phy_class)); diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c index f3d7c1da299f..1bf393239d40 100644 --- a/drivers/power/supply/power_supply_core.c +++ b/drivers/power/supply/power_supply_core.c @@ -1462,7 +1462,7 @@ EXPORT_SYMBOL_GPL(power_supply_get_drvdata); static int __init power_supply_class_init(void) { - power_supply_class = class_create(THIS_MODULE, "power_supply"); + power_supply_class = class_create("power_supply"); if (IS_ERR(power_supply_class)) return PTR_ERR(power_supply_class); diff --git a/drivers/pps/pps.c b/drivers/pps/pps.c index 22a65ad4e46e..5d19baae6a38 100644 --- a/drivers/pps/pps.c +++ b/drivers/pps/pps.c @@ -456,7 +456,7 @@ static int __init pps_init(void) { int err; - pps_class = class_create(THIS_MODULE, "pps"); + pps_class = class_create("pps"); if (IS_ERR(pps_class)) { pr_err("failed to allocate class\n"); return PTR_ERR(pps_class); diff --git a/drivers/ptp/ptp_clock.c b/drivers/ptp/ptp_clock.c index 62d4d29e7c05..790f9250b381 100644 --- a/drivers/ptp/ptp_clock.c +++ b/drivers/ptp/ptp_clock.c @@ -460,7 +460,7 @@ static int __init ptp_init(void) { int err; - ptp_class = class_create(THIS_MODULE, "ptp"); + ptp_class = class_create("ptp"); if (IS_ERR(ptp_class)) { pr_err("ptp: failed to allocate class\n"); return PTR_ERR(ptp_class); diff --git a/drivers/rapidio/devices/rio_mport_cdev.c b/drivers/rapidio/devices/rio_mport_cdev.c index 43db495f1986..deb96c3160a7 100644 --- a/drivers/rapidio/devices/rio_mport_cdev.c +++ b/drivers/rapidio/devices/rio_mport_cdev.c @@ -2603,7 +2603,7 @@ static int __init mport_init(void) int ret; /* Create device class needed by udev */ - dev_class = class_create(THIS_MODULE, DRV_NAME); + dev_class = class_create(DRV_NAME); if (IS_ERR(dev_class)) { rmcd_error("Unable to create " DRV_NAME " class"); return PTR_ERR(dev_class); diff --git a/drivers/rapidio/rio_cm.c b/drivers/rapidio/rio_cm.c index db4c265287ae..acaf9cda014c 100644 --- a/drivers/rapidio/rio_cm.c +++ b/drivers/rapidio/rio_cm.c @@ -2297,7 +2297,7 @@ static int __init riocm_init(void) int ret; /* Create device class needed by udev */ - dev_class = class_create(THIS_MODULE, DRV_NAME); + dev_class = class_create(DRV_NAME); if (IS_ERR(dev_class)) { riocm_error("Cannot create " DRV_NAME " class"); return PTR_ERR(dev_class); diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c index a2207c0cf432..5039df757127 100644 --- a/drivers/rpmsg/rpmsg_core.c +++ b/drivers/rpmsg/rpmsg_core.c @@ -694,7 +694,7 @@ static int __init rpmsg_init(void) { int ret; - rpmsg_class = class_create(THIS_MODULE, "rpmsg"); + rpmsg_class = class_create("rpmsg"); if (IS_ERR(rpmsg_class)) { pr_err("failed to create rpmsg class\n"); return PTR_ERR(rpmsg_class); diff --git a/drivers/rtc/class.c b/drivers/rtc/class.c index e5b7b48cffac..edfd942f8c54 100644 --- a/drivers/rtc/class.c +++ b/drivers/rtc/class.c @@ -475,7 +475,7 @@ EXPORT_SYMBOL_GPL(devm_rtc_device_register); static int __init rtc_init(void) { - rtc_class = class_create(THIS_MODULE, "rtc"); + rtc_class = class_create("rtc"); if (IS_ERR(rtc_class)) { pr_err("couldn't create class\n"); return PTR_ERR(rtc_class); diff --git a/drivers/s390/char/hmcdrv_dev.c b/drivers/s390/char/hmcdrv_dev.c index cb8fdf057eca..8d50c894711f 100644 --- a/drivers/s390/char/hmcdrv_dev.c +++ b/drivers/s390/char/hmcdrv_dev.c @@ -308,7 +308,7 @@ int hmcdrv_dev_init(void) * /proc/devices), but not under /dev nor /sys/devices/virtual. So * we have to create an associated class (see /sys/class). */ - hmcdrv_dev_class = class_create(THIS_MODULE, HMCDRV_DEV_CLASS); + hmcdrv_dev_class = class_create(HMCDRV_DEV_CLASS); if (IS_ERR(hmcdrv_dev_class)) { rc = PTR_ERR(hmcdrv_dev_class); diff --git a/drivers/s390/char/raw3270.c b/drivers/s390/char/raw3270.c index 09d7570d3b7d..7115c0f85650 100644 --- a/drivers/s390/char/raw3270.c +++ b/drivers/s390/char/raw3270.c @@ -1319,7 +1319,7 @@ static int raw3270_init(void) if (rc == 0) { /* Create attributes for early (= console) device. */ mutex_lock(&raw3270_mutex); - class3270 = class_create(THIS_MODULE, "3270"); + class3270 = class_create("3270"); list_for_each_entry(rp, &raw3270_devices, list) { get_device(&rp->cdev->dev); raw3270_create_attributes(rp); diff --git a/drivers/s390/char/tape_class.c b/drivers/s390/char/tape_class.c index c21dc68e05a0..277a0f903d11 100644 --- a/drivers/s390/char/tape_class.c +++ b/drivers/s390/char/tape_class.c @@ -117,7 +117,7 @@ EXPORT_SYMBOL(unregister_tape_dev); static int __init tape_init(void) { - tape_class = class_create(THIS_MODULE, "tape390"); + tape_class = class_create("tape390"); return 0; } diff --git a/drivers/s390/char/vmlogrdr.c b/drivers/s390/char/vmlogrdr.c index ed970ecfafdf..6946ba9a9de2 100644 --- a/drivers/s390/char/vmlogrdr.c +++ b/drivers/s390/char/vmlogrdr.c @@ -699,7 +699,7 @@ static int vmlogrdr_register_driver(void) if (ret) goto out_iucv; - vmlogrdr_class = class_create(THIS_MODULE, "vmlogrdr"); + vmlogrdr_class = class_create("vmlogrdr"); if (IS_ERR(vmlogrdr_class)) { ret = PTR_ERR(vmlogrdr_class); vmlogrdr_class = NULL; diff --git a/drivers/s390/char/vmur.c b/drivers/s390/char/vmur.c index 131293f7f152..82efdd20ad01 100644 --- a/drivers/s390/char/vmur.c +++ b/drivers/s390/char/vmur.c @@ -1022,7 +1022,7 @@ static int __init ur_init(void) debug_set_level(vmur_dbf, 6); - vmur_class = class_create(THIS_MODULE, "vmur"); + vmur_class = class_create("vmur"); if (IS_ERR(vmur_class)) { rc = PTR_ERR(vmur_class); goto fail_free_dbf; diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 6fe05bb82c77..582ac301d315 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -2171,7 +2171,7 @@ static int __init zcdn_init(void) int rc; /* create a new class 'zcrypt' */ - zcrypt_class = class_create(THIS_MODULE, ZCRYPT_NAME); + zcrypt_class = class_create(ZCRYPT_NAME); if (IS_ERR(zcrypt_class)) { rc = PTR_ERR(zcrypt_class); goto out_class_create_failed; diff --git a/drivers/sbus/char/oradax.c b/drivers/sbus/char/oradax.c index e300cf26bc2a..75701b165e91 100644 --- a/drivers/sbus/char/oradax.c +++ b/drivers/sbus/char/oradax.c @@ -323,7 +323,7 @@ static int __init dax_attach(void) goto done; } - cl = class_create(THIS_MODULE, DAX_NAME); + cl = class_create(DAX_NAME); if (IS_ERR(cl)) { dax_err("class_create failed"); ret = PTR_ERR(cl); diff --git a/drivers/scsi/ch.c b/drivers/scsi/ch.c index 72fe6df78bc5..ac648bb8f7e7 100644 --- a/drivers/scsi/ch.c +++ b/drivers/scsi/ch.c @@ -995,7 +995,7 @@ static int __init init_ch_module(void) int rc; printk(KERN_INFO "SCSI Media Changer driver v" VERSION " \n"); - ch_sysfs_class = class_create(THIS_MODULE, "scsi_changer"); + ch_sysfs_class = class_create("scsi_changer"); if (IS_ERR(ch_sysfs_class)) { rc = PTR_ERR(ch_sysfs_class); return rc; diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c index 395b00b942f7..debd36974119 100644 --- a/drivers/scsi/cxlflash/main.c +++ b/drivers/scsi/cxlflash/main.c @@ -3880,7 +3880,7 @@ static int cxlflash_class_init(void) cxlflash_major = MAJOR(devno); - cxlflash_class = class_create(THIS_MODULE, "cxlflash"); + cxlflash_class = class_create("cxlflash"); if (IS_ERR(cxlflash_class)) { rc = PTR_ERR(cxlflash_class); pr_err("%s: class_create failed rc=%d\n", __func__, rc); diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c index 836ddc476764..7d57cc92532c 100644 --- a/drivers/scsi/pmcraid.c +++ b/drivers/scsi/pmcraid.c @@ -5346,7 +5346,7 @@ static int __init pmcraid_init(void) } pmcraid_major = MAJOR(dev); - pmcraid_class = class_create(THIS_MODULE, PMCRAID_DEVFILE); + pmcraid_class = class_create(PMCRAID_DEVFILE); if (IS_ERR(pmcraid_class)) { error = PTR_ERR(pmcraid_class); diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index a91049213203..4997f880d4a4 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -1677,7 +1677,7 @@ init_sg(void) SG_MAX_DEVS, "sg"); if (rc) return rc; - sg_sysfs_class = class_create(THIS_MODULE, "scsi_generic"); + sg_sysfs_class = class_create("scsi_generic"); if ( IS_ERR(sg_sysfs_class) ) { rc = PTR_ERR(sg_sysfs_class); goto err_out; diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index 5a038c667401..539e70c7daed 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -877,7 +877,7 @@ static int __init spidev_init(void) if (status < 0) return status; - spidev_class = class_create(THIS_MODULE, "spidev"); + spidev_class = class_create("spidev"); if (IS_ERR(spidev_class)) { unregister_chrdev(SPIDEV_MAJOR, spidev_spi_driver.driver.name); return PTR_ERR(spidev_class); diff --git a/drivers/staging/fieldbus/anybuss/arcx-anybus.c b/drivers/staging/fieldbus/anybuss/arcx-anybus.c index 9af2e63050d1..369400ec30d8 100644 --- a/drivers/staging/fieldbus/anybuss/arcx-anybus.c +++ b/drivers/staging/fieldbus/anybuss/arcx-anybus.c @@ -352,7 +352,7 @@ static int __init controller_init(void) { int err; - controller_class = class_create(THIS_MODULE, "arcx_anybus_controller"); + controller_class = class_create("arcx_anybus_controller"); if (IS_ERR(controller_class)) return PTR_ERR(controller_class); err = platform_driver_register(&controller_driver); diff --git a/drivers/staging/greybus/authentication.c b/drivers/staging/greybus/authentication.c index 297e69f011c7..7e01790a4659 100644 --- a/drivers/staging/greybus/authentication.c +++ b/drivers/staging/greybus/authentication.c @@ -402,7 +402,7 @@ int cap_init(void) { int ret; - cap_class = class_create(THIS_MODULE, "gb_authenticate"); + cap_class = class_create("gb_authenticate"); if (IS_ERR(cap_class)) return PTR_ERR(cap_class); diff --git a/drivers/staging/greybus/fw-management.c b/drivers/staging/greybus/fw-management.c index 3342b84597da..cd9141e4b794 100644 --- a/drivers/staging/greybus/fw-management.c +++ b/drivers/staging/greybus/fw-management.c @@ -696,7 +696,7 @@ int fw_mgmt_init(void) { int ret; - fw_mgmt_class = class_create(THIS_MODULE, "gb_fw_mgmt"); + fw_mgmt_class = class_create("gb_fw_mgmt"); if (IS_ERR(fw_mgmt_class)) return PTR_ERR(fw_mgmt_class); diff --git a/drivers/staging/greybus/raw.c b/drivers/staging/greybus/raw.c index 2a375f407d38..8bca8cb12cc6 100644 --- a/drivers/staging/greybus/raw.c +++ b/drivers/staging/greybus/raw.c @@ -340,7 +340,7 @@ static int raw_init(void) dev_t dev; int retval; - raw_class = class_create(THIS_MODULE, "gb_raw"); + raw_class = class_create("gb_raw"); if (IS_ERR(raw_class)) { retval = PTR_ERR(raw_class); goto error_class; diff --git a/drivers/staging/pi433/pi433_if.c b/drivers/staging/pi433/pi433_if.c index b59f6a4cb611..f08fdf06d566 100644 --- a/drivers/staging/pi433/pi433_if.c +++ b/drivers/staging/pi433/pi433_if.c @@ -1400,7 +1400,7 @@ static int __init pi433_init(void) if (status < 0) return status; - pi433_class = class_create(THIS_MODULE, "pi433"); + pi433_class = class_create("pi433"); if (IS_ERR(pi433_class)) { unregister_chrdev(MAJOR(pi433_dev), pi433_spi_driver.driver.name); diff --git a/drivers/staging/vme_user/vme_user.c b/drivers/staging/vme_user/vme_user.c index 4e533c0bfe6d..b9367b575d00 100644 --- a/drivers/staging/vme_user/vme_user.c +++ b/drivers/staging/vme_user/vme_user.c @@ -614,7 +614,7 @@ static int vme_user_probe(struct vme_dev *vdev) } /* Create sysfs entries - on udev systems this creates the dev files */ - vme_user_sysfs_class = class_create(THIS_MODULE, driver_name); + vme_user_sysfs_class = class_create(driver_name); if (IS_ERR(vme_user_sysfs_class)) { dev_err(&vdev->dev, "Error creating vme_user class.\n"); err = PTR_ERR(vme_user_sysfs_class); diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index 452cbb8ad484..0eb342de0b00 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -1226,7 +1226,7 @@ static int __init tee_init(void) { int rc; - tee_class = class_create(THIS_MODULE, "tee"); + tee_class = class_create("tee"); if (IS_ERR(tee_class)) { pr_err("couldn't create class\n"); return PTR_ERR(tee_class); diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 36fb945fdad4..1382d9050ce8 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -3512,7 +3512,7 @@ static char *tty_devnode(const struct device *dev, umode_t *mode) static int __init tty_class_init(void) { - tty_class = class_create(THIS_MODULE, "tty"); + tty_class = class_create("tty"); if (IS_ERR(tty_class)) return PTR_ERR(tty_class); tty_class->devnode = tty_devnode; diff --git a/drivers/tty/vt/vc_screen.c b/drivers/tty/vt/vc_screen.c index 1dc07f9214d5..498ba9c0ee93 100644 --- a/drivers/tty/vt/vc_screen.c +++ b/drivers/tty/vt/vc_screen.c @@ -804,7 +804,7 @@ int __init vcs_init(void) if (register_chrdev(VCS_MAJOR, "vcs", &vcs_fops)) panic("unable to get major %d for vcs device", VCS_MAJOR); - vc_class = class_create(THIS_MODULE, "vc"); + vc_class = class_create("vc"); device_create(vc_class, NULL, MKDEV(VCS_MAJOR, 0), NULL, "vcs"); device_create(vc_class, NULL, MKDEV(VCS_MAJOR, 64), NULL, "vcsu"); diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 57a5c23b51d4..5496bf4b76ec 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -4241,7 +4241,7 @@ static int __init vtconsole_class_init(void) { int i; - vtconsole_class = class_create(THIS_MODULE, "vtconsole"); + vtconsole_class = class_create("vtconsole"); if (IS_ERR(vtconsole_class)) { pr_warn("Unable to create vt console class; errno = %ld\n", PTR_ERR(vtconsole_class)); diff --git a/drivers/usb/core/file.c b/drivers/usb/core/file.c index da7d88e069e6..c4ed3310e069 100644 --- a/drivers/usb/core/file.c +++ b/drivers/usb/core/file.c @@ -88,7 +88,7 @@ static int init_usb_class(void) } kref_init(&usb_class->kref); - usb_class->class = class_create(THIS_MODULE, "usbmisc"); + usb_class->class = class_create("usbmisc"); if (IS_ERR(usb_class->class)) { result = PTR_ERR(usb_class->class); printk(KERN_ERR "class_create failed for usb devices\n"); diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c index a8da3b4a2855..9f6b10134121 100644 --- a/drivers/usb/gadget/function/f_hid.c +++ b/drivers/usb/gadget/function/f_hid.c @@ -1325,7 +1325,7 @@ int ghid_setup(struct usb_gadget *g, int count) int status; dev_t dev; - hidg_class = class_create(THIS_MODULE, "hidg"); + hidg_class = class_create("hidg"); if (IS_ERR(hidg_class)) { status = PTR_ERR(hidg_class); hidg_class = NULL; diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c index 4903d761a872..28db3e336e7d 100644 --- a/drivers/usb/gadget/function/f_printer.c +++ b/drivers/usb/gadget/function/f_printer.c @@ -1512,7 +1512,7 @@ static int gprinter_setup(int count) int status; dev_t devt; - usb_gadget_class = class_create(THIS_MODULE, "usb_printer_gadget"); + usb_gadget_class = class_create("usb_printer_gadget"); if (IS_ERR(usb_gadget_class)) { status = PTR_ERR(usb_gadget_class); usb_gadget_class = NULL; diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index 23b0629a8774..df930b041c33 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -1758,7 +1758,7 @@ static int __init usb_udc_init(void) { int rc; - udc_class = class_create(THIS_MODULE, "udc"); + udc_class = class_create("udc"); if (IS_ERR(udc_class)) { pr_err("failed to create udc class --> %ld\n", PTR_ERR(udc_class)); diff --git a/drivers/usb/mon/mon_bin.c b/drivers/usb/mon/mon_bin.c index abb1cd35d8a6..952c56789258 100644 --- a/drivers/usb/mon/mon_bin.c +++ b/drivers/usb/mon/mon_bin.c @@ -1379,7 +1379,7 @@ int __init mon_bin_init(void) { int rc; - mon_bin_class = class_create(THIS_MODULE, "usbmon"); + mon_bin_class = class_create("usbmon"); if (IS_ERR(mon_bin_class)) { rc = PTR_ERR(mon_bin_class); goto err_class; diff --git a/drivers/usb/roles/class.c b/drivers/usb/roles/class.c index 56814ef80c24..0395bd5dbd3e 100644 --- a/drivers/usb/roles/class.c +++ b/drivers/usb/roles/class.c @@ -392,7 +392,7 @@ EXPORT_SYMBOL_GPL(usb_role_switch_get_drvdata); static int __init usb_roles_init(void) { - role_class = class_create(THIS_MODULE, "usb_role"); + role_class = class_create("usb_role"); return PTR_ERR_OR_ZERO(role_class); } subsys_initcall(usb_roles_init); diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index 0c3b48616a9f..c421b83f6fa2 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -1793,7 +1793,7 @@ static int vduse_init(void) int ret; struct device *dev; - vduse_class = class_create(THIS_MODULE, "vduse"); + vduse_class = class_create("vduse"); if (IS_ERR(vduse_class)) return PTR_ERR(vduse_class); diff --git a/drivers/vfio/group.c b/drivers/vfio/group.c index 27d5ba7cf9dc..fc75c1000d74 100644 --- a/drivers/vfio/group.c +++ b/drivers/vfio/group.c @@ -878,7 +878,7 @@ int __init vfio_group_init(void) return ret; /* /dev/vfio/$GROUP */ - vfio.class = class_create(THIS_MODULE, "vfio"); + vfio.class = class_create("vfio"); if (IS_ERR(vfio.class)) { ret = PTR_ERR(vfio.class); goto err_group_class; diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c index 43bd6b76e2b6..9ae671b98592 100644 --- a/drivers/vfio/vfio_main.c +++ b/drivers/vfio/vfio_main.c @@ -1408,7 +1408,7 @@ static int __init vfio_init(void) goto err_virqfd; /* /sys/class/vfio-dev/vfioX */ - vfio.device_class = class_create(THIS_MODULE, "vfio-dev"); + vfio.device_class = class_create("vfio-dev"); if (IS_ERR(vfio.device_class)) { ret = PTR_ERR(vfio.device_class); goto err_dev_class; diff --git a/drivers/video/backlight/backlight.c b/drivers/video/backlight/backlight.c index 6eea72aa8dbf..9a885d398c22 100644 --- a/drivers/video/backlight/backlight.c +++ b/drivers/video/backlight/backlight.c @@ -751,7 +751,7 @@ static void __exit backlight_class_exit(void) static int __init backlight_class_init(void) { - backlight_class = class_create(THIS_MODULE, "backlight"); + backlight_class = class_create("backlight"); if (IS_ERR(backlight_class)) { pr_warn("Unable to create backlight class; errno = %ld\n", PTR_ERR(backlight_class)); diff --git a/drivers/video/backlight/lcd.c b/drivers/video/backlight/lcd.c index db56e465aaff..77c5cb2a44e2 100644 --- a/drivers/video/backlight/lcd.c +++ b/drivers/video/backlight/lcd.c @@ -323,7 +323,7 @@ static void __exit lcd_class_exit(void) static int __init lcd_class_init(void) { - lcd_class = class_create(THIS_MODULE, "lcd"); + lcd_class = class_create("lcd"); if (IS_ERR(lcd_class)) { pr_warn("Unable to create backlight class; errno = %ld\n", PTR_ERR(lcd_class)); diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c index 875541ff185b..b073795d99d0 100644 --- a/drivers/video/fbdev/core/fbmem.c +++ b/drivers/video/fbdev/core/fbmem.c @@ -1749,7 +1749,7 @@ fbmem_init(void) goto err_chrdev; } - fb_class = class_create(THIS_MODULE, "graphics"); + fb_class = class_create("graphics"); if (IS_ERR(fb_class)) { ret = PTR_ERR(fb_class); pr_warn("Unable to create fb class; errno = %d\n", ret); diff --git a/fs/coda/psdev.c b/fs/coda/psdev.c index b39580ad4ce5..3c3148588491 100644 --- a/fs/coda/psdev.c +++ b/fs/coda/psdev.c @@ -361,7 +361,7 @@ static int __init init_coda_psdev(void) __func__, CODA_PSDEV_MAJOR); return -EIO; } - coda_psdev_class = class_create(THIS_MODULE, "coda"); + coda_psdev_class = class_create("coda"); if (IS_ERR(coda_psdev_class)) { err = PTR_ERR(coda_psdev_class); goto out_chrdev; diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 179a5c5e28fd..91e89e68177e 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -623,7 +623,7 @@ static int __init cuse_init(void) /* CUSE is not prepared for FUSE_DEV_IOC_CLONE */ cuse_channel_fops.unlocked_ioctl = NULL; - cuse_class = class_create(THIS_MODULE, "cuse"); + cuse_class = class_create("cuse"); if (IS_ERR(cuse_class)) return PTR_ERR(cuse_class); diff --git a/fs/pstore/pmsg.c b/fs/pstore/pmsg.c index ab82e5f05346..aa178304bc03 100644 --- a/fs/pstore/pmsg.c +++ b/fs/pstore/pmsg.c @@ -64,7 +64,7 @@ void pstore_register_pmsg(void) goto err; } - pmsg_class = class_create(THIS_MODULE, PMSG_NAME); + pmsg_class = class_create(PMSG_NAME); if (IS_ERR(pmsg_class)) { pr_err("device class file already in use\n"); goto err_class; diff --git a/include/linux/device/class.h b/include/linux/device/class.h index bf736f14f0c1..cda598fc5fa0 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -256,7 +256,6 @@ extern void class_destroy(struct class *cls); /** * class_create - create a struct class structure - * @owner: dummy pointer, does nothing, will be removed soon. * @name: pointer to a string for the name of this class. * * This is used to create a struct class pointer that can then be used @@ -267,7 +266,7 @@ extern void class_destroy(struct class *cls); * Note, the pointer created here is to be destroyed when finished by * making a call to class_destroy(). */ -#define class_create(owner, name) \ +#define class_create(name) \ ({ \ static struct lock_class_key __key; \ __class_create(name, &__key); \ diff --git a/mm/backing-dev.c b/mm/backing-dev.c index a53b9360b72e..ad011308cebe 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -347,7 +347,7 @@ ATTRIBUTE_GROUPS(bdi_dev); static __init int bdi_class_init(void) { - bdi_class = class_create(THIS_MODULE, "bdi"); + bdi_class = class_create("bdi"); if (IS_ERR(bdi_class)) return PTR_ERR(bdi_class); diff --git a/net/bluetooth/hci_sysfs.c b/net/bluetooth/hci_sysfs.c index 08542dfc2dc5..2934d7f4d564 100644 --- a/net/bluetooth/hci_sysfs.c +++ b/net/bluetooth/hci_sysfs.c @@ -112,7 +112,7 @@ void hci_init_sysfs(struct hci_dev *hdev) int __init bt_sysfs_init(void) { - bt_class = class_create(THIS_MODULE, "bluetooth"); + bt_class = class_create("bluetooth"); return PTR_ERR_OR_ZERO(bt_class); } diff --git a/net/netfilter/xt_IDLETIMER.c b/net/netfilter/xt_IDLETIMER.c index 8d36303f3935..db720efa811d 100644 --- a/net/netfilter/xt_IDLETIMER.c +++ b/net/netfilter/xt_IDLETIMER.c @@ -490,7 +490,7 @@ static int __init idletimer_tg_init(void) { int err; - idletimer_tg_class = class_create(THIS_MODULE, "xt_idletimer"); + idletimer_tg_class = class_create("xt_idletimer"); err = PTR_ERR(idletimer_tg_class); if (IS_ERR(idletimer_tg_class)) { pr_debug("couldn't register device class\n"); diff --git a/samples/vfio-mdev/mbochs.c b/samples/vfio-mdev/mbochs.c index e54eb752e1ba..a3d3249639d0 100644 --- a/samples/vfio-mdev/mbochs.c +++ b/samples/vfio-mdev/mbochs.c @@ -1418,7 +1418,7 @@ static int __init mbochs_dev_init(void) if (ret) goto err_cdev; - mbochs_class = class_create(THIS_MODULE, MBOCHS_CLASS_NAME); + mbochs_class = class_create(MBOCHS_CLASS_NAME); if (IS_ERR(mbochs_class)) { pr_err("Error: failed to register mbochs_dev class\n"); ret = PTR_ERR(mbochs_class); diff --git a/samples/vfio-mdev/mdpy.c b/samples/vfio-mdev/mdpy.c index e8400fdab71d..ef1630fc5a91 100644 --- a/samples/vfio-mdev/mdpy.c +++ b/samples/vfio-mdev/mdpy.c @@ -705,7 +705,7 @@ static int __init mdpy_dev_init(void) if (ret) goto err_cdev; - mdpy_class = class_create(THIS_MODULE, MDPY_CLASS_NAME); + mdpy_class = class_create(MDPY_CLASS_NAME); if (IS_ERR(mdpy_class)) { pr_err("Error: failed to register mdpy_dev class\n"); ret = PTR_ERR(mdpy_class); diff --git a/samples/vfio-mdev/mtty.c b/samples/vfio-mdev/mtty.c index e887de672c52..0b6c386e620d 100644 --- a/samples/vfio-mdev/mtty.c +++ b/samples/vfio-mdev/mtty.c @@ -1316,7 +1316,7 @@ static int __init mtty_dev_init(void) if (ret) goto err_cdev; - mtty_dev.vd_class = class_create(THIS_MODULE, MTTY_CLASS_NAME); + mtty_dev.vd_class = class_create(MTTY_CLASS_NAME); if (IS_ERR(mtty_dev.vd_class)) { pr_err("Error: failed to register mtty_dev class\n"); diff --git a/sound/sound_core.c b/sound/sound_core.c index 3e7dd6fcb7cf..4f6911274d56 100644 --- a/sound/sound_core.c +++ b/sound/sound_core.c @@ -45,7 +45,7 @@ static int __init init_soundcore(void) if (rc) return rc; - sound_class = class_create(THIS_MODULE, "sound"); + sound_class = class_create("sound"); if (IS_ERR(sound_class)) { cleanup_oss_soundcore(); return PTR_ERR(sound_class); diff --git a/tools/testing/nvdimm/test/ndtest.c b/tools/testing/nvdimm/test/ndtest.c index 01ceb98c15a0..3eba10c1e3e8 100644 --- a/tools/testing/nvdimm/test/ndtest.c +++ b/tools/testing/nvdimm/test/ndtest.c @@ -921,7 +921,7 @@ static __init int ndtest_init(void) nfit_test_setup(ndtest_resource_lookup, NULL); - ndtest_dimm_class = class_create(THIS_MODULE, "nfit_test_dimm"); + ndtest_dimm_class = class_create("nfit_test_dimm"); if (IS_ERR(ndtest_dimm_class)) { rc = PTR_ERR(ndtest_dimm_class); goto err_register; diff --git a/tools/testing/nvdimm/test/nfit.c b/tools/testing/nvdimm/test/nfit.c index c75abb497a1a..45b1c072e349 100644 --- a/tools/testing/nvdimm/test/nfit.c +++ b/tools/testing/nvdimm/test/nfit.c @@ -3282,7 +3282,7 @@ static __init int nfit_test_init(void) if (!nfit_wq) return -ENOMEM; - nfit_test_dimm = class_create(THIS_MODULE, "nfit_test_dimm"); + nfit_test_dimm = class_create("nfit_test_dimm"); if (IS_ERR(nfit_test_dimm)) { rc = PTR_ERR(nfit_test_dimm); goto err_register; -- cgit v1.2.3 From a2fd6e42e4fb661e183977072022b7dd8dd65d90 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:18:36 +0100 Subject: driver core: class: make class_dev_iter_init() options const class_dev_iter_init() does not modify the struct class or the struct device passed into it, so mark them as const * to enforce that. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313181843.1207845-5-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 4 ++-- include/linux/device/class.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index 9439c6c7466f..5a60e8895165 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -284,8 +284,8 @@ EXPORT_SYMBOL_GPL(class_destroy); * otherwise if it is NULL, the iteration starts at the beginning of * the list. */ -void class_dev_iter_init(struct class_dev_iter *iter, struct class *class, - struct device *start, const struct device_type *type) +void class_dev_iter_init(struct class_dev_iter *iter, const struct class *class, + const struct device *start, const struct device_type *type) { struct klist_node *start_knode = NULL; diff --git a/include/linux/device/class.h b/include/linux/device/class.h index cda598fc5fa0..1f5cfae8db88 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -103,8 +103,8 @@ void class_compat_remove_link(struct class_compat *cls, struct device *dev, struct device *device_link); extern void class_dev_iter_init(struct class_dev_iter *iter, - struct class *class, - struct device *start, + const struct class *class, + const struct device *start, const struct device_type *type); extern struct device *class_dev_iter_next(struct class_dev_iter *iter); extern void class_dev_iter_exit(struct class_dev_iter *iter); -- cgit v1.2.3 From 69df024ebbf8b1d4d8f0808b5700688fed9e45e9 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:18:37 +0100 Subject: driver core: class: make class_for_each_device() options const class_for_each_device() does not modify the struct class or the struct device passed into it, so mark them as const * to enforce that. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313181843.1207845-6-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 2 +- include/linux/device/class.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index 5a60e8895165..4937d660c571 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -355,7 +355,7 @@ EXPORT_SYMBOL_GPL(class_dev_iter_exit); * @fn is allowed to do anything including calling back into class * code. There's no locking restriction. */ -int class_for_each_device(struct class *class, struct device *start, +int class_for_each_device(const struct class *class, const struct device *start, void *data, int (*fn)(struct device *, void *)) { struct class_dev_iter iter; diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 1f5cfae8db88..fdbcd487e508 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -109,7 +109,7 @@ extern void class_dev_iter_init(struct class_dev_iter *iter, extern struct device *class_dev_iter_next(struct class_dev_iter *iter); extern void class_dev_iter_exit(struct class_dev_iter *iter); -extern int class_for_each_device(struct class *class, struct device *start, +extern int class_for_each_device(const struct class *class, const struct device *start, void *data, int (*fn)(struct device *dev, void *data)); extern struct device *class_find_device(struct class *class, -- cgit v1.2.3 From cf41015ea8d3f2ae451d9b7e39ef42569caeb875 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:18:38 +0100 Subject: driver core: class: make class_find_device*() options const The class_find_device*() functions do not modify the struct class or the struct device passed into it, so mark them as const * to enforce that. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313181843.1207845-7-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 2 +- include/linux/device/class.h | 25 ++++++++++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index 4937d660c571..52ba0187e66d 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -402,7 +402,7 @@ EXPORT_SYMBOL_GPL(class_for_each_device); * @match is allowed to do anything including calling back into class * code. There's no locking restriction. */ -struct device *class_find_device(struct class *class, struct device *start, +struct device *class_find_device(const struct class *class, const struct device *start, const void *data, int (*match)(struct device *, const void *)) { diff --git a/include/linux/device/class.h b/include/linux/device/class.h index fdbcd487e508..dfa8958105e7 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -112,8 +112,8 @@ extern void class_dev_iter_exit(struct class_dev_iter *iter); extern int class_for_each_device(const struct class *class, const struct device *start, void *data, int (*fn)(struct device *dev, void *data)); -extern struct device *class_find_device(struct class *class, - struct device *start, const void *data, +extern struct device *class_find_device(const struct class *class, + const struct device *start, const void *data, int (*match)(struct device *, const void *)); /** @@ -122,7 +122,7 @@ extern struct device *class_find_device(struct class *class, * @class: class type * @name: name of the device to match */ -static inline struct device *class_find_device_by_name(struct class *class, +static inline struct device *class_find_device_by_name(const struct class *class, const char *name) { return class_find_device(class, NULL, name, device_match_name); @@ -134,8 +134,8 @@ static inline struct device *class_find_device_by_name(struct class *class, * @class: class type * @np: of_node of the device to match. */ -static inline struct device * -class_find_device_by_of_node(struct class *class, const struct device_node *np) +static inline struct device *class_find_device_by_of_node(const struct class *class, + const struct device_node *np) { return class_find_device(class, NULL, np, device_match_of_node); } @@ -146,9 +146,8 @@ class_find_device_by_of_node(struct class *class, const struct device_node *np) * @class: class type * @fwnode: fwnode of the device to match. */ -static inline struct device * -class_find_device_by_fwnode(struct class *class, - const struct fwnode_handle *fwnode) +static inline struct device *class_find_device_by_fwnode(const struct class *class, + const struct fwnode_handle *fwnode) { return class_find_device(class, NULL, fwnode, device_match_fwnode); } @@ -159,7 +158,7 @@ class_find_device_by_fwnode(struct class *class, * @class: class type * @devt: device type of the device to match. */ -static inline struct device *class_find_device_by_devt(struct class *class, +static inline struct device *class_find_device_by_devt(const struct class *class, dev_t devt) { return class_find_device(class, NULL, &devt, device_match_devt); @@ -173,14 +172,14 @@ struct acpi_device; * @class: class type * @adev: ACPI_COMPANION device to match. */ -static inline struct device * -class_find_device_by_acpi_dev(struct class *class, const struct acpi_device *adev) +static inline struct device *class_find_device_by_acpi_dev(const struct class *class, + const struct acpi_device *adev) { return class_find_device(class, NULL, adev, device_match_acpi_dev); } #else -static inline struct device * -class_find_device_by_acpi_dev(struct class *class, const void *adev) +static inline struct device *class_find_device_by_acpi_dev(const struct class *class, + const void *adev) { return NULL; } -- cgit v1.2.3 From 80842a92907bd68454591f9c7a73778f130ac38f Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:18:39 +0100 Subject: driver core: class: make class_create/remove_file*() options const The class_create_file*() and class_remove_file*() functions do not modify the struct class at all, so mark them as const * to enforce that. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313181843.1207845-8-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 4 ++-- include/linux/device/class.h | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index 52ba0187e66d..3d65221b0dcb 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -87,7 +87,7 @@ static const struct kobj_type class_ktype = { static struct kset *class_kset; -int class_create_file_ns(struct class *cls, const struct class_attribute *attr, +int class_create_file_ns(const struct class *cls, const struct class_attribute *attr, const void *ns) { int error; @@ -101,7 +101,7 @@ int class_create_file_ns(struct class *cls, const struct class_attribute *attr, } EXPORT_SYMBOL_GPL(class_create_file_ns); -void class_remove_file_ns(struct class *cls, const struct class_attribute *attr, +void class_remove_file_ns(const struct class *cls, const struct class_attribute *attr, const void *ns) { if (cls) diff --git a/include/linux/device/class.h b/include/linux/device/class.h index dfa8958105e7..75c1451fcc63 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -200,20 +200,20 @@ struct class_attribute { #define CLASS_ATTR_WO(_name) \ struct class_attribute class_attr_##_name = __ATTR_WO(_name) -extern int __must_check class_create_file_ns(struct class *class, +extern int __must_check class_create_file_ns(const struct class *class, const struct class_attribute *attr, const void *ns); -extern void class_remove_file_ns(struct class *class, +extern void class_remove_file_ns(const struct class *class, const struct class_attribute *attr, const void *ns); -static inline int __must_check class_create_file(struct class *class, - const struct class_attribute *attr) +static inline int __must_check class_create_file(const struct class *class, + const struct class_attribute *attr) { return class_create_file_ns(class, attr, NULL); } -static inline void class_remove_file(struct class *class, +static inline void class_remove_file(const struct class *class, const struct class_attribute *attr) { return class_remove_file_ns(class, attr, NULL); -- cgit v1.2.3 From d2fff09656369d5465585ad2c72d8ba8ada758b5 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:18:40 +0100 Subject: driver core: device: make device_destroy() take a const class * device_destroy() does not modify the struct class passed into it, so mark it as const to enforce this rule. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313181843.1207845-9-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/core.c | 2 +- include/linux/device.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index 57076837b33e..f3b7040ef9b6 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -4383,7 +4383,7 @@ EXPORT_SYMBOL_GPL(device_create_with_groups); * This call unregisters and cleans up a device that was created with a * call to device_create(). */ -void device_destroy(struct class *class, dev_t devt) +void device_destroy(const struct class *class, dev_t devt) { struct device *dev; diff --git a/include/linux/device.h b/include/linux/device.h index 12dc08aa5c0f..2562451b875b 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1021,7 +1021,7 @@ __printf(6, 7) struct device * device_create_with_groups(struct class *cls, struct device *parent, dev_t devt, void *drvdata, const struct attribute_group **groups, const char *fmt, ...); -void device_destroy(struct class *cls, dev_t devt); +void device_destroy(const struct class *cls, dev_t devt); int __must_check device_add_groups(struct device *dev, const struct attribute_group **groups); -- cgit v1.2.3 From 9fa120fbd507e58af3494b8765427fdc8181e1eb Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:18:42 +0100 Subject: driver core: device: mark struct class in struct device as constant The pointer to a struct class in a struct device should never be used to change anything in that class. So mark it as constant to enforce this requirement. This requires a few minor changes to some internal driver core functions to enforce the const * being used here now. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313181843.1207845-11-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/core.c | 10 +++++----- include/linux/device.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index f3b7040ef9b6..0970db630839 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -2820,7 +2820,7 @@ EXPORT_SYMBOL_GPL(devm_device_add_groups); static int device_add_attrs(struct device *dev) { - struct class *class = dev->class; + const struct class *class = dev->class; const struct device_type *type = dev->type; int error; @@ -2887,7 +2887,7 @@ static int device_add_attrs(struct device *dev) static void device_remove_attrs(struct device *dev) { - struct class *class = dev->class; + const struct class *class = dev->class; const struct device_type *type = dev->type; if (dev->physical_location) { @@ -3120,7 +3120,7 @@ struct kobject *virtual_device_parent(struct device *dev) struct class_dir { struct kobject kobj; - struct class *class; + const struct class *class; }; #define to_class_dir(obj) container_of(obj, struct class_dir, kobj) @@ -3145,7 +3145,7 @@ static const struct kobj_type class_dir_ktype = { }; static struct kobject * -class_dir_create_and_add(struct class *class, struct kobject *parent_kobj) +class_dir_create_and_add(const struct class *class, struct kobject *parent_kobj) { struct class_dir *dir; int retval; @@ -4580,7 +4580,7 @@ static int device_attrs_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid) { struct kobject *kobj = &dev->kobj; - struct class *class = dev->class; + const struct class *class = dev->class; const struct device_type *type = dev->type; int error; diff --git a/include/linux/device.h b/include/linux/device.h index 2562451b875b..ea07b95b06e5 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -631,7 +631,7 @@ struct device { spinlock_t devres_lock; struct list_head devres_head; - struct class *class; + const struct class *class; const struct attribute_group **groups; /* optional groups */ void (*release)(struct device *dev); -- cgit v1.2.3 From 2bd5c63978b771462da10d8771f56a1e656501d9 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:18:43 +0100 Subject: driver core: device: make device_create*() take a const struct class * The functions device_create() and device_create_with_groups() do not modify the struct class passed into it, so enforce this by changing the function parameters to be struct const class. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313181843.1207845-12-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/core.c | 6 +++--- include/linux/device.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index 0970db630839..f1889b9cab45 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -4253,7 +4253,7 @@ static void device_create_release(struct device *dev) } static __printf(6, 0) struct device * -device_create_groups_vargs(struct class *class, struct device *parent, +device_create_groups_vargs(const struct class *class, struct device *parent, dev_t devt, void *drvdata, const struct attribute_group **groups, const char *fmt, va_list args) @@ -4317,7 +4317,7 @@ error: * Note: the struct class passed to this function must have previously * been created with a call to class_create(). */ -struct device *device_create(struct class *class, struct device *parent, +struct device *device_create(const struct class *class, struct device *parent, dev_t devt, void *drvdata, const char *fmt, ...) { va_list vargs; @@ -4358,7 +4358,7 @@ EXPORT_SYMBOL_GPL(device_create); * Note: the struct class passed to this function must have previously * been created with a call to class_create(). */ -struct device *device_create_with_groups(struct class *class, +struct device *device_create_with_groups(const struct class *class, struct device *parent, dev_t devt, void *drvdata, const struct attribute_group **groups, diff --git a/include/linux/device.h b/include/linux/device.h index ea07b95b06e5..862397ae520d 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1015,10 +1015,10 @@ bool device_is_bound(struct device *dev); * Easy functions for dynamically creating devices on the fly */ __printf(5, 6) struct device * -device_create(struct class *cls, struct device *parent, dev_t devt, +device_create(const struct class *cls, struct device *parent, dev_t devt, void *drvdata, const char *fmt, ...); __printf(6, 7) struct device * -device_create_with_groups(struct class *cls, struct device *parent, dev_t devt, +device_create_with_groups(const struct class *cls, struct device *parent, dev_t devt, void *drvdata, const struct attribute_group **groups, const char *fmt, ...); void device_destroy(const struct class *cls, dev_t devt); -- cgit v1.2.3 From 7ce93729091dff24e6a1c3578b58b36f4b1cf10a Mon Sep 17 00:00:00 2001 From: Jason Baron Date: Fri, 10 Mar 2023 16:27:28 -0500 Subject: dyndbg: cleanup dynamic usage in ib_srp.c Currently, in dynamic_debug.h we only provide DEFINE_DYNAMIC_DEBUG_METADATA() and DYNAMIC_DEBUG_BRANCH() definitions if CONFIG_DYNAMIC_CORE is enabled. Thus, drivers such as infiniband srp (see: drivers/infiniband/ulp/srp/ib_srp.c) must provide their own definitions for !CONFIG_DYNAMIC_CORE. Thus, let's move this !CONFIG_DYNAMIC_CORE case into dynamic_debug.h. However, the dynamic debug interfaces should really only be defined if CONFIG_DYNAMIC_DEBUG is set or CONFIG_DYNAMIC_CORE is set along with DYNAMIC_DEBUG_MODULE, (see: Documentation/admin-guide/dynamic-debug-howto.rst). Thus, the undefined case becomes: !((CONFIG_DYNAMIC_DEBUG || (CONFIG_DYNAMIC_CORE && DYNAMIC_DEBUG_MODULE)). With those changes in place, we can remove the !CONFIG_DYNAMIC_CORE case from ib_srp.c This change was prompted by a build breakeage in ib_srp.c stemming from the inclusion of dynamic_debug.h unconditionally in module.h, due to commit 7deabd674988 ("dyndbg: use the module notifier callbacks"). In that case, if we have CONFIG_DYNAMIC_CORE=y and CONFIG_DYNAMIC_DEBUG=n then the definitions for DEFINE_DYNAMIC_DEBUG_METADATA() and DYNAMIC_DEBUG_BRANCH() are defined once in ib_srp.c and then again in the dynamic_debug.h. This had been working prior to the above referenced commit because dynamic_debug.h was only pulled into ib_srp.c conditinally via printk.h if CONFIG_DYNAMIC_DEBUG was set. Also, the exported functions in lib/dynamic_debug.c itself may not have a prototype if CONFIG_DYNAMIC_DEBUG=n and CONFIG_DYNAMIC_CORE=y. This would trigger the -Wmissing-prototypes warning. The exported functions are behind (include/linux/dynamic_debug.h): if defined(CONFIG_DYNAMIC_DEBUG) || \ (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) Thus, by adding -DDYNAMIC_CONFIG_MODULE to the lib/Makefile we can ensure that the exported functions have a prototype in all cases, since lib/dynamic_debug.c is built whenever CONFIG_DYNAMIC_DEBUG_CORE=y. Fixes: 7deabd674988 ("dyndbg: use the module notifier callbacks") Reported-by: kernel test robot Link: https://lore.kernel.org/oe-kbuild-all/202303071444.sIbZTDCy-lkp@intel.com/ Signed-off-by: Jason Baron [mcgrof: adjust commit log, and remove urldefense from URL] Signed-off-by: Luis Chamberlain --- drivers/infiniband/ulp/srp/ib_srp.c | 5 ---- include/linux/dynamic_debug.h | 59 +++++++++++++++++++++++-------------- lib/Makefile | 3 ++ 3 files changed, 40 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index df21b30b7735..40ba2c6927c6 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -62,11 +62,6 @@ MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator"); MODULE_LICENSE("Dual BSD/GPL"); -#if !defined(CONFIG_DYNAMIC_DEBUG) -#define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt) -#define DYNAMIC_DEBUG_BRANCH(descriptor) false -#endif - static unsigned int srp_sg_tablesize; static unsigned int cmd_sg_entries; static unsigned int indirect_sg_entries; diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h index f401414341fb..061dd84d09f3 100644 --- a/include/linux/dynamic_debug.h +++ b/include/linux/dynamic_debug.h @@ -128,14 +128,16 @@ struct ddebug_class_param { const struct ddebug_class_map *map; }; -#if defined(CONFIG_DYNAMIC_DEBUG_CORE) +/* + * pr_debug() and friends are globally enabled or modules have selectively + * enabled them. + */ +#if defined(CONFIG_DYNAMIC_DEBUG) || \ + (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) extern __printf(2, 3) void __dynamic_pr_debug(struct _ddebug *descriptor, const char *fmt, ...); -extern int ddebug_dyndbg_module_param_cb(char *param, char *val, - const char *modname); - struct device; extern __printf(3, 4) @@ -284,10 +286,6 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor, KERN_DEBUG, prefix_str, prefix_type, \ rowsize, groupsize, buf, len, ascii) -struct kernel_param; -int param_set_dyndbg_classes(const char *instr, const struct kernel_param *kp); -int param_get_dyndbg_classes(char *buffer, const struct kernel_param *kp); - /* for test only, generally expect drm.debug style macro wrappers */ #define __pr_debug_cls(cls, fmt, ...) do { \ BUILD_BUG_ON_MSG(!__builtin_constant_p(cls), \ @@ -295,23 +293,14 @@ int param_get_dyndbg_classes(char *buffer, const struct kernel_param *kp); dynamic_pr_debug_cls(cls, fmt, ##__VA_ARGS__); \ } while (0) -#else /* !CONFIG_DYNAMIC_DEBUG_CORE */ +#else /* !(CONFIG_DYNAMIC_DEBUG || (CONFIG_DYNAMIC_DEBUG_CORE && DYNAMIC_DEBUG_MODULE)) */ #include #include #include -static inline int ddebug_dyndbg_module_param_cb(char *param, char *val, - const char *modname) -{ - if (!strcmp(param, "dyndbg")) { - /* avoid pr_warn(), which wants pr_fmt() fully defined */ - printk(KERN_WARNING "dyndbg param is supported only in " - "CONFIG_DYNAMIC_DEBUG builds\n"); - return 0; /* allow and ignore */ - } - return -EINVAL; -} +#define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt) +#define DYNAMIC_DEBUG_BRANCH(descriptor) false #define dynamic_pr_debug(fmt, ...) \ do { if (0) printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); } while (0) @@ -324,14 +313,40 @@ static inline int ddebug_dyndbg_module_param_cb(char *param, char *val, rowsize, groupsize, buf, len, ascii); \ } while (0) +#endif /* CONFIG_DYNAMIC_DEBUG || (CONFIG_DYNAMIC_DEBUG_CORE && DYNAMIC_DEBUG_MODULE) */ + + +#ifdef CONFIG_DYNAMIC_DEBUG_CORE + +extern int ddebug_dyndbg_module_param_cb(char *param, char *val, + const char *modname); +struct kernel_param; +int param_set_dyndbg_classes(const char *instr, const struct kernel_param *kp); +int param_get_dyndbg_classes(char *buffer, const struct kernel_param *kp); + +#else + +static inline int ddebug_dyndbg_module_param_cb(char *param, char *val, + const char *modname) +{ + if (!strcmp(param, "dyndbg")) { + /* avoid pr_warn(), which wants pr_fmt() fully defined */ + printk(KERN_WARNING "dyndbg param is supported only in " + "CONFIG_DYNAMIC_DEBUG builds\n"); + return 0; /* allow and ignore */ + } + return -EINVAL; +} + struct kernel_param; static inline int param_set_dyndbg_classes(const char *instr, const struct kernel_param *kp) { return 0; } static inline int param_get_dyndbg_classes(char *buffer, const struct kernel_param *kp) { return 0; } -#endif /* !CONFIG_DYNAMIC_DEBUG_CORE */ +#endif + extern const struct kernel_param_ops param_ops_dyndbg_classes; -#endif +#endif /* _DYNAMIC_DEBUG_H */ diff --git a/lib/Makefile b/lib/Makefile index baf2821f7a00..7afcd85f78f6 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -231,6 +231,9 @@ lib-$(CONFIG_GENERIC_BUG) += bug.o obj-$(CONFIG_HAVE_ARCH_TRACEHOOK) += syscall.o obj-$(CONFIG_DYNAMIC_DEBUG_CORE) += dynamic_debug.o +#ensure exported functions have prototypes +CFLAGS_dynamic_debug.o := -DDYNAMIC_DEBUG_MODULE + obj-$(CONFIG_SYMBOLIC_ERRNAME) += errname.o obj-$(CONFIG_NLATTR) += nlattr.o -- cgit v1.2.3 From 3703bd54cd37e7875f51ece8df8c85c184e40bba Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 8 Mar 2023 15:38:46 +0800 Subject: kallsyms: Delete an unused parameter related to {module_}kallsyms_on_each_symbol() The parameter 'struct module *' in the hook function associated with {module_}kallsyms_on_each_symbol() is no longer used. Delete it. Suggested-by: Petr Mladek Signed-off-by: Zhen Lei Reviewed-by: Vincenzo Palazzo Acked-by: Jiri Olsa Acked-by: Steven Rostedt (Google) Signed-off-by: Luis Chamberlain --- include/linux/kallsyms.h | 7 +++---- include/linux/module.h | 6 ++---- kernel/kallsyms.c | 5 ++--- kernel/kallsyms_selftest.c | 6 +++--- kernel/livepatch/core.c | 3 +-- kernel/module/kallsyms.c | 5 ++--- kernel/trace/ftrace.c | 3 +-- 7 files changed, 14 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h index 0065209cc004..fe3c9993b5bf 100644 --- a/include/linux/kallsyms.h +++ b/include/linux/kallsyms.h @@ -67,8 +67,7 @@ static inline void *dereference_symbol_descriptor(void *ptr) #ifdef CONFIG_KALLSYMS unsigned long kallsyms_sym_address(int idx); -int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, - unsigned long), +int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), void *data); int kallsyms_on_each_match_symbol(int (*fn)(void *, unsigned long), const char *name, void *data); @@ -166,8 +165,8 @@ static inline bool kallsyms_show_value(const struct cred *cred) return false; } -static inline int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, - unsigned long), void *data) +static inline int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), + void *data) { return -EOPNOTSUPP; } diff --git a/include/linux/module.h b/include/linux/module.h index c3b357196470..d2d900077bde 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -965,13 +965,11 @@ static inline bool module_sig_ok(struct module *module) #if defined(CONFIG_MODULES) && defined(CONFIG_KALLSYMS) int module_kallsyms_on_each_symbol(const char *modname, - int (*fn)(void *, const char *, - struct module *, unsigned long), + int (*fn)(void *, const char *, unsigned long), void *data); #else static inline int module_kallsyms_on_each_symbol(const char *modname, - int (*fn)(void *, const char *, - struct module *, unsigned long), + int (*fn)(void *, const char *, unsigned long), void *data) { return -EOPNOTSUPP; diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 83f499182c9a..77747391f49b 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -288,8 +288,7 @@ unsigned long kallsyms_lookup_name(const char *name) * Iterate over all symbols in vmlinux. For symbols from modules use * module_kallsyms_on_each_symbol instead. */ -int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, - unsigned long), +int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), void *data) { char namebuf[KSYM_NAME_LEN]; @@ -299,7 +298,7 @@ int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, for (i = 0, off = 0; i < kallsyms_num_syms; i++) { off = kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf)); - ret = fn(data, namebuf, NULL, kallsyms_sym_address(i)); + ret = fn(data, namebuf, kallsyms_sym_address(i)); if (ret != 0) return ret; cond_resched(); diff --git a/kernel/kallsyms_selftest.c b/kernel/kallsyms_selftest.c index bfbc12da3326..a2e3745d15c4 100644 --- a/kernel/kallsyms_selftest.c +++ b/kernel/kallsyms_selftest.c @@ -95,7 +95,7 @@ static struct test_item test_items[] = { static char stub_name[KSYM_NAME_LEN]; -static int stat_symbol_len(void *data, const char *name, struct module *mod, unsigned long addr) +static int stat_symbol_len(void *data, const char *name, unsigned long addr) { *(u32 *)data += strlen(name); @@ -154,7 +154,7 @@ static void test_kallsyms_compression_ratio(void) pr_info(" ---------------------------------------------------------\n"); } -static int lookup_name(void *data, const char *name, struct module *mod, unsigned long addr) +static int lookup_name(void *data, const char *name, unsigned long addr) { u64 t0, t1, t; struct test_stat *stat = (struct test_stat *)data; @@ -207,7 +207,7 @@ static bool match_cleanup_name(const char *s, const char *name) return !strncmp(s, name, len); } -static int find_symbol(void *data, const char *name, struct module *mod, unsigned long addr) +static int find_symbol(void *data, const char *name, unsigned long addr) { struct test_stat *stat = (struct test_stat *)data; diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 4bd2d5e10f20..1de2c40cc378 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -142,8 +142,7 @@ static int klp_match_callback(void *data, unsigned long addr) return 0; } -static int klp_find_callback(void *data, const char *name, - struct module *mod, unsigned long addr) +static int klp_find_callback(void *data, const char *name, unsigned long addr) { struct klp_find_arg *args = data; diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index 8e89f459ffdd..3320586584f1 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -503,8 +503,7 @@ unsigned long module_kallsyms_lookup_name(const char *name) } int module_kallsyms_on_each_symbol(const char *modname, - int (*fn)(void *, const char *, - struct module *, unsigned long), + int (*fn)(void *, const char *, unsigned long), void *data) { struct module *mod; @@ -533,7 +532,7 @@ int module_kallsyms_on_each_symbol(const char *modname, continue; ret = fn(data, kallsyms_symbol_name(kallsyms, i), - mod, kallsyms_symbol_value(sym)); + kallsyms_symbol_value(sym)); if (ret != 0) goto out; } diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 29baa97d0d53..76caca8f496a 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -8391,8 +8391,7 @@ struct kallsyms_data { * and returns 1 in case we resolved all the requested symbols, * 0 otherwise. */ -static int kallsyms_callback(void *data, const char *name, - struct module *mod, unsigned long addr) +static int kallsyms_callback(void *data, const char *name, unsigned long addr) { struct kallsyms_data *args = data; const char **sym; -- cgit v1.2.3 From 419405c92299d793b95053aa54d95e2d3f45a1a4 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 6 Mar 2023 08:56:49 +0100 Subject: interconnect: drop racy registration API Now that all interconnect drivers have been converted to the new provider registration API, the old racy interface can be removed. Reviewed-by: Konrad Dybcio Signed-off-by: Johan Hovold Link: https://lore.kernel.org/r/20230306075651.2449-22-johan+linaro@kernel.org Signed-off-by: Georgi Djakov --- drivers/interconnect/core.c | 16 ---------------- include/linux/interconnect-provider.h | 11 ----------- 2 files changed, 27 deletions(-) (limited to 'include/linux') diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c index 4175a42bf748..da456329c524 100644 --- a/drivers/interconnect/core.c +++ b/drivers/interconnect/core.c @@ -1084,22 +1084,6 @@ void icc_provider_deregister(struct icc_provider *provider) } EXPORT_SYMBOL_GPL(icc_provider_deregister); -int icc_provider_add(struct icc_provider *provider) -{ - icc_provider_init(provider); - - return icc_provider_register(provider); -} -EXPORT_SYMBOL_GPL(icc_provider_add); - -void icc_provider_del(struct icc_provider *provider) -{ - WARN_ON(!list_empty(&provider->nodes)); - - icc_provider_deregister(provider); -} -EXPORT_SYMBOL_GPL(icc_provider_del); - static const struct of_device_id __maybe_unused ignore_list[] = { { .compatible = "qcom,sc7180-ipa-virt" }, { .compatible = "qcom,sc8180x-ipa-virt" }, diff --git a/include/linux/interconnect-provider.h b/include/linux/interconnect-provider.h index d12cd18aab3f..b9af9016a95e 100644 --- a/include/linux/interconnect-provider.h +++ b/include/linux/interconnect-provider.h @@ -125,8 +125,6 @@ int icc_nodes_remove(struct icc_provider *provider); void icc_provider_init(struct icc_provider *provider); int icc_provider_register(struct icc_provider *provider); void icc_provider_deregister(struct icc_provider *provider); -int icc_provider_add(struct icc_provider *provider); -void icc_provider_del(struct icc_provider *provider); struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec); void icc_sync_state(struct device *dev); @@ -179,15 +177,6 @@ static inline int icc_provider_register(struct icc_provider *provider) static inline void icc_provider_deregister(struct icc_provider *provider) { } -static inline int icc_provider_add(struct icc_provider *provider) -{ - return -ENOTSUPP; -} - -static inline void icc_provider_del(struct icc_provider *provider) -{ -} - static inline struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec) { return ERR_PTR(-ENOTSUPP); -- cgit v1.2.3 From d54c1fd4a51e8fbc7f9da86b0cd338a4f7cd2bb2 Mon Sep 17 00:00:00 2001 From: Qin Jian Date: Mon, 19 Dec 2022 09:51:30 +0800 Subject: clk: Add Sunplus SP7021 clock driver Add clock driver for Sunplus SP7021 SoC. Signed-off-by: Qin Jian Link: https://lore.kernel.org/r/20221219015130.42621-1-qinjian@cqplus1.com Signed-off-by: Stephen Boyd --- MAINTAINERS | 1 + drivers/clk/Kconfig | 10 + drivers/clk/Makefile | 1 + drivers/clk/clk-sp7021.c | 713 +++++++++++++++++++++++++++++++++++++++++++ include/linux/clk-provider.h | 19 ++ 5 files changed, 744 insertions(+) create mode 100644 drivers/clk/clk-sp7021.c (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 8d5bc223f305..f72107a6e86f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2849,6 +2849,7 @@ F: Documentation/devicetree/bindings/reset/sunplus,reset.yaml F: arch/arm/boot/dts/sunplus-sp7021*.dts* F: arch/arm/configs/sp7021_*defconfig F: arch/arm/mach-sunplus/ +F: drivers/clk/clk-sp7021.c F: drivers/irqchip/irq-sp7021-intc.c F: drivers/reset/reset-sunplus.c F: include/dt-bindings/clock/sunplus,sp7021-clkc.h diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index b6c5bf69a2b2..54e01bb2f568 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -436,6 +436,16 @@ config COMMON_CLK_K210 help Support for the Canaan Kendryte K210 RISC-V SoC clocks. +config COMMON_CLK_SP7021 + tristate "Clock driver for Sunplus SP7021 SoC" + depends on SOC_SP7021 || COMPILE_TEST + default SOC_SP7021 + help + This driver supports the Sunplus SP7021 SoC clocks. + It implements SP7021 PLLs/gate. + Not all features of the PLL are currently supported + by the driver. + source "drivers/clk/actions/Kconfig" source "drivers/clk/analogbits/Kconfig" source "drivers/clk/baikal-t1/Kconfig" diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index e3ca0d058a25..7bbf77acf181 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -65,6 +65,7 @@ obj-$(CONFIG_COMMON_CLK_SI5351) += clk-si5351.o obj-$(CONFIG_COMMON_CLK_SI514) += clk-si514.o obj-$(CONFIG_COMMON_CLK_SI544) += clk-si544.o obj-$(CONFIG_COMMON_CLK_SI570) += clk-si570.o +obj-$(CONFIG_COMMON_CLK_SP7021) += clk-sp7021.o obj-$(CONFIG_COMMON_CLK_STM32F) += clk-stm32f4.o obj-$(CONFIG_COMMON_CLK_STM32H7) += clk-stm32h7.o obj-$(CONFIG_COMMON_CLK_STM32MP157) += clk-stm32mp1.o diff --git a/drivers/clk/clk-sp7021.c b/drivers/clk/clk-sp7021.c new file mode 100644 index 000000000000..8fec14120105 --- /dev/null +++ b/drivers/clk/clk-sp7021.c @@ -0,0 +1,713 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* + * Copyright (C) Sunplus Technology Co., Ltd. + * All rights reserved. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* speical div_width values for PLLTV/PLLA */ +#define DIV_TV 33 +#define DIV_A 34 + +/* PLLTV parameters */ +enum { + SEL_FRA, + SDM_MOD, + PH_SEL, + NFRA, + DIVR, + DIVN, + DIVM, + P_MAX +}; + +#define MASK_SEL_FRA GENMASK(1, 1) +#define MASK_SDM_MOD GENMASK(2, 2) +#define MASK_PH_SEL GENMASK(4, 4) +#define MASK_NFRA GENMASK(12, 6) +#define MASK_DIVR GENMASK(8, 7) +#define MASK_DIVN GENMASK(7, 0) +#define MASK_DIVM GENMASK(14, 8) + +/* HIWORD_MASK FIELD_PREP */ +#define HWM_FIELD_PREP(mask, value) \ +({ \ + u32 _m = mask; \ + (_m << 16) | FIELD_PREP(_m, value); \ +}) + +struct sp_pll { + struct clk_hw hw; + void __iomem *reg; + spinlock_t lock; /* lock for reg */ + int div_shift; + int div_width; + int pd_bit; /* power down bit idx */ + int bp_bit; /* bypass bit idx */ + unsigned long brate; /* base rate, TODO: replace brate with muldiv */ + u32 p[P_MAX]; /* for hold PLLTV/PLLA parameters */ +}; + +#define to_sp_pll(_hw) container_of(_hw, struct sp_pll, hw) + +struct sp_clk_gate_info { + u16 reg; /* reg_index_shift */ + u16 ext_parent; /* parent is extclk */ +}; + +static const struct sp_clk_gate_info sp_clk_gates[] = { + { 0x02 }, + { 0x05 }, + { 0x06 }, + { 0x07 }, + { 0x09 }, + { 0x0b, 1 }, + { 0x0f, 1 }, + { 0x14 }, + { 0x15 }, + { 0x16 }, + { 0x17 }, + { 0x18, 1 }, + { 0x19, 1 }, + { 0x1a, 1 }, + { 0x1b, 1 }, + { 0x1c, 1 }, + { 0x1d, 1 }, + { 0x1e }, + { 0x1f, 1 }, + { 0x20 }, + { 0x21 }, + { 0x22 }, + { 0x23 }, + { 0x24 }, + { 0x25 }, + { 0x26 }, + { 0x2a }, + { 0x2b }, + { 0x2d }, + { 0x2e }, + { 0x30 }, + { 0x31 }, + { 0x32 }, + { 0x33 }, + { 0x3d }, + { 0x3e }, + { 0x3f }, + { 0x42 }, + { 0x44 }, + { 0x4b }, + { 0x4c }, + { 0x4d }, + { 0x4e }, + { 0x4f }, + { 0x50 }, + { 0x55 }, + { 0x60 }, + { 0x61 }, + { 0x6a }, + { 0x73 }, + { 0x86 }, + { 0x8a }, + { 0x8b }, + { 0x8d }, + { 0x8e }, + { 0x8f }, + { 0x90 }, + { 0x92 }, + { 0x93 }, + { 0x95 }, + { 0x96 }, + { 0x97 }, + { 0x98 }, + { 0x99 }, +}; + +#define _M 1000000UL +#define F_27M (27 * _M) + +/*********************************** PLL_TV **********************************/ + +/* TODO: set proper FVCO range */ +#define FVCO_MIN (100 * _M) +#define FVCO_MAX (200 * _M) + +#define F_MIN (FVCO_MIN / 8) +#define F_MAX (FVCO_MAX) + +static long plltv_integer_div(struct sp_pll *clk, unsigned long freq) +{ + /* valid m values: 27M must be divisible by m */ + static const u32 m_table[] = { + 1, 2, 3, 4, 5, 6, 8, 9, 10, 12, 15, 16, 18, 20, 24, 25, 27, 30, 32 + }; + u32 m, n, r; + unsigned long fvco, nf; + long ret; + + freq = clamp(freq, F_MIN, F_MAX); + + /* DIVR 0~3 */ + for (r = 0; r <= 3; r++) { + fvco = freq << r; + if (fvco <= FVCO_MAX) + break; + } + + /* DIVM */ + for (m = 0; m < ARRAY_SIZE(m_table); m++) { + nf = fvco * m_table[m]; + n = nf / F_27M; + if ((n * F_27M) == nf) + break; + } + if (m >= ARRAY_SIZE(m_table)) { + ret = -EINVAL; + goto err_not_found; + } + + /* save parameters */ + clk->p[SEL_FRA] = 0; + clk->p[DIVR] = r; + clk->p[DIVN] = n; + clk->p[DIVM] = m_table[m]; + + return freq; + +err_not_found: + pr_err("%s: %s freq:%lu not found a valid setting\n", + __func__, clk_hw_get_name(&clk->hw), freq); + + return ret; +} + +/* parameters for PLLTV fractional divider */ +static const u32 pt[][5] = { + /* conventional fractional */ + { + 1, /* factor */ + 5, /* 5 * p0 (nint) */ + 1, /* 1 * p0 */ + F_27M, /* F_27M / p0 */ + 1, /* p0 / p2 */ + }, + /* phase rotation */ + { + 10, /* factor */ + 54, /* 5.4 * p0 (nint) */ + 2, /* 0.2 * p0 */ + F_27M / 10, /* F_27M / p0 */ + 5, /* p0 / p2 */ + }, +}; + +static const u32 sdm_mod_vals[] = { 91, 55 }; + +static long plltv_fractional_div(struct sp_pll *clk, unsigned long freq) +{ + u32 m, r; + u32 nint, nfra; + u32 df_quotient_min = 210000000; + u32 df_remainder_min = 0; + unsigned long fvco, nf, f, fout = 0; + int sdm, ph; + + freq = clamp(freq, F_MIN, F_MAX); + + /* DIVR 0~3 */ + for (r = 0; r <= 3; r++) { + fvco = freq << r; + if (fvco <= FVCO_MAX) + break; + } + f = F_27M >> r; + + /* PH_SEL */ + for (ph = ARRAY_SIZE(pt) - 1; ph >= 0; ph--) { + const u32 *pp = pt[ph]; + + /* SDM_MOD */ + for (sdm = 0; sdm < ARRAY_SIZE(sdm_mod_vals); sdm++) { + u32 mod = sdm_mod_vals[sdm]; + + /* DIVM 1~32 */ + for (m = 1; m <= 32; m++) { + u32 df; /* diff freq */ + u32 df_quotient, df_remainder; + + nf = fvco * m; + nint = nf / pp[3]; + + if (nint < pp[1]) + continue; + if (nint > pp[1]) + break; + + nfra = (((nf % pp[3]) * mod * pp[4]) + (F_27M / 2)) / F_27M; + if (nfra) { + u32 df0 = f * (nint + pp[2]) / pp[0]; + u32 df1 = f * (mod - nfra) / mod / pp[4]; + + df = df0 - df1; + } else { + df = f * (nint) / pp[0]; + } + + df_quotient = df / m; + df_remainder = ((df % m) * 1000) / m; + + if (freq > df_quotient) { + df_quotient = freq - df_quotient - 1; + df_remainder = 1000 - df_remainder; + } else { + df_quotient = df_quotient - freq; + } + + if (df_quotient_min > df_quotient || + (df_quotient_min == df_quotient && + df_remainder_min > df_remainder)) { + /* found a closer freq, save parameters */ + clk->p[SEL_FRA] = 1; + clk->p[SDM_MOD] = sdm; + clk->p[PH_SEL] = ph; + clk->p[NFRA] = nfra; + clk->p[DIVR] = r; + clk->p[DIVM] = m; + + fout = df / m; + df_quotient_min = df_quotient; + df_remainder_min = df_remainder; + } + } + } + } + + if (!fout) { + pr_err("%s: %s freq:%lu not found a valid setting\n", + __func__, clk_hw_get_name(&clk->hw), freq); + return -EINVAL; + } + + return fout; +} + +static long plltv_div(struct sp_pll *clk, unsigned long freq) +{ + if (freq % 100) + return plltv_fractional_div(clk, freq); + + return plltv_integer_div(clk, freq); +} + +static int plltv_set_rate(struct sp_pll *clk) +{ + unsigned long flags; + u32 r0, r1, r2; + + r0 = BIT(clk->bp_bit + 16); + r0 |= HWM_FIELD_PREP(MASK_SEL_FRA, clk->p[SEL_FRA]); + r0 |= HWM_FIELD_PREP(MASK_SDM_MOD, clk->p[SDM_MOD]); + r0 |= HWM_FIELD_PREP(MASK_PH_SEL, clk->p[PH_SEL]); + r0 |= HWM_FIELD_PREP(MASK_NFRA, clk->p[NFRA]); + + r1 = HWM_FIELD_PREP(MASK_DIVR, clk->p[DIVR]); + + r2 = HWM_FIELD_PREP(MASK_DIVN, clk->p[DIVN] - 1); + r2 |= HWM_FIELD_PREP(MASK_DIVM, clk->p[DIVM] - 1); + + spin_lock_irqsave(&clk->lock, flags); + writel(r0, clk->reg); + writel(r1, clk->reg + 4); + writel(r2, clk->reg + 8); + spin_unlock_irqrestore(&clk->lock, flags); + + return 0; +} + +/*********************************** PLL_A ***********************************/ + +/* from Q628_PLLs_REG_setting.xlsx */ +static const struct { + u32 rate; + u32 regs[5]; +} pa[] = { + { + .rate = 135475200, + .regs = { + 0x4801, + 0x02df, + 0x248f, + 0x0211, + 0x33e9 + } + }, + { + .rate = 147456000, + .regs = { + 0x4801, + 0x1adf, + 0x2490, + 0x0349, + 0x33e9 + } + }, + { + .rate = 196608000, + .regs = { + 0x4801, + 0x42ef, + 0x2495, + 0x01c6, + 0x33e9 + } + }, +}; + +static int plla_set_rate(struct sp_pll *clk) +{ + const u32 *pp = pa[clk->p[0]].regs; + unsigned long flags; + int i; + + spin_lock_irqsave(&clk->lock, flags); + for (i = 0; i < ARRAY_SIZE(pa->regs); i++) + writel(0xffff0000 | pp[i], clk->reg + (i * 4)); + spin_unlock_irqrestore(&clk->lock, flags); + + return 0; +} + +static long plla_round_rate(struct sp_pll *clk, unsigned long rate) +{ + int i = ARRAY_SIZE(pa); + + while (--i) { + if (rate >= pa[i].rate) + break; + } + clk->p[0] = i; + + return pa[i].rate; +} + +/********************************** SP_PLL ***********************************/ + +static long sp_pll_calc_div(struct sp_pll *clk, unsigned long rate) +{ + u32 fbdiv; + u32 max = 1 << clk->div_width; + + fbdiv = DIV_ROUND_CLOSEST(rate, clk->brate); + if (fbdiv > max) + fbdiv = max; + + return fbdiv; +} + +static long sp_pll_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *prate) +{ + struct sp_pll *clk = to_sp_pll(hw); + long ret; + + if (rate == *prate) { + ret = *prate; /* bypass */ + } else if (clk->div_width == DIV_A) { + ret = plla_round_rate(clk, rate); + } else if (clk->div_width == DIV_TV) { + ret = plltv_div(clk, rate); + if (ret < 0) + ret = *prate; + } else { + ret = sp_pll_calc_div(clk, rate) * clk->brate; + } + + return ret; +} + +static unsigned long sp_pll_recalc_rate(struct clk_hw *hw, + unsigned long prate) +{ + struct sp_pll *clk = to_sp_pll(hw); + u32 reg = readl(clk->reg); + unsigned long ret; + + if (reg & BIT(clk->bp_bit)) { + ret = prate; /* bypass */ + } else if (clk->div_width == DIV_A) { + ret = pa[clk->p[0]].rate; + } else if (clk->div_width == DIV_TV) { + u32 m, r, reg2; + + r = FIELD_GET(MASK_DIVR, readl(clk->reg + 4)); + reg2 = readl(clk->reg + 8); + m = FIELD_GET(MASK_DIVM, reg2) + 1; + + if (reg & MASK_SEL_FRA) { + /* fractional divider */ + u32 sdm = FIELD_GET(MASK_SDM_MOD, reg); + u32 ph = FIELD_GET(MASK_PH_SEL, reg); + u32 nfra = FIELD_GET(MASK_NFRA, reg); + const u32 *pp = pt[ph]; + unsigned long r0, r1; + + ret = prate >> r; + r0 = ret * (pp[1] + pp[2]) / pp[0]; + r1 = ret * (sdm_mod_vals[sdm] - nfra) / sdm_mod_vals[sdm] / pp[4]; + ret = (r0 - r1) / m; + } else { + /* integer divider */ + u32 n = FIELD_GET(MASK_DIVN, reg2) + 1; + + ret = (prate / m * n) >> r; + } + } else { + u32 fbdiv = ((reg >> clk->div_shift) & ((1 << clk->div_width) - 1)) + 1; + + ret = clk->brate * fbdiv; + } + + return ret; +} + +static int sp_pll_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long prate) +{ + struct sp_pll *clk = to_sp_pll(hw); + unsigned long flags; + u32 reg; + + reg = BIT(clk->bp_bit + 16); /* HIWORD_MASK */ + + if (rate == prate) { + reg |= BIT(clk->bp_bit); /* bypass */ + } else if (clk->div_width == DIV_A) { + return plla_set_rate(clk); + } else if (clk->div_width == DIV_TV) { + return plltv_set_rate(clk); + } else if (clk->div_width) { + u32 fbdiv = sp_pll_calc_div(clk, rate); + u32 mask = GENMASK(clk->div_shift + clk->div_width - 1, clk->div_shift); + + reg |= mask << 16; + reg |= ((fbdiv - 1) << clk->div_shift) & mask; + } + + spin_lock_irqsave(&clk->lock, flags); + writel(reg, clk->reg); + spin_unlock_irqrestore(&clk->lock, flags); + + return 0; +} + +static int sp_pll_enable(struct clk_hw *hw) +{ + struct sp_pll *clk = to_sp_pll(hw); + + writel(BIT(clk->pd_bit + 16) | BIT(clk->pd_bit), clk->reg); + + return 0; +} + +static void sp_pll_disable(struct clk_hw *hw) +{ + struct sp_pll *clk = to_sp_pll(hw); + + writel(BIT(clk->pd_bit + 16), clk->reg); +} + +static int sp_pll_is_enabled(struct clk_hw *hw) +{ + struct sp_pll *clk = to_sp_pll(hw); + + return readl(clk->reg) & BIT(clk->pd_bit); +} + +static const struct clk_ops sp_pll_ops = { + .enable = sp_pll_enable, + .disable = sp_pll_disable, + .is_enabled = sp_pll_is_enabled, + .round_rate = sp_pll_round_rate, + .recalc_rate = sp_pll_recalc_rate, + .set_rate = sp_pll_set_rate +}; + +static const struct clk_ops sp_pll_sub_ops = { + .enable = sp_pll_enable, + .disable = sp_pll_disable, + .is_enabled = sp_pll_is_enabled, + .recalc_rate = sp_pll_recalc_rate, +}; + +static struct clk_hw *sp_pll_register(struct device *dev, const char *name, + const struct clk_parent_data *parent_data, + void __iomem *reg, int pd_bit, int bp_bit, + unsigned long brate, int shift, int width, + unsigned long flags) +{ + struct sp_pll *pll; + struct clk_hw *hw; + struct clk_init_data initd = { + .name = name, + .parent_data = parent_data, + .ops = (bp_bit >= 0) ? &sp_pll_ops : &sp_pll_sub_ops, + .num_parents = 1, + .flags = flags, + }; + int ret; + + pll = devm_kzalloc(dev, sizeof(*pll), GFP_KERNEL); + if (!pll) + return ERR_PTR(-ENOMEM); + + pll->hw.init = &initd; + pll->reg = reg; + pll->pd_bit = pd_bit; + pll->bp_bit = bp_bit; + pll->brate = brate; + pll->div_shift = shift; + pll->div_width = width; + spin_lock_init(&pll->lock); + + hw = &pll->hw; + ret = devm_clk_hw_register(dev, hw); + if (ret) + return ERR_PTR(ret); + + return hw; +} + +#define PLLA_CTL (pll_base + 0x1c) +#define PLLE_CTL (pll_base + 0x30) +#define PLLF_CTL (pll_base + 0x34) +#define PLLTV_CTL (pll_base + 0x38) + +static int sp7021_clk_probe(struct platform_device *pdev) +{ + static const u32 sp_clken[] = { + 0x67ef, 0x03ff, 0xff03, 0xfff0, 0x0004, /* G0.1~5 */ + 0x0000, 0x8000, 0xffff, 0x0040, 0x0000, /* G0.6~10 */ + }; + static struct clk_parent_data pd_ext, pd_sys, pd_e; + struct device *dev = &pdev->dev; + void __iomem *clk_base, *pll_base, *sys_base; + struct clk_hw_onecell_data *clk_data; + struct clk_hw **hws; + int i; + + clk_base = devm_platform_ioremap_resource(pdev, 0); + if (!clk_base) + return -ENXIO; + pll_base = devm_platform_ioremap_resource(pdev, 1); + if (!pll_base) + return -ENXIO; + sys_base = devm_platform_ioremap_resource(pdev, 2); + if (!sys_base) + return -ENXIO; + + /* enable default clks */ + for (i = 0; i < ARRAY_SIZE(sp_clken); i++) + writel((sp_clken[i] << 16) | sp_clken[i], clk_base + i * 4); + + clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, CLK_MAX), + GFP_KERNEL); + if (!clk_data) + return -ENOMEM; + + hws = clk_data->hws; + pd_ext.index = 0; + + /* PLLs */ + hws[PLL_A] = sp_pll_register(dev, "plla", &pd_ext, PLLA_CTL, + 11, 12, 27000000, 0, DIV_A, 0); + if (IS_ERR(hws[PLL_A])) + return PTR_ERR(hws[PLL_A]); + + hws[PLL_E] = sp_pll_register(dev, "plle", &pd_ext, PLLE_CTL, + 6, 2, 50000000, 0, 0, 0); + if (IS_ERR(hws[PLL_E])) + return PTR_ERR(hws[PLL_E]); + pd_e.hw = hws[PLL_E]; + hws[PLL_E_2P5] = sp_pll_register(dev, "plle_2p5", &pd_e, PLLE_CTL, + 13, -1, 2500000, 0, 0, 0); + if (IS_ERR(hws[PLL_E_2P5])) + return PTR_ERR(hws[PLL_E_2P5]); + hws[PLL_E_25] = sp_pll_register(dev, "plle_25", &pd_e, PLLE_CTL, + 12, -1, 25000000, 0, 0, 0); + if (IS_ERR(hws[PLL_E_25])) + return PTR_ERR(hws[PLL_E_25]); + hws[PLL_E_112P5] = sp_pll_register(dev, "plle_112p5", &pd_e, PLLE_CTL, + 11, -1, 112500000, 0, 0, 0); + if (IS_ERR(hws[PLL_E_112P5])) + return PTR_ERR(hws[PLL_E_112P5]); + + hws[PLL_F] = sp_pll_register(dev, "pllf", &pd_ext, PLLF_CTL, + 0, 10, 13500000, 1, 4, 0); + if (IS_ERR(hws[PLL_F])) + return PTR_ERR(hws[PLL_F]); + + hws[PLL_TV] = sp_pll_register(dev, "plltv", &pd_ext, PLLTV_CTL, + 0, 15, 27000000, 0, DIV_TV, 0); + if (IS_ERR(hws[PLL_TV])) + return PTR_ERR(hws[PLL_TV]); + hws[PLL_TV_A] = devm_clk_hw_register_divider(dev, "plltv_a", "plltv", 0, + PLLTV_CTL + 4, 5, 1, + CLK_DIVIDER_POWER_OF_TWO, + &to_sp_pll(hws[PLL_TV])->lock); + if (IS_ERR(hws[PLL_TV_A])) + return PTR_ERR(hws[PLL_TV_A]); + + /* system clock, should not be disabled */ + hws[PLL_SYS] = sp_pll_register(dev, "pllsys", &pd_ext, sys_base, + 10, 9, 13500000, 0, 4, CLK_IS_CRITICAL); + if (IS_ERR(hws[PLL_SYS])) + return PTR_ERR(hws[PLL_SYS]); + pd_sys.hw = hws[PLL_SYS]; + + /* gates */ + for (i = 0; i < ARRAY_SIZE(sp_clk_gates); i++) { + char name[10]; + u32 j = sp_clk_gates[i].reg; + struct clk_parent_data *pd = sp_clk_gates[i].ext_parent ? &pd_ext : &pd_sys; + + sprintf(name, "%02d_0x%02x", i, j); + hws[i] = devm_clk_hw_register_gate_parent_data(dev, name, pd, 0, + clk_base + (j >> 4) * 4, + j & 0x0f, + CLK_GATE_HIWORD_MASK, + NULL); + if (IS_ERR(hws[i])) + return PTR_ERR(hws[i]); + } + + clk_data->num = CLK_MAX; + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data); +} + +static const struct of_device_id sp7021_clk_dt_ids[] = { + { .compatible = "sunplus,sp7021-clkc" }, + { } +}; +MODULE_DEVICE_TABLE(of, sp7021_clk_dt_ids); + +static struct platform_driver sp7021_clk_driver = { + .probe = sp7021_clk_probe, + .driver = { + .name = "sp7021-clk", + .of_match_table = sp7021_clk_dt_ids, + }, +}; +module_platform_driver(sp7021_clk_driver); + +MODULE_AUTHOR("Sunplus Technology"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Clock driver for Sunplus SP7021 SoC"); diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index 842e72a5348f..0e3bc3eb9911 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -608,6 +608,25 @@ struct clk *clk_register_gate(struct device *dev, const char *name, __devm_clk_hw_register_gate((dev), NULL, (name), (parent_name), NULL, \ NULL, (flags), (reg), (bit_idx), \ (clk_gate_flags), (lock)) +/** + * devm_clk_hw_register_gate_parent_data - register a gate clock with the + * clock framework + * @dev: device that is registering this clock + * @name: name of this clock + * @parent_data: parent clk data + * @flags: framework-specific flags for this clock + * @reg: register address to control gating of this clock + * @bit_idx: which bit in the register controls gating of this clock + * @clk_gate_flags: gate-specific flags for this clock + * @lock: shared register lock for this clock + */ +#define devm_clk_hw_register_gate_parent_data(dev, name, parent_data, flags, \ + reg, bit_idx, clk_gate_flags, \ + lock) \ + __devm_clk_hw_register_gate((dev), NULL, (name), NULL, NULL, \ + (parent_data), (flags), (reg), (bit_idx), \ + (clk_gate_flags), (lock)) + void clk_unregister_gate(struct clk *clk); void clk_hw_unregister_gate(struct clk_hw *hw); int clk_gate_is_enabled(struct clk_hw *hw); -- cgit v1.2.3 From 2d337b7158f8c38dacf8394028ab87b8a25ed707 Mon Sep 17 00:00:00 2001 From: ZhangPeng Date: Wed, 1 Mar 2023 10:06:27 +0000 Subject: userfaultfd: move unprivileged_userfaultfd sysctl to its own file The sysctl_unprivileged_userfaultfd is part of userfaultfd, move it to its own file. Signed-off-by: ZhangPeng Signed-off-by: Luis Chamberlain --- fs/userfaultfd.c | 20 +++++++++++++++++++- include/linux/userfaultfd_k.h | 2 -- kernel/sysctl.c | 11 ----------- 3 files changed, 19 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 44d1ee429eb0..d01f803a6b11 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -32,7 +32,22 @@ #include #include -int sysctl_unprivileged_userfaultfd __read_mostly; +static int sysctl_unprivileged_userfaultfd __read_mostly; + +#ifdef CONFIG_SYSCTL +static struct ctl_table vm_userfaultfd_table[] = { + { + .procname = "unprivileged_userfaultfd", + .data = &sysctl_unprivileged_userfaultfd, + .maxlen = sizeof(sysctl_unprivileged_userfaultfd), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { } +}; +#endif static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly; @@ -2178,6 +2193,9 @@ static int __init userfaultfd_init(void) 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, init_once_userfaultfd_ctx); +#ifdef CONFIG_SYSCTL + register_sysctl_init("vm", vm_userfaultfd_table); +#endif return 0; } __initcall(userfaultfd_init); diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 3767f18114ef..fff49fec0258 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -36,8 +36,6 @@ #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) -extern int sysctl_unprivileged_userfaultfd; - extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); /* diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 1c240d2c99bc..c14552a662ae 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2438,17 +2438,6 @@ static struct ctl_table vm_table[] = { .extra1 = (void *)&mmap_rnd_compat_bits_min, .extra2 = (void *)&mmap_rnd_compat_bits_max, }, -#endif -#ifdef CONFIG_USERFAULTFD - { - .procname = "unprivileged_userfaultfd", - .data = &sysctl_unprivileged_userfaultfd, - .maxlen = sizeof(sysctl_unprivileged_userfaultfd), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, #endif { } }; -- cgit v1.2.3 From 962de54828c5bb100eb8de881b79ed9c8b7468c0 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 9 Mar 2023 20:20:11 +0800 Subject: mm: hugetlb: move hugeltb sysctls to its own file This moves all hugetlb sysctls to its own file, also kill an useless hugetlb_treat_movable_handler() defination. Signed-off-by: Kefeng Wang Reviewed-by: Luis Chamberlain Reviewed-by: Mike Kravetz Reviewed-by: Muchun Song Signed-off-by: Luis Chamberlain --- include/linux/hugetlb.h | 8 -------- kernel/sysctl.c | 32 ------------------------------- mm/hugetlb.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 48 insertions(+), 43 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 7c977d234aba..4056b05d81ed 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -124,14 +124,6 @@ void hugepage_put_subpool(struct hugepage_subpool *spool); void hugetlb_dup_vma_private(struct vm_area_struct *vma); void clear_vma_resv_huge_pages(struct vm_area_struct *vma); -int hugetlb_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *); -int hugetlb_overcommit_handler(struct ctl_table *, int, void *, size_t *, - loff_t *); -int hugetlb_treat_movable_handler(struct ctl_table *, int, void *, size_t *, - loff_t *); -int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int, void *, size_t *, - loff_t *); - int move_hugetlb_page_tables(struct vm_area_struct *vma, struct vm_area_struct *new_vma, unsigned long old_addr, unsigned long new_addr, diff --git a/kernel/sysctl.c b/kernel/sysctl.c index c14552a662ae..ce0297acf97c 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2140,38 +2140,6 @@ static struct ctl_table vm_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, -#endif -#ifdef CONFIG_HUGETLB_PAGE - { - .procname = "nr_hugepages", - .data = NULL, - .maxlen = sizeof(unsigned long), - .mode = 0644, - .proc_handler = hugetlb_sysctl_handler, - }, -#ifdef CONFIG_NUMA - { - .procname = "nr_hugepages_mempolicy", - .data = NULL, - .maxlen = sizeof(unsigned long), - .mode = 0644, - .proc_handler = &hugetlb_mempolicy_sysctl_handler, - }, -#endif - { - .procname = "hugetlb_shm_group", - .data = &sysctl_hugetlb_shm_group, - .maxlen = sizeof(gid_t), - .mode = 0644, - .proc_handler = proc_dointvec, - }, - { - .procname = "nr_overcommit_hugepages", - .data = NULL, - .maxlen = sizeof(unsigned long), - .mode = 0644, - .proc_handler = hugetlb_overcommit_handler, - }, #endif { .procname = "lowmem_reserve_ratio", diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 07abcb6eb203..0ee77c55e44e 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4202,6 +4202,12 @@ static void __init hugetlb_sysfs_init(void) hugetlb_register_all_nodes(); } +#ifdef CONFIG_SYSCTL +static void hugetlb_sysctl_init(void); +#else +static inline void hugetlb_sysctl_init(void) { } +#endif + static int __init hugetlb_init(void) { int i; @@ -4257,6 +4263,7 @@ static int __init hugetlb_init(void) hugetlb_sysfs_init(); hugetlb_cgroup_file_init(); + hugetlb_sysctl_init(); #ifdef CONFIG_SMP num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus()); @@ -4588,7 +4595,7 @@ out: return ret; } -int hugetlb_sysctl_handler(struct ctl_table *table, int write, +static int hugetlb_sysctl_handler(struct ctl_table *table, int write, void *buffer, size_t *length, loff_t *ppos) { @@ -4597,7 +4604,7 @@ int hugetlb_sysctl_handler(struct ctl_table *table, int write, } #ifdef CONFIG_NUMA -int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, +static int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, void *buffer, size_t *length, loff_t *ppos) { return hugetlb_sysctl_handler_common(true, table, write, @@ -4605,7 +4612,7 @@ int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write, } #endif /* CONFIG_NUMA */ -int hugetlb_overcommit_handler(struct ctl_table *table, int write, +static int hugetlb_overcommit_handler(struct ctl_table *table, int write, void *buffer, size_t *length, loff_t *ppos) { struct hstate *h = &default_hstate; @@ -4634,6 +4641,44 @@ out: return ret; } +static struct ctl_table hugetlb_table[] = { + { + .procname = "nr_hugepages", + .data = NULL, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = hugetlb_sysctl_handler, + }, +#ifdef CONFIG_NUMA + { + .procname = "nr_hugepages_mempolicy", + .data = NULL, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = &hugetlb_mempolicy_sysctl_handler, + }, +#endif + { + .procname = "hugetlb_shm_group", + .data = &sysctl_hugetlb_shm_group, + .maxlen = sizeof(gid_t), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "nr_overcommit_hugepages", + .data = NULL, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = hugetlb_overcommit_handler, + }, + { } +}; + +static void hugetlb_sysctl_init(void) +{ + register_sysctl_init("vm", hugetlb_table); +} #endif /* CONFIG_SYSCTL */ void hugetlb_report_meminfo(struct seq_file *m) -- cgit v1.2.3 From e11b521a7b69c2621bb2e5920bb96f6d2facdc7e Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 24 Jan 2023 09:56:53 -0500 Subject: ftrace: Show a list of all functions that have ever been enabled When debugging a crash that appears to be related to ftrace, but not for sure, it is useful to know if a function was ever enabled by ftrace or not. It could be that a BPF program was attached to it, or possibly a live patch. We are having crashes in the field where this information is not always known. But having ftrace set a flag if a function has ever been attached since boot up helps tremendously in trying to know if a crash had to do with something using ftrace. For analyzing crashes, the use of a kdump image can have access to the flags. When looking at issues where the kernel did not panic, the touched_functions file can simply be used. Link: https://lore.kernel.org/linux-trace-kernel/20230124095653.6fd1640e@gandalf.local.home Cc: Masami Hiramatsu Cc: Catalin Marinas Tested-by: Mark Rutland Tested-by: Chris Li Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 5 ++++- kernel/trace/ftrace.c | 51 +++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 931f3d904529..327046f1278d 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -548,6 +548,7 @@ bool is_ftrace_trampoline(unsigned long addr); * DIRECT - there is a direct function to call * CALL_OPS - the record can use callsite-specific ops * CALL_OPS_EN - the function is set up to use callsite-specific ops + * TOUCHED - A callback was added since boot up * * When a new ftrace_ops is registered and wants a function to save * pt_regs, the rec->flags REGS is set. When the function has been @@ -567,9 +568,10 @@ enum { FTRACE_FL_DIRECT_EN = (1UL << 23), FTRACE_FL_CALL_OPS = (1UL << 22), FTRACE_FL_CALL_OPS_EN = (1UL << 21), + FTRACE_FL_TOUCHED = (1UL << 20), }; -#define FTRACE_REF_MAX_SHIFT 21 +#define FTRACE_REF_MAX_SHIFT 20 #define FTRACE_REF_MAX ((1UL << FTRACE_REF_MAX_SHIFT) - 1) #define ftrace_rec_count(rec) ((rec)->flags & FTRACE_REF_MAX) @@ -628,6 +630,7 @@ enum { FTRACE_ITER_PROBE = (1 << 4), FTRACE_ITER_MOD = (1 << 5), FTRACE_ITER_ENABLED = (1 << 6), + FTRACE_ITER_TOUCHED = (1 << 7), }; void arch_ftrace_update_code(int command); diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 3b46dba3f69b..db8532a4d5c8 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -45,6 +45,9 @@ #include "trace_output.h" #include "trace_stat.h" +/* Flags that do not get reset */ +#define FTRACE_NOCLEAR_FLAGS (FTRACE_FL_DISABLED | FTRACE_FL_TOUCHED) + #define FTRACE_INVALID_FUNCTION "__ftrace_invalid_address__" #define FTRACE_WARN_ON(cond) \ @@ -2256,7 +2259,7 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) flag ^= rec->flags & FTRACE_FL_ENABLED; if (update) { - rec->flags |= FTRACE_FL_ENABLED; + rec->flags |= FTRACE_FL_ENABLED | FTRACE_FL_TOUCHED; if (flag & FTRACE_FL_REGS) { if (rec->flags & FTRACE_FL_REGS) rec->flags |= FTRACE_FL_REGS_EN; @@ -2326,7 +2329,7 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) if (update) { /* If there's no more users, clear all flags */ if (!ftrace_rec_count(rec)) - rec->flags &= FTRACE_FL_DISABLED; + rec->flags &= FTRACE_NOCLEAR_FLAGS; else /* * Just disable the record, but keep the ops TRAMP @@ -3147,7 +3150,7 @@ int ftrace_shutdown(struct ftrace_ops *ops, int command) struct dyn_ftrace *rec; do_for_each_ftrace_rec(pg, rec) { - if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED)) + if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_NOCLEAR_FLAGS)) pr_warn(" %pS flags:%lx\n", (void *)rec->ip, rec->flags); } while_for_each_ftrace_rec(); @@ -3598,7 +3601,10 @@ t_func_next(struct seq_file *m, loff_t *pos) !ftrace_lookup_ip(iter->hash, rec->ip)) || ((iter->flags & FTRACE_ITER_ENABLED) && - !(rec->flags & FTRACE_FL_ENABLED))) { + !(rec->flags & FTRACE_FL_ENABLED)) || + + ((iter->flags & FTRACE_ITER_TOUCHED) && + !(rec->flags & FTRACE_FL_TOUCHED))) { rec = NULL; goto retry; @@ -3857,7 +3863,7 @@ static int t_show(struct seq_file *m, void *v) return 0; } - if (iter->flags & FTRACE_ITER_ENABLED) { + if (iter->flags & (FTRACE_ITER_ENABLED | FTRACE_ITER_TOUCHED)) { struct ftrace_ops *ops; seq_printf(m, " (%ld)%s%s%s%s", @@ -3959,6 +3965,31 @@ ftrace_enabled_open(struct inode *inode, struct file *file) return 0; } +static int +ftrace_touched_open(struct inode *inode, struct file *file) +{ + struct ftrace_iterator *iter; + + /* + * This shows us what functions have ever been enabled + * (traced, direct, patched, etc). Not sure if we want lockdown + * to hide such critical information for an admin. + * Although, perhaps it can show information we don't + * want people to see, but if something had traced + * something, we probably want to know about it. + */ + + iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); + if (!iter) + return -ENOMEM; + + iter->pg = ftrace_pages_start; + iter->flags = FTRACE_ITER_TOUCHED; + iter->ops = &global_ops; + + return 0; +} + /** * ftrace_regex_open - initialize function tracer filter files * @ops: The ftrace_ops that hold the hash filters @@ -5872,6 +5903,13 @@ static const struct file_operations ftrace_enabled_fops = { .release = seq_release_private, }; +static const struct file_operations ftrace_touched_fops = { + .open = ftrace_touched_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + static const struct file_operations ftrace_filter_fops = { .open = ftrace_filter_open, .read = seq_read, @@ -6336,6 +6374,9 @@ static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) trace_create_file("enabled_functions", TRACE_MODE_READ, d_tracer, NULL, &ftrace_enabled_fops); + trace_create_file("touched_functions", TRACE_MODE_READ, + d_tracer, NULL, &ftrace_touched_fops); + ftrace_create_filter_files(&global_ops, d_tracer); #ifdef CONFIG_FUNCTION_GRAPH_TRACER -- cgit v1.2.3 From 0f74f8b6675cc36d689abb4d9b3d75ab4049b7d7 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Sun, 19 Mar 2023 00:33:07 +0100 Subject: i3c: Make i3c_master_unregister() return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function returned zero unconditionally. Switch the return type to void and simplify the callers accordingly. Signed-off-by: Uwe Kleine-König Reviewed-by: Miquel Raynal Link: https://lore.kernel.org/r/20230318233311.265186-2-u.kleine-koenig@pengutronix.de Signed-off-by: Alexandre Belloni --- drivers/i3c/master.c | 6 +----- drivers/i3c/master/dw-i3c-master.c | 5 +---- drivers/i3c/master/i3c-master-cdns.c | 5 +---- drivers/i3c/master/mipi-i3c-hci/core.c | 4 +++- drivers/i3c/master/svc-i3c-master.c | 5 +---- include/linux/i3c/master.h | 2 +- 6 files changed, 8 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index 54e4c34b4a22..04d6d54d2ab8 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -2695,17 +2695,13 @@ EXPORT_SYMBOL_GPL(i3c_master_register); * @master: master used to send frames on the bus * * Basically undo everything done in i3c_master_register(). - * - * Return: 0 in case of success, a negative error code otherwise. */ -int i3c_master_unregister(struct i3c_master_controller *master) +void i3c_master_unregister(struct i3c_master_controller *master) { i3c_master_i2c_adapter_cleanup(master); i3c_master_unregister_i3c_devs(master); i3c_master_bus_cleanup(master); device_unregister(&master->dev); - - return 0; } EXPORT_SYMBOL_GPL(i3c_master_unregister); diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index 4859dd75388d..8d04b3c72b0e 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -1185,11 +1185,8 @@ err_disable_core_clk: static int dw_i3c_remove(struct platform_device *pdev) { struct dw_i3c_master *master = platform_get_drvdata(pdev); - int ret; - ret = i3c_master_unregister(&master->base); - if (ret) - return ret; + i3c_master_unregister(&master->base); reset_control_assert(master->core_rst); diff --git a/drivers/i3c/master/i3c-master-cdns.c b/drivers/i3c/master/i3c-master-cdns.c index 5b37ffe5ad5b..454925c5e097 100644 --- a/drivers/i3c/master/i3c-master-cdns.c +++ b/drivers/i3c/master/i3c-master-cdns.c @@ -1665,11 +1665,8 @@ err_disable_pclk: static int cdns_i3c_master_remove(struct platform_device *pdev) { struct cdns_i3c_master *master = platform_get_drvdata(pdev); - int ret; - ret = i3c_master_unregister(&master->base); - if (ret) - return ret; + i3c_master_unregister(&master->base); clk_disable_unprepare(master->sysclk); clk_disable_unprepare(master->pclk); diff --git a/drivers/i3c/master/mipi-i3c-hci/core.c b/drivers/i3c/master/mipi-i3c-hci/core.c index 6aef5ce43cc1..f9bc58366a72 100644 --- a/drivers/i3c/master/mipi-i3c-hci/core.c +++ b/drivers/i3c/master/mipi-i3c-hci/core.c @@ -769,7 +769,9 @@ static int i3c_hci_remove(struct platform_device *pdev) { struct i3c_hci *hci = platform_get_drvdata(pdev); - return i3c_master_unregister(&hci->master); + i3c_master_unregister(&hci->master); + + return 0; } static const __maybe_unused struct of_device_id i3c_hci_of_match[] = { diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c index d6e9ed74cdcf..e5476d04b403 100644 --- a/drivers/i3c/master/svc-i3c-master.c +++ b/drivers/i3c/master/svc-i3c-master.c @@ -1572,11 +1572,8 @@ err_disable_clks: static int svc_i3c_master_remove(struct platform_device *pdev) { struct svc_i3c_master *master = platform_get_drvdata(pdev); - int ret; - ret = i3c_master_unregister(&master->base); - if (ret) - return ret; + i3c_master_unregister(&master->base); pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_disable(&pdev->dev); diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h index a12cda9dc715..0b52da4f2346 100644 --- a/include/linux/i3c/master.h +++ b/include/linux/i3c/master.h @@ -542,7 +542,7 @@ int i3c_master_register(struct i3c_master_controller *master, struct device *parent, const struct i3c_master_controller_ops *ops, bool secondary); -int i3c_master_unregister(struct i3c_master_controller *master); +void i3c_master_unregister(struct i3c_master_controller *master); /** * i3c_dev_get_master_data() - get master private data attached to an I3C -- cgit v1.2.3 From e3ff7c609f39671d1aaff4fb4a8594e14f3e03f8 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Fri, 24 Feb 2023 08:50:00 -0800 Subject: livepatch,sched: Add livepatch task switching to cond_resched() There have been reports [1][2] of live patches failing to complete within a reasonable amount of time due to CPU-bound kthreads. Fix it by patching tasks in cond_resched(). There are four different flavors of cond_resched(), depending on the kernel configuration. Hook into all of them. A more elegant solution might be to use a preempt notifier. However, non-ORC unwinders can't unwind a preempted task reliably. [1] https://lore.kernel.org/lkml/20220507174628.2086373-1-song@kernel.org/ [2] https://lkml.kernel.org/lkml/20230120-vhost-klp-switching-v1-0-7c2b65519c43@kernel.org Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Petr Mladek Tested-by: Seth Forshee (DigitalOcean) Link: https://lore.kernel.org/r/4ae981466b7814ec221014fc2554b2f86f3fb70b.1677257135.git.jpoimboe@kernel.org --- include/linux/livepatch.h | 1 + include/linux/livepatch_sched.h | 29 +++++++++++ include/linux/sched.h | 20 ++++++-- kernel/livepatch/core.c | 1 + kernel/livepatch/transition.c | 107 ++++++++++++++++++++++++++++++++++------ kernel/sched/core.c | 64 +++++++++++++++++++++--- 6 files changed, 194 insertions(+), 28 deletions(-) create mode 100644 include/linux/livepatch_sched.h (limited to 'include/linux') diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index 293e29960c6e..9b9b38e89563 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -13,6 +13,7 @@ #include #include #include +#include #if IS_ENABLED(CONFIG_LIVEPATCH) diff --git a/include/linux/livepatch_sched.h b/include/linux/livepatch_sched.h new file mode 100644 index 000000000000..013794fb5da0 --- /dev/null +++ b/include/linux/livepatch_sched.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _LINUX_LIVEPATCH_SCHED_H_ +#define _LINUX_LIVEPATCH_SCHED_H_ + +#include +#include + +#ifdef CONFIG_LIVEPATCH + +void __klp_sched_try_switch(void); + +#if !defined(CONFIG_PREEMPT_DYNAMIC) || !defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) + +DECLARE_STATIC_KEY_FALSE(klp_sched_try_switch_key); + +static __always_inline void klp_sched_try_switch(void) +{ + if (static_branch_unlikely(&klp_sched_try_switch_key)) + __klp_sched_try_switch(); +} + +#endif /* !CONFIG_PREEMPT_DYNAMIC || !CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ + +#else /* !CONFIG_LIVEPATCH */ +static inline void klp_sched_try_switch(void) {} +static inline void __klp_sched_try_switch(void) {} +#endif /* CONFIG_LIVEPATCH */ + +#endif /* _LINUX_LIVEPATCH_SCHED_H_ */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 63d242164b1a..6d654eb4cabd 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -36,6 +36,7 @@ #include #include #include +#include #include /* task_struct member predeclarations (sorted alphabetically): */ @@ -2070,6 +2071,9 @@ extern int __cond_resched(void); #if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) +void sched_dynamic_klp_enable(void); +void sched_dynamic_klp_disable(void); + DECLARE_STATIC_CALL(cond_resched, __cond_resched); static __always_inline int _cond_resched(void) @@ -2078,6 +2082,7 @@ static __always_inline int _cond_resched(void) } #elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) + extern int dynamic_cond_resched(void); static __always_inline int _cond_resched(void) @@ -2085,20 +2090,25 @@ static __always_inline int _cond_resched(void) return dynamic_cond_resched(); } -#else +#else /* !CONFIG_PREEMPTION */ static inline int _cond_resched(void) { + klp_sched_try_switch(); return __cond_resched(); } -#endif /* CONFIG_PREEMPT_DYNAMIC */ +#endif /* PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ -#else +#else /* CONFIG_PREEMPTION && !CONFIG_PREEMPT_DYNAMIC */ -static inline int _cond_resched(void) { return 0; } +static inline int _cond_resched(void) +{ + klp_sched_try_switch(); + return 0; +} -#endif /* !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) */ +#endif /* !CONFIG_PREEMPTION || CONFIG_PREEMPT_DYNAMIC */ #define cond_resched() ({ \ __might_resched(__FILE__, __LINE__, 0); \ diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 4bd2d5e10f20..eea7c8ec6e05 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -33,6 +33,7 @@ * * - klp_ftrace_handler() * - klp_update_patch_state() + * - __klp_sched_try_switch() */ DEFINE_MUTEX(klp_mutex); diff --git a/kernel/livepatch/transition.c b/kernel/livepatch/transition.c index 824e2e3f07dd..e9fd83a02228 100644 --- a/kernel/livepatch/transition.c +++ b/kernel/livepatch/transition.c @@ -9,6 +9,7 @@ #include #include +#include #include "core.h" #include "patch.h" #include "transition.h" @@ -26,6 +27,25 @@ static int klp_target_state = KLP_UNDEFINED; static unsigned int klp_signals_cnt; +/* + * When a livepatch is in progress, enable klp stack checking in + * cond_resched(). This helps CPU-bound kthreads get patched. + */ +#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) + +#define klp_cond_resched_enable() sched_dynamic_klp_enable() +#define klp_cond_resched_disable() sched_dynamic_klp_disable() + +#else /* !CONFIG_PREEMPT_DYNAMIC || !CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ + +DEFINE_STATIC_KEY_FALSE(klp_sched_try_switch_key); +EXPORT_SYMBOL(klp_sched_try_switch_key); + +#define klp_cond_resched_enable() static_branch_enable(&klp_sched_try_switch_key) +#define klp_cond_resched_disable() static_branch_disable(&klp_sched_try_switch_key) + +#endif /* CONFIG_PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ + /* * This work can be performed periodically to finish patching or unpatching any * "straggler" tasks which failed to transition in the first attempt. @@ -174,8 +194,8 @@ void klp_update_patch_state(struct task_struct *task) * barrier (smp_rmb) for two cases: * * 1) Enforce the order of the TIF_PATCH_PENDING read and the - * klp_target_state read. The corresponding write barrier is in - * klp_init_transition(). + * klp_target_state read. The corresponding write barriers are in + * klp_init_transition() and klp_reverse_transition(). * * 2) Enforce the order of the TIF_PATCH_PENDING read and a future read * of func->transition, if klp_ftrace_handler() is called later on @@ -343,6 +363,44 @@ static bool klp_try_switch_task(struct task_struct *task) return !ret; } +void __klp_sched_try_switch(void) +{ + if (likely(!klp_patch_pending(current))) + return; + + /* + * This function is called from cond_resched() which is called in many + * places throughout the kernel. Using the klp_mutex here might + * deadlock. + * + * Instead, disable preemption to prevent racing with other callers of + * klp_try_switch_task(). Thanks to task_call_func() they won't be + * able to switch this task while it's running. + */ + preempt_disable(); + + /* + * Make sure current didn't get patched between the above check and + * preempt_disable(). + */ + if (unlikely(!klp_patch_pending(current))) + goto out; + + /* + * Enforce the order of the TIF_PATCH_PENDING read above and the + * klp_target_state read in klp_try_switch_task(). The corresponding + * write barriers are in klp_init_transition() and + * klp_reverse_transition(). + */ + smp_rmb(); + + klp_try_switch_task(current); + +out: + preempt_enable(); +} +EXPORT_SYMBOL(__klp_sched_try_switch); + /* * Sends a fake signal to all non-kthread tasks with TIF_PATCH_PENDING set. * Kthreads with TIF_PATCH_PENDING set are woken up. @@ -449,7 +507,8 @@ void klp_try_complete_transition(void) return; } - /* we're done, now cleanup the data structures */ + /* Done! Now cleanup the data structures. */ + klp_cond_resched_disable(); patch = klp_transition_patch; klp_complete_transition(); @@ -501,6 +560,8 @@ void klp_start_transition(void) set_tsk_thread_flag(task, TIF_PATCH_PENDING); } + klp_cond_resched_enable(); + klp_signals_cnt = 0; } @@ -556,8 +617,9 @@ void klp_init_transition(struct klp_patch *patch, int state) * see a func in transition with a task->patch_state of KLP_UNDEFINED. * * Also enforce the order of the klp_target_state write and future - * TIF_PATCH_PENDING writes to ensure klp_update_patch_state() doesn't - * set a task->patch_state to KLP_UNDEFINED. + * TIF_PATCH_PENDING writes to ensure klp_update_patch_state() and + * __klp_sched_try_switch() don't set a task->patch_state to + * KLP_UNDEFINED. */ smp_wmb(); @@ -593,14 +655,10 @@ void klp_reverse_transition(void) klp_target_state == KLP_PATCHED ? "patching to unpatching" : "unpatching to patching"); - klp_transition_patch->enabled = !klp_transition_patch->enabled; - - klp_target_state = !klp_target_state; - /* * Clear all TIF_PATCH_PENDING flags to prevent races caused by - * klp_update_patch_state() running in parallel with - * klp_start_transition(). + * klp_update_patch_state() or __klp_sched_try_switch() running in + * parallel with the reverse transition. */ read_lock(&tasklist_lock); for_each_process_thread(g, task) @@ -610,9 +668,28 @@ void klp_reverse_transition(void) for_each_possible_cpu(cpu) clear_tsk_thread_flag(idle_task(cpu), TIF_PATCH_PENDING); - /* Let any remaining calls to klp_update_patch_state() complete */ + /* + * Make sure all existing invocations of klp_update_patch_state() and + * __klp_sched_try_switch() see the cleared TIF_PATCH_PENDING before + * starting the reverse transition. + */ klp_synchronize_transition(); + /* + * All patching has stopped, now re-initialize the global variables to + * prepare for the reverse transition. + */ + klp_transition_patch->enabled = !klp_transition_patch->enabled; + klp_target_state = !klp_target_state; + + /* + * Enforce the order of the klp_target_state write and the + * TIF_PATCH_PENDING writes in klp_start_transition() to ensure + * klp_update_patch_state() and __klp_sched_try_switch() don't set + * task->patch_state to the wrong value. + */ + smp_wmb(); + klp_start_transition(); } @@ -626,9 +703,9 @@ void klp_copy_process(struct task_struct *child) * the task flag up to date with the parent here. * * The operation is serialized against all klp_*_transition() - * operations by the tasklist_lock. The only exception is - * klp_update_patch_state(current), but we cannot race with - * that because we are current. + * operations by the tasklist_lock. The only exceptions are + * klp_update_patch_state(current) and __klp_sched_try_switch(), but we + * cannot race with them because we are current. */ if (test_tsk_thread_flag(current, TIF_PATCH_PENDING)) set_tsk_thread_flag(child, TIF_PATCH_PENDING); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 5ddd9610be56..b9616f153946 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8525,6 +8525,7 @@ EXPORT_STATIC_CALL_TRAMP(might_resched); static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); int __sched dynamic_cond_resched(void) { + klp_sched_try_switch(); if (!static_branch_unlikely(&sk_dynamic_cond_resched)) return 0; return __cond_resched(); @@ -8673,13 +8674,17 @@ int sched_dynamic_mode(const char *str) #error "Unsupported PREEMPT_DYNAMIC mechanism" #endif -void sched_dynamic_update(int mode) +DEFINE_MUTEX(sched_dynamic_mutex); +static bool klp_override; + +static void __sched_dynamic_update(int mode) { /* * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in * the ZERO state, which is invalid. */ - preempt_dynamic_enable(cond_resched); + if (!klp_override) + preempt_dynamic_enable(cond_resched); preempt_dynamic_enable(might_resched); preempt_dynamic_enable(preempt_schedule); preempt_dynamic_enable(preempt_schedule_notrace); @@ -8687,36 +8692,79 @@ void sched_dynamic_update(int mode) switch (mode) { case preempt_dynamic_none: - preempt_dynamic_enable(cond_resched); + if (!klp_override) + preempt_dynamic_enable(cond_resched); preempt_dynamic_disable(might_resched); preempt_dynamic_disable(preempt_schedule); preempt_dynamic_disable(preempt_schedule_notrace); preempt_dynamic_disable(irqentry_exit_cond_resched); - pr_info("Dynamic Preempt: none\n"); + if (mode != preempt_dynamic_mode) + pr_info("Dynamic Preempt: none\n"); break; case preempt_dynamic_voluntary: - preempt_dynamic_enable(cond_resched); + if (!klp_override) + preempt_dynamic_enable(cond_resched); preempt_dynamic_enable(might_resched); preempt_dynamic_disable(preempt_schedule); preempt_dynamic_disable(preempt_schedule_notrace); preempt_dynamic_disable(irqentry_exit_cond_resched); - pr_info("Dynamic Preempt: voluntary\n"); + if (mode != preempt_dynamic_mode) + pr_info("Dynamic Preempt: voluntary\n"); break; case preempt_dynamic_full: - preempt_dynamic_disable(cond_resched); + if (!klp_override) + preempt_dynamic_disable(cond_resched); preempt_dynamic_disable(might_resched); preempt_dynamic_enable(preempt_schedule); preempt_dynamic_enable(preempt_schedule_notrace); preempt_dynamic_enable(irqentry_exit_cond_resched); - pr_info("Dynamic Preempt: full\n"); + if (mode != preempt_dynamic_mode) + pr_info("Dynamic Preempt: full\n"); break; } preempt_dynamic_mode = mode; } +void sched_dynamic_update(int mode) +{ + mutex_lock(&sched_dynamic_mutex); + __sched_dynamic_update(mode); + mutex_unlock(&sched_dynamic_mutex); +} + +#ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL + +static int klp_cond_resched(void) +{ + __klp_sched_try_switch(); + return __cond_resched(); +} + +void sched_dynamic_klp_enable(void) +{ + mutex_lock(&sched_dynamic_mutex); + + klp_override = true; + static_call_update(cond_resched, klp_cond_resched); + + mutex_unlock(&sched_dynamic_mutex); +} + +void sched_dynamic_klp_disable(void) +{ + mutex_lock(&sched_dynamic_mutex); + + klp_override = false; + __sched_dynamic_update(preempt_dynamic_mode); + + mutex_unlock(&sched_dynamic_mutex); +} + +#endif /* CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ + static int __init setup_preempt_mode(char *str) { int mode = sched_dynamic_mode(str); -- cgit v1.2.3 From 6e2a3a324aab9d76e6b864eb6ebc60b197b5141a Mon Sep 17 00:00:00 2001 From: Or Har-Toov Date: Sun, 19 Mar 2023 14:59:30 +0200 Subject: net/mlx5: Expose bits for enabling out-of-order by default Add needed HW bits for enabling out-of-order by default and use go_back_n when out-of-order is not needed. Signed-off-by: Or Har-Toov Signed-off-by: Leon Romanovsky Link: https://lore.kernel.org/r/75d6dfe263989a05c08c43406132b336ea12d00a.1679230449.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- include/linux/mlx5/mlx5_ifc.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 66d76e97a087..62039b147432 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1077,7 +1077,9 @@ struct mlx5_ifc_roce_cap_bits { u8 sw_r_roce_src_udp_port[0x1]; u8 fl_rc_qp_when_roce_disabled[0x1]; u8 fl_rc_qp_when_roce_enabled[0x1]; - u8 reserved_at_7[0x17]; + u8 reserved_at_7[0x1]; + u8 qp_ooo_transmit_default[0x1]; + u8 reserved_at_9[0x15]; u8 qp_ts_format[0x2]; u8 reserved_at_20[0x60]; @@ -1493,7 +1495,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_b0[0x1]; u8 uplink_follow[0x1]; u8 ts_cqe_to_dest_cqn[0x1]; - u8 reserved_at_b3[0x7]; + u8 reserved_at_b3[0x6]; + u8 go_back_n[0x1]; u8 shampo[0x1]; u8 reserved_at_bb[0x5]; @@ -3261,7 +3264,8 @@ struct mlx5_ifc_qpc_bits { u8 log_rq_stride[0x3]; u8 no_sq[0x1]; u8 log_sq_size[0x4]; - u8 reserved_at_55[0x3]; + u8 reserved_at_55[0x1]; + u8 retry_mode[0x2]; u8 ts_format[0x2]; u8 reserved_at_5a[0x1]; u8 rlky[0x1]; -- cgit v1.2.3 From 9cc61e5fbd619eea0401f519e7bac72fe3d4d1e8 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:29:04 +0100 Subject: driver core: bus: move dev_root out of struct bus_type Now that all accesses of dev_root is through the bus_get_dev_root() call, move the pointer out of struct bus_type and into the private dynamic structure, subsys_private. With this change, there is no modifiable portions of struct bus_type so it can be marked as a constant structure and moved to read-only memory. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313182918.1312597-22-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/base.h | 2 ++ drivers/base/bus.c | 28 ++++++++++++++++++++++------ include/linux/device/bus.h | 2 -- 3 files changed, 24 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/base.h b/drivers/base/base.h index b055eba1ec30..f1034e27e651 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -27,6 +27,7 @@ * on this bus. * @bus - pointer back to the struct bus_type that this structure is associated * with. + * @dev_root: Default device to use as the parent. * * @glue_dirs - "glue" directory to put in-between the parent device to * avoid namespace conflicts @@ -49,6 +50,7 @@ struct subsys_private { struct blocking_notifier_head bus_notifier; unsigned int drivers_autoprobe:1; struct bus_type *bus; + struct device *dev_root; struct kset glue_dirs; struct class *class; diff --git a/drivers/base/bus.c b/drivers/base/bus.c index dd4b82d7510f..91a6b6b1fc49 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -935,8 +935,8 @@ void bus_unregister(const struct bus_type *bus) return; pr_debug("bus: '%s': unregistering\n", bus->name); - if (bus->dev_root) - device_unregister(bus->dev_root); + if (sp->dev_root) + device_unregister(sp->dev_root); bus_kobj = &sp->subsys.kobj; sysfs_remove_groups(bus_kobj, bus->bus_groups); @@ -1198,6 +1198,7 @@ static int subsys_register(struct bus_type *subsys, const struct attribute_group **groups, struct kobject *parent_of_root) { + struct subsys_private *sp; struct device *dev; int err; @@ -1205,6 +1206,12 @@ static int subsys_register(struct bus_type *subsys, if (err < 0) return err; + sp = bus_to_subsys(subsys); + if (!sp) { + err = -EINVAL; + goto err_sp; + } + dev = kzalloc(sizeof(struct device), GFP_KERNEL); if (!dev) { err = -ENOMEM; @@ -1223,7 +1230,8 @@ static int subsys_register(struct bus_type *subsys, if (err < 0) goto err_dev_reg; - subsys->dev_root = dev; + sp->dev_root = dev; + subsys_put(sp); return 0; err_dev_reg: @@ -1232,6 +1240,8 @@ err_dev_reg: err_name: kfree(dev); err_dev: + subsys_put(sp); +err_sp: bus_unregister(subsys); return err; } @@ -1349,9 +1359,15 @@ bool bus_is_registered(const struct bus_type *bus) */ struct device *bus_get_dev_root(const struct bus_type *bus) { - if (bus) - return get_device(bus->dev_root); - return NULL; + struct subsys_private *sp = bus_to_subsys(bus); + struct device *dev_root; + + if (!sp) + return NULL; + + dev_root = get_device(sp->dev_root); + subsys_put(sp); + return dev_root; } EXPORT_SYMBOL_GPL(bus_get_dev_root); diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index 6ce32ef4b8fd..c258e8770285 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -26,7 +26,6 @@ struct fwnode_handle; * * @name: The name of the bus. * @dev_name: Used for subsystems to enumerate devices like ("foo%u", dev->id). - * @dev_root: Default device to use as the parent. * @bus_groups: Default attributes of the bus. * @dev_groups: Default attributes of the devices on the bus. * @drv_groups: Default attributes of the device drivers on the bus. @@ -82,7 +81,6 @@ struct fwnode_handle; struct bus_type { const char *name; const char *dev_name; - struct device *dev_root; const struct attribute_group **bus_groups; const struct attribute_group **dev_groups; const struct attribute_group **drv_groups; -- cgit v1.2.3 From 75cff725d9566699a670a02b3cfd1c6e9e9ed53e Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:29:05 +0100 Subject: driver core: bus: mark the struct bus_type for sysfs callbacks as constant struct bus_type should never be modified in a sysfs callback as there is nothing in the structure to modify, and frankly, the structure is almost never used in a sysfs callback, so mark it as constant to allow struct bus_type to be moved to read-only memory. Cc: "David S. Miller" Cc: "James E.J. Bottomley" Cc: "K. Y. Srinivasan" Cc: Alexander Gordeev Cc: Alexandre Bounine Cc: Alison Schofield Cc: Ben Widawsky Cc: Dexuan Cui Cc: Eric Dumazet Cc: Haiyang Zhang Cc: Hannes Reinecke Cc: Harald Freudenberger Cc: Heiko Carstens Cc: Hu Haowen Cc: Jakub Kicinski Cc: Jens Axboe Cc: Jonathan Corbet Cc: Laurentiu Tudor Cc: Matt Porter Cc: Michael Ellerman Cc: Paolo Abeni Cc: Stuart Yoder Cc: Vasily Gorbik Cc: Vishal Verma Cc: Yanteng Si Acked-by: Ilya Dryomov # rbd Acked-by: Ira Weiny # cxl Reviewed-by: Alex Shi Acked-by: Iwona Winiarska Acked-by: Dan Williams Acked-by: Bjorn Helgaas # pci Acked-by: Wei Liu Acked-by: Martin K. Petersen # scsi Link: https://lore.kernel.org/r/20230313182918.1312597-23-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/driver-model/bus.rst | 4 +-- Documentation/filesystems/sysfs.rst | 4 +-- .../translations/zh_CN/filesystems/sysfs.txt | 4 +-- .../translations/zh_TW/filesystems/sysfs.txt | 4 +-- arch/powerpc/platforms/pseries/ibmebus.c | 4 +-- arch/powerpc/platforms/pseries/vio.c | 8 ++--- drivers/ata/pata_parport/pata_parport.c | 6 ++-- drivers/base/bus.c | 8 ++--- drivers/block/rbd.c | 34 ++++++++---------- drivers/bus/fsl-mc/fsl-mc-bus.c | 6 ++-- drivers/cxl/core/port.c | 2 +- drivers/hv/vmbus_drv.c | 2 +- drivers/net/netdevsim/bus.c | 4 +-- drivers/pci/pci-sysfs.c | 2 +- drivers/pci/pci.c | 4 +-- drivers/peci/sysfs.c | 2 +- drivers/rapidio/rio-sysfs.c | 2 +- drivers/s390/crypto/ap_bus.c | 42 +++++++++++----------- drivers/scsi/fcoe/fcoe_sysfs.c | 8 ++--- drivers/scsi/fcoe/fcoe_transport.c | 6 ++-- include/linux/device/bus.h | 4 +-- include/scsi/libfcoe.h | 6 ++-- 22 files changed, 78 insertions(+), 88 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/driver-model/bus.rst b/Documentation/driver-api/driver-model/bus.rst index 016b15a6e8ea..9709ab62a468 100644 --- a/Documentation/driver-api/driver-model/bus.rst +++ b/Documentation/driver-api/driver-model/bus.rst @@ -125,8 +125,8 @@ Exporting Attributes struct bus_attribute { struct attribute attr; - ssize_t (*show)(struct bus_type *, char * buf); - ssize_t (*store)(struct bus_type *, const char * buf, size_t count); + ssize_t (*show)(const struct bus_type *, char * buf); + ssize_t (*store)(const struct bus_type *, const char * buf, size_t count); }; Bus drivers can export attributes using the BUS_ATTR_RW macro that works diff --git a/Documentation/filesystems/sysfs.rst b/Documentation/filesystems/sysfs.rst index f8187d466b97..c32993bc83c7 100644 --- a/Documentation/filesystems/sysfs.rst +++ b/Documentation/filesystems/sysfs.rst @@ -373,8 +373,8 @@ Structure:: struct bus_attribute { struct attribute attr; - ssize_t (*show)(struct bus_type *, char * buf); - ssize_t (*store)(struct bus_type *, const char * buf, size_t count); + ssize_t (*show)(const struct bus_type *, char * buf); + ssize_t (*store)(const struct bus_type *, const char * buf, size_t count); }; Declaring:: diff --git a/Documentation/translations/zh_CN/filesystems/sysfs.txt b/Documentation/translations/zh_CN/filesystems/sysfs.txt index 046cc1d52058..547062759e60 100644 --- a/Documentation/translations/zh_CN/filesystems/sysfs.txt +++ b/Documentation/translations/zh_CN/filesystems/sysfs.txt @@ -329,8 +329,8 @@ void device_remove_file(struct device *dev, const struct device_attribute * attr struct bus_attribute { struct attribute attr; - ssize_t (*show)(struct bus_type *, char * buf); - ssize_t (*store)(struct bus_type *, const char * buf, size_t count); + ssize_t (*show)(const struct bus_type *, char * buf); + ssize_t (*store)(const struct bus_type *, const char * buf, size_t count); }; 声明: diff --git a/Documentation/translations/zh_TW/filesystems/sysfs.txt b/Documentation/translations/zh_TW/filesystems/sysfs.txt index acd677f19d4f..280824cc7e5d 100644 --- a/Documentation/translations/zh_TW/filesystems/sysfs.txt +++ b/Documentation/translations/zh_TW/filesystems/sysfs.txt @@ -332,8 +332,8 @@ void device_remove_file(struct device *dev, const struct device_attribute * attr struct bus_attribute { struct attribute attr; - ssize_t (*show)(struct bus_type *, char * buf); - ssize_t (*store)(struct bus_type *, const char * buf, size_t count); + ssize_t (*show)(const struct bus_type *, char * buf); + ssize_t (*store)(const struct bus_type *, const char * buf, size_t count); }; 聲明: diff --git a/arch/powerpc/platforms/pseries/ibmebus.c b/arch/powerpc/platforms/pseries/ibmebus.c index bb9c18682783..44703f13985b 100644 --- a/arch/powerpc/platforms/pseries/ibmebus.c +++ b/arch/powerpc/platforms/pseries/ibmebus.c @@ -267,7 +267,7 @@ static char *ibmebus_chomp(const char *in, size_t count) return out; } -static ssize_t probe_store(struct bus_type *bus, const char *buf, size_t count) +static ssize_t probe_store(const struct bus_type *bus, const char *buf, size_t count) { struct device_node *dn = NULL; struct device *dev; @@ -305,7 +305,7 @@ out: } static BUS_ATTR_WO(probe); -static ssize_t remove_store(struct bus_type *bus, const char *buf, size_t count) +static ssize_t remove_store(const struct bus_type *bus, const char *buf, size_t count) { struct device *dev; char *path; diff --git a/arch/powerpc/platforms/pseries/vio.c b/arch/powerpc/platforms/pseries/vio.c index 770df9351aaa..bf7aff6390be 100644 --- a/arch/powerpc/platforms/pseries/vio.c +++ b/arch/powerpc/platforms/pseries/vio.c @@ -1006,7 +1006,7 @@ ATTRIBUTE_GROUPS(vio_cmo_dev); /* sysfs bus functions and data structures for CMO */ #define viobus_cmo_rd_attr(name) \ -static ssize_t cmo_bus_##name##_show(struct bus_type *bt, char *buf) \ +static ssize_t cmo_bus_##name##_show(const struct bus_type *bt, char *buf) \ { \ return sprintf(buf, "%lu\n", vio_cmo.name); \ } \ @@ -1015,7 +1015,7 @@ static struct bus_attribute bus_attr_cmo_bus_##name = \ #define viobus_cmo_pool_rd_attr(name, var) \ static ssize_t \ -cmo_##name##_##var##_show(struct bus_type *bt, char *buf) \ +cmo_##name##_##var##_show(const struct bus_type *bt, char *buf) \ { \ return sprintf(buf, "%lu\n", vio_cmo.name.var); \ } \ @@ -1030,12 +1030,12 @@ viobus_cmo_pool_rd_attr(reserve, size); viobus_cmo_pool_rd_attr(excess, size); viobus_cmo_pool_rd_attr(excess, free); -static ssize_t cmo_high_show(struct bus_type *bt, char *buf) +static ssize_t cmo_high_show(const struct bus_type *bt, char *buf) { return sprintf(buf, "%lu\n", vio_cmo.high); } -static ssize_t cmo_high_store(struct bus_type *bt, const char *buf, +static ssize_t cmo_high_store(const struct bus_type *bt, const char *buf, size_t count) { unsigned long flags; diff --git a/drivers/ata/pata_parport/pata_parport.c b/drivers/ata/pata_parport/pata_parport.c index 294a266a0dda..64d1bde26940 100644 --- a/drivers/ata/pata_parport/pata_parport.c +++ b/drivers/ata/pata_parport/pata_parport.c @@ -554,8 +554,7 @@ void pata_parport_unregister_driver(struct pi_protocol *pr) } EXPORT_SYMBOL_GPL(pata_parport_unregister_driver); -static ssize_t new_device_store(struct bus_type *bus, const char *buf, - size_t count) +static ssize_t new_device_store(const struct bus_type *bus, const char *buf, size_t count) { char port[12] = "auto"; char protocol[8] = "auto"; @@ -630,8 +629,7 @@ static void pi_remove_one(struct device *dev) /* pata_parport_dev_release will do kfree(pi) */ } -static ssize_t delete_device_store(struct bus_type *bus, const char *buf, - size_t count) +static ssize_t delete_device_store(const struct bus_type *bus, const char *buf, size_t count) { struct device *dev; diff --git a/drivers/base/bus.c b/drivers/base/bus.c index 91a6b6b1fc49..819ab745fa9f 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -274,7 +274,7 @@ static ssize_t bind_store(struct device_driver *drv, const char *buf, } static DRIVER_ATTR_IGNORE_LOCKDEP(bind, 0200, NULL, bind_store); -static ssize_t drivers_autoprobe_show(struct bus_type *bus, char *buf) +static ssize_t drivers_autoprobe_show(const struct bus_type *bus, char *buf) { struct subsys_private *sp = bus_to_subsys(bus); int ret; @@ -287,7 +287,7 @@ static ssize_t drivers_autoprobe_show(struct bus_type *bus, char *buf) return ret; } -static ssize_t drivers_autoprobe_store(struct bus_type *bus, +static ssize_t drivers_autoprobe_store(const struct bus_type *bus, const char *buf, size_t count) { struct subsys_private *sp = bus_to_subsys(bus); @@ -304,7 +304,7 @@ static ssize_t drivers_autoprobe_store(struct bus_type *bus, return count; } -static ssize_t drivers_probe_store(struct bus_type *bus, +static ssize_t drivers_probe_store(const struct bus_type *bus, const char *buf, size_t count) { struct device *dev; @@ -808,7 +808,7 @@ static void klist_devices_put(struct klist_node *n) put_device(dev); } -static ssize_t bus_uevent_store(struct bus_type *bus, +static ssize_t bus_uevent_store(const struct bus_type *bus, const char *buf, size_t count) { struct subsys_private *sp = bus_to_subsys(bus); diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index 5cb008b9700a..84ad3b17956f 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -491,12 +491,12 @@ static bool single_major = true; module_param(single_major, bool, 0444); MODULE_PARM_DESC(single_major, "Use a single major number for all rbd devices (default: true)"); -static ssize_t add_store(struct bus_type *bus, const char *buf, size_t count); -static ssize_t remove_store(struct bus_type *bus, const char *buf, +static ssize_t add_store(const struct bus_type *bus, const char *buf, size_t count); +static ssize_t remove_store(const struct bus_type *bus, const char *buf, size_t count); -static ssize_t add_single_major_store(struct bus_type *bus, const char *buf, +static ssize_t add_single_major_store(const struct bus_type *bus, const char *buf, size_t count); -static ssize_t remove_single_major_store(struct bus_type *bus, const char *buf, +static ssize_t remove_single_major_store(const struct bus_type *bus, const char *buf, size_t count); static int rbd_dev_image_probe(struct rbd_device *rbd_dev, int depth); @@ -538,7 +538,7 @@ static bool rbd_is_lock_owner(struct rbd_device *rbd_dev) return is_lock_owner; } -static ssize_t supported_features_show(struct bus_type *bus, char *buf) +static ssize_t supported_features_show(const struct bus_type *bus, char *buf) { return sprintf(buf, "0x%llx\n", RBD_FEATURES_SUPPORTED); } @@ -6967,9 +6967,7 @@ err_out_format: return ret; } -static ssize_t do_rbd_add(struct bus_type *bus, - const char *buf, - size_t count) +static ssize_t do_rbd_add(const char *buf, size_t count) { struct rbd_device *rbd_dev = NULL; struct ceph_options *ceph_opts = NULL; @@ -7081,18 +7079,18 @@ err_out_args: goto out; } -static ssize_t add_store(struct bus_type *bus, const char *buf, size_t count) +static ssize_t add_store(const struct bus_type *bus, const char *buf, size_t count) { if (single_major) return -EINVAL; - return do_rbd_add(bus, buf, count); + return do_rbd_add(buf, count); } -static ssize_t add_single_major_store(struct bus_type *bus, const char *buf, +static ssize_t add_single_major_store(const struct bus_type *bus, const char *buf, size_t count) { - return do_rbd_add(bus, buf, count); + return do_rbd_add(buf, count); } static void rbd_dev_remove_parent(struct rbd_device *rbd_dev) @@ -7122,9 +7120,7 @@ static void rbd_dev_remove_parent(struct rbd_device *rbd_dev) } } -static ssize_t do_rbd_remove(struct bus_type *bus, - const char *buf, - size_t count) +static ssize_t do_rbd_remove(const char *buf, size_t count) { struct rbd_device *rbd_dev = NULL; struct list_head *tmp; @@ -7196,18 +7192,18 @@ static ssize_t do_rbd_remove(struct bus_type *bus, return count; } -static ssize_t remove_store(struct bus_type *bus, const char *buf, size_t count) +static ssize_t remove_store(const struct bus_type *bus, const char *buf, size_t count) { if (single_major) return -EINVAL; - return do_rbd_remove(bus, buf, count); + return do_rbd_remove(buf, count); } -static ssize_t remove_single_major_store(struct bus_type *bus, const char *buf, +static ssize_t remove_single_major_store(const struct bus_type *bus, const char *buf, size_t count) { - return do_rbd_remove(bus, buf, count); + return do_rbd_remove(buf, count); } /* diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 36cb091a33b4..653e2d4c116f 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -231,7 +231,7 @@ exit: return 0; } -static ssize_t rescan_store(struct bus_type *bus, +static ssize_t rescan_store(const struct bus_type *bus, const char *buf, size_t count) { unsigned long val; @@ -284,7 +284,7 @@ exit: return 0; } -static ssize_t autorescan_store(struct bus_type *bus, +static ssize_t autorescan_store(const struct bus_type *bus, const char *buf, size_t count) { bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_set_autorescan); @@ -292,7 +292,7 @@ static ssize_t autorescan_store(struct bus_type *bus, return count; } -static ssize_t autorescan_show(struct bus_type *bus, char *buf) +static ssize_t autorescan_show(const struct bus_type *bus, char *buf) { bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_get_autorescan); return strlen(buf); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 8ee6b6e2e2a4..66333cd6248e 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -1927,7 +1927,7 @@ bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd) EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL); /* for user tooling to ensure port disable work has completed */ -static ssize_t flush_store(struct bus_type *bus, const char *buf, size_t count) +static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count) { if (sysfs_streq(buf, "1")) { flush_workqueue(cxl_bus_wq); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index d24dd65b33d4..513adba09f56 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -684,7 +684,7 @@ static const struct attribute_group vmbus_dev_group = { __ATTRIBUTE_GROUPS(vmbus_dev); /* Set up the attribute for /sys/bus/vmbus/hibernation */ -static ssize_t hibernation_show(struct bus_type *bus, char *buf) +static ssize_t hibernation_show(const struct bus_type *bus, char *buf) { return sprintf(buf, "%d\n", !!hv_is_hibernation_supported()); } diff --git a/drivers/net/netdevsim/bus.c b/drivers/net/netdevsim/bus.c index 0052968e881e..0787ad252dd9 100644 --- a/drivers/net/netdevsim/bus.c +++ b/drivers/net/netdevsim/bus.c @@ -132,7 +132,7 @@ static struct nsim_bus_dev * nsim_bus_dev_new(unsigned int id, unsigned int port_count, unsigned int num_queues); static ssize_t -new_device_store(struct bus_type *bus, const char *buf, size_t count) +new_device_store(const struct bus_type *bus, const char *buf, size_t count) { unsigned int id, port_count, num_queues; struct nsim_bus_dev *nsim_bus_dev; @@ -186,7 +186,7 @@ static BUS_ATTR_WO(new_device); static void nsim_bus_dev_del(struct nsim_bus_dev *nsim_bus_dev); static ssize_t -del_device_store(struct bus_type *bus, const char *buf, size_t count) +del_device_store(const struct bus_type *bus, const char *buf, size_t count) { struct nsim_bus_dev *nsim_bus_dev, *tmp; unsigned int id; diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index dd0d9d9bc509..ab32a91f287b 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -428,7 +428,7 @@ static ssize_t msi_bus_store(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RW(msi_bus); -static ssize_t rescan_store(struct bus_type *bus, const char *buf, size_t count) +static ssize_t rescan_store(const struct bus_type *bus, const char *buf, size_t count) { unsigned long val; struct pci_bus *b = NULL; diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 7a67611dc5f4..45c3bb039f21 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -6679,7 +6679,7 @@ void pci_reassigndev_resource_alignment(struct pci_dev *dev) } } -static ssize_t resource_alignment_show(struct bus_type *bus, char *buf) +static ssize_t resource_alignment_show(const struct bus_type *bus, char *buf) { size_t count = 0; @@ -6691,7 +6691,7 @@ static ssize_t resource_alignment_show(struct bus_type *bus, char *buf) return count; } -static ssize_t resource_alignment_store(struct bus_type *bus, +static ssize_t resource_alignment_store(const struct bus_type *bus, const char *buf, size_t count) { char *param, *old, *end; diff --git a/drivers/peci/sysfs.c b/drivers/peci/sysfs.c index db9ef05776e3..c04244075794 100644 --- a/drivers/peci/sysfs.c +++ b/drivers/peci/sysfs.c @@ -15,7 +15,7 @@ static int rescan_controller(struct device *dev, void *data) return peci_controller_scan_devices(to_peci_controller(dev)); } -static ssize_t rescan_store(struct bus_type *bus, const char *buf, size_t count) +static ssize_t rescan_store(const struct bus_type *bus, const char *buf, size_t count) { bool res; int ret; diff --git a/drivers/rapidio/rio-sysfs.c b/drivers/rapidio/rio-sysfs.c index f7679602498e..90d391210533 100644 --- a/drivers/rapidio/rio-sysfs.c +++ b/drivers/rapidio/rio-sysfs.c @@ -286,7 +286,7 @@ const struct attribute_group *rio_dev_groups[] = { NULL, }; -static ssize_t scan_store(struct bus_type *bus, const char *buf, size_t count) +static ssize_t scan_store(const struct bus_type *bus, const char *buf, size_t count) { long val; int rc; diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index f4cc1720156f..5a99e0b18289 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -1166,12 +1166,12 @@ EXPORT_SYMBOL(ap_parse_mask_str); * AP bus attributes. */ -static ssize_t ap_domain_show(struct bus_type *bus, char *buf) +static ssize_t ap_domain_show(const struct bus_type *bus, char *buf) { return scnprintf(buf, PAGE_SIZE, "%d\n", ap_domain_index); } -static ssize_t ap_domain_store(struct bus_type *bus, +static ssize_t ap_domain_store(const struct bus_type *bus, const char *buf, size_t count) { int domain; @@ -1193,7 +1193,7 @@ static ssize_t ap_domain_store(struct bus_type *bus, static BUS_ATTR_RW(ap_domain); -static ssize_t ap_control_domain_mask_show(struct bus_type *bus, char *buf) +static ssize_t ap_control_domain_mask_show(const struct bus_type *bus, char *buf) { if (!ap_qci_info) /* QCI not supported */ return scnprintf(buf, PAGE_SIZE, "not supported\n"); @@ -1208,7 +1208,7 @@ static ssize_t ap_control_domain_mask_show(struct bus_type *bus, char *buf) static BUS_ATTR_RO(ap_control_domain_mask); -static ssize_t ap_usage_domain_mask_show(struct bus_type *bus, char *buf) +static ssize_t ap_usage_domain_mask_show(const struct bus_type *bus, char *buf) { if (!ap_qci_info) /* QCI not supported */ return scnprintf(buf, PAGE_SIZE, "not supported\n"); @@ -1223,7 +1223,7 @@ static ssize_t ap_usage_domain_mask_show(struct bus_type *bus, char *buf) static BUS_ATTR_RO(ap_usage_domain_mask); -static ssize_t ap_adapter_mask_show(struct bus_type *bus, char *buf) +static ssize_t ap_adapter_mask_show(const struct bus_type *bus, char *buf) { if (!ap_qci_info) /* QCI not supported */ return scnprintf(buf, PAGE_SIZE, "not supported\n"); @@ -1238,7 +1238,7 @@ static ssize_t ap_adapter_mask_show(struct bus_type *bus, char *buf) static BUS_ATTR_RO(ap_adapter_mask); -static ssize_t ap_interrupts_show(struct bus_type *bus, char *buf) +static ssize_t ap_interrupts_show(const struct bus_type *bus, char *buf) { return scnprintf(buf, PAGE_SIZE, "%d\n", ap_irq_flag ? 1 : 0); @@ -1246,12 +1246,12 @@ static ssize_t ap_interrupts_show(struct bus_type *bus, char *buf) static BUS_ATTR_RO(ap_interrupts); -static ssize_t config_time_show(struct bus_type *bus, char *buf) +static ssize_t config_time_show(const struct bus_type *bus, char *buf) { return scnprintf(buf, PAGE_SIZE, "%d\n", ap_config_time); } -static ssize_t config_time_store(struct bus_type *bus, +static ssize_t config_time_store(const struct bus_type *bus, const char *buf, size_t count) { int time; @@ -1265,12 +1265,12 @@ static ssize_t config_time_store(struct bus_type *bus, static BUS_ATTR_RW(config_time); -static ssize_t poll_thread_show(struct bus_type *bus, char *buf) +static ssize_t poll_thread_show(const struct bus_type *bus, char *buf) { return scnprintf(buf, PAGE_SIZE, "%d\n", ap_poll_kthread ? 1 : 0); } -static ssize_t poll_thread_store(struct bus_type *bus, +static ssize_t poll_thread_store(const struct bus_type *bus, const char *buf, size_t count) { int flag, rc; @@ -1289,12 +1289,12 @@ static ssize_t poll_thread_store(struct bus_type *bus, static BUS_ATTR_RW(poll_thread); -static ssize_t poll_timeout_show(struct bus_type *bus, char *buf) +static ssize_t poll_timeout_show(const struct bus_type *bus, char *buf) { return scnprintf(buf, PAGE_SIZE, "%llu\n", poll_timeout); } -static ssize_t poll_timeout_store(struct bus_type *bus, const char *buf, +static ssize_t poll_timeout_store(const struct bus_type *bus, const char *buf, size_t count) { unsigned long long time; @@ -1318,21 +1318,21 @@ static ssize_t poll_timeout_store(struct bus_type *bus, const char *buf, static BUS_ATTR_RW(poll_timeout); -static ssize_t ap_max_domain_id_show(struct bus_type *bus, char *buf) +static ssize_t ap_max_domain_id_show(const struct bus_type *bus, char *buf) { return scnprintf(buf, PAGE_SIZE, "%d\n", ap_max_domain_id); } static BUS_ATTR_RO(ap_max_domain_id); -static ssize_t ap_max_adapter_id_show(struct bus_type *bus, char *buf) +static ssize_t ap_max_adapter_id_show(const struct bus_type *bus, char *buf) { return scnprintf(buf, PAGE_SIZE, "%d\n", ap_max_adapter_id); } static BUS_ATTR_RO(ap_max_adapter_id); -static ssize_t apmask_show(struct bus_type *bus, char *buf) +static ssize_t apmask_show(const struct bus_type *bus, char *buf) { int rc; @@ -1393,7 +1393,7 @@ static int apmask_commit(unsigned long *newapm) return 0; } -static ssize_t apmask_store(struct bus_type *bus, const char *buf, +static ssize_t apmask_store(const struct bus_type *bus, const char *buf, size_t count) { int rc, changes = 0; @@ -1425,7 +1425,7 @@ done: static BUS_ATTR_RW(apmask); -static ssize_t aqmask_show(struct bus_type *bus, char *buf) +static ssize_t aqmask_show(const struct bus_type *bus, char *buf) { int rc; @@ -1486,7 +1486,7 @@ static int aqmask_commit(unsigned long *newaqm) return 0; } -static ssize_t aqmask_store(struct bus_type *bus, const char *buf, +static ssize_t aqmask_store(const struct bus_type *bus, const char *buf, size_t count) { int rc, changes = 0; @@ -1518,13 +1518,13 @@ done: static BUS_ATTR_RW(aqmask); -static ssize_t scans_show(struct bus_type *bus, char *buf) +static ssize_t scans_show(const struct bus_type *bus, char *buf) { return scnprintf(buf, PAGE_SIZE, "%llu\n", atomic64_read(&ap_scan_bus_count)); } -static ssize_t scans_store(struct bus_type *bus, const char *buf, +static ssize_t scans_store(const struct bus_type *bus, const char *buf, size_t count) { AP_DBF_INFO("%s force AP bus rescan\n", __func__); @@ -1536,7 +1536,7 @@ static ssize_t scans_store(struct bus_type *bus, const char *buf, static BUS_ATTR_RW(scans); -static ssize_t bindings_show(struct bus_type *bus, char *buf) +static ssize_t bindings_show(const struct bus_type *bus, char *buf) { int rc; unsigned int apqns, n; diff --git a/drivers/scsi/fcoe/fcoe_sysfs.c b/drivers/scsi/fcoe/fcoe_sysfs.c index 6260aa5ea6af..e17957f8085c 100644 --- a/drivers/scsi/fcoe/fcoe_sysfs.c +++ b/drivers/scsi/fcoe/fcoe_sysfs.c @@ -659,17 +659,17 @@ static const struct device_type fcoe_fcf_device_type = { .release = fcoe_fcf_device_release, }; -static ssize_t ctlr_create_store(struct bus_type *bus, const char *buf, +static ssize_t ctlr_create_store(const struct bus_type *bus, const char *buf, size_t count) { - return fcoe_ctlr_create_store(bus, buf, count); + return fcoe_ctlr_create_store(buf, count); } static BUS_ATTR_WO(ctlr_create); -static ssize_t ctlr_destroy_store(struct bus_type *bus, const char *buf, +static ssize_t ctlr_destroy_store(const struct bus_type *bus, const char *buf, size_t count) { - return fcoe_ctlr_destroy_store(bus, buf, count); + return fcoe_ctlr_destroy_store(buf, count); } static BUS_ATTR_WO(ctlr_destroy); diff --git a/drivers/scsi/fcoe/fcoe_transport.c b/drivers/scsi/fcoe/fcoe_transport.c index 62341c6353a7..46b0bf237be1 100644 --- a/drivers/scsi/fcoe/fcoe_transport.c +++ b/drivers/scsi/fcoe/fcoe_transport.c @@ -745,8 +745,7 @@ static int libfcoe_device_notification(struct notifier_block *notifier, return NOTIFY_OK; } -ssize_t fcoe_ctlr_create_store(struct bus_type *bus, - const char *buf, size_t count) +ssize_t fcoe_ctlr_create_store(const char *buf, size_t count) { struct net_device *netdev = NULL; struct fcoe_transport *ft = NULL; @@ -808,8 +807,7 @@ out_nodev: return count; } -ssize_t fcoe_ctlr_destroy_store(struct bus_type *bus, - const char *buf, size_t count) +ssize_t fcoe_ctlr_destroy_store(const char *buf, size_t count) { int rc = -ENODEV; struct net_device *netdev = NULL; diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index c258e8770285..78c875386c06 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -118,8 +118,8 @@ extern int __must_check bus_rescan_devices(struct bus_type *bus); struct bus_attribute { struct attribute attr; - ssize_t (*show)(struct bus_type *bus, char *buf); - ssize_t (*store)(struct bus_type *bus, const char *buf, size_t count); + ssize_t (*show)(const struct bus_type *bus, char *buf); + ssize_t (*store)(const struct bus_type *bus, const char *buf, size_t count); }; #define BUS_ATTR_RW(_name) \ diff --git a/include/scsi/libfcoe.h b/include/scsi/libfcoe.h index 279782156373..8300ef1a982e 100644 --- a/include/scsi/libfcoe.h +++ b/include/scsi/libfcoe.h @@ -397,10 +397,8 @@ int fcoe_transport_attach(struct fcoe_transport *ft); int fcoe_transport_detach(struct fcoe_transport *ft); /* sysfs store handler for ctrl_control interface */ -ssize_t fcoe_ctlr_create_store(struct bus_type *bus, - const char *buf, size_t count); -ssize_t fcoe_ctlr_destroy_store(struct bus_type *bus, - const char *buf, size_t count); +ssize_t fcoe_ctlr_create_store(const char *buf, size_t count); +ssize_t fcoe_ctlr_destroy_store(const char *buf, size_t count); #endif /* _LIBFCOE_H */ -- cgit v1.2.3 From 00c4a3c47da761b4ba5d09c46b6fc77af5759081 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:29:06 +0100 Subject: driver core: bus: constantify bus_register() bus_register() is now safe to take a constant * to bus_type, so make that change and mark the subsys_private bus_type * constant as well. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313182918.1312597-24-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/base.h | 2 +- drivers/base/bus.c | 2 +- include/linux/device/bus.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/base.h b/drivers/base/base.h index f1034e27e651..09c6682d16b5 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -49,7 +49,7 @@ struct subsys_private { struct klist klist_drivers; struct blocking_notifier_head bus_notifier; unsigned int drivers_autoprobe:1; - struct bus_type *bus; + const struct bus_type *bus; struct device *dev_root; struct kset glue_dirs; diff --git a/drivers/base/bus.c b/drivers/base/bus.c index 819ab745fa9f..f739a2a79e59 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -841,7 +841,7 @@ static struct bus_attribute bus_attr_uevent = __ATTR(uevent, 0200, NULL, * infrastructure, then register the children subsystems it has: * the devices and drivers that belong to the subsystem. */ -int bus_register(struct bus_type *bus) +int bus_register(const struct bus_type *bus) { int retval; struct subsys_private *priv; diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index 78c875386c06..c30c4748c170 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -110,7 +110,7 @@ struct bus_type { bool need_parent_lock; }; -extern int __must_check bus_register(struct bus_type *bus); +extern int __must_check bus_register(const struct bus_type *bus); extern void bus_unregister(const struct bus_type *bus); -- cgit v1.2.3 From 9622b9f282e0f0d37683b21575b683039169e397 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:29:07 +0100 Subject: driver core: bus: constify bus_rescan_devices() The bus_rescan_devices() function was missed in the previous change of the bus_for_each* constant pointer changes, so fix it up now to take a const * to struct bus_type. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313182918.1312597-25-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/bus.c | 2 +- include/linux/device/bus.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/bus.c b/drivers/base/bus.c index f739a2a79e59..ced61fad390e 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -769,7 +769,7 @@ static int __must_check bus_rescan_devices_helper(struct device *dev, * attached and rescan it against existing drivers to see if it matches * any by calling device_attach() for the unbound devices. */ -int bus_rescan_devices(struct bus_type *bus) +int bus_rescan_devices(const struct bus_type *bus) { return bus_for_each_dev(bus, NULL, NULL, bus_rescan_devices_helper); } diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index c30c4748c170..b517b82d4723 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -114,7 +114,7 @@ extern int __must_check bus_register(const struct bus_type *bus); extern void bus_unregister(const struct bus_type *bus); -extern int __must_check bus_rescan_devices(struct bus_type *bus); +extern int __must_check bus_rescan_devices(const struct bus_type *bus); struct bus_attribute { struct attribute attr; -- cgit v1.2.3 From 7c06be04251a0b3349868622a2111a54cbfad8d4 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:29:08 +0100 Subject: driver core: bus: constify driver_find() The driver_find() function can now take a const * to bus_type, not just a * so fix that up. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313182918.1312597-26-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/bus.c | 2 +- include/linux/device/driver.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/bus.c b/drivers/base/bus.c index ced61fad390e..8fea26c22521 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -1307,7 +1307,7 @@ EXPORT_SYMBOL_GPL(subsys_virtual_register); * from being unregistered or unloaded while the caller is using it. * The caller is responsible for preventing this. */ -struct device_driver *driver_find(const char *name, struct bus_type *bus) +struct device_driver *driver_find(const char *name, const struct bus_type *bus) { struct subsys_private *sp = bus_to_subsys(bus); struct kobject *k; diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index 50d0a416a5e7..ceb0e477c2c8 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -126,7 +126,7 @@ extern int __must_check driver_register(struct device_driver *drv); extern void driver_unregister(struct device_driver *drv); extern struct device_driver *driver_find(const char *name, - struct bus_type *bus); + const struct bus_type *bus); extern int driver_probe_done(void); extern void wait_for_device_probe(void); void __init wait_for_init_devices_probe(void); -- cgit v1.2.3 From c28dd08ef713d2127c5bad3f3e0e93d6ec0309a2 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:29:10 +0100 Subject: driver core: make the bus_type in struct device_driver constant The pointer to struct bus_type in struct device_driver should only be pointing to something that can never change now that the driver core has fixed up the previously writable fields. So mark it as a constant pointer to enforce this and move forward with the goal of moving bus_type into read-only memory. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313182918.1312597-28-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- include/linux/device/driver.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index ceb0e477c2c8..0f22a6f46f8c 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -95,7 +95,7 @@ enum probe_type { */ struct device_driver { const char *name; - struct bus_type *bus; + const struct bus_type *bus; struct module *owner; const char *mod_name; /* used for built-in modules */ -- cgit v1.2.3 From b18d0a0f92a8fe9e56d812808184c8c4b9f18f92 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:29:16 +0100 Subject: iommu: make the pointer to struct bus_type constant A number of iommu functions take a struct bus_type * and never modify the data passed in, so make them all const * as that is what the driver core is expecting to have passed into as well. This is a step toward making all struct bus_type pointers constant in the kernel. Cc: Will Deacon Cc: Robin Murphy Cc: iommu@lists.linux.dev Acked-by: Joerg Roedel Link: https://lore.kernel.org/r/20230313182918.1312597-34-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/iommu/iommu.c | 14 +++++++------- include/linux/iommu.h | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 10db680acaed..0b5e181998c9 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -89,7 +89,7 @@ static int iommu_bus_notifier(struct notifier_block *nb, unsigned long action, void *data); static int iommu_alloc_default_domain(struct iommu_group *group, struct device *dev); -static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, +static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus, unsigned type); static int __iommu_attach_device(struct iommu_domain *domain, struct device *dev); @@ -1631,7 +1631,7 @@ static int iommu_get_def_domain_type(struct device *dev) return 0; } -static int iommu_group_alloc_default_domain(struct bus_type *bus, +static int iommu_group_alloc_default_domain(const struct bus_type *bus, struct iommu_group *group, unsigned int type) { @@ -1777,7 +1777,7 @@ static int probe_get_default_domain_type(struct device *dev, void *data) return 0; } -static void probe_alloc_default_domain(struct bus_type *bus, +static void probe_alloc_default_domain(const struct bus_type *bus, struct iommu_group *group) { struct __group_domain_type gtype; @@ -1832,7 +1832,7 @@ static int iommu_group_create_direct_mappings(struct iommu_group *group) iommu_do_create_direct_mappings); } -int bus_iommu_probe(struct bus_type *bus) +int bus_iommu_probe(const struct bus_type *bus) { struct iommu_group *group, *next; LIST_HEAD(group_list); @@ -1876,7 +1876,7 @@ int bus_iommu_probe(struct bus_type *bus) return ret; } -bool iommu_present(struct bus_type *bus) +bool iommu_present(const struct bus_type *bus) { return bus->iommu_ops != NULL; } @@ -1951,7 +1951,7 @@ void iommu_set_fault_handler(struct iommu_domain *domain, } EXPORT_SYMBOL_GPL(iommu_set_fault_handler); -static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, +static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus, unsigned type) { struct iommu_domain *domain; @@ -1976,7 +1976,7 @@ static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, return domain; } -struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) +struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus) { return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED); } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 6595454d4f48..0fd4e6734d5b 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -455,11 +455,11 @@ static inline const struct iommu_ops *dev_iommu_ops(struct device *dev) return dev->iommu->iommu_dev->ops; } -extern int bus_iommu_probe(struct bus_type *bus); -extern bool iommu_present(struct bus_type *bus); +extern int bus_iommu_probe(const struct bus_type *bus); +extern bool iommu_present(const struct bus_type *bus); extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap); extern bool iommu_group_has_isolated_msi(struct iommu_group *group); -extern struct iommu_domain *iommu_domain_alloc(struct bus_type *bus); +extern struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus); extern struct iommu_group *iommu_group_get_by_id(int id); extern void iommu_domain_free(struct iommu_domain *domain); extern int iommu_attach_device(struct iommu_domain *domain, @@ -732,7 +732,7 @@ struct iommu_device {}; struct iommu_fault_param {}; struct iommu_iotlb_gather {}; -static inline bool iommu_present(struct bus_type *bus) +static inline bool iommu_present(const struct bus_type *bus) { return false; } @@ -742,7 +742,7 @@ static inline bool device_iommu_capable(struct device *dev, enum iommu_cap cap) return false; } -static inline struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) +static inline struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus) { return NULL; } -- cgit v1.2.3 From d492cc2573a08352c48a66d3e3f312a15fb3f363 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:29:17 +0100 Subject: driver core: device.h: make struct bus_type a const * Now that all users who accessed the bus_type structure in struct device are properly using it as a const *, mark it as such so that no one can modify it going forward anymore. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230313182918.1312597-35-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- include/linux/device.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/device.h b/include/linux/device.h index 862397ae520d..3b23772d3bbb 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -566,7 +566,7 @@ struct device { const char *init_name; /* initial name of the device */ const struct device_type *type; - struct bus_type *bus; /* type of bus device is on */ + const struct bus_type *bus; /* type of bus device is on */ struct device_driver *driver; /* which driver has allocated this device */ void *platform_data; /* Platform specific data, device -- cgit v1.2.3 From 9d11b13402d1b80f7f3ca5061d75f15cf8002555 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 13 Mar 2023 19:29:18 +0100 Subject: USB: mark all struct bus_type as const Now that the driver core can properly handle constant struct bus_type, move all of the USB subsystem struct bus_type structures as const, placing them into read-only memory which can not be modified at runtime. Cc: Johan Hovold Cc: Evan Green Cc: Alan Stern Cc: linux-usb@vger.kernel.org Acked-by: Heikki Krogerus Link: https://lore.kernel.org/r/20230313182918.1312597-36-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/usb/common/ulpi.c | 2 +- drivers/usb/core/driver.c | 2 +- drivers/usb/core/usb.h | 2 +- drivers/usb/gadget/udc/core.c | 4 ++-- drivers/usb/serial/bus.c | 2 +- drivers/usb/typec/bus.c | 2 +- drivers/usb/typec/bus.h | 2 +- include/linux/usb/serial.h | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c index a98b2108376a..8305a5dfb910 100644 --- a/drivers/usb/common/ulpi.c +++ b/drivers/usb/common/ulpi.c @@ -90,7 +90,7 @@ static void ulpi_remove(struct device *dev) drv->remove(to_ulpi_dev(dev)); } -static struct bus_type ulpi_bus = { +static const struct bus_type ulpi_bus = { .name = "ulpi", .match = ulpi_match, .uevent = ulpi_uevent, diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index a0e076c6f3a4..f58a0299fb3b 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -2025,7 +2025,7 @@ int usb_disable_usb2_hardware_lpm(struct usb_device *udev) #endif /* CONFIG_PM */ -struct bus_type usb_bus_type = { +const struct bus_type usb_bus_type = { .name = "usb", .match = usb_device_match, .uevent = usb_uevent, diff --git a/drivers/usb/core/usb.h b/drivers/usb/core/usb.h index 0eac7d4285d1..cd434af259c3 100644 --- a/drivers/usb/core/usb.h +++ b/drivers/usb/core/usb.h @@ -140,7 +140,7 @@ static inline int usb_disable_usb2_hardware_lpm(struct usb_device *udev) #endif -extern struct bus_type usb_bus_type; +extern const struct bus_type usb_bus_type; extern struct mutex usb_port_peer_mutex; extern struct device_type usb_device_type; extern struct device_type usb_if_device_type; diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index df930b041c33..d919f0743a0a 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -26,7 +26,7 @@ static DEFINE_IDA(gadget_id_numbers); -static struct bus_type gadget_bus_type; +static const struct bus_type gadget_bus_type; /** * struct usb_udc - describes one usb device controller @@ -1747,7 +1747,7 @@ static int usb_udc_uevent(const struct device *dev, struct kobj_uevent_env *env) return 0; } -static struct bus_type gadget_bus_type = { +static const struct bus_type gadget_bus_type = { .name = "gadget", .probe = gadget_bind_driver, .remove = gadget_unbind_driver, diff --git a/drivers/usb/serial/bus.c b/drivers/usb/serial/bus.c index 9e38142acd38..3eb8dc3a1a8f 100644 --- a/drivers/usb/serial/bus.c +++ b/drivers/usb/serial/bus.c @@ -144,7 +144,7 @@ static void free_dynids(struct usb_serial_driver *drv) spin_unlock(&drv->dynids.lock); } -struct bus_type usb_serial_bus_type = { +const struct bus_type usb_serial_bus_type = { .name = "usb-serial", .match = usb_serial_device_match, .probe = usb_serial_device_probe, diff --git a/drivers/usb/typec/bus.c b/drivers/usb/typec/bus.c index 098f0efaa58d..fe5b9a2e61f5 100644 --- a/drivers/usb/typec/bus.c +++ b/drivers/usb/typec/bus.c @@ -431,7 +431,7 @@ static void typec_remove(struct device *dev) adev->ops = NULL; } -struct bus_type typec_bus = { +const struct bus_type typec_bus = { .name = "typec", .dev_groups = typec_groups, .match = typec_match, diff --git a/drivers/usb/typec/bus.h b/drivers/usb/typec/bus.h index c89168857417..643b8c81786d 100644 --- a/drivers/usb/typec/bus.h +++ b/drivers/usb/typec/bus.h @@ -28,7 +28,7 @@ struct altmode { #define to_altmode(d) container_of(d, struct altmode, adev) -extern struct bus_type typec_bus; +extern const struct bus_type typec_bus; extern const struct device_type typec_altmode_dev_type; #define is_typec_altmode(_dev_) (_dev_->type == &typec_altmode_dev_type) diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h index f7bfedb740f5..7eeb5f9c4f0d 100644 --- a/include/linux/usb/serial.h +++ b/include/linux/usb/serial.h @@ -378,7 +378,7 @@ void usb_serial_handle_dcd_change(struct usb_serial_port *usb_port, int usb_serial_bus_register(struct usb_serial_driver *device); void usb_serial_bus_deregister(struct usb_serial_driver *device); -extern struct bus_type usb_serial_bus_type; +extern const struct bus_type usb_serial_bus_type; extern struct tty_driver *usb_serial_tty_driver; static inline void usb_serial_debug_data(struct device *dev, -- cgit v1.2.3 From 1fb1ea0d9cb8359ae9d6f67f22666c74d5b9f47d Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 10 Mar 2023 19:07:47 +0200 Subject: mei: Move uuid.h to the MEI namespace There is only a single user of the UUID uAPI, let's make it part of that user. The way it's done is to prevent compilation time breakage for the user space that does #include In the future MEI user space tools can switch over to use mei_uuid.h. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20230310170747.22782-1-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- MAINTAINERS | 1 + drivers/misc/mei/bus-fixup.c | 2 +- drivers/misc/mei/hdcp/mei_hdcp.c | 2 +- drivers/misc/mei/hw.h | 2 +- drivers/misc/mei/main.c | 1 - drivers/misc/mei/pxp/mei_pxp.c | 2 +- include/linux/mod_devicetable.h | 1 + include/linux/uuid.h | 3 --- include/uapi/linux/mei.h | 2 +- include/uapi/linux/mei_uuid.h | 29 +++++++++++++++++++++++++++++ include/uapi/linux/uuid.h | 31 +------------------------------ 11 files changed, 37 insertions(+), 39 deletions(-) create mode 100644 include/uapi/linux/mei_uuid.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 7ee30a356e89..619c30ec36dc 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10459,6 +10459,7 @@ F: drivers/watchdog/mei_wdt.c F: include/linux/mei_aux.h F: include/linux/mei_cl_bus.h F: include/uapi/linux/mei.h +F: include/uapi/linux/mei_uuid.h F: include/uapi/linux/uuid.h F: samples/mei/* diff --git a/drivers/misc/mei/bus-fixup.c b/drivers/misc/mei/bus-fixup.c index 211536109308..31e3c74ca1f1 100644 --- a/drivers/misc/mei/bus-fixup.c +++ b/drivers/misc/mei/bus-fixup.c @@ -9,8 +9,8 @@ #include #include #include -#include +#include #include #include "mei_dev.h" diff --git a/drivers/misc/mei/hdcp/mei_hdcp.c b/drivers/misc/mei/hdcp/mei_hdcp.c index e0dcd5c114db..45e3d4d27797 100644 --- a/drivers/misc/mei/hdcp/mei_hdcp.c +++ b/drivers/misc/mei/hdcp/mei_hdcp.c @@ -18,7 +18,7 @@ #include #include -#include +#include #include #include #include diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h index 319418ddf4fb..e910302fcd1f 100644 --- a/drivers/misc/mei/hw.h +++ b/drivers/misc/mei/hw.h @@ -7,7 +7,7 @@ #ifndef _MEI_HW_TYPES_H_ #define _MEI_HW_TYPES_H_ -#include +#include /* * Timeouts in Seconds diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c index 632d4ae21e46..c64291741d73 100644 --- a/drivers/misc/mei/main.c +++ b/drivers/misc/mei/main.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/misc/mei/pxp/mei_pxp.c b/drivers/misc/mei/pxp/mei_pxp.c index 7ee1fa7b1cb3..3bf560bbdee0 100644 --- a/drivers/misc/mei/pxp/mei_pxp.c +++ b/drivers/misc/mei/pxp/mei_pxp.c @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include #include diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index 549590e9c644..35203ca3fc37 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -9,6 +9,7 @@ #define LINUX_MOD_DEVICETABLE_H #ifdef __KERNEL__ +#include #include #include typedef unsigned long kernel_ulong_t; diff --git a/include/linux/uuid.h b/include/linux/uuid.h index 6b1a3efa1e0b..43d4a79b273d 100644 --- a/include/linux/uuid.h +++ b/include/linux/uuid.h @@ -107,7 +107,4 @@ extern const u8 uuid_index[16]; int guid_parse(const char *uuid, guid_t *u); int uuid_parse(const char *uuid, uuid_t *u); -/* MEI UUID type, don't use anywhere else */ -#include - #endif diff --git a/include/uapi/linux/mei.h b/include/uapi/linux/mei.h index 4f3638489d01..6e57743628c0 100644 --- a/include/uapi/linux/mei.h +++ b/include/uapi/linux/mei.h @@ -7,7 +7,7 @@ #ifndef _LINUX_MEI_H #define _LINUX_MEI_H -#include +#include /* * This IOCTL is used to associate the current file descriptor with a diff --git a/include/uapi/linux/mei_uuid.h b/include/uapi/linux/mei_uuid.h new file mode 100644 index 000000000000..676ebe12d623 --- /dev/null +++ b/include/uapi/linux/mei_uuid.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * MEI UUID definition + * + * Copyright (C) 2010, Intel Corp. + * Huang Ying + */ + +#ifndef _UAPI_LINUX_MEI_UUID_H_ +#define _UAPI_LINUX_MEI_UUID_H_ + +#include + +typedef struct { + __u8 b[16]; +} uuid_le; + +#define UUID_LE(a, b, c, d0, d1, d2, d3, d4, d5, d6, d7) \ +((uuid_le) \ +{{ (a) & 0xff, ((a) >> 8) & 0xff, ((a) >> 16) & 0xff, ((a) >> 24) & 0xff, \ + (b) & 0xff, ((b) >> 8) & 0xff, \ + (c) & 0xff, ((c) >> 8) & 0xff, \ + (d0), (d1), (d2), (d3), (d4), (d5), (d6), (d7) }}) + +#define NULL_UUID_LE \ + UUID_LE(0x00000000, 0x0000, 0x0000, 0x00, 0x00, 0x00, 0x00, \ + 0x00, 0x00, 0x00, 0x00) + +#endif /* _UAPI_LINUX_MEI_UUID_H_ */ diff --git a/include/uapi/linux/uuid.h b/include/uapi/linux/uuid.h index 96ac684a4b2f..8443738f4bb2 100644 --- a/include/uapi/linux/uuid.h +++ b/include/uapi/linux/uuid.h @@ -1,30 +1 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* DO NOT USE in new code! This is solely for MEI due to legacy reasons */ -/* - * MEI UUID definition - * - * Copyright (C) 2010, Intel Corp. - * Huang Ying - */ - -#ifndef _UAPI_LINUX_UUID_H_ -#define _UAPI_LINUX_UUID_H_ - -#include - -typedef struct { - __u8 b[16]; -} uuid_le; - -#define UUID_LE(a, b, c, d0, d1, d2, d3, d4, d5, d6, d7) \ -((uuid_le) \ -{{ (a) & 0xff, ((a) >> 8) & 0xff, ((a) >> 16) & 0xff, ((a) >> 24) & 0xff, \ - (b) & 0xff, ((b) >> 8) & 0xff, \ - (c) & 0xff, ((c) >> 8) & 0xff, \ - (d0), (d1), (d2), (d3), (d4), (d5), (d6), (d7) }}) - -#define NULL_UUID_LE \ - UUID_LE(0x00000000, 0x0000, 0x0000, 0x00, 0x00, 0x00, 0x00, \ - 0x00, 0x00, 0x00, 0x00) - -#endif /* _UAPI_LINUX_UUID_H_ */ +#include -- cgit v1.2.3 From f7515d9fe8fc4b80754cd4d98a5fcaee84adeebb Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:07 -0800 Subject: objtool: Add objtool_types.h Reduce the amount of header sync churn by splitting the shared objtool.h types into a new file. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/dec622720851210ceafa12d4f4c5f9e73c832152.1677683419.git.jpoimboe@kernel.org --- MAINTAINERS | 2 +- include/linux/objtool.h | 42 +------- include/linux/objtool_types.h | 48 +++++++++ tools/include/linux/objtool.h | 200 ------------------------------------ tools/include/linux/objtool_types.h | 48 +++++++++ tools/objtool/check.c | 2 +- tools/objtool/orc_dump.c | 2 +- tools/objtool/orc_gen.c | 2 +- tools/objtool/sync-check.sh | 2 +- 9 files changed, 102 insertions(+), 246 deletions(-) create mode 100644 include/linux/objtool_types.h delete mode 100644 tools/include/linux/objtool.h create mode 100644 tools/include/linux/objtool_types.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 8d5bc223f305..36e9e642c05e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15090,8 +15090,8 @@ OBJTOOL M: Josh Poimboeuf M: Peter Zijlstra S: Supported +F: include/linux/objtool*.h F: tools/objtool/ -F: include/linux/objtool.h OCELOT ETHERNET SWITCH DRIVER M: Vladimir Oltean diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 9ac3df3fccf0..8375792acfc0 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -2,47 +2,7 @@ #ifndef _LINUX_OBJTOOL_H #define _LINUX_OBJTOOL_H -#ifndef __ASSEMBLY__ - -#include - -/* - * This struct is used by asm and inline asm code to manually annotate the - * location of registers on the stack. - */ -struct unwind_hint { - u32 ip; - s16 sp_offset; - u8 sp_reg; - u8 type; - u8 signal; - u8 end; -}; -#endif - -/* - * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP - * (the caller's SP right before it made the call). Used for all callable - * functions, i.e. all C code and all callable asm functions. - * - * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset - * points to a fully populated pt_regs from a syscall, interrupt, or exception. - * - * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that - * sp_reg+sp_offset points to the iret return frame. - * - * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. - * Useful for code which doesn't have an ELF function annotation. - * - * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. - */ -#define UNWIND_HINT_TYPE_CALL 0 -#define UNWIND_HINT_TYPE_REGS 1 -#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 -#define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_ENTRY 4 -#define UNWIND_HINT_TYPE_SAVE 5 -#define UNWIND_HINT_TYPE_RESTORE 6 +#include #ifdef CONFIG_OBJTOOL diff --git a/include/linux/objtool_types.h b/include/linux/objtool_types.h new file mode 100644 index 000000000000..8513537a30ed --- /dev/null +++ b/include/linux/objtool_types.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_OBJTOOL_TYPES_H +#define _LINUX_OBJTOOL_TYPES_H + +#ifndef __ASSEMBLY__ + +#include + +/* + * This struct is used by asm and inline asm code to manually annotate the + * location of registers on the stack. + */ +struct unwind_hint { + u32 ip; + s16 sp_offset; + u8 sp_reg; + u8 type; + u8 signal; + u8 end; +}; + +#endif /* __ASSEMBLY__ */ + +/* + * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP + * (the caller's SP right before it made the call). Used for all callable + * functions, i.e. all C code and all callable asm functions. + * + * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset + * points to a fully populated pt_regs from a syscall, interrupt, or exception. + * + * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that + * sp_reg+sp_offset points to the iret return frame. + * + * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. + * Useful for code which doesn't have an ELF function annotation. + * + * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. + */ +#define UNWIND_HINT_TYPE_CALL 0 +#define UNWIND_HINT_TYPE_REGS 1 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +#define UNWIND_HINT_TYPE_FUNC 3 +#define UNWIND_HINT_TYPE_ENTRY 4 +#define UNWIND_HINT_TYPE_SAVE 5 +#define UNWIND_HINT_TYPE_RESTORE 6 + +#endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/tools/include/linux/objtool.h b/tools/include/linux/objtool.h deleted file mode 100644 index 9ac3df3fccf0..000000000000 --- a/tools/include/linux/objtool.h +++ /dev/null @@ -1,200 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_OBJTOOL_H -#define _LINUX_OBJTOOL_H - -#ifndef __ASSEMBLY__ - -#include - -/* - * This struct is used by asm and inline asm code to manually annotate the - * location of registers on the stack. - */ -struct unwind_hint { - u32 ip; - s16 sp_offset; - u8 sp_reg; - u8 type; - u8 signal; - u8 end; -}; -#endif - -/* - * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP - * (the caller's SP right before it made the call). Used for all callable - * functions, i.e. all C code and all callable asm functions. - * - * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset - * points to a fully populated pt_regs from a syscall, interrupt, or exception. - * - * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that - * sp_reg+sp_offset points to the iret return frame. - * - * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. - * Useful for code which doesn't have an ELF function annotation. - * - * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. - */ -#define UNWIND_HINT_TYPE_CALL 0 -#define UNWIND_HINT_TYPE_REGS 1 -#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 -#define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_ENTRY 4 -#define UNWIND_HINT_TYPE_SAVE 5 -#define UNWIND_HINT_TYPE_RESTORE 6 - -#ifdef CONFIG_OBJTOOL - -#include - -#ifndef __ASSEMBLY__ - -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ - "987: \n\t" \ - ".pushsection .discard.unwind_hints\n\t" \ - /* struct unwind_hint */ \ - ".long 987b - .\n\t" \ - ".short " __stringify(sp_offset) "\n\t" \ - ".byte " __stringify(sp_reg) "\n\t" \ - ".byte " __stringify(type) "\n\t" \ - ".byte " __stringify(signal) "\n\t" \ - ".byte " __stringify(end) "\n\t" \ - ".balign 4 \n\t" \ - ".popsection\n\t" - -/* - * This macro marks the given function's stack frame as "non-standard", which - * tells objtool to ignore the function when doing stack metadata validation. - * It should only be used in special cases where you're 100% sure it won't - * affect the reliability of frame pointers and kernel stack traces. - * - * For more information, see tools/objtool/Documentation/objtool.txt. - */ -#define STACK_FRAME_NON_STANDARD(func) \ - static void __used __section(".discard.func_stack_frame_non_standard") \ - *__func_stack_frame_non_standard_##func = func - -/* - * STACK_FRAME_NON_STANDARD_FP() is a frame-pointer-specific function ignore - * for the case where a function is intentionally missing frame pointer setup, - * but otherwise needs objtool/ORC coverage when frame pointers are disabled. - */ -#ifdef CONFIG_FRAME_POINTER -#define STACK_FRAME_NON_STANDARD_FP(func) STACK_FRAME_NON_STANDARD(func) -#else -#define STACK_FRAME_NON_STANDARD_FP(func) -#endif - -#define ANNOTATE_NOENDBR \ - "986: \n\t" \ - ".pushsection .discard.noendbr\n\t" \ - _ASM_PTR " 986b\n\t" \ - ".popsection\n\t" - -#define ASM_REACHABLE \ - "998:\n\t" \ - ".pushsection .discard.reachable\n\t" \ - ".long 998b - .\n\t" \ - ".popsection\n\t" - -#else /* __ASSEMBLY__ */ - -/* - * This macro indicates that the following intra-function call is valid. - * Any non-annotated intra-function call will cause objtool to issue a warning. - */ -#define ANNOTATE_INTRA_FUNCTION_CALL \ - 999: \ - .pushsection .discard.intra_function_calls; \ - .long 999b; \ - .popsection; - -/* - * In asm, there are two kinds of code: normal C-type callable functions and - * the rest. The normal callable functions can be called by other code, and - * don't do anything unusual with the stack. Such normal callable functions - * are annotated with the ENTRY/ENDPROC macros. Most asm code falls in this - * category. In this case, no special debugging annotations are needed because - * objtool can automatically generate the ORC data for the ORC unwinder to read - * at runtime. - * - * Anything which doesn't fall into the above category, such as syscall and - * interrupt handlers, tends to not be called directly by other functions, and - * often does unusual non-C-function-type things with the stack pointer. Such - * code needs to be annotated such that objtool can understand it. The - * following CFI hint macros are for this type of code. - * - * These macros provide hints to objtool about the state of the stack at each - * instruction. Objtool starts from the hints and follows the code flow, - * making automatic CFI adjustments when it sees pushes and pops, filling out - * the debuginfo as necessary. It will also warn if it sees any - * inconsistencies. - */ -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 -.Lunwind_hint_ip_\@: - .pushsection .discard.unwind_hints - /* struct unwind_hint */ - .long .Lunwind_hint_ip_\@ - . - .short \sp_offset - .byte \sp_reg - .byte \type - .byte \signal - .byte \end - .balign 4 - .popsection -.endm - -.macro STACK_FRAME_NON_STANDARD func:req - .pushsection .discard.func_stack_frame_non_standard, "aw" - _ASM_PTR \func - .popsection -.endm - -.macro STACK_FRAME_NON_STANDARD_FP func:req -#ifdef CONFIG_FRAME_POINTER - STACK_FRAME_NON_STANDARD \func -#endif -.endm - -.macro ANNOTATE_NOENDBR -.Lhere_\@: - .pushsection .discard.noendbr - .quad .Lhere_\@ - .popsection -.endm - -.macro REACHABLE -.Lhere_\@: - .pushsection .discard.reachable - .long .Lhere_\@ - . - .popsection -.endm - -#endif /* __ASSEMBLY__ */ - -#else /* !CONFIG_OBJTOOL */ - -#ifndef __ASSEMBLY__ - -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ - "\n\t" -#define STACK_FRAME_NON_STANDARD(func) -#define STACK_FRAME_NON_STANDARD_FP(func) -#define ANNOTATE_NOENDBR -#define ASM_REACHABLE -#else -#define ANNOTATE_INTRA_FUNCTION_CALL -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 -.endm -.macro STACK_FRAME_NON_STANDARD func:req -.endm -.macro ANNOTATE_NOENDBR -.endm -.macro REACHABLE -.endm -#endif - -#endif /* CONFIG_OBJTOOL */ - -#endif /* _LINUX_OBJTOOL_H */ diff --git a/tools/include/linux/objtool_types.h b/tools/include/linux/objtool_types.h new file mode 100644 index 000000000000..8513537a30ed --- /dev/null +++ b/tools/include/linux/objtool_types.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_OBJTOOL_TYPES_H +#define _LINUX_OBJTOOL_TYPES_H + +#ifndef __ASSEMBLY__ + +#include + +/* + * This struct is used by asm and inline asm code to manually annotate the + * location of registers on the stack. + */ +struct unwind_hint { + u32 ip; + s16 sp_offset; + u8 sp_reg; + u8 type; + u8 signal; + u8 end; +}; + +#endif /* __ASSEMBLY__ */ + +/* + * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP + * (the caller's SP right before it made the call). Used for all callable + * functions, i.e. all C code and all callable asm functions. + * + * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset + * points to a fully populated pt_regs from a syscall, interrupt, or exception. + * + * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that + * sp_reg+sp_offset points to the iret return frame. + * + * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. + * Useful for code which doesn't have an ELF function annotation. + * + * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. + */ +#define UNWIND_HINT_TYPE_CALL 0 +#define UNWIND_HINT_TYPE_REGS 1 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +#define UNWIND_HINT_TYPE_FUNC 3 +#define UNWIND_HINT_TYPE_ENTRY 4 +#define UNWIND_HINT_TYPE_SAVE 5 +#define UNWIND_HINT_TYPE_RESTORE 6 + +#endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 37c36dc1d868..efc2baa02883 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include #include diff --git a/tools/objtool/orc_dump.c b/tools/objtool/orc_dump.c index 2d8ebdcd1db3..9f6c528c26f2 100644 --- a/tools/objtool/orc_dump.c +++ b/tools/objtool/orc_dump.c @@ -4,7 +4,7 @@ */ #include -#include +#include #include #include #include diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c index 57a4527d5988..f49630a21e0f 100644 --- a/tools/objtool/orc_gen.c +++ b/tools/objtool/orc_gen.c @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include diff --git a/tools/objtool/sync-check.sh b/tools/objtool/sync-check.sh index 105a291ff8e7..81d120d05442 100755 --- a/tools/objtool/sync-check.sh +++ b/tools/objtool/sync-check.sh @@ -6,7 +6,7 @@ if [ -z "$SRCARCH" ]; then exit 1 fi -FILES="include/linux/objtool.h" +FILES="include/linux/objtool_types.h" if [ "$SRCARCH" = "x86" ]; then FILES="$FILES -- cgit v1.2.3 From 1c0c1faf5692c18c127d044ecc0cc92c7bab3477 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:08 -0800 Subject: objtool: Use relative pointers for annotations They produce the needed relocations while using half the space. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/bed05c64e28200220c9b1754a2f3ce71f73076ea.1677683419.git.jpoimboe@kernel.org --- arch/x86/include/asm/nospec-branch.h | 6 +++--- include/linux/objtool.h | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 3ef70e54a858..78ed1546b775 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -194,9 +194,9 @@ * builds. */ .macro ANNOTATE_RETPOLINE_SAFE - .Lannotate_\@: +.Lhere_\@: .pushsection .discard.retpoline_safe - _ASM_PTR .Lannotate_\@ + .long .Lhere_\@ - . .popsection .endm @@ -318,7 +318,7 @@ #define ANNOTATE_RETPOLINE_SAFE \ "999:\n\t" \ ".pushsection .discard.retpoline_safe\n\t" \ - _ASM_PTR " 999b\n\t" \ + ".long 999b - .\n\t" \ ".popsection\n\t" typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE]; diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 8375792acfc0..2b0258d273fc 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -49,7 +49,7 @@ #define ANNOTATE_NOENDBR \ "986: \n\t" \ ".pushsection .discard.noendbr\n\t" \ - _ASM_PTR " 986b\n\t" \ + ".long 986b - .\n\t" \ ".popsection\n\t" #define ASM_REACHABLE \ @@ -67,7 +67,7 @@ #define ANNOTATE_INTRA_FUNCTION_CALL \ 999: \ .pushsection .discard.intra_function_calls; \ - .long 999b; \ + .long 999b - .; \ .popsection; /* @@ -92,10 +92,10 @@ * inconsistencies. */ .macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 -.Lunwind_hint_ip_\@: +.Lhere_\@: .pushsection .discard.unwind_hints /* struct unwind_hint */ - .long .Lunwind_hint_ip_\@ - . + .long .Lhere_\@ - . .short \sp_offset .byte \sp_reg .byte \type @@ -107,7 +107,7 @@ .macro STACK_FRAME_NON_STANDARD func:req .pushsection .discard.func_stack_frame_non_standard, "aw" - _ASM_PTR \func + .long \func - . .popsection .endm @@ -120,7 +120,7 @@ .macro ANNOTATE_NOENDBR .Lhere_\@: .pushsection .discard.noendbr - .quad .Lhere_\@ + .long .Lhere_\@ - . .popsection .endm -- cgit v1.2.3 From d88ebba45dfe67114a6ac8c6514f2c65b6ed64c7 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:09 -0800 Subject: objtool: Change UNWIND_HINT() argument order The most important argument is 'type', make that one first. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/d994f8c29376c5618c75698df28fc03b52d3a868.1677683419.git.jpoimboe@kernel.org --- arch/x86/include/asm/unwind_hints.h | 2 +- include/linux/objtool.h | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index e7c71750b309..97b392217c0f 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -67,7 +67,7 @@ #else #define UNWIND_HINT_FUNC \ - UNWIND_HINT(ORC_REG_SP, 8, UNWIND_HINT_TYPE_FUNC, 0, 0) + UNWIND_HINT(UNWIND_HINT_TYPE_FUNC, ORC_REG_SP, 8, 0, 0) #endif /* __ASSEMBLY__ */ diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 2b0258d273fc..725d7f0b6748 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -10,7 +10,7 @@ #ifndef __ASSEMBLY__ -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ +#define UNWIND_HINT(type, sp_reg, sp_offset, signal, end) \ "987: \n\t" \ ".pushsection .discard.unwind_hints\n\t" \ /* struct unwind_hint */ \ @@ -137,8 +137,7 @@ #ifndef __ASSEMBLY__ -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ - "\n\t" +#define UNWIND_HINT(type, sp_reg, sp_offset, signal, end) "\n\t" #define STACK_FRAME_NON_STANDARD(func) #define STACK_FRAME_NON_STANDARD_FP(func) #define ANNOTATE_NOENDBR -- cgit v1.2.3 From f902cfdd46aedd2afb3e8033223312dbf5fbb675 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:10 -0800 Subject: x86,objtool: Introduce ORC_TYPE_* Unwind hints and ORC entry types are two distinct things. Separate them out more explicitly. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/cc879d38fff8a43f8f7beb2fd56e35a5a384d7cd.1677683419.git.jpoimboe@kernel.org --- arch/x86/include/asm/orc_types.h | 4 ++++ arch/x86/kernel/unwind_orc.c | 12 ++++++------ include/linux/objtool_types.h | 1 + tools/arch/x86/include/asm/orc_types.h | 4 ++++ tools/include/linux/objtool_types.h | 1 + tools/objtool/orc_dump.c | 7 +++---- tools/objtool/orc_gen.c | 19 +++++++++++++++++-- 7 files changed, 36 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h index 1343a62106de..b4d4ec78589e 100644 --- a/arch/x86/include/asm/orc_types.h +++ b/arch/x86/include/asm/orc_types.h @@ -39,6 +39,10 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 +#define ORC_TYPE_CALL 0 +#define ORC_TYPE_REGS 1 +#define ORC_TYPE_REGS_PARTIAL 2 + #ifndef __ASSEMBLY__ #include diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c index 37307b40f8da..8348ac581de3 100644 --- a/arch/x86/kernel/unwind_orc.c +++ b/arch/x86/kernel/unwind_orc.c @@ -133,7 +133,7 @@ static struct orc_entry null_orc_entry = { .sp_offset = sizeof(long), .sp_reg = ORC_REG_SP, .bp_reg = ORC_REG_UNDEFINED, - .type = UNWIND_HINT_TYPE_CALL + .type = ORC_TYPE_CALL }; #ifdef CONFIG_CALL_THUNKS @@ -153,7 +153,7 @@ static struct orc_entry *orc_callthunk_find(unsigned long ip) /* Fake frame pointer entry -- used as a fallback for generated code */ static struct orc_entry orc_fp_entry = { - .type = UNWIND_HINT_TYPE_CALL, + .type = ORC_TYPE_CALL, .sp_reg = ORC_REG_BP, .sp_offset = 16, .bp_reg = ORC_REG_PREV_SP, @@ -554,7 +554,7 @@ bool unwind_next_frame(struct unwind_state *state) /* Find IP, SP and possibly regs: */ switch (orc->type) { - case UNWIND_HINT_TYPE_CALL: + case ORC_TYPE_CALL: ip_p = sp - sizeof(long); if (!deref_stack_reg(state, ip_p, &state->ip)) @@ -567,7 +567,7 @@ bool unwind_next_frame(struct unwind_state *state) state->prev_regs = NULL; break; - case UNWIND_HINT_TYPE_REGS: + case ORC_TYPE_REGS: if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) { orc_warn_current("can't access registers at %pB\n", (void *)orig_ip); @@ -590,13 +590,13 @@ bool unwind_next_frame(struct unwind_state *state) state->full_regs = true; break; - case UNWIND_HINT_TYPE_REGS_PARTIAL: + case ORC_TYPE_REGS_PARTIAL: if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) { orc_warn_current("can't access iret registers at %pB\n", (void *)orig_ip); goto err; } - /* See UNWIND_HINT_TYPE_REGS case comment. */ + /* See ORC_TYPE_REGS case comment. */ state->ip = unwind_recover_rethook(state, state->ip, (unsigned long *)(state->sp - sizeof(long))); diff --git a/include/linux/objtool_types.h b/include/linux/objtool_types.h index 8513537a30ed..9a83468c0039 100644 --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -40,6 +40,7 @@ struct unwind_hint { #define UNWIND_HINT_TYPE_CALL 0 #define UNWIND_HINT_TYPE_REGS 1 #define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +/* The below hint types don't have corresponding ORC types */ #define UNWIND_HINT_TYPE_FUNC 3 #define UNWIND_HINT_TYPE_ENTRY 4 #define UNWIND_HINT_TYPE_SAVE 5 diff --git a/tools/arch/x86/include/asm/orc_types.h b/tools/arch/x86/include/asm/orc_types.h index 1343a62106de..b4d4ec78589e 100644 --- a/tools/arch/x86/include/asm/orc_types.h +++ b/tools/arch/x86/include/asm/orc_types.h @@ -39,6 +39,10 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 +#define ORC_TYPE_CALL 0 +#define ORC_TYPE_REGS 1 +#define ORC_TYPE_REGS_PARTIAL 2 + #ifndef __ASSEMBLY__ #include diff --git a/tools/include/linux/objtool_types.h b/tools/include/linux/objtool_types.h index 8513537a30ed..9a83468c0039 100644 --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -40,6 +40,7 @@ struct unwind_hint { #define UNWIND_HINT_TYPE_CALL 0 #define UNWIND_HINT_TYPE_REGS 1 #define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +/* The below hint types don't have corresponding ORC types */ #define UNWIND_HINT_TYPE_FUNC 3 #define UNWIND_HINT_TYPE_ENTRY 4 #define UNWIND_HINT_TYPE_SAVE 5 diff --git a/tools/objtool/orc_dump.c b/tools/objtool/orc_dump.c index 9f6c528c26f2..97ecbb8b9034 100644 --- a/tools/objtool/orc_dump.c +++ b/tools/objtool/orc_dump.c @@ -4,7 +4,6 @@ */ #include -#include #include #include #include @@ -39,11 +38,11 @@ static const char *reg_name(unsigned int reg) static const char *orc_type_name(unsigned int type) { switch (type) { - case UNWIND_HINT_TYPE_CALL: + case ORC_TYPE_CALL: return "call"; - case UNWIND_HINT_TYPE_REGS: + case ORC_TYPE_REGS: return "regs"; - case UNWIND_HINT_TYPE_REGS_PARTIAL: + case ORC_TYPE_REGS_PARTIAL: return "regs (partial)"; default: return "?"; diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c index f49630a21e0f..e85bbb996f6c 100644 --- a/tools/objtool/orc_gen.c +++ b/tools/objtool/orc_gen.c @@ -26,6 +26,22 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, return 0; } + switch (cfi->type) { + case UNWIND_HINT_TYPE_CALL: + orc->type = ORC_TYPE_CALL; + break; + case UNWIND_HINT_TYPE_REGS: + orc->type = ORC_TYPE_REGS; + break; + case UNWIND_HINT_TYPE_REGS_PARTIAL: + orc->type = ORC_TYPE_REGS_PARTIAL; + break; + default: + WARN_FUNC("unknown unwind hint type %d", + insn->sec, insn->offset, cfi->type); + return -1; + } + orc->end = cfi->end; orc->signal = cfi->signal; @@ -83,7 +99,6 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, orc->sp_offset = cfi->cfa.offset; orc->bp_offset = bp->offset; - orc->type = cfi->type; return 0; } @@ -151,7 +166,7 @@ int orc_create(struct objtool_file *file) struct orc_entry null = { .sp_reg = ORC_REG_UNDEFINED, .bp_reg = ORC_REG_UNDEFINED, - .type = UNWIND_HINT_TYPE_CALL, + .type = ORC_TYPE_CALL, }; /* Build a deduplicated list of ORC entries: */ -- cgit v1.2.3 From 4708ea14bef314fc901857eefd65678236a9f2d9 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:11 -0800 Subject: x86,objtool: Separate unret validation from unwind hints The ENTRY unwind hint type is serving double duty as both an empty unwind hint and an unret validation annotation. Unret validation is unrelated to unwinding. Separate it out into its own annotation. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/ff7448d492ea21b86d8a90264b105fbd0d751077.1677683419.git.jpoimboe@kernel.org --- arch/x86/entry/entry_64.S | 14 ++++---- arch/x86/include/asm/nospec-branch.h | 8 ++--- arch/x86/include/asm/unwind_hints.h | 8 ++++- arch/x86/kernel/head_64.S | 5 --- include/linux/objtool.h | 16 +++++++++ include/linux/objtool_types.h | 5 ++- tools/include/linux/objtool_types.h | 5 ++- tools/objtool/check.c | 65 ++++++++++++++++++++++++----------- tools/objtool/include/objtool/check.h | 4 +-- 9 files changed, 85 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index eccc3431e515..5b93eb7db0ab 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -388,9 +388,9 @@ SYM_CODE_START(\asmsym) .if \vector == X86_TRAP_BP /* #BP advances %rip to the next instruction */ - UNWIND_HINT_IRET_REGS offset=\has_error_code*8 signal=0 + UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8 signal=0 .else - UNWIND_HINT_IRET_REGS offset=\has_error_code*8 + UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8 .endif ENDBR @@ -461,7 +461,7 @@ SYM_CODE_END(\asmsym) */ .macro idtentry_mce_db vector asmsym cfunc SYM_CODE_START(\asmsym) - UNWIND_HINT_IRET_REGS + UNWIND_HINT_IRET_ENTRY ENDBR ASM_CLAC cld @@ -518,7 +518,7 @@ SYM_CODE_END(\asmsym) */ .macro idtentry_vc vector asmsym cfunc SYM_CODE_START(\asmsym) - UNWIND_HINT_IRET_REGS + UNWIND_HINT_IRET_ENTRY ENDBR ASM_CLAC cld @@ -582,7 +582,7 @@ SYM_CODE_END(\asmsym) */ .macro idtentry_df vector asmsym cfunc SYM_CODE_START(\asmsym) - UNWIND_HINT_IRET_REGS offset=8 + UNWIND_HINT_IRET_ENTRY offset=8 ENDBR ASM_CLAC cld @@ -1107,7 +1107,7 @@ SYM_CODE_START(error_entry) FENCE_SWAPGS_KERNEL_ENTRY CALL_DEPTH_ACCOUNT leaq 8(%rsp), %rax /* return pt_regs pointer */ - ANNOTATE_UNRET_END + VALIDATE_UNRET_END RET .Lbstep_iret: @@ -1153,7 +1153,7 @@ SYM_CODE_END(error_return) * when PAGE_TABLE_ISOLATION is in use. Do not clobber. */ SYM_CODE_START(asm_exc_nmi) - UNWIND_HINT_IRET_REGS + UNWIND_HINT_IRET_ENTRY ENDBR /* diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 78ed1546b775..edb2b0cb8efe 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -210,8 +210,8 @@ * Abuse ANNOTATE_RETPOLINE_SAFE on a NOP to indicate UNRET_END, should * eventually turn into it's own annotation. */ -.macro ANNOTATE_UNRET_END -#ifdef CONFIG_DEBUG_ENTRY +.macro VALIDATE_UNRET_END +#if defined(CONFIG_NOINSTR_VALIDATION) && defined(CONFIG_CPU_UNRET_ENTRY) ANNOTATE_RETPOLINE_SAFE nop #endif @@ -286,7 +286,7 @@ .macro UNTRAIN_RET #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \ defined(CONFIG_CALL_DEPTH_TRACKING) - ANNOTATE_UNRET_END + VALIDATE_UNRET_END ALTERNATIVE_3 "", \ CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \ "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ @@ -297,7 +297,7 @@ .macro UNTRAIN_RET_FROM_CALL #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \ defined(CONFIG_CALL_DEPTH_TRACKING) - ANNOTATE_UNRET_END + VALIDATE_UNRET_END ALTERNATIVE_3 "", \ CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \ "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index 97b392217c0f..4c0f28d665eb 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -12,7 +12,8 @@ .endm .macro UNWIND_HINT_ENTRY - UNWIND_HINT type=UNWIND_HINT_TYPE_ENTRY end=1 + VALIDATE_UNRET_BEGIN + UNWIND_HINT_EMPTY .endm .macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 partial=0 signal=1 @@ -52,6 +53,11 @@ UNWIND_HINT_REGS base=\base offset=\offset partial=1 signal=\signal .endm +.macro UNWIND_HINT_IRET_ENTRY base=%rsp offset=0 signal=1 + VALIDATE_UNRET_BEGIN + UNWIND_HINT_IRET_REGS base=\base offset=\offset signal=\signal +.endm + .macro UNWIND_HINT_FUNC UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=8 type=UNWIND_HINT_TYPE_FUNC .endm diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 222efd4a09bc..ee3ed15ee1f8 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -390,8 +390,6 @@ SYM_CODE_START_NOALIGN(vc_boot_ghcb) UNWIND_HINT_IRET_REGS offset=8 ENDBR - ANNOTATE_UNRET_END - /* Build pt_regs */ PUSH_AND_CLEAR_REGS @@ -451,7 +449,6 @@ SYM_CODE_END(early_idt_handler_array) SYM_CODE_START_LOCAL(early_idt_handler_common) UNWIND_HINT_IRET_REGS offset=16 - ANNOTATE_UNRET_END /* * The stack is the hardware frame, an error code or zero, and the * vector number. @@ -501,8 +498,6 @@ SYM_CODE_START_NOALIGN(vc_no_ghcb) UNWIND_HINT_IRET_REGS offset=8 ENDBR - ANNOTATE_UNRET_END - /* Build pt_regs */ PUSH_AND_CLEAR_REGS diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 725d7f0b6748..5aa475118820 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -124,6 +124,22 @@ .popsection .endm +/* + * Use objtool to validate the entry requirement that all code paths do + * VALIDATE_UNRET_END before RET. + * + * NOTE: The macro must be used at the beginning of a global symbol, otherwise + * it will be ignored. + */ +.macro VALIDATE_UNRET_BEGIN +#if defined(CONFIG_NOINSTR_VALIDATION) && defined(CONFIG_CPU_UNRET_ENTRY) +.Lhere_\@: + .pushsection .discard.validate_unret + .long .Lhere_\@ - . + .popsection +#endif +.endm + .macro REACHABLE .Lhere_\@: .pushsection .discard.reachable diff --git a/include/linux/objtool_types.h b/include/linux/objtool_types.h index 9a83468c0039..9787ad0f2ef4 100644 --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -42,8 +42,7 @@ struct unwind_hint { #define UNWIND_HINT_TYPE_REGS_PARTIAL 2 /* The below hint types don't have corresponding ORC types */ #define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_ENTRY 4 -#define UNWIND_HINT_TYPE_SAVE 5 -#define UNWIND_HINT_TYPE_RESTORE 6 +#define UNWIND_HINT_TYPE_SAVE 4 +#define UNWIND_HINT_TYPE_RESTORE 5 #endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/tools/include/linux/objtool_types.h b/tools/include/linux/objtool_types.h index 9a83468c0039..9787ad0f2ef4 100644 --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -42,8 +42,7 @@ struct unwind_hint { #define UNWIND_HINT_TYPE_REGS_PARTIAL 2 /* The below hint types don't have corresponding ORC types */ #define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_ENTRY 4 -#define UNWIND_HINT_TYPE_SAVE 5 -#define UNWIND_HINT_TYPE_RESTORE 6 +#define UNWIND_HINT_TYPE_SAVE 4 +#define UNWIND_HINT_TYPE_RESTORE 5 #endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/tools/objtool/check.c b/tools/objtool/check.c index efc2baa02883..10be80b3fe67 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2307,16 +2307,9 @@ static int read_unwind_hints(struct objtool_file *file) WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR", insn->sec, insn->offset); } - - insn->entry = 1; } } - if (hint->type == UNWIND_HINT_TYPE_ENTRY) { - hint->type = UNWIND_HINT_TYPE_CALL; - insn->entry = 1; - } - if (hint->type == UNWIND_HINT_TYPE_FUNC) { insn->cfi = &func_cfi; continue; @@ -2449,6 +2442,34 @@ static int read_instr_hints(struct objtool_file *file) return 0; } +static int read_validate_unret_hints(struct objtool_file *file) +{ + struct section *sec; + struct instruction *insn; + struct reloc *reloc; + + sec = find_section_by_name(file->elf, ".rela.discard.validate_unret"); + if (!sec) + return 0; + + list_for_each_entry(reloc, &sec->reloc_list, list) { + if (reloc->sym->type != STT_SECTION) { + WARN("unexpected relocation symbol type in %s", sec->name); + return -1; + } + + insn = find_insn(file, reloc->sym->sec, reloc->addend); + if (!insn) { + WARN("bad .discard.instr_end entry"); + return -1; + } + insn->unret = 1; + } + + return 0; +} + + static int read_intra_function_calls(struct objtool_file *file) { struct instruction *insn; @@ -2667,6 +2688,10 @@ static int decode_sections(struct objtool_file *file) if (ret) return ret; + ret = read_validate_unret_hints(file); + if (ret) + return ret; + return 0; } @@ -3863,10 +3888,10 @@ static int validate_unwind_hints(struct objtool_file *file, struct section *sec) /* * Validate rethunk entry constraint: must untrain RET before the first RET. * - * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes + * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes * before an actual RET instruction. */ -static int validate_entry(struct objtool_file *file, struct instruction *insn) +static int validate_unret(struct objtool_file *file, struct instruction *insn) { struct instruction *next, *dest; int ret, warnings = 0; @@ -3874,10 +3899,10 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) for (;;) { next = next_insn_to_validate(file, insn); - if (insn->visited & VISITED_ENTRY) + if (insn->visited & VISITED_UNRET) return 0; - insn->visited |= VISITED_ENTRY; + insn->visited |= VISITED_UNRET; if (!insn->ignore_alts && insn->alts) { struct alternative *alt; @@ -3887,7 +3912,7 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) if (alt->skip_orig) skip_orig = true; - ret = validate_entry(file, alt->insn); + ret = validate_unret(file, alt->insn); if (ret) { if (opts.backtrace) BT_FUNC("(alt)", insn); @@ -3915,7 +3940,7 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) insn->sec, insn->offset); return -1; } - ret = validate_entry(file, insn->jump_dest); + ret = validate_unret(file, insn->jump_dest); if (ret) { if (opts.backtrace) { BT_FUNC("(branch%s)", insn, @@ -3940,7 +3965,7 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) return -1; } - ret = validate_entry(file, dest); + ret = validate_unret(file, dest); if (ret) { if (opts.backtrace) BT_FUNC("(call)", insn); @@ -3976,19 +4001,19 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) } /* - * Validate that all branches starting at 'insn->entry' encounter UNRET_END - * before RET. + * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter + * VALIDATE_UNRET_END before RET. */ -static int validate_unret(struct objtool_file *file) +static int validate_unrets(struct objtool_file *file) { struct instruction *insn; int ret, warnings = 0; for_each_insn(file, insn) { - if (!insn->entry) + if (!insn->unret) continue; - ret = validate_entry(file, insn); + ret = validate_unret(file, insn); if (ret < 0) { WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset); return ret; @@ -4607,7 +4632,7 @@ int check(struct objtool_file *file) * Must be after validate_branch() and friends, it plays * further games with insn->visited. */ - ret = validate_unret(file); + ret = validate_unrets(file); if (ret < 0) return ret; warnings += ret; diff --git a/tools/objtool/include/objtool/check.h b/tools/objtool/include/objtool/check.h index 3e7c7004f7df..daa46f1f0965 100644 --- a/tools/objtool/include/objtool/check.h +++ b/tools/objtool/include/objtool/check.h @@ -61,7 +61,7 @@ struct instruction { restore : 1, retpoline_safe : 1, noendbr : 1, - entry : 1, + unret : 1, visited : 4, no_reloc : 1; /* 10 bit hole */ @@ -92,7 +92,7 @@ static inline struct symbol *insn_func(struct instruction *insn) #define VISITED_BRANCH 0x01 #define VISITED_BRANCH_UACCESS 0x02 #define VISITED_BRANCH_MASK 0x03 -#define VISITED_ENTRY 0x04 +#define VISITED_UNRET 0x04 static inline bool is_static_jump(struct instruction *insn) { -- cgit v1.2.3 From fb799447ae2974a07907906dff5bd4b9e47b7123 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 1 Mar 2023 07:13:12 -0800 Subject: x86,objtool: Split UNWIND_HINT_EMPTY in two Mark reported that the ORC unwinder incorrectly marks an unwind as reliable when the unwind terminates prematurely in the dark corners of return_to_handler() due to lack of information about the next frame. The problem is UNWIND_HINT_EMPTY is used in two different situations: 1) The end of the kernel stack unwind before hitting user entry, boot code, or fork entry 2) A blind spot in ORC coverage where the unwinder has to bail due to lack of information about the next frame The ORC unwinder has no way to tell the difference between the two. When it encounters an undefined stack state with 'end=1', it blindly marks the stack reliable, which can break the livepatch consistency model. Fix it by splitting UNWIND_HINT_EMPTY into UNWIND_HINT_UNDEFINED and UNWIND_HINT_END_OF_STACK. Reported-by: Mark Rutland Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Acked-by: Steven Rostedt (Google) Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/fd6212c8b450d3564b855e1cb48404d6277b4d9f.1677683419.git.jpoimboe@kernel.org --- Documentation/livepatch/reliable-stacktrace.rst | 2 +- arch/x86/entry/entry_64.S | 12 +++++------ arch/x86/include/asm/orc_types.h | 14 ++++++------- arch/x86/include/asm/unwind_hints.h | 12 +++++++---- arch/x86/kernel/ftrace_64.S | 2 +- arch/x86/kernel/head_64.S | 12 +++++------ arch/x86/kernel/relocate_kernel_64.S | 10 ++++----- arch/x86/kernel/unwind_orc.c | 15 ++++++-------- arch/x86/lib/retpoline.S | 6 +++--- arch/x86/platform/pvh/head.S | 2 +- arch/x86/xen/xen-asm.S | 4 ++-- arch/x86/xen/xen-head.S | 4 ++-- include/linux/objtool.h | 10 ++++----- include/linux/objtool_types.h | 27 ++++++++++++++++--------- scripts/sorttable.h | 2 +- tools/arch/x86/include/asm/orc_types.h | 14 ++++++------- tools/include/linux/objtool_types.h | 27 ++++++++++++++++--------- tools/objtool/check.c | 2 +- tools/objtool/orc_dump.c | 8 ++++++-- tools/objtool/orc_gen.c | 26 ++++++++++++------------ 20 files changed, 116 insertions(+), 95 deletions(-) (limited to 'include/linux') diff --git a/Documentation/livepatch/reliable-stacktrace.rst b/Documentation/livepatch/reliable-stacktrace.rst index 67459d2ca2af..d56bb706172f 100644 --- a/Documentation/livepatch/reliable-stacktrace.rst +++ b/Documentation/livepatch/reliable-stacktrace.rst @@ -183,7 +183,7 @@ trampoline or return trampoline. For example, considering the x86_64 .. code-block:: none SYM_CODE_START(return_to_handler) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED subq $24, %rsp /* Save the return values */ diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 5b93eb7db0ab..45e135be2a9f 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -205,7 +205,7 @@ syscall_return_via_sysret: */ movq %rsp, %rdi movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK pushq RSP-RDI(%rdi) /* RSP */ pushq (%rdi) /* RDI */ @@ -286,7 +286,7 @@ SYM_FUNC_END(__switch_to_asm) .pushsection .text, "ax" __FUNC_ALIGN SYM_CODE_START_NOALIGN(ret_from_fork) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR // copy_thread CALL_DEPTH_ACCOUNT movq %rax, %rdi @@ -303,7 +303,7 @@ SYM_CODE_START_NOALIGN(ret_from_fork) 1: /* kernel thread */ - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK movq %r12, %rdi CALL_NOSPEC rbx /* @@ -643,7 +643,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) */ movq %rsp, %rdi movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* Copy the IRET frame to the trampoline stack. */ pushq 6*8(%rdi) /* SS */ @@ -869,7 +869,7 @@ SYM_CODE_END(exc_xen_hypervisor_callback) */ __FUNC_ALIGN SYM_CODE_START_NOALIGN(xen_failsafe_callback) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ENDBR movl %ds, %ecx cmpw %cx, 0x10(%rsp) @@ -1520,7 +1520,7 @@ SYM_CODE_END(asm_exc_nmi) * MSRs to fully disable 32-bit SYSCALL. */ SYM_CODE_START(ignore_sysret) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ENDBR mov $-ENOSYS, %eax sysretl diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h index b4d4ec78589e..46d7e06763c9 100644 --- a/arch/x86/include/asm/orc_types.h +++ b/arch/x86/include/asm/orc_types.h @@ -39,9 +39,11 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 -#define ORC_TYPE_CALL 0 -#define ORC_TYPE_REGS 1 -#define ORC_TYPE_REGS_PARTIAL 2 +#define ORC_TYPE_UNDEFINED 0 +#define ORC_TYPE_END_OF_STACK 1 +#define ORC_TYPE_CALL 2 +#define ORC_TYPE_REGS 3 +#define ORC_TYPE_REGS_PARTIAL 4 #ifndef __ASSEMBLY__ #include @@ -60,16 +62,14 @@ struct orc_entry { #if defined(__LITTLE_ENDIAN_BITFIELD) unsigned sp_reg:4; unsigned bp_reg:4; - unsigned type:2; + unsigned type:3; unsigned signal:1; - unsigned end:1; #elif defined(__BIG_ENDIAN_BITFIELD) unsigned bp_reg:4; unsigned sp_reg:4; unsigned unused:4; - unsigned end:1; unsigned signal:1; - unsigned type:2; + unsigned type:3; #endif } __packed; diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index 4c0f28d665eb..01cb9692b160 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -7,13 +7,17 @@ #ifdef __ASSEMBLY__ -.macro UNWIND_HINT_EMPTY - UNWIND_HINT type=UNWIND_HINT_TYPE_CALL end=1 +.macro UNWIND_HINT_END_OF_STACK + UNWIND_HINT type=UNWIND_HINT_TYPE_END_OF_STACK +.endm + +.macro UNWIND_HINT_UNDEFINED + UNWIND_HINT type=UNWIND_HINT_TYPE_UNDEFINED .endm .macro UNWIND_HINT_ENTRY VALIDATE_UNRET_BEGIN - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK .endm .macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 partial=0 signal=1 @@ -73,7 +77,7 @@ #else #define UNWIND_HINT_FUNC \ - UNWIND_HINT(UNWIND_HINT_TYPE_FUNC, ORC_REG_SP, 8, 0, 0) + UNWIND_HINT(UNWIND_HINT_TYPE_FUNC, ORC_REG_SP, 8, 0) #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 1265ad519249..0387732e9c3f 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -340,7 +340,7 @@ STACK_FRAME_NON_STANDARD_FP(__fentry__) #ifdef CONFIG_FUNCTION_GRAPH_TRACER SYM_CODE_START(return_to_handler) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR subq $16, %rsp diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index ee3ed15ee1f8..8d8d25c64827 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -42,7 +42,7 @@ L3_START_KERNEL = pud_index(__START_KERNEL_map) __HEAD .code64 SYM_CODE_START_NOALIGN(startup_64) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, * and someone has loaded an identity mapped page table @@ -105,7 +105,7 @@ SYM_CODE_START_NOALIGN(startup_64) lretq .Lon_kernel_cs: - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* Sanitize CPU configuration */ call verify_cpu @@ -127,7 +127,7 @@ SYM_CODE_START_NOALIGN(startup_64) SYM_CODE_END(startup_64) SYM_CODE_START(secondary_startup_64) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR /* * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, @@ -156,7 +156,7 @@ SYM_CODE_START(secondary_startup_64) * verify_cpu() above to make sure NX is enabled. */ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR /* @@ -238,7 +238,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) ANNOTATE_RETPOLINE_SAFE jmp *%rax 1: - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR // above /* @@ -371,7 +371,7 @@ SYM_CODE_END(secondary_startup_64) */ SYM_CODE_START(start_cpu0) ANNOTATE_NOENDBR - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK movq initial_stack(%rip), %rsp jmp .Ljump_to_C_code SYM_CODE_END(start_cpu0) diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index 4a73351f87f8..56cab1bb25f5 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -43,7 +43,7 @@ .code64 SYM_CODE_START_NOALIGN(relocate_range) SYM_CODE_START_NOALIGN(relocate_kernel) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR /* * %rdi indirection_page @@ -113,7 +113,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel) SYM_CODE_END(relocate_kernel) SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* set return address to 0 if not preserving context */ pushq $0 /* store the start address on the stack */ @@ -231,7 +231,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR // RET target, above movq RSP(%r8), %rsp movq CR4(%r8), %rax @@ -256,8 +256,8 @@ SYM_CODE_END(virtual_mapped) /* Do the copies */ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) - UNWIND_HINT_EMPTY - movq %rdi, %rcx /* Put the page_list in %rcx */ + UNWIND_HINT_END_OF_STACK + movq %rdi, %rcx /* Put the page_list in %rcx */ xorl %edi, %edi xorl %esi, %esi jmp 1f diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c index 8348ac581de3..3ac50b7298d1 100644 --- a/arch/x86/kernel/unwind_orc.c +++ b/arch/x86/kernel/unwind_orc.c @@ -158,7 +158,6 @@ static struct orc_entry orc_fp_entry = { .sp_offset = 16, .bp_reg = ORC_REG_PREV_SP, .bp_offset = -16, - .end = 0, }; static struct orc_entry *orc_find(unsigned long ip) @@ -250,13 +249,13 @@ static int orc_sort_cmp(const void *_a, const void *_b) return -1; /* - * The "weak" section terminator entries need to always be on the left + * The "weak" section terminator entries need to always be first * to ensure the lookup code skips them in favor of real entries. * These terminator entries exist to handle any gaps created by * whitelisted .o files which didn't get objtool generation. */ orc_a = cur_orc_table + (a - cur_orc_ip_table); - return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; + return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; } void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size, @@ -474,14 +473,12 @@ bool unwind_next_frame(struct unwind_state *state) */ orc = &orc_fp_entry; state->error = true; - } - - /* End-of-stack check for kernel threads: */ - if (orc->sp_reg == ORC_REG_UNDEFINED) { - if (!orc->end) + } else { + if (orc->type == ORC_TYPE_UNDEFINED) goto err; - goto the_end; + if (orc->type == ORC_TYPE_END_OF_STACK) + goto the_end; } state->signal = orc->signal; diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index 5f61c65322be..27ef53fab6bd 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -33,7 +33,7 @@ .align RETPOLINE_THUNK_SIZE SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR ALTERNATIVE_2 __stringify(RETPOLINE \reg), \ @@ -75,7 +75,7 @@ SYM_CODE_END(__x86_indirect_thunk_array) .align RETPOLINE_THUNK_SIZE SYM_INNER_LABEL(__x86_indirect_call_thunk_\reg, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR CALL_DEPTH_ACCOUNT @@ -103,7 +103,7 @@ SYM_CODE_END(__x86_indirect_call_thunk_array) .align RETPOLINE_THUNK_SIZE SYM_INNER_LABEL(__x86_indirect_jump_thunk_\reg, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR POLINE \reg ANNOTATE_UNRET_SAFE diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index 7fe564eaf228..c4365a05ab83 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -50,7 +50,7 @@ #define PVH_DS_SEL (PVH_GDT_ENTRY_DS * 8) SYM_CODE_START_LOCAL(pvh_start_xen) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK cld lgdt (_pa(gdt)) diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S index 4a184f6e4e4d..08f1ceb9eb81 100644 --- a/arch/x86/xen/xen-asm.S +++ b/arch/x86/xen/xen-asm.S @@ -165,7 +165,7 @@ xen_pv_trap asm_exc_xen_hypervisor_callback SYM_CODE_START(xen_early_idt_handler_array) i = 0 .rept NUM_EXCEPTION_VECTORS - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ENDBR pop %rcx pop %r11 @@ -193,7 +193,7 @@ hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 * rsp->rax } */ SYM_CODE_START(xen_iret) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR pushq $0 jmp hypercall_iret diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S index e36ea4268bd2..6eafdd17c242 100644 --- a/arch/x86/xen/xen-head.S +++ b/arch/x86/xen/xen-head.S @@ -45,7 +45,7 @@ SYM_CODE_END(hypercall_page) #ifdef CONFIG_XEN_PV __INIT SYM_CODE_START(startup_xen) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR cld @@ -71,7 +71,7 @@ SYM_CODE_END(startup_xen) #ifdef CONFIG_XEN_PV_SMP .pushsection .text SYM_CODE_START(asm_cpu_bringup_and_idle) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ENDBR call cpu_bringup_and_idle diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 5aa475118820..03f82c2c2ebf 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -10,7 +10,7 @@ #ifndef __ASSEMBLY__ -#define UNWIND_HINT(type, sp_reg, sp_offset, signal, end) \ +#define UNWIND_HINT(type, sp_reg, sp_offset, signal) \ "987: \n\t" \ ".pushsection .discard.unwind_hints\n\t" \ /* struct unwind_hint */ \ @@ -19,7 +19,6 @@ ".byte " __stringify(sp_reg) "\n\t" \ ".byte " __stringify(type) "\n\t" \ ".byte " __stringify(signal) "\n\t" \ - ".byte " __stringify(end) "\n\t" \ ".balign 4 \n\t" \ ".popsection\n\t" @@ -91,7 +90,7 @@ * the debuginfo as necessary. It will also warn if it sees any * inconsistencies. */ -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 +.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 .Lhere_\@: .pushsection .discard.unwind_hints /* struct unwind_hint */ @@ -100,7 +99,6 @@ .byte \sp_reg .byte \type .byte \signal - .byte \end .balign 4 .popsection .endm @@ -153,14 +151,14 @@ #ifndef __ASSEMBLY__ -#define UNWIND_HINT(type, sp_reg, sp_offset, signal, end) "\n\t" +#define UNWIND_HINT(type, sp_reg, sp_offset, signal) "\n\t" #define STACK_FRAME_NON_STANDARD(func) #define STACK_FRAME_NON_STANDARD_FP(func) #define ANNOTATE_NOENDBR #define ASM_REACHABLE #else #define ANNOTATE_INTRA_FUNCTION_CALL -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 +.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 .endm .macro STACK_FRAME_NON_STANDARD func:req .endm diff --git a/include/linux/objtool_types.h b/include/linux/objtool_types.h index 9787ad0f2ef4..453a4f4ef39d 100644 --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -16,12 +16,18 @@ struct unwind_hint { u8 sp_reg; u8 type; u8 signal; - u8 end; }; #endif /* __ASSEMBLY__ */ /* + * UNWIND_HINT_TYPE_UNDEFINED: A blind spot in ORC coverage which can result in + * a truncated and unreliable stack unwind. + * + * UNWIND_HINT_TYPE_END_OF_STACK: The end of the kernel stack unwind before + * hitting user entry, boot code, or fork entry (when there are no pt_regs + * available). + * * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP * (the caller's SP right before it made the call). Used for all callable * functions, i.e. all C code and all callable asm functions. @@ -32,17 +38,20 @@ struct unwind_hint { * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that * sp_reg+sp_offset points to the iret return frame. * - * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. + * UNWIND_HINT_TYPE_FUNC: Generate the unwind metadata of a callable function. * Useful for code which doesn't have an ELF function annotation. * - * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. + * UNWIND_HINT_TYPE_{SAVE,RESTORE}: Save the unwind metadata at a certain + * location so that it can be restored later. */ -#define UNWIND_HINT_TYPE_CALL 0 -#define UNWIND_HINT_TYPE_REGS 1 -#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +#define UNWIND_HINT_TYPE_UNDEFINED 0 +#define UNWIND_HINT_TYPE_END_OF_STACK 1 +#define UNWIND_HINT_TYPE_CALL 2 +#define UNWIND_HINT_TYPE_REGS 3 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 4 /* The below hint types don't have corresponding ORC types */ -#define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_SAVE 4 -#define UNWIND_HINT_TYPE_RESTORE 5 +#define UNWIND_HINT_TYPE_FUNC 5 +#define UNWIND_HINT_TYPE_SAVE 6 +#define UNWIND_HINT_TYPE_RESTORE 7 #endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/scripts/sorttable.h b/scripts/sorttable.h index deb7c1d3e979..7bd0184380d3 100644 --- a/scripts/sorttable.h +++ b/scripts/sorttable.h @@ -128,7 +128,7 @@ static int orc_sort_cmp(const void *_a, const void *_b) * whitelisted .o files which didn't get objtool generation. */ orc_a = g_orc_table + (a - g_orc_ip_table); - return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; + return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; } static void *sort_orctable(void *arg) diff --git a/tools/arch/x86/include/asm/orc_types.h b/tools/arch/x86/include/asm/orc_types.h index b4d4ec78589e..46d7e06763c9 100644 --- a/tools/arch/x86/include/asm/orc_types.h +++ b/tools/arch/x86/include/asm/orc_types.h @@ -39,9 +39,11 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 -#define ORC_TYPE_CALL 0 -#define ORC_TYPE_REGS 1 -#define ORC_TYPE_REGS_PARTIAL 2 +#define ORC_TYPE_UNDEFINED 0 +#define ORC_TYPE_END_OF_STACK 1 +#define ORC_TYPE_CALL 2 +#define ORC_TYPE_REGS 3 +#define ORC_TYPE_REGS_PARTIAL 4 #ifndef __ASSEMBLY__ #include @@ -60,16 +62,14 @@ struct orc_entry { #if defined(__LITTLE_ENDIAN_BITFIELD) unsigned sp_reg:4; unsigned bp_reg:4; - unsigned type:2; + unsigned type:3; unsigned signal:1; - unsigned end:1; #elif defined(__BIG_ENDIAN_BITFIELD) unsigned bp_reg:4; unsigned sp_reg:4; unsigned unused:4; - unsigned end:1; unsigned signal:1; - unsigned type:2; + unsigned type:3; #endif } __packed; diff --git a/tools/include/linux/objtool_types.h b/tools/include/linux/objtool_types.h index 9787ad0f2ef4..453a4f4ef39d 100644 --- a/tools/include/linux/objtool_types.h +++ b/tools/include/linux/objtool_types.h @@ -16,12 +16,18 @@ struct unwind_hint { u8 sp_reg; u8 type; u8 signal; - u8 end; }; #endif /* __ASSEMBLY__ */ /* + * UNWIND_HINT_TYPE_UNDEFINED: A blind spot in ORC coverage which can result in + * a truncated and unreliable stack unwind. + * + * UNWIND_HINT_TYPE_END_OF_STACK: The end of the kernel stack unwind before + * hitting user entry, boot code, or fork entry (when there are no pt_regs + * available). + * * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP * (the caller's SP right before it made the call). Used for all callable * functions, i.e. all C code and all callable asm functions. @@ -32,17 +38,20 @@ struct unwind_hint { * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that * sp_reg+sp_offset points to the iret return frame. * - * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. + * UNWIND_HINT_TYPE_FUNC: Generate the unwind metadata of a callable function. * Useful for code which doesn't have an ELF function annotation. * - * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. + * UNWIND_HINT_TYPE_{SAVE,RESTORE}: Save the unwind metadata at a certain + * location so that it can be restored later. */ -#define UNWIND_HINT_TYPE_CALL 0 -#define UNWIND_HINT_TYPE_REGS 1 -#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 +#define UNWIND_HINT_TYPE_UNDEFINED 0 +#define UNWIND_HINT_TYPE_END_OF_STACK 1 +#define UNWIND_HINT_TYPE_CALL 2 +#define UNWIND_HINT_TYPE_REGS 3 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 4 /* The below hint types don't have corresponding ORC types */ -#define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_SAVE 4 -#define UNWIND_HINT_TYPE_RESTORE 5 +#define UNWIND_HINT_TYPE_FUNC 5 +#define UNWIND_HINT_TYPE_SAVE 6 +#define UNWIND_HINT_TYPE_RESTORE 7 #endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 10be80b3fe67..cae6ac6ff246 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2243,6 +2243,7 @@ static void set_func_state(struct cfi_state *state) memcpy(&state->regs, &initial_func_cfi.regs, CFI_NUM_REGS * sizeof(struct cfi_reg)); state->stack_size = initial_func_cfi.cfa.offset; + state->type = UNWIND_HINT_TYPE_CALL; } static int read_unwind_hints(struct objtool_file *file) @@ -2327,7 +2328,6 @@ static int read_unwind_hints(struct objtool_file *file) cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); cfi.type = hint->type; cfi.signal = hint->signal; - cfi.end = hint->end; insn->cfi = cfi_hash_find_or_add(&cfi); } diff --git a/tools/objtool/orc_dump.c b/tools/objtool/orc_dump.c index 97ecbb8b9034..0e183bb1c720 100644 --- a/tools/objtool/orc_dump.c +++ b/tools/objtool/orc_dump.c @@ -38,6 +38,10 @@ static const char *reg_name(unsigned int reg) static const char *orc_type_name(unsigned int type) { switch (type) { + case ORC_TYPE_UNDEFINED: + return "(und)"; + case ORC_TYPE_END_OF_STACK: + return "end"; case ORC_TYPE_CALL: return "call"; case ORC_TYPE_REGS: @@ -201,6 +205,7 @@ int orc_dump(const char *_objname) printf("%llx:", (unsigned long long)(orc_ip_addr + (i * sizeof(int)) + orc_ip[i])); } + printf("type:%s", orc_type_name(orc[i].type)); printf(" sp:"); @@ -210,8 +215,7 @@ int orc_dump(const char *_objname) print_reg(orc[i].bp_reg, bswap_if_needed(&dummy_elf, orc[i].bp_offset)); - printf(" type:%s signal:%d end:%d\n", - orc_type_name(orc[i].type), orc[i].signal, orc[i].end); + printf(" signal:%d\n", orc[i].signal); } elf_end(elf); diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c index e85bbb996f6c..b327f9ccfe73 100644 --- a/tools/objtool/orc_gen.c +++ b/tools/objtool/orc_gen.c @@ -21,12 +21,22 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, memset(orc, 0, sizeof(*orc)); if (!cfi) { - orc->end = 0; - orc->sp_reg = ORC_REG_UNDEFINED; + /* + * This is usually either unreachable nops/traps (which don't + * trigger unreachable instruction warnings), or + * STACK_FRAME_NON_STANDARD functions. + */ + orc->type = ORC_TYPE_UNDEFINED; return 0; } switch (cfi->type) { + case UNWIND_HINT_TYPE_UNDEFINED: + orc->type = ORC_TYPE_UNDEFINED; + return 0; + case UNWIND_HINT_TYPE_END_OF_STACK: + orc->type = ORC_TYPE_END_OF_STACK; + return 0; case UNWIND_HINT_TYPE_CALL: orc->type = ORC_TYPE_CALL; break; @@ -42,14 +52,8 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, return -1; } - orc->end = cfi->end; orc->signal = cfi->signal; - if (cfi->cfa.base == CFI_UNDEFINED) { - orc->sp_reg = ORC_REG_UNDEFINED; - return 0; - } - switch (cfi->cfa.base) { case CFI_SP: orc->sp_reg = ORC_REG_SP; @@ -163,11 +167,7 @@ int orc_create(struct objtool_file *file) struct orc_list_entry *entry; struct list_head orc_list; - struct orc_entry null = { - .sp_reg = ORC_REG_UNDEFINED, - .bp_reg = ORC_REG_UNDEFINED, - .type = ORC_TYPE_CALL, - }; + struct orc_entry null = { .type = ORC_TYPE_UNDEFINED }; /* Build a deduplicated list of ORC entries: */ INIT_LIST_HEAD(&orc_list); -- cgit v1.2.3 From f530b531fb9e05eb134cfc334242799ea974d111 Mon Sep 17 00:00:00 2001 From: Mathias Krause Date: Fri, 17 Feb 2023 20:33:36 +0100 Subject: KVM: Shrink struct kvm_mmu_memory_cache Move the 'capacity' member around to make use of the padding hole on 64 bit systems instead of introducing yet another one. This allows us to save 8 bytes per instance for 64 bit builds of which, e.g., x86's struct kvm_vcpu_arch has a few. Signed-off-by: Mathias Krause Link: https://lore.kernel.org/r/20230217193336.15278-3-minipli@grsecurity.net Signed-off-by: Sean Christopherson --- include/linux/kvm_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h index 2728d49bbdf6..6f4737d5046a 100644 --- a/include/linux/kvm_types.h +++ b/include/linux/kvm_types.h @@ -91,11 +91,11 @@ struct gfn_to_pfn_cache { * is topped up (__kvm_mmu_topup_memory_cache()). */ struct kvm_mmu_memory_cache { - int nobjs; gfp_t gfp_zero; gfp_t gfp_custom; struct kmem_cache *kmem_cache; int capacity; + int nobjs; void **objects; }; #endif -- cgit v1.2.3 From 704bc669e1dda3eb8f6d5cb462b21e85558a3912 Mon Sep 17 00:00:00 2001 From: Jungseung Lee Date: Mon, 20 Mar 2023 12:29:05 +0900 Subject: workqueue: Introduce show_freezable_workqueues Currently show_all_workqueue is called if freeze fails at the time of freeze the workqueues, which shows the status of all workqueues and of all worker pools. In this cases we may only need to dump state of only workqueues that are freezable and busy. This patch defines show_freezable_workqueues, which uses show_one_workqueue, a granular function that shows the state of individual workqueues, so that dump only the state of freezable workqueues at that time. tj: Minor message adjustment. Signed-off-by: Jungseung Lee Signed-off-by: Tejun Heo --- include/linux/workqueue.h | 1 + kernel/power/process.c | 2 +- kernel/workqueue.c | 26 ++++++++++++++++++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index ac551b8ee7d9..3992c994787f 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -472,6 +472,7 @@ extern unsigned int work_busy(struct work_struct *work); extern __printf(1, 2) void set_worker_desc(const char *fmt, ...); extern void print_worker_info(const char *log_lvl, struct task_struct *task); extern void show_all_workqueues(void); +extern void show_freezable_workqueues(void); extern void show_one_workqueue(struct workqueue_struct *wq); extern void wq_worker_comm(char *buf, size_t size, struct task_struct *task); diff --git a/kernel/power/process.c b/kernel/power/process.c index 6c1c7e566d35..cae81a87cc91 100644 --- a/kernel/power/process.c +++ b/kernel/power/process.c @@ -93,7 +93,7 @@ static int try_to_freeze_tasks(bool user_only) todo - wq_busy, wq_busy); if (wq_busy) - show_all_workqueues(); + show_freezable_workqueues(); if (!wakeup || pm_debug_messages_on) { read_lock(&tasklist_lock); diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 044b4eee760b..d78935e4fb5d 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -5065,8 +5065,7 @@ next_pool: /** * show_all_workqueues - dump workqueue state * - * Called from a sysrq handler or try_to_freeze_tasks() and prints out - * all busy workqueues and pools. + * Called from a sysrq handler and prints out all busy workqueues and pools. */ void show_all_workqueues(void) { @@ -5087,6 +5086,29 @@ void show_all_workqueues(void) rcu_read_unlock(); } +/** + * show_freezable_workqueues - dump freezable workqueue state + * + * Called from try_to_freeze_tasks() and prints out all freezable workqueues + * still busy. + */ +void show_freezable_workqueues(void) +{ + struct workqueue_struct *wq; + + rcu_read_lock(); + + pr_info("Showing freezable workqueues that are still busy:\n"); + + list_for_each_entry_rcu(wq, &workqueues, list) { + if (!(wq->flags & WQ_FREEZABLE)) + continue; + show_one_workqueue(wq); + } + + rcu_read_unlock(); +} + /* used to show worker information through /proc/PID/{comm,stat,status} */ void wq_worker_comm(char *buf, size_t size, struct task_struct *task) { -- cgit v1.2.3 From 8e4645226b4931e96d55546a1fb3863aa50b5e62 Mon Sep 17 00:00:00 2001 From: Haifeng Xu Date: Tue, 28 Feb 2023 08:35:37 +0000 Subject: cpuset: Clean up cpuset_node_allowed Commit 002f290627c2 ("cpuset: use static key better and convert to new API") has used __cpuset_node_allowed() instead of cpuset_node_allowed() to check whether we can allocate on a memory node. Now this function isn't used by anyone, so we can do the follow things to clean up it. 1. remove unused codes 2. rename __cpuset_node_allowed() to cpuset_node_allowed() 3. update comments in mm/page_alloc.c Suggested-by: Waiman Long Signed-off-by: Haifeng Xu Acked-by: Waiman Long Signed-off-by: Tejun Heo --- include/linux/cpuset.h | 16 ++-------------- kernel/cgroup/cpuset.c | 4 ++-- mm/page_alloc.c | 4 ++-- 3 files changed, 6 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index d58e0476ee8e..980b76a1237e 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -80,18 +80,11 @@ extern nodemask_t cpuset_mems_allowed(struct task_struct *p); void cpuset_init_current_mems_allowed(void); int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask); -extern bool __cpuset_node_allowed(int node, gfp_t gfp_mask); - -static inline bool cpuset_node_allowed(int node, gfp_t gfp_mask) -{ - if (cpusets_enabled()) - return __cpuset_node_allowed(node, gfp_mask); - return true; -} +extern bool cpuset_node_allowed(int node, gfp_t gfp_mask); static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) { - return __cpuset_node_allowed(zone_to_nid(z), gfp_mask); + return cpuset_node_allowed(zone_to_nid(z), gfp_mask); } static inline bool cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) @@ -223,11 +216,6 @@ static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) return 1; } -static inline bool cpuset_node_allowed(int node, gfp_t gfp_mask) -{ - return true; -} - static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) { return true; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 636f1c682ac0..0241b07d6f21 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -3831,7 +3831,7 @@ static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) } /* - * __cpuset_node_allowed - Can we allocate on a memory node? + * cpuset_node_allowed - Can we allocate on a memory node? * @node: is this an allowed node? * @gfp_mask: memory allocation flags * @@ -3870,7 +3870,7 @@ static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) * GFP_KERNEL - any node in enclosing hardwalled cpuset ok * GFP_USER - only nodes in current tasks mems allowed ok. */ -bool __cpuset_node_allowed(int node, gfp_t gfp_mask) +bool cpuset_node_allowed(int node, gfp_t gfp_mask) { struct cpuset *cs; /* current cpuset ancestors */ bool allowed; /* is allocation in zone z allowed? */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ac1fc986af44..ea4e39b06475 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4217,7 +4217,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, retry: /* * Scan zonelist, looking for a zone with enough free. - * See also __cpuset_node_allowed() comment in kernel/cgroup/cpuset.c. + * See also cpuset_node_allowed() comment in kernel/cgroup/cpuset.c. */ no_fallback = alloc_flags & ALLOC_NOFRAGMENT; z = ac->preferred_zoneref; @@ -4891,7 +4891,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order) /* * Ignore cpuset mems for non-blocking __GFP_HIGH (probably * GFP_ATOMIC) rather than fail, see the comment for - * __cpuset_node_allowed(). + * cpuset_node_allowed(). */ if (alloc_flags & ALLOC_MIN_RESERVE) alloc_flags &= ~ALLOC_CPUSET; -- cgit v1.2.3 From 4c8c3c7f70a6779d30f5492acbc9978f4636fe7a Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Tue, 7 Mar 2023 14:35:56 +0000 Subject: treewide: Trace IPIs sent via smp_send_reschedule() To be able to trace invocations of smp_send_reschedule(), rename the arch-specific definitions of it to arch_smp_send_reschedule() and wrap it into an smp_send_reschedule() that contains a tracepoint. Changes to include the declaration of the tracepoint were driven by the following coccinelle script: @func_use@ @@ smp_send_reschedule(...); @include@ @@ #include @no_include depends on func_use && !include@ @@ #include <...> + + #include [csky bits] [riscv bits] Signed-off-by: Valentin Schneider Signed-off-by: Peter Zijlstra (Intel) Acked-by: Guo Ren Acked-by: Palmer Dabbelt Link: https://lore.kernel.org/r/20230307143558.294354-6-vschneid@redhat.com --- arch/alpha/kernel/smp.c | 2 +- arch/arc/kernel/smp.c | 2 +- arch/arm/kernel/smp.c | 2 +- arch/arm/mach-actions/platsmp.c | 2 ++ arch/arm64/kernel/smp.c | 2 +- arch/csky/kernel/smp.c | 2 +- arch/hexagon/kernel/smp.c | 2 +- arch/ia64/kernel/smp.c | 4 ++-- arch/loongarch/kernel/smp.c | 4 ++-- arch/mips/include/asm/smp.h | 2 +- arch/mips/kernel/rtlx-cmp.c | 2 ++ arch/openrisc/kernel/smp.c | 2 +- arch/parisc/kernel/smp.c | 4 ++-- arch/powerpc/kernel/smp.c | 6 ++++-- arch/powerpc/kvm/book3s_hv.c | 3 +++ arch/powerpc/platforms/powernv/subcore.c | 2 ++ arch/riscv/kernel/smp.c | 4 ++-- arch/s390/kernel/smp.c | 2 +- arch/sh/kernel/smp.c | 2 +- arch/sparc/kernel/smp_32.c | 2 +- arch/sparc/kernel/smp_64.c | 2 +- arch/x86/include/asm/smp.h | 2 +- arch/x86/kvm/svm/svm.c | 4 ++++ arch/x86/kvm/x86.c | 2 ++ arch/xtensa/kernel/smp.c | 2 +- include/linux/smp.h | 11 +++++++++-- virt/kvm/kvm_main.c | 3 +++ 27 files changed, 53 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c index 0ede4b044e86..7439b2377df5 100644 --- a/arch/alpha/kernel/smp.c +++ b/arch/alpha/kernel/smp.c @@ -562,7 +562,7 @@ handle_ipi(struct pt_regs *regs) } void -smp_send_reschedule(int cpu) +arch_smp_send_reschedule(int cpu) { #ifdef DEBUG_IPI_MSG if (cpu == hard_smp_processor_id()) diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c index ad93fe6e4b77..409cfa4675b4 100644 --- a/arch/arc/kernel/smp.c +++ b/arch/arc/kernel/smp.c @@ -292,7 +292,7 @@ static void ipi_send_msg(const struct cpumask *callmap, enum ipi_msg_type msg) ipi_send_msg_one(cpu, msg); } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { ipi_send_msg_one(cpu, IPI_RESCHEDULE); } diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 5edf09237b2f..b350bfc9d1f8 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -746,7 +746,7 @@ void __init set_smp_ipi_range(int ipi_base, int n) ipi_setup(smp_processor_id()); } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); } diff --git a/arch/arm/mach-actions/platsmp.c b/arch/arm/mach-actions/platsmp.c index f26618b43514..7b208e96fbb6 100644 --- a/arch/arm/mach-actions/platsmp.c +++ b/arch/arm/mach-actions/platsmp.c @@ -20,6 +20,8 @@ #include #include +#include + #define OWL_CPU1_ADDR 0x50 #define OWL_CPU1_FLAG 0x5c diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 438c16fc4463..66f2745062dd 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -976,7 +976,7 @@ void __init set_smp_ipi_range(int ipi_base, int n) ipi_setup(smp_processor_id()); } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); } diff --git a/arch/csky/kernel/smp.c b/arch/csky/kernel/smp.c index b45d1073307f..be77383acb5f 100644 --- a/arch/csky/kernel/smp.c +++ b/arch/csky/kernel/smp.c @@ -140,7 +140,7 @@ void smp_send_stop(void) on_each_cpu(ipi_stop, NULL, 1); } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); } diff --git a/arch/hexagon/kernel/smp.c b/arch/hexagon/kernel/smp.c index 4ba93e59370c..4e8bee25b8c6 100644 --- a/arch/hexagon/kernel/smp.c +++ b/arch/hexagon/kernel/smp.c @@ -217,7 +217,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) } } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { send_ipi(cpumask_of(cpu), IPI_RESCHEDULE); } diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c index e2cc59db86bc..ea4f009a232b 100644 --- a/arch/ia64/kernel/smp.c +++ b/arch/ia64/kernel/smp.c @@ -220,11 +220,11 @@ kdump_smp_send_init(void) * Called with preemption disabled. */ void -smp_send_reschedule (int cpu) +arch_smp_send_reschedule (int cpu) { ia64_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0); } -EXPORT_SYMBOL_GPL(smp_send_reschedule); +EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); /* * Called with preemption disabled. diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index 8c6e227cb29d..83225610a148 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -155,11 +155,11 @@ void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int action) * it goes straight through and wastes no time serializing * anything. Worst case is that we lose a reschedule ... */ -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { loongson_send_ipi_single(cpu, SMP_RESCHEDULE); } -EXPORT_SYMBOL_GPL(smp_send_reschedule); +EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); irqreturn_t loongson_ipi_interrupt(int irq, void *dev) { diff --git a/arch/mips/include/asm/smp.h b/arch/mips/include/asm/smp.h index 5d9ff61004ca..9806e79895d9 100644 --- a/arch/mips/include/asm/smp.h +++ b/arch/mips/include/asm/smp.h @@ -66,7 +66,7 @@ extern void calculate_cpu_foreign_map(void); * it goes straight through and wastes no time serializing * anything. Worst case is that we lose a reschedule ... */ -static inline void smp_send_reschedule(int cpu) +static inline void arch_smp_send_reschedule(int cpu) { extern const struct plat_smp_ops *mp_ops; /* private */ diff --git a/arch/mips/kernel/rtlx-cmp.c b/arch/mips/kernel/rtlx-cmp.c index d26dcc4b46e7..e991cc936c1c 100644 --- a/arch/mips/kernel/rtlx-cmp.c +++ b/arch/mips/kernel/rtlx-cmp.c @@ -17,6 +17,8 @@ #include #include +#include + static int major; static void rtlx_interrupt(void) diff --git a/arch/openrisc/kernel/smp.c b/arch/openrisc/kernel/smp.c index e1419095a6f0..0a7a059e2dff 100644 --- a/arch/openrisc/kernel/smp.c +++ b/arch/openrisc/kernel/smp.c @@ -173,7 +173,7 @@ void handle_IPI(unsigned int ipi_msg) } } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); } diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c index 7dbd92cafae3..b7fc859fa87d 100644 --- a/arch/parisc/kernel/smp.c +++ b/arch/parisc/kernel/smp.c @@ -246,8 +246,8 @@ void kgdb_roundup_cpus(void) inline void smp_send_stop(void) { send_IPI_allbutself(IPI_CPU_STOP); } -void -smp_send_reschedule(int cpu) { send_IPI_single(cpu, IPI_RESCHEDULE); } +void +arch_smp_send_reschedule(int cpu) { send_IPI_single(cpu, IPI_RESCHEDULE); } void smp_send_all_nop(void) diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 6b90f10a6c81..35f101ccb540 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -61,6 +61,8 @@ #include #include +#include + #ifdef DEBUG #include #define DBG(fmt...) udbg_printf(fmt) @@ -364,12 +366,12 @@ static inline void do_message_pass(int cpu, int msg) #endif } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { if (likely(smp_ops)) do_message_pass(cpu, PPC_MSG_RESCHEDULE); } -EXPORT_SYMBOL_GPL(smp_send_reschedule); +EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); void arch_send_call_function_single_ipi(int cpu) { diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 6ba68dd6190b..3b70b5f80bd5 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -80,6 +81,8 @@ #include #include +#include + #include "book3s.h" #include "book3s_hv.h" diff --git a/arch/powerpc/platforms/powernv/subcore.c b/arch/powerpc/platforms/powernv/subcore.c index 7e98b00ea2e8..c53c4c797768 100644 --- a/arch/powerpc/platforms/powernv/subcore.c +++ b/arch/powerpc/platforms/powernv/subcore.c @@ -20,6 +20,8 @@ #include #include +#include + #include "subcore.h" #include "powernv.h" diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 8c3b59f1f9b8..42e9656a1db2 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -328,8 +328,8 @@ bool smp_crash_stop_failed(void) } #endif -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { send_ipi_single(cpu, IPI_RESCHEDULE); } -EXPORT_SYMBOL_GPL(smp_send_reschedule); +EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index d4888453bbf8..a710319f97e9 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -553,7 +553,7 @@ void arch_send_call_function_single_ipi(int cpu) * it goes straight through and wastes no time serializing * anything. Worst case is that we lose a reschedule ... */ -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { pcpu_ec_call(pcpu_devices + cpu, ec_schedule); } diff --git a/arch/sh/kernel/smp.c b/arch/sh/kernel/smp.c index 65924d9ec245..5cf35a774dc7 100644 --- a/arch/sh/kernel/smp.c +++ b/arch/sh/kernel/smp.c @@ -256,7 +256,7 @@ void __init smp_cpus_done(unsigned int max_cpus) (bogosum / (5000/HZ)) % 100); } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { mp_ops->send_ipi(cpu, SMP_MSG_RESCHEDULE); } diff --git a/arch/sparc/kernel/smp_32.c b/arch/sparc/kernel/smp_32.c index ad8094d955eb..87eaa7719fa2 100644 --- a/arch/sparc/kernel/smp_32.c +++ b/arch/sparc/kernel/smp_32.c @@ -120,7 +120,7 @@ void cpu_panic(void) struct linux_prom_registers smp_penguin_ctable = { 0 }; -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { /* * CPU model dependent way of implementing IPI generation targeting diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index a55295d1b924..e5964d1d8b37 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -1430,7 +1430,7 @@ static unsigned long send_cpu_poke(int cpu) return hv_err; } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { if (cpu == smp_processor_id()) { WARN_ON_ONCE(preemptible()); diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index b4dbb20dab1a..f9757123d8fa 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -98,7 +98,7 @@ static inline void play_dead(void) smp_ops.play_dead(); } -static inline void smp_send_reschedule(int cpu) +static inline void arch_smp_send_reschedule(int cpu) { smp_ops.smp_send_reschedule(cpu); } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 252e7f37e4e2..424fcdba4c78 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -41,6 +42,9 @@ #include #include + +#include + #include "trace.h" #include "svm.h" diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 7713420abab0..07ba937bdb6f 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -60,7 +60,9 @@ #include #include #include +#include +#include #include #include diff --git a/arch/xtensa/kernel/smp.c b/arch/xtensa/kernel/smp.c index 4dc109dd6214..d95907b8e4d3 100644 --- a/arch/xtensa/kernel/smp.c +++ b/arch/xtensa/kernel/smp.c @@ -389,7 +389,7 @@ void arch_send_call_function_single_ipi(int cpu) send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC); } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); } diff --git a/include/linux/smp.h b/include/linux/smp.h index a80ab58ae3f1..c036a2228d8d 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -125,8 +125,15 @@ extern void smp_send_stop(void); /* * sends a 'reschedule' event to another CPU: */ -extern void smp_send_reschedule(int cpu); - +extern void arch_smp_send_reschedule(int cpu); +/* + * scheduler_ipi() is inline so can't be passed as callback reason, but the + * callsite IP should be sufficient for root-causing IPIs sent from here. + */ +#define smp_send_reschedule(cpu) ({ \ + trace_ipi_send_cpumask(cpumask_of(cpu), _RET_IP_, NULL); \ + arch_smp_send_reschedule(cpu); \ +}) /* * Prepare machine for booting other CPUs. diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index d255964ec331..7d1889629bfe 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -62,11 +62,14 @@ #include "kvm_mm.h" #include "vfio.h" +#include + #define CREATE_TRACE_POINTS #include #include + /* Worst case buffer size needed for holding an integer. */ #define ITOA_MAX_LEN 12 -- cgit v1.2.3 From 68e2d17c9eb311ab59aeb6d0c38aad8985fa2596 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 22 Mar 2023 11:28:36 +0100 Subject: trace: Add trace_ipi_send_cpu() Because copying cpumasks around when targeting a single CPU is a bit daft... Tested-and-reviewed-by: Valentin Schneider Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20230322103004.GA571242%40hirez.programming.kicks-ass.net --- include/linux/smp.h | 6 +++--- include/trace/events/ipi.h | 22 ++++++++++++++++++++++ kernel/irq_work.c | 6 ++---- kernel/sched/core.c | 1 + kernel/smp.c | 4 ++-- 5 files changed, 30 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/smp.h b/include/linux/smp.h index c036a2228d8d..ed8f344ba627 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -130,9 +130,9 @@ extern void arch_smp_send_reschedule(int cpu); * scheduler_ipi() is inline so can't be passed as callback reason, but the * callsite IP should be sufficient for root-causing IPIs sent from here. */ -#define smp_send_reschedule(cpu) ({ \ - trace_ipi_send_cpumask(cpumask_of(cpu), _RET_IP_, NULL); \ - arch_smp_send_reschedule(cpu); \ +#define smp_send_reschedule(cpu) ({ \ + trace_ipi_send_cpu(cpu, _RET_IP_, NULL); \ + arch_smp_send_reschedule(cpu); \ }) /* diff --git a/include/trace/events/ipi.h b/include/trace/events/ipi.h index b1125dc27682..3de9bfc982ce 100644 --- a/include/trace/events/ipi.h +++ b/include/trace/events/ipi.h @@ -35,6 +35,28 @@ TRACE_EVENT(ipi_raise, TP_printk("target_mask=%s (%s)", __get_bitmask(target_cpus), __entry->reason) ); +TRACE_EVENT(ipi_send_cpu, + + TP_PROTO(const unsigned int cpu, unsigned long callsite, void *callback), + + TP_ARGS(cpu, callsite, callback), + + TP_STRUCT__entry( + __field(unsigned int, cpu) + __field(void *, callsite) + __field(void *, callback) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->callsite = (void *)callsite; + __entry->callback = callback; + ), + + TP_printk("cpu=%u callsite=%pS callback=%pS", + __entry->cpu, __entry->callsite, __entry->callback) +); + TRACE_EVENT(ipi_send_cpumask, TP_PROTO(const struct cpumask *cpumask, unsigned long callsite, void *callback), diff --git a/kernel/irq_work.c b/kernel/irq_work.c index c33e88e32a67..2f4fb336dda1 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c @@ -78,10 +78,8 @@ void __weak arch_irq_work_raise(void) static __always_inline void irq_work_raise(struct irq_work *work) { - if (trace_ipi_send_cpumask_enabled() && arch_irq_work_has_interrupt()) - trace_ipi_send_cpumask(cpumask_of(smp_processor_id()), - _RET_IP_, - work->func); + if (trace_ipi_send_cpu_enabled() && arch_irq_work_has_interrupt()) + trace_ipi_send_cpu(smp_processor_id(), _RET_IP_, work->func); arch_irq_work_raise(); } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b0a48cfc0a22..ad40755ddc11 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -96,6 +96,7 @@ #include "../../io_uring/io-wq.h" #include "../smpboot.h" +EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); /* diff --git a/kernel/smp.c b/kernel/smp.c index 37e9613a0889..43f0796ecdb2 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -107,7 +107,7 @@ static __always_inline void send_call_function_single_ipi(int cpu, smp_call_func_t func) { if (call_function_single_prep_ipi(cpu)) { - trace_ipi_send_cpumask(cpumask_of(cpu), _RET_IP_, func); + trace_ipi_send_cpu(cpu, _RET_IP_, func); arch_send_call_function_single_ipi(cpu); } } @@ -346,7 +346,7 @@ void __smp_call_single_queue(int cpu, struct llist_node *node) * even if we haven't sent the smp_call IPI yet (e.g. the stopper * executes migration_cpu_stop() on the remote CPU). */ - if (trace_ipi_send_cpumask_enabled()) { + if (trace_ipi_send_cpu_enabled()) { call_single_data_t *csd; smp_call_func_t func; -- cgit v1.2.3 From 0a392354dbc3ff748e0856a75592fe8d0fdc7674 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 24 Mar 2023 09:26:47 +0000 Subject: device property: constify fwnode_get_phy_mode() argument fwnode_get_phy_mode() does not modify the fwnode argument, merely using it to obtain the phy-mode property value. Therefore, it can be made const. Signed-off-by: Russell King (Oracle) Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/E1pfdh9-00EQ8t-HB@rmk-PC.armlinux.org.uk Signed-off-by: Greg Kroah-Hartman --- drivers/base/property.c | 2 +- include/linux/property.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/property.c b/drivers/base/property.c index 3fc25e568598..35b8f3a4e24b 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -935,7 +935,7 @@ EXPORT_SYMBOL_GPL(device_get_dma_attr); * 'phy-connection-type', and return its index in phy_modes table, or errno in * error case. */ -int fwnode_get_phy_mode(struct fwnode_handle *fwnode) +int fwnode_get_phy_mode(const struct fwnode_handle *fwnode) { const char *pm; int err, i; diff --git a/include/linux/property.h b/include/linux/property.h index 0a29db15ff34..110634f5db2e 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -396,7 +396,7 @@ enum dev_dma_attr device_get_dma_attr(const struct device *dev); const void *device_get_match_data(const struct device *dev); int device_get_phy_mode(struct device *dev); -int fwnode_get_phy_mode(struct fwnode_handle *fwnode); +int fwnode_get_phy_mode(const struct fwnode_handle *fwnode); void __iomem *fwnode_iomap(struct fwnode_handle *fwnode, int index); -- cgit v1.2.3 From 5b9ff0ba11042096bfb396e506fa9038e6a61de7 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 24 Mar 2023 13:27:20 +0200 Subject: device property: Constify a few fwnode APIs The fwnode parameter is not altered in the following APIs: - fwnode_get_next_parent_dev() - fwnode_is_ancestor_of() - fwnode_graph_get_endpoint_count() so constify them. Reported-by: Russell King (Oracle) Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20230324112720.71315-1-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/property.c | 6 +++--- include/linux/property.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/property.c b/drivers/base/property.c index 35b8f3a4e24b..6a908e9c30f8 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -648,7 +648,7 @@ EXPORT_SYMBOL_GPL(fwnode_get_next_parent); * * Return: a pointer to the device of the @fwnode's closest ancestor. */ -struct device *fwnode_get_next_parent_dev(struct fwnode_handle *fwnode) +struct device *fwnode_get_next_parent_dev(const struct fwnode_handle *fwnode) { struct fwnode_handle *parent; struct device *dev; @@ -718,7 +718,7 @@ EXPORT_SYMBOL_GPL(fwnode_get_nth_parent); * * Return: true if @ancestor is an ancestor of @child. Otherwise, returns false. */ -bool fwnode_is_ancestor_of(struct fwnode_handle *ancestor, struct fwnode_handle *child) +bool fwnode_is_ancestor_of(const struct fwnode_handle *ancestor, const struct fwnode_handle *child) { struct fwnode_handle *parent; @@ -1235,7 +1235,7 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_endpoint_by_id); * If FWNODE_GRAPH_DEVICE_DISABLED flag is specified, also unconnected endpoints * and endpoints connected to disabled devices are counted. */ -unsigned int fwnode_graph_get_endpoint_count(struct fwnode_handle *fwnode, +unsigned int fwnode_graph_get_endpoint_count(const struct fwnode_handle *fwnode, unsigned long flags) { struct fwnode_handle *ep; diff --git a/include/linux/property.h b/include/linux/property.h index 110634f5db2e..4a536548606b 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -105,11 +105,11 @@ struct fwnode_handle *fwnode_get_next_parent(struct fwnode_handle *fwnode); for (parent = fwnode_get_parent(fwnode); parent; \ parent = fwnode_get_next_parent(parent)) -struct device *fwnode_get_next_parent_dev(struct fwnode_handle *fwnode); +struct device *fwnode_get_next_parent_dev(const struct fwnode_handle *fwnode); unsigned int fwnode_count_parents(const struct fwnode_handle *fwn); struct fwnode_handle *fwnode_get_nth_parent(struct fwnode_handle *fwn, unsigned int depth); -bool fwnode_is_ancestor_of(struct fwnode_handle *ancestor, struct fwnode_handle *child); +bool fwnode_is_ancestor_of(const struct fwnode_handle *ancestor, const struct fwnode_handle *child); struct fwnode_handle *fwnode_get_next_child_node( const struct fwnode_handle *fwnode, struct fwnode_handle *child); struct fwnode_handle *fwnode_get_next_available_child_node( @@ -433,7 +433,7 @@ static inline bool fwnode_graph_is_endpoint(const struct fwnode_handle *fwnode) struct fwnode_handle * fwnode_graph_get_endpoint_by_id(const struct fwnode_handle *fwnode, u32 port, u32 endpoint, unsigned long flags); -unsigned int fwnode_graph_get_endpoint_count(struct fwnode_handle *fwnode, +unsigned int fwnode_graph_get_endpoint_count(const struct fwnode_handle *fwnode, unsigned long flags); #define fwnode_graph_for_each_endpoint(fwnode, child) \ -- cgit v1.2.3 From dcfbb67e48a2becfce7990386e985b9c45098ee5 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 24 Mar 2023 11:01:31 +0100 Subject: driver core: class: use lock_class_key already present in struct subsys_private In commit 37e98d9bedb5 ("driver core: bus: move lock_class_key into dynamic structure"), we moved the lock_class_key into the internal structure shared by busses and classes, but only used it for buses. Move the class code to use this structure as it is already present and being allocated, instead of the statically allocated on-the-stack variable that class_create() was using as part of a macro wrapper around the core function call. Reviewed-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230324100132.1633647-1-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 15 +++++++++------ include/linux/device/class.h | 36 ++---------------------------------- 2 files changed, 11 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index 3d65221b0dcb..dbaeb79ae917 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -154,9 +154,10 @@ static void class_remove_groups(struct class *cls, return sysfs_remove_groups(&cls->p->subsys.kobj, groups); } -int __class_register(struct class *cls, struct lock_class_key *key) +int class_register(struct class *cls) { struct subsys_private *cp; + struct lock_class_key *key; int error; pr_debug("device class '%s': registering\n", cls->name); @@ -167,6 +168,8 @@ int __class_register(struct class *cls, struct lock_class_key *key) klist_init(&cp->klist_devices, klist_class_dev_get, klist_class_dev_put); INIT_LIST_HEAD(&cp->interfaces); kset_init(&cp->glue_dirs); + key = &cp->lock_key; + lockdep_register_key(key); __mutex_init(&cp->mutex, "subsys mutex", key); error = kobject_set_name(&cp->subsys.kobj, "%s", cls->name); if (error) { @@ -201,7 +204,7 @@ err_out: cls->p = NULL; return error; } -EXPORT_SYMBOL_GPL(__class_register); +EXPORT_SYMBOL_GPL(class_register); void class_unregister(struct class *cls) { @@ -218,7 +221,7 @@ static void class_create_release(struct class *cls) } /** - * __class_create - create a struct class structure + * class_create - create a struct class structure * @name: pointer to a string for the name of this class. * @key: the lock_class_key for this class; used by mutex lock debugging * @@ -230,7 +233,7 @@ static void class_create_release(struct class *cls) * Note, the pointer created here is to be destroyed when finished by * making a call to class_destroy(). */ -struct class *__class_create(const char *name, struct lock_class_key *key) +struct class *class_create(const char *name) { struct class *cls; int retval; @@ -244,7 +247,7 @@ struct class *__class_create(const char *name, struct lock_class_key *key) cls->name = name; cls->class_release = class_create_release; - retval = __class_register(cls, key); + retval = class_register(cls); if (retval) goto error; @@ -254,7 +257,7 @@ error: kfree(cls); return ERR_PTR(retval); } -EXPORT_SYMBOL_GPL(__class_create); +EXPORT_SYMBOL_GPL(class_create); /** * class_destroy - destroys a struct class structure diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 75c1451fcc63..03d2f99f84c5 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -82,18 +82,9 @@ struct class_dev_iter { extern struct kobject *sysfs_dev_block_kobj; extern struct kobject *sysfs_dev_char_kobj; -extern int __must_check __class_register(struct class *class, - struct lock_class_key *key); +extern int __must_check class_register(struct class *class); extern void class_unregister(struct class *class); -/* This is a #define to keep the compiler from merging different - * instances of the __key variable */ -#define class_register(class) \ -({ \ - static struct lock_class_key __key; \ - __class_register(class, &__key); \ -}) - struct class_compat; struct class_compat *class_compat_register(const char *name); void class_compat_unregister(struct class_compat *cls); @@ -246,30 +237,7 @@ struct class_interface { extern int __must_check class_interface_register(struct class_interface *); extern void class_interface_unregister(struct class_interface *); -extern struct class * __must_check __class_create(const char *name, - struct lock_class_key *key); +extern struct class * __must_check class_create(const char *name); extern void class_destroy(struct class *cls); -/* This is a #define to keep the compiler from merging different - * instances of the __key variable */ - -/** - * class_create - create a struct class structure - * @name: pointer to a string for the name of this class. - * - * This is used to create a struct class pointer that can then be used - * in calls to device_create(). - * - * Returns &struct class pointer on success, or ERR_PTR() on error. - * - * Note, the pointer created here is to be destroyed when finished by - * making a call to class_destroy(). - */ -#define class_create(name) \ -({ \ - static struct lock_class_key __key; \ - __class_create(name, &__key); \ -}) - - #endif /* _DEVICE_CLASS_H_ */ -- cgit v1.2.3 From 43718dca48429b9eb5b82cdd41e2a1f162a02fb9 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 24 Mar 2023 11:01:32 +0100 Subject: driver core: class.h: remove extern from function prototypes The kernel coding style does not require 'extern' in function prototypes in .h files, so remove them from include/linux/device/class.h as they are not needed. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230324100132.1633647-2-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- include/linux/device/class.h | 46 +++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 03d2f99f84c5..1dc7706cb42d 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -82,8 +82,9 @@ struct class_dev_iter { extern struct kobject *sysfs_dev_block_kobj; extern struct kobject *sysfs_dev_char_kobj; -extern int __must_check class_register(struct class *class); -extern void class_unregister(struct class *class); + +int __must_check class_register(struct class *class); +void class_unregister(struct class *class); struct class_compat; struct class_compat *class_compat_register(const char *name); @@ -93,19 +94,15 @@ int class_compat_create_link(struct class_compat *cls, struct device *dev, void class_compat_remove_link(struct class_compat *cls, struct device *dev, struct device *device_link); -extern void class_dev_iter_init(struct class_dev_iter *iter, - const struct class *class, - const struct device *start, - const struct device_type *type); -extern struct device *class_dev_iter_next(struct class_dev_iter *iter); -extern void class_dev_iter_exit(struct class_dev_iter *iter); +void class_dev_iter_init(struct class_dev_iter *iter, const struct class *class, + const struct device *start, const struct device_type *type); +struct device *class_dev_iter_next(struct class_dev_iter *iter); +void class_dev_iter_exit(struct class_dev_iter *iter); -extern int class_for_each_device(const struct class *class, const struct device *start, - void *data, - int (*fn)(struct device *dev, void *data)); -extern struct device *class_find_device(const struct class *class, - const struct device *start, const void *data, - int (*match)(struct device *, const void *)); +int class_for_each_device(const struct class *class, const struct device *start, void *data, + int (*fn)(struct device *dev, void *data)); +struct device *class_find_device(const struct class *class, const struct device *start, + const void *data, int (*match)(struct device *, const void *)); /** * class_find_device_by_name - device iterator for locating a particular device @@ -191,12 +188,10 @@ struct class_attribute { #define CLASS_ATTR_WO(_name) \ struct class_attribute class_attr_##_name = __ATTR_WO(_name) -extern int __must_check class_create_file_ns(const struct class *class, - const struct class_attribute *attr, - const void *ns); -extern void class_remove_file_ns(const struct class *class, - const struct class_attribute *attr, - const void *ns); +int __must_check class_create_file_ns(const struct class *class, const struct class_attribute *attr, + const void *ns); +void class_remove_file_ns(const struct class *class, const struct class_attribute *attr, + const void *ns); static inline int __must_check class_create_file(const struct class *class, const struct class_attribute *attr) @@ -223,8 +218,7 @@ struct class_attribute_string { struct class_attribute_string class_attr_##_name = \ _CLASS_ATTR_STRING(_name, _mode, _str) -extern ssize_t show_class_attr_string(struct class *class, struct class_attribute *attr, - char *buf); +ssize_t show_class_attr_string(struct class *class, struct class_attribute *attr, char *buf); struct class_interface { struct list_head node; @@ -234,10 +228,10 @@ struct class_interface { void (*remove_dev) (struct device *, struct class_interface *); }; -extern int __must_check class_interface_register(struct class_interface *); -extern void class_interface_unregister(struct class_interface *); +int __must_check class_interface_register(struct class_interface *); +void class_interface_unregister(struct class_interface *); -extern struct class * __must_check class_create(const char *name); -extern void class_destroy(struct class *cls); +struct class * __must_check class_create(const char *name); +void class_destroy(struct class *cls); #endif /* _DEVICE_CLASS_H_ */ -- cgit v1.2.3 From f43243c66e5e9ad839d235f82a58e73a7e7612af Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 24 Mar 2023 13:27:06 +0100 Subject: driver core: device.h: remove extern from function prototypes The kernel coding style does not require 'extern' in function prototypes in .h files, so remove them from include/linux/device.h as they are not needed. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230324122711.2664537-1-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- include/linux/device.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/device.h b/include/linux/device.h index 3b23772d3bbb..472dd24d4823 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1088,8 +1088,7 @@ void device_link_remove(void *consumer, struct device *supplier); void device_links_supplier_sync_state_pause(void); void device_links_supplier_sync_state_resume(void); -extern __printf(3, 4) -int dev_err_probe(const struct device *dev, int err, const char *fmt, ...); +__printf(3, 4) int dev_err_probe(const struct device *dev, int err, const char *fmt, ...); /* Create alias, so I can be autoloaded. */ #define MODULE_ALIAS_CHARDEV(major,minor) \ -- cgit v1.2.3 From 0d62b79fd8083ae2400f31e38f8d63db3bd59bff Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 24 Mar 2023 13:27:07 +0100 Subject: driver core: bus.h: remove extern from function prototypes The kernel coding style does not require 'extern' in function prototypes in .h files, so remove them from include/linux/device/bus.h as they are not needed. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230324122711.2664537-2-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- include/linux/device/bus.h | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index b517b82d4723..65a5e2c0f04d 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -110,11 +110,11 @@ struct bus_type { bool need_parent_lock; }; -extern int __must_check bus_register(const struct bus_type *bus); +int __must_check bus_register(const struct bus_type *bus); -extern void bus_unregister(const struct bus_type *bus); +void bus_unregister(const struct bus_type *bus); -extern int __must_check bus_rescan_devices(const struct bus_type *bus); +int __must_check bus_rescan_devices(const struct bus_type *bus); struct bus_attribute { struct attribute attr; @@ -244,10 +244,8 @@ void bus_sort_breadthfirst(struct bus_type *bus, */ struct notifier_block; -extern int bus_register_notifier(const struct bus_type *bus, - struct notifier_block *nb); -extern int bus_unregister_notifier(const struct bus_type *bus, - struct notifier_block *nb); +int bus_register_notifier(const struct bus_type *bus, struct notifier_block *nb); +int bus_unregister_notifier(const struct bus_type *bus, struct notifier_block *nb); /** * enum bus_notifier_event - Bus Notifier events that have happened @@ -279,7 +277,7 @@ enum bus_notifier_event { BUS_NOTIFY_DRIVER_NOT_BOUND, }; -extern struct kset *bus_get_kset(const struct bus_type *bus); +struct kset *bus_get_kset(const struct bus_type *bus); struct device *bus_get_dev_root(const struct bus_type *bus); #endif -- cgit v1.2.3 From 8a2b9c84c708cf2a99499d5a685a642af7b68c37 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 24 Mar 2023 13:27:08 +0100 Subject: driver core: driver.h: remove extern from function prototypes The kernel coding style does not require 'extern' in function prototypes in .h files, so remove them from include/linux/device/driver.h as they are not needed. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230324122711.2664537-3-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- include/linux/device/driver.h | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index 0f22a6f46f8c..c244267a6744 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -122,13 +122,12 @@ struct device_driver { }; -extern int __must_check driver_register(struct device_driver *drv); -extern void driver_unregister(struct device_driver *drv); +int __must_check driver_register(struct device_driver *drv); +void driver_unregister(struct device_driver *drv); -extern struct device_driver *driver_find(const char *name, - const struct bus_type *bus); -extern int driver_probe_done(void); -extern void wait_for_device_probe(void); +struct device_driver *driver_find(const char *name, const struct bus_type *bus); +int driver_probe_done(void); +void wait_for_device_probe(void); void __init wait_for_init_devices_probe(void); /* sysfs interface for exporting driver attributes */ @@ -147,18 +146,15 @@ struct driver_attribute { #define DRIVER_ATTR_WO(_name) \ struct driver_attribute driver_attr_##_name = __ATTR_WO(_name) -extern int __must_check driver_create_file(struct device_driver *driver, - const struct driver_attribute *attr); -extern void driver_remove_file(struct device_driver *driver, - const struct driver_attribute *attr); +int __must_check driver_create_file(struct device_driver *driver, + const struct driver_attribute *attr); +void driver_remove_file(struct device_driver *driver, + const struct driver_attribute *attr); int driver_set_override(struct device *dev, const char **override, const char *s, size_t len); -extern int __must_check driver_for_each_device(struct device_driver *drv, - struct device *start, - void *data, - int (*fn)(struct device *dev, - void *)); +int __must_check driver_for_each_device(struct device_driver *drv, struct device *start, + void *data, int (*fn)(struct device *dev, void *)); struct device *driver_find_device(struct device_driver *drv, struct device *start, const void *data, int (*match)(struct device *dev, const void *data)); -- cgit v1.2.3 From 44650f33d3bd76dd968cb8880c8eedd465030cd5 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 24 Mar 2023 13:27:11 +0100 Subject: kobject.h remove extern from function prototypes The kernel coding style does not require 'extern' in function prototypes in .h files, so remove them from include/linux/kobject.h as they are not needed. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230324122711.2664537-6-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- include/linux/kobject.h | 59 ++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kobject.h b/include/linux/kobject.h index bdab370a24f4..c392c811d9ad 100644 --- a/include/linux/kobject.h +++ b/include/linux/kobject.h @@ -79,43 +79,37 @@ struct kobject { unsigned int uevent_suppress:1; }; -extern __printf(2, 3) -int kobject_set_name(struct kobject *kobj, const char *name, ...); -extern __printf(2, 0) -int kobject_set_name_vargs(struct kobject *kobj, const char *fmt, - va_list vargs); +__printf(2, 3) int kobject_set_name(struct kobject *kobj, const char *name, ...); +__printf(2, 0) int kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list vargs); static inline const char *kobject_name(const struct kobject *kobj) { return kobj->name; } -extern void kobject_init(struct kobject *kobj, const struct kobj_type *ktype); -extern __printf(3, 4) __must_check -int kobject_add(struct kobject *kobj, struct kobject *parent, - const char *fmt, ...); -extern __printf(4, 5) __must_check -int kobject_init_and_add(struct kobject *kobj, - const struct kobj_type *ktype, struct kobject *parent, - const char *fmt, ...); +void kobject_init(struct kobject *kobj, const struct kobj_type *ktype); +__printf(3, 4) __must_check int kobject_add(struct kobject *kobj, + struct kobject *parent, + const char *fmt, ...); +__printf(4, 5) __must_check int kobject_init_and_add(struct kobject *kobj, + const struct kobj_type *ktype, + struct kobject *parent, + const char *fmt, ...); -extern void kobject_del(struct kobject *kobj); +void kobject_del(struct kobject *kobj); -extern struct kobject * __must_check kobject_create_and_add(const char *name, - struct kobject *parent); +struct kobject * __must_check kobject_create_and_add(const char *name, struct kobject *parent); -extern int __must_check kobject_rename(struct kobject *, const char *new_name); -extern int __must_check kobject_move(struct kobject *, struct kobject *); +int __must_check kobject_rename(struct kobject *, const char *new_name); +int __must_check kobject_move(struct kobject *, struct kobject *); -extern struct kobject *kobject_get(struct kobject *kobj); -extern struct kobject * __must_check kobject_get_unless_zero( - struct kobject *kobj); -extern void kobject_put(struct kobject *kobj); +struct kobject *kobject_get(struct kobject *kobj); +struct kobject * __must_check kobject_get_unless_zero(struct kobject *kobj); +void kobject_put(struct kobject *kobj); -extern const void *kobject_namespace(const struct kobject *kobj); -extern void kobject_get_ownership(const struct kobject *kobj, - kuid_t *uid, kgid_t *gid); -extern char *kobject_get_path(const struct kobject *kobj, gfp_t flag); +const void *kobject_namespace(const struct kobject *kobj); +void kobject_get_ownership(const struct kobject *kobj, kuid_t *uid, kgid_t *gid); +char *kobject_get_path(const struct kobject *kobj, gfp_t flag); struct kobj_type { void (*release)(struct kobject *kobj); @@ -176,12 +170,11 @@ struct kset { const struct kset_uevent_ops *uevent_ops; } __randomize_layout; -extern void kset_init(struct kset *kset); -extern int __must_check kset_register(struct kset *kset); -extern void kset_unregister(struct kset *kset); -extern struct kset * __must_check kset_create_and_add(const char *name, - const struct kset_uevent_ops *u, - struct kobject *parent_kobj); +void kset_init(struct kset *kset); +int __must_check kset_register(struct kset *kset); +void kset_unregister(struct kset *kset); +struct kset * __must_check kset_create_and_add(const char *name, const struct kset_uevent_ops *u, + struct kobject *parent_kobj); static inline struct kset *to_kset(struct kobject *kobj) { @@ -203,7 +196,7 @@ static inline const struct kobj_type *get_ktype(const struct kobject *kobj) return kobj->ktype; } -extern struct kobject *kset_find_obj(struct kset *, const char *); +struct kobject *kset_find_obj(struct kset *, const char *); /* The global /sys/kernel/ kobject for people to chain off of */ extern struct kobject *kernel_kobj; -- cgit v1.2.3 From b0d237087c674c43df76c1a0bc2737592f3038f4 Mon Sep 17 00:00:00 2001 From: Jun Miao Date: Thu, 23 Feb 2023 13:28:51 +0800 Subject: KVM: Fix comments that refer to the non-existent install_new_memslots() Fix stale comments that were left behind when install_new_memslots() was replaced by kvm_swap_active_memslots() as part of the scalable memslots rework. Fixes: a54d806688fe ("KVM: Keep memslots in tree-based structures instead of array-based ones") Signed-off-by: Jun Miao Link: https://lore.kernel.org/r/20230223052851.1054799-1-jun.miao@intel.com Signed-off-by: Sean Christopherson --- Documentation/virt/kvm/locking.rst | 2 +- include/linux/kvm_host.h | 4 ++-- virt/kvm/kvm_main.c | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/Documentation/virt/kvm/locking.rst b/Documentation/virt/kvm/locking.rst index 14c4e9fa501d..8c77554e4896 100644 --- a/Documentation/virt/kvm/locking.rst +++ b/Documentation/virt/kvm/locking.rst @@ -21,7 +21,7 @@ The acquisition orders for mutexes are as follows: - kvm->mn_active_invalidate_count ensures that pairs of invalidate_range_start() and invalidate_range_end() callbacks use the same memslots array. kvm->slots_lock and kvm->slots_arch_lock - are taken on the waiting side in install_new_memslots, so MMU notifiers + are taken on the waiting side when modifying memslots, so MMU notifiers must not take either kvm->slots_lock or kvm->slots_arch_lock. For SRCU: diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 90edc16d37e5..9696c2fb30e9 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -58,7 +58,7 @@ /* * Bit 63 of the memslot generation number is an "update in-progress flag", - * e.g. is temporarily set for the duration of install_new_memslots(). + * e.g. is temporarily set for the duration of kvm_swap_active_memslots(). * This flag effectively creates a unique generation number that is used to * mark cached memslot data, e.g. MMIO accesses, as potentially being stale, * i.e. may (or may not) have come from the previous memslots generation. @@ -713,7 +713,7 @@ struct kvm { * use by the VM. To be used under the slots_lock (above) or in a * kvm->srcu critical section where acquiring the slots_lock would * lead to deadlock with the synchronize_srcu in - * install_new_memslots. + * kvm_swap_active_memslots(). */ struct mutex slots_arch_lock; struct mm_struct *mm; /* userspace tied to this vm */ diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 8f0a7da37e32..d1abb331ea68 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1298,7 +1298,7 @@ static void kvm_destroy_vm(struct kvm *kvm) * At this point, pending calls to invalidate_range_start() * have completed but no more MMU notifiers will run, so * mn_active_invalidate_count may remain unbalanced. - * No threads can be waiting in install_new_memslots as the + * No threads can be waiting in kvm_swap_active_memslots() as the * last reference on KVM has been dropped, but freeing * memslots would deadlock without this manual intervention. */ @@ -1742,13 +1742,13 @@ static void kvm_invalidate_memslot(struct kvm *kvm, kvm_arch_flush_shadow_memslot(kvm, old); kvm_arch_guest_memory_reclaimed(kvm); - /* Was released by kvm_swap_active_memslots, reacquire. */ + /* Was released by kvm_swap_active_memslots(), reacquire. */ mutex_lock(&kvm->slots_arch_lock); /* * Copy the arch-specific field of the newly-installed slot back to the * old slot as the arch data could have changed between releasing - * slots_arch_lock in install_new_memslots() and re-acquiring the lock + * slots_arch_lock in kvm_swap_active_memslots() and re-acquiring the lock * above. Writers are required to retrieve memslots *after* acquiring * slots_arch_lock, thus the active slot's data is guaranteed to be fresh. */ @@ -1810,11 +1810,11 @@ static int kvm_set_memslot(struct kvm *kvm, int r; /* - * Released in kvm_swap_active_memslots. + * Released in kvm_swap_active_memslots(). * - * Must be held from before the current memslots are copied until - * after the new memslots are installed with rcu_assign_pointer, - * then released before the synchronize srcu in kvm_swap_active_memslots. + * Must be held from before the current memslots are copied until after + * the new memslots are installed with rcu_assign_pointer, then + * released before the synchronize srcu in kvm_swap_active_memslots(). * * When modifying memslots outside of the slots_lock, must be held * before reading the pointer to the current memslots until after all -- cgit v1.2.3 From 009455205e68ef017a429dc818b92c4a84b867c4 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 24 Mar 2023 10:08:14 +0100 Subject: driver core: bus: move documentation for lock_key to proper location. In commit 37e98d9bedb5 ("driver core: bus: move lock_class_key into dynamic structure"), the lock_key variable moved out of struct bus_type and into struct subsys_private, yet the documentation for it did not move. Fix that up and place the documentation comment in the correct location. Cc: "Rafael J. Wysocki" Fixes: 37e98d9bedb5 ("driver core: bus: move lock_class_key into dynamic structure") Link: https://lore.kernel.org/r/20230324090814.386654-1-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/base.h | 1 + include/linux/device/bus.h | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/base/base.h b/drivers/base/base.h index a2d3a1d0fa9b..d3e081dc4b13 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -33,6 +33,7 @@ * avoid namespace conflicts * @class - pointer back to the struct class that this structure is associated * with. + * @lock_key: Lock class key for use by the lock validator * * This structure is the one that is the actual kobject allowing struct * bus_type/class to be statically allocated safely. Nothing outside of the diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index 65a5e2c0f04d..ae10c4322754 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -65,7 +65,6 @@ struct fwnode_handle; * @iommu_ops: IOMMU specific operations for this bus, used to attach IOMMU * driver implementations to a bus and allow the driver to do * bus-specific setup - * @lock_key: Lock class key for use by the lock validator * @need_parent_lock: When probing or removing a device on this bus, the * device core should lock the device's parent. * -- cgit v1.2.3 From 579d472b379983498a6a7d9f7b1dd74c20afdc50 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 27 Mar 2023 16:01:50 +0300 Subject: device property: Remove unused struct net_device forward declaration There is no users in the property.h for the struct net_device. Remove the latter for good. Signed-off-by: Andy Shevchenko Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230327130150.84114-2-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- include/linux/property.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/property.h b/include/linux/property.h index 4a536548606b..59f452198c64 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -16,7 +16,6 @@ #include struct device; -struct net_device; enum dev_prop_type { DEV_PROP_U8, -- cgit v1.2.3 From 5c9a27df4eb9a402770d5547af255a765e1c10ac Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 27 Mar 2023 18:03:19 +0200 Subject: driver core: move sysfs_dev_char_kobj out of class.h The structure sysfs_dev_char_kobj is local only to the driver core code, so move it out of the global class.h file and into the internal base.h file as no one else should be touching this symbol. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230327160319.513974-1-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/base.h | 3 +++ include/linux/device/class.h | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/base/base.h b/drivers/base/base.h index d3e081dc4b13..2867ca4ee4ce 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -189,6 +189,9 @@ const char *device_get_devnode(const struct device *dev, umode_t *mode, extern struct kset *devices_kset; void devices_kset_move_last(struct device *dev); +/* /sys/dev/char directory */ +extern struct kobject *sysfs_dev_char_kobj; + #if defined(CONFIG_MODULES) && defined(CONFIG_SYSFS) void module_add_driver(struct module *mod, struct device_driver *drv); void module_remove_driver(struct device_driver *drv); diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 1dc7706cb42d..d3960733c0fa 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -81,7 +81,6 @@ struct class_dev_iter { }; extern struct kobject *sysfs_dev_block_kobj; -extern struct kobject *sysfs_dev_char_kobj; int __must_check class_register(struct class *class); void class_unregister(struct class *class); -- cgit v1.2.3 From 517d4927aabe488144863e72b52bb3e506fecd34 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Sat, 25 Mar 2023 09:45:26 +0100 Subject: driver core: bus: constify class_unregister/destroy() The class_unregister() and class_destroy() function should be taking a const * to struct class, not just a *, so fix that up. Cc: "Rafael J. Wysocki" Link: https://lore.kernel.org/r/20230325084526.3622123-1-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 8 ++++---- include/linux/device/class.h | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index 0f8938a17144..8ae91e118827 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -142,13 +142,13 @@ static void klist_class_dev_put(struct klist_node *n) put_device(dev); } -static int class_add_groups(struct class *cls, +static int class_add_groups(const struct class *cls, const struct attribute_group **groups) { return sysfs_create_groups(&cls->p->subsys.kobj, groups); } -static void class_remove_groups(struct class *cls, +static void class_remove_groups(const struct class *cls, const struct attribute_group **groups) { return sysfs_remove_groups(&cls->p->subsys.kobj, groups); @@ -206,7 +206,7 @@ err_out: } EXPORT_SYMBOL_GPL(class_register); -void class_unregister(struct class *cls) +void class_unregister(const struct class *cls) { pr_debug("device class '%s': unregistering\n", cls->name); class_remove_groups(cls, cls->class_groups); @@ -265,7 +265,7 @@ EXPORT_SYMBOL_GPL(class_create); * Note, the pointer to be destroyed must have been created with a call * to class_create(). */ -void class_destroy(struct class *cls) +void class_destroy(const struct class *cls) { if (IS_ERR_OR_NULL(cls)) return; diff --git a/include/linux/device/class.h b/include/linux/device/class.h index d3960733c0fa..73436c578d37 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -83,7 +83,7 @@ struct class_dev_iter { extern struct kobject *sysfs_dev_block_kobj; int __must_check class_register(struct class *class); -void class_unregister(struct class *class); +void class_unregister(const struct class *class); struct class_compat; struct class_compat *class_compat_register(const char *name); @@ -231,6 +231,6 @@ int __must_check class_interface_register(struct class_interface *); void class_interface_unregister(struct class_interface *); struct class * __must_check class_create(const char *name); -void class_destroy(struct class *cls); +void class_destroy(const struct class *cls); #endif /* _DEVICE_CLASS_H_ */ -- cgit v1.2.3 From 76d0de5729c0569c4071e7f21fcab394e502f03a Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 2 Feb 2023 00:56:01 +0900 Subject: fprobe: Pass entry_data to handlers Pass the private entry_data to the entry and exit handlers so that they can share the context data, something like saved function arguments etc. User must specify the private entry_data size by @entry_data_size field before registering the fprobe. Link: https://lkml.kernel.org/r/167526696173.433354.17408372048319432574.stgit@mhiramat.roam.corp.google.com Cc: Florent Revest Cc: Mark Rutland Cc: Will Deacon Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/fprobe.h | 8 ++++++-- kernel/trace/bpf_trace.c | 2 +- kernel/trace/fprobe.c | 21 ++++++++++++++------- lib/test_fprobe.c | 6 ++++-- samples/fprobe/fprobe_example.c | 6 ++++-- 5 files changed, 29 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fprobe.h b/include/linux/fprobe.h index 1c2bde0ead73..e0d4e6136249 100644 --- a/include/linux/fprobe.h +++ b/include/linux/fprobe.h @@ -13,6 +13,7 @@ * @nmissed: The counter for missing events. * @flags: The status flag. * @rethook: The rethook data structure. (internal data) + * @entry_data_size: The private data storage size. * @entry_handler: The callback function for function entry. * @exit_handler: The callback function for function exit. */ @@ -29,9 +30,12 @@ struct fprobe { unsigned long nmissed; unsigned int flags; struct rethook *rethook; + size_t entry_data_size; - void (*entry_handler)(struct fprobe *fp, unsigned long entry_ip, struct pt_regs *regs); - void (*exit_handler)(struct fprobe *fp, unsigned long entry_ip, struct pt_regs *regs); + void (*entry_handler)(struct fprobe *fp, unsigned long entry_ip, + struct pt_regs *regs, void *entry_data); + void (*exit_handler)(struct fprobe *fp, unsigned long entry_ip, + struct pt_regs *regs, void *entry_data); }; /* This fprobe is soft-disabled. */ diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index e8da032bb6fc..fa403c323501 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2646,7 +2646,7 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, static void kprobe_multi_link_handler(struct fprobe *fp, unsigned long fentry_ip, - struct pt_regs *regs) + struct pt_regs *regs, void *data) { struct bpf_kprobe_multi_link *link; diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index e8143e368074..fa25d09c9d57 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -17,14 +17,16 @@ struct fprobe_rethook_node { struct rethook_node node; unsigned long entry_ip; + char data[]; }; static void fprobe_handler(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct ftrace_regs *fregs) { struct fprobe_rethook_node *fpr; - struct rethook_node *rh; + struct rethook_node *rh = NULL; struct fprobe *fp; + void *entry_data = NULL; int bit; fp = container_of(ops, struct fprobe, ops); @@ -37,9 +39,6 @@ static void fprobe_handler(unsigned long ip, unsigned long parent_ip, return; } - if (fp->entry_handler) - fp->entry_handler(fp, ip, ftrace_get_regs(fregs)); - if (fp->exit_handler) { rh = rethook_try_get(fp->rethook); if (!rh) { @@ -48,9 +47,16 @@ static void fprobe_handler(unsigned long ip, unsigned long parent_ip, } fpr = container_of(rh, struct fprobe_rethook_node, node); fpr->entry_ip = ip; - rethook_hook(rh, ftrace_get_regs(fregs), true); + if (fp->entry_data_size) + entry_data = fpr->data; } + if (fp->entry_handler) + fp->entry_handler(fp, ip, ftrace_get_regs(fregs), entry_data); + + if (rh) + rethook_hook(rh, ftrace_get_regs(fregs), true); + out: ftrace_test_recursion_unlock(bit); } @@ -81,7 +87,8 @@ static void fprobe_exit_handler(struct rethook_node *rh, void *data, fpr = container_of(rh, struct fprobe_rethook_node, node); - fp->exit_handler(fp, fpr->entry_ip, regs); + fp->exit_handler(fp, fpr->entry_ip, regs, + fp->entry_data_size ? (void *)fpr->data : NULL); } NOKPROBE_SYMBOL(fprobe_exit_handler); @@ -146,7 +153,7 @@ static int fprobe_init_rethook(struct fprobe *fp, int num) for (i = 0; i < size; i++) { struct fprobe_rethook_node *node; - node = kzalloc(sizeof(*node), GFP_KERNEL); + node = kzalloc(sizeof(*node) + fp->entry_data_size, GFP_KERNEL); if (!node) { rethook_free(fp->rethook); fp->rethook = NULL; diff --git a/lib/test_fprobe.c b/lib/test_fprobe.c index 1fb56cf5e5ce..e4f65d114ed2 100644 --- a/lib/test_fprobe.c +++ b/lib/test_fprobe.c @@ -30,7 +30,8 @@ static noinline u32 fprobe_selftest_target2(u32 value) return (value / div_factor) + 1; } -static notrace void fp_entry_handler(struct fprobe *fp, unsigned long ip, struct pt_regs *regs) +static notrace void fp_entry_handler(struct fprobe *fp, unsigned long ip, + struct pt_regs *regs, void *data) { KUNIT_EXPECT_FALSE(current_test, preemptible()); /* This can be called on the fprobe_selftest_target and the fprobe_selftest_target2 */ @@ -39,7 +40,8 @@ static notrace void fp_entry_handler(struct fprobe *fp, unsigned long ip, struct entry_val = (rand1 / div_factor); } -static notrace void fp_exit_handler(struct fprobe *fp, unsigned long ip, struct pt_regs *regs) +static notrace void fp_exit_handler(struct fprobe *fp, unsigned long ip, + struct pt_regs *regs, void *data) { unsigned long ret = regs_return_value(regs); diff --git a/samples/fprobe/fprobe_example.c b/samples/fprobe/fprobe_example.c index e22da8573116..dd794990ad7e 100644 --- a/samples/fprobe/fprobe_example.c +++ b/samples/fprobe/fprobe_example.c @@ -48,7 +48,8 @@ static void show_backtrace(void) stack_trace_print(stacks, len, 24); } -static void sample_entry_handler(struct fprobe *fp, unsigned long ip, struct pt_regs *regs) +static void sample_entry_handler(struct fprobe *fp, unsigned long ip, + struct pt_regs *regs, void *data) { if (use_trace) /* @@ -63,7 +64,8 @@ static void sample_entry_handler(struct fprobe *fp, unsigned long ip, struct pt_ show_backtrace(); } -static void sample_exit_handler(struct fprobe *fp, unsigned long ip, struct pt_regs *regs) +static void sample_exit_handler(struct fprobe *fp, unsigned long ip, struct pt_regs *regs, + void *data) { unsigned long rip = instruction_pointer(regs); -- cgit v1.2.3 From 59a7a298565aa0ce44ce8e4fbcbb89a19730013a Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 2 Feb 2023 00:56:19 +0900 Subject: fprobe: Add nr_maxactive to specify rethook_node pool size Add nr_maxactive to specify rethook_node pool size. This means the maximum number of actively running target functions concurrently for probing by exit_handler. Note that if the running function is preempted or sleep, it is still counted as 'active'. Link: https://lkml.kernel.org/r/167526697917.433354.17779774988245113106.stgit@mhiramat.roam.corp.google.com Cc: Florent Revest Cc: Mark Rutland Cc: Will Deacon Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/fprobe.h | 2 ++ kernel/trace/fprobe.c | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fprobe.h b/include/linux/fprobe.h index e0d4e6136249..678f741a7b33 100644 --- a/include/linux/fprobe.h +++ b/include/linux/fprobe.h @@ -14,6 +14,7 @@ * @flags: The status flag. * @rethook: The rethook data structure. (internal data) * @entry_data_size: The private data storage size. + * @nr_maxactive: The max number of active functions. * @entry_handler: The callback function for function entry. * @exit_handler: The callback function for function exit. */ @@ -31,6 +32,7 @@ struct fprobe { unsigned int flags; struct rethook *rethook; size_t entry_data_size; + int nr_maxactive; void (*entry_handler)(struct fprobe *fp, unsigned long entry_ip, struct pt_regs *regs, void *entry_data); diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index fa25d09c9d57..f222848571f2 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -143,7 +143,10 @@ static int fprobe_init_rethook(struct fprobe *fp, int num) } /* Initialize rethook if needed */ - size = num * num_possible_cpus() * 2; + if (fp->nr_maxactive) + size = fp->nr_maxactive; + else + size = num * num_possible_cpus() * 2; if (size < 0) return -E2BIG; -- cgit v1.2.3 From 39d954200bf6ad503c722e44d0be80c7b826fa42 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Thu, 2 Feb 2023 00:56:38 +0900 Subject: fprobe: Skip exit_handler if entry_handler returns !0 Skip hooking function return and calling exit_handler if the entry_handler() returns !0. Link: https://lkml.kernel.org/r/167526699798.433354.10998365726830117303.stgit@mhiramat.roam.corp.google.com Cc: Florent Revest Cc: Mark Rutland Cc: Will Deacon Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/fprobe.h | 4 ++-- kernel/trace/bpf_trace.c | 15 +++++++++++++-- kernel/trace/fprobe.c | 14 +++++++++----- lib/test_fprobe.c | 7 +++++-- samples/fprobe/fprobe_example.c | 5 +++-- 5 files changed, 32 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fprobe.h b/include/linux/fprobe.h index 678f741a7b33..47fefc7f363b 100644 --- a/include/linux/fprobe.h +++ b/include/linux/fprobe.h @@ -34,8 +34,8 @@ struct fprobe { size_t entry_data_size; int nr_maxactive; - void (*entry_handler)(struct fprobe *fp, unsigned long entry_ip, - struct pt_regs *regs, void *entry_data); + int (*entry_handler)(struct fprobe *fp, unsigned long entry_ip, + struct pt_regs *regs, void *entry_data); void (*exit_handler)(struct fprobe *fp, unsigned long entry_ip, struct pt_regs *regs, void *entry_data); }; diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index fa403c323501..d804172b709c 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -2644,12 +2644,23 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, return err; } -static void +static int kprobe_multi_link_handler(struct fprobe *fp, unsigned long fentry_ip, struct pt_regs *regs, void *data) { struct bpf_kprobe_multi_link *link; + link = container_of(fp, struct bpf_kprobe_multi_link, fp); + kprobe_multi_link_prog_run(link, get_entry_ip(fentry_ip), regs); + return 0; +} + +static void +kprobe_multi_link_exit_handler(struct fprobe *fp, unsigned long fentry_ip, + struct pt_regs *regs, void *data) +{ + struct bpf_kprobe_multi_link *link; + link = container_of(fp, struct bpf_kprobe_multi_link, fp); kprobe_multi_link_prog_run(link, get_entry_ip(fentry_ip), regs); } @@ -2848,7 +2859,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr goto error; if (flags & BPF_F_KPROBE_MULTI_RETURN) - link->fp.exit_handler = kprobe_multi_link_handler; + link->fp.exit_handler = kprobe_multi_link_exit_handler; else link->fp.entry_handler = kprobe_multi_link_handler; diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index f222848571f2..9abb3905bc8e 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -27,7 +27,7 @@ static void fprobe_handler(unsigned long ip, unsigned long parent_ip, struct rethook_node *rh = NULL; struct fprobe *fp; void *entry_data = NULL; - int bit; + int bit, ret; fp = container_of(ops, struct fprobe, ops); if (fprobe_disabled(fp)) @@ -52,11 +52,15 @@ static void fprobe_handler(unsigned long ip, unsigned long parent_ip, } if (fp->entry_handler) - fp->entry_handler(fp, ip, ftrace_get_regs(fregs), entry_data); - - if (rh) - rethook_hook(rh, ftrace_get_regs(fregs), true); + ret = fp->entry_handler(fp, ip, ftrace_get_regs(fregs), entry_data); + /* If entry_handler returns !0, nmissed is not counted. */ + if (rh) { + if (ret) + rethook_recycle(rh); + else + rethook_hook(rh, ftrace_get_regs(fregs), true); + } out: ftrace_test_recursion_unlock(bit); } diff --git a/lib/test_fprobe.c b/lib/test_fprobe.c index 4b37d7022f35..9fa2ac9eda83 100644 --- a/lib/test_fprobe.c +++ b/lib/test_fprobe.c @@ -37,7 +37,7 @@ static noinline u32 fprobe_selftest_nest_target(u32 value, u32 (*nest)(u32)) return nest(value + 2); } -static notrace void fp_entry_handler(struct fprobe *fp, unsigned long ip, +static notrace int fp_entry_handler(struct fprobe *fp, unsigned long ip, struct pt_regs *regs, void *data) { KUNIT_EXPECT_FALSE(current_test, preemptible()); @@ -51,6 +51,8 @@ static notrace void fp_entry_handler(struct fprobe *fp, unsigned long ip, *(u32 *)data = entry_val; } else KUNIT_EXPECT_NULL(current_test, data); + + return 0; } static notrace void fp_exit_handler(struct fprobe *fp, unsigned long ip, @@ -74,10 +76,11 @@ static notrace void fp_exit_handler(struct fprobe *fp, unsigned long ip, KUNIT_EXPECT_NULL(current_test, data); } -static notrace void nest_entry_handler(struct fprobe *fp, unsigned long ip, +static notrace int nest_entry_handler(struct fprobe *fp, unsigned long ip, struct pt_regs *regs, void *data) { KUNIT_EXPECT_FALSE(current_test, preemptible()); + return 0; } static notrace void nest_exit_handler(struct fprobe *fp, unsigned long ip, diff --git a/samples/fprobe/fprobe_example.c b/samples/fprobe/fprobe_example.c index dd794990ad7e..4efc8feb6277 100644 --- a/samples/fprobe/fprobe_example.c +++ b/samples/fprobe/fprobe_example.c @@ -48,8 +48,8 @@ static void show_backtrace(void) stack_trace_print(stacks, len, 24); } -static void sample_entry_handler(struct fprobe *fp, unsigned long ip, - struct pt_regs *regs, void *data) +static int sample_entry_handler(struct fprobe *fp, unsigned long ip, + struct pt_regs *regs, void *data) { if (use_trace) /* @@ -62,6 +62,7 @@ static void sample_entry_handler(struct fprobe *fp, unsigned long ip, nhit++; if (stackdump) show_backtrace(); + return 0; } static void sample_exit_handler(struct fprobe *fp, unsigned long ip, struct pt_regs *regs, -- cgit v1.2.3 From d155df53f31068c3340733d586eb9b3ddfd70fc5 Mon Sep 17 00:00:00 2001 From: Ma Wupeng Date: Fri, 17 Feb 2023 10:56:15 +0800 Subject: x86/mm/pat: clear VM_PAT if copy_p4d_range failed Syzbot reports a warning in untrack_pfn(). Digging into the root we found that this is due to memory allocation failure in pmd_alloc_one. And this failure is produced due to failslab. In copy_page_range(), memory alloaction for pmd failed. During the error handling process in copy_page_range(), mmput() is called to remove all vmas. While untrack_pfn this empty pfn, warning happens. Here's a simplified flow: dup_mm dup_mmap copy_page_range copy_p4d_range copy_pud_range copy_pmd_range pmd_alloc __pmd_alloc pmd_alloc_one page = alloc_pages(gfp, 0); if (!page) return NULL; mmput exit_mmap unmap_vmas unmap_single_vma untrack_pfn follow_phys WARN_ON_ONCE(1); Since this vma is not generate successfully, we can clear flag VM_PAT. In this case, untrack_pfn() will not be called while cleaning this vma. Function untrack_pfn_moved() has also been renamed to fit the new logic. Link: https://lkml.kernel.org/r/20230217025615.1595558-1-mawupeng1@huawei.com Signed-off-by: Ma Wupeng Reported-by: Cc: Andy Lutomirski Cc: Borislav Petkov Cc: Dave Hansen Cc: Peter Zijlstra Cc: Suresh Siddha Cc: Toshi Kani Signed-off-by: Andrew Morton --- arch/x86/mm/pat/memtype.c | 12 ++++++++---- include/linux/pgtable.h | 7 ++++--- mm/memory.c | 1 + mm/mremap.c | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/mm/pat/memtype.c b/arch/x86/mm/pat/memtype.c index 46a00aa858b6..de10800cd4dd 100644 --- a/arch/x86/mm/pat/memtype.c +++ b/arch/x86/mm/pat/memtype.c @@ -1073,11 +1073,15 @@ void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn, } /* - * untrack_pfn_moved is called, while mremapping a pfnmap for a new region, - * with the old vma after its pfnmap page table has been removed. The new - * vma has a new pfnmap to the same pfn & cache type with VM_PAT set. + * untrack_pfn_clear is called if the following situation fits: + * + * 1) while mremapping a pfnmap for a new region, with the old vma after + * its pfnmap page table has been removed. The new vma has a new pfnmap + * to the same pfn & cache type with VM_PAT set. + * 2) while duplicating vm area, the new vma fails to copy the pgtable from + * old vma. */ -void untrack_pfn_moved(struct vm_area_struct *vma) +void untrack_pfn_clear(struct vm_area_struct *vma) { vm_flags_clear(vma, VM_PAT); } diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index c63cd44777ec..9dc936bc77d1 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1191,9 +1191,10 @@ static inline void untrack_pfn(struct vm_area_struct *vma, } /* - * untrack_pfn_moved is called while mremapping a pfnmap for a new region. + * untrack_pfn_clear is called while mremapping a pfnmap for a new region + * or fails to copy pgtable during duplicate vm area. */ -static inline void untrack_pfn_moved(struct vm_area_struct *vma) +static inline void untrack_pfn_clear(struct vm_area_struct *vma) { } #else @@ -1205,7 +1206,7 @@ extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, extern int track_pfn_copy(struct vm_area_struct *vma); extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn, unsigned long size, bool mm_wr_locked); -extern void untrack_pfn_moved(struct vm_area_struct *vma); +extern void untrack_pfn_clear(struct vm_area_struct *vma); #endif #ifdef CONFIG_MMU diff --git a/mm/memory.c b/mm/memory.c index f456f3b5049c..bfa3100ec5a3 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1290,6 +1290,7 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) continue; if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd, addr, next))) { + untrack_pfn_clear(dst_vma); ret = -ENOMEM; break; } diff --git a/mm/mremap.c b/mm/mremap.c index 411a85682b58..1ddf7beb62e9 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -683,7 +683,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, /* Tell pfnmap has moved from this vma */ if (unlikely(vma->vm_flags & VM_PFNMAP)) - untrack_pfn_moved(vma); + untrack_pfn_clear(vma); if (unlikely(!err && (flags & MREMAP_DONTUNMAP))) { /* We always clear VM_LOCKED[ONFAULT] on the old vma */ -- cgit v1.2.3 From 9a52b2f32a0942047348b30f866b846da5fcf4e3 Mon Sep 17 00:00:00 2001 From: "T.J. Alumbaugh" Date: Tue, 14 Feb 2023 03:54:44 +0000 Subject: mm: multi-gen LRU: clean up sysfs code This patch cleans up the sysfs code. Specifically, 1. use sysfs_emit(), 2. use __ATTR_RW(), and 3. constify multi-gen LRU struct attribute_group. Link: https://lkml.kernel.org/r/20230214035445.1250139-1-talumbau@google.com Signed-off-by: T.J. Alumbaugh Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 2 +- mm/vmscan.c | 22 +++++++++------------- 2 files changed, 10 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 9fb1b03b83b2..bf8786d45b31 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1369,7 +1369,7 @@ typedef struct pglist_data { #ifdef CONFIG_LRU_GEN /* kswap mm walk data */ - struct lru_gen_mm_walk mm_walk; + struct lru_gen_mm_walk mm_walk; /* lru_gen_folio list */ struct lru_gen_memcg memcg_lru; #endif diff --git a/mm/vmscan.c b/mm/vmscan.c index 9c1c5e8b24b8..7ac7d1ec13e0 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -5663,14 +5663,14 @@ unlock: * sysfs interface ******************************************************************************/ -static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +static ssize_t min_ttl_ms_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); + return sysfs_emit(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); } /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, - const char *buf, size_t len) +static ssize_t min_ttl_ms_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t len) { unsigned int msecs; @@ -5682,11 +5682,9 @@ static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, return len; } -static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( - min_ttl_ms, 0644, show_min_ttl, store_min_ttl -); +static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR_RW(min_ttl_ms); -static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { unsigned int caps = 0; @@ -5703,7 +5701,7 @@ static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, c } /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ -static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, +static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t len) { int i; @@ -5730,9 +5728,7 @@ static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, return len; } -static struct kobj_attribute lru_gen_enabled_attr = __ATTR( - enabled, 0644, show_enabled, store_enabled -); +static struct kobj_attribute lru_gen_enabled_attr = __ATTR_RW(enabled); static struct attribute *lru_gen_attrs[] = { &lru_gen_min_ttl_attr.attr, @@ -5740,7 +5736,7 @@ static struct attribute *lru_gen_attrs[] = { NULL }; -static struct attribute_group lru_gen_attr_group = { +static const struct attribute_group lru_gen_attr_group = { .name = "lru_gen", .attrs = lru_gen_attrs, }; -- cgit v1.2.3 From aa464ba9a1e444d5ef95bb63ee3b2ef26fc96ed7 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 3 Feb 2023 17:18:34 +1000 Subject: lazy tlb: introduce lazy tlb mm refcount helper functions Add explicit _lazy_tlb annotated functions for lazy tlb mm refcounting. This makes the lazy tlb mm references more obvious, and allows the refcounting scheme to be modified in later changes. There is no functional change with this patch. Link: https://lkml.kernel.org/r/20230203071837.1136453-3-npiggin@gmail.com Signed-off-by: Nicholas Piggin Acked-by: Linus Torvalds Cc: Andy Lutomirski Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: Michael Ellerman Cc: Nadav Amit Cc: Peter Zijlstra Cc: Rik van Riel Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm/mach-rpc/ecard.c | 2 +- arch/powerpc/kernel/smp.c | 2 +- arch/powerpc/mm/book3s64/radix_tlb.c | 4 ++-- fs/exec.c | 2 +- include/linux/sched/mm.h | 16 ++++++++++++++++ kernel/cpu.c | 2 +- kernel/exit.c | 2 +- kernel/kthread.c | 12 ++++++++++-- kernel/sched/core.c | 15 ++++++++------- 9 files changed, 41 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/mach-rpc/ecard.c b/arch/arm/mach-rpc/ecard.c index 53813f9464a2..c30df1097c52 100644 --- a/arch/arm/mach-rpc/ecard.c +++ b/arch/arm/mach-rpc/ecard.c @@ -253,7 +253,7 @@ static int ecard_init_mm(void) current->mm = mm; current->active_mm = mm; activate_mm(active_mm, mm); - mmdrop(active_mm); + mmdrop_lazy_tlb(active_mm); ecard_init_pgtables(mm); return 0; } diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 6b90f10a6c81..7db6b3faea65 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -1611,7 +1611,7 @@ void start_secondary(void *unused) if (IS_ENABLED(CONFIG_PPC32)) setup_kup(); - mmgrab(&init_mm); + mmgrab_lazy_tlb(&init_mm); current->active_mm = &init_mm; smp_store_cpu_info(cpu); diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index e50bc5fc7ddf..ce804b7bf84e 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -797,10 +797,10 @@ void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush) if (current->active_mm == mm) { WARN_ON_ONCE(current->mm != NULL); /* Is a kernel thread and is using mm as the lazy tlb */ - mmgrab(&init_mm); + mmgrab_lazy_tlb(&init_mm); current->active_mm = &init_mm; switch_mm_irqs_off(mm, &init_mm, current); - mmdrop(mm); + mmdrop_lazy_tlb(mm); } /* diff --git a/fs/exec.c b/fs/exec.c index 7c44d0c65b1b..87cf3a2f0e9a 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1034,7 +1034,7 @@ static int exec_mmap(struct mm_struct *mm) mmput(old_mm); return 0; } - mmdrop(active_mm); + mmdrop_lazy_tlb(active_mm); return 0; } diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 2a243616f222..5376caf6fcf3 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -79,6 +79,22 @@ static inline void mmdrop_sched(struct mm_struct *mm) } #endif +/* Helpers for lazy TLB mm refcounting */ +static inline void mmgrab_lazy_tlb(struct mm_struct *mm) +{ + mmgrab(mm); +} + +static inline void mmdrop_lazy_tlb(struct mm_struct *mm) +{ + mmdrop(mm); +} + +static inline void mmdrop_lazy_tlb_sched(struct mm_struct *mm) +{ + mmdrop_sched(mm); +} + /** * mmget() - Pin the address space associated with a &struct mm_struct. * @mm: The address space to pin. diff --git a/kernel/cpu.c b/kernel/cpu.c index 6c0a92ca6bb5..189895288d9d 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -623,7 +623,7 @@ static int finish_cpu(unsigned int cpu) */ if (mm != &init_mm) idle->active_mm = &init_mm; - mmdrop(mm); + mmdrop_lazy_tlb(mm); return 0; } diff --git a/kernel/exit.c b/kernel/exit.c index f2afdb0add7c..86902cb5ab78 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -537,7 +537,7 @@ static void exit_mm(void) return; sync_mm_rss(mm); mmap_read_lock(mm); - mmgrab(mm); + mmgrab_lazy_tlb(mm); BUG_ON(mm != current->active_mm); /* more a memory barrier than a real lock */ task_lock(current); diff --git a/kernel/kthread.c b/kernel/kthread.c index 1f1b60f1a746..470708c205e8 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -1415,6 +1415,11 @@ void kthread_use_mm(struct mm_struct *mm) WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD)); WARN_ON_ONCE(tsk->mm); + /* + * It is possible for mm to be the same as tsk->active_mm, but + * we must still mmgrab(mm) and mmdrop_lazy_tlb(active_mm), + * because these references are not equivalent. + */ mmgrab(mm); task_lock(tsk); @@ -1438,9 +1443,9 @@ void kthread_use_mm(struct mm_struct *mm) * memory barrier after storing to tsk->mm, before accessing * user-space memory. A full memory barrier for membarrier * {PRIVATE,GLOBAL}_EXPEDITED is implicitly provided by - * mmdrop(). + * mmdrop_lazy_tlb(). */ - mmdrop(active_mm); + mmdrop_lazy_tlb(active_mm); } EXPORT_SYMBOL_GPL(kthread_use_mm); @@ -1468,10 +1473,13 @@ void kthread_unuse_mm(struct mm_struct *mm) local_irq_disable(); tsk->mm = NULL; membarrier_update_current_mm(NULL); + mmgrab_lazy_tlb(mm); /* active_mm is still 'mm' */ enter_lazy_tlb(mm, tsk); local_irq_enable(); task_unlock(tsk); + + mmdrop(mm); } EXPORT_SYMBOL_GPL(kthread_unuse_mm); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0d18c3969f90..143e46bd2a68 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5203,13 +5203,14 @@ static struct rq *finish_task_switch(struct task_struct *prev) * rq->curr, before returning to userspace, so provide them here: * * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly - * provided by mmdrop(), + * provided by mmdrop_lazy_tlb(), * - a sync_core for SYNC_CORE. */ if (mm) { membarrier_mm_sync_core_before_usermode(mm); - mmdrop_sched(mm); + mmdrop_lazy_tlb_sched(mm); } + if (unlikely(prev_state == TASK_DEAD)) { if (prev->sched_class->task_dead) prev->sched_class->task_dead(prev); @@ -5266,9 +5267,9 @@ context_switch(struct rq *rq, struct task_struct *prev, /* * kernel -> kernel lazy + transfer active - * user -> kernel lazy + mmgrab() active + * user -> kernel lazy + mmgrab_lazy_tlb() active * - * kernel -> user switch + mmdrop() active + * kernel -> user switch + mmdrop_lazy_tlb() active * user -> user switch */ if (!next->mm) { // to kernel @@ -5276,7 +5277,7 @@ context_switch(struct rq *rq, struct task_struct *prev, next->active_mm = prev->active_mm; if (prev->mm) // from user - mmgrab(prev->active_mm); + mmgrab_lazy_tlb(prev->active_mm); else prev->active_mm = NULL; } else { // to user @@ -5293,7 +5294,7 @@ context_switch(struct rq *rq, struct task_struct *prev, lru_gen_use_mm(next->mm); if (!prev->mm) { // from kernel - /* will mmdrop() in finish_task_switch(). */ + /* will mmdrop_lazy_tlb() in finish_task_switch(). */ rq->prev_mm = prev->active_mm; prev->active_mm = NULL; } @@ -9935,7 +9936,7 @@ void __init sched_init(void) /* * The boot idle thread does lazy MMU switching as well: */ - mmgrab(&init_mm); + mmgrab_lazy_tlb(&init_mm); enter_lazy_tlb(&init_mm, current); /* -- cgit v1.2.3 From 88e3009b5283bbd41447f0352d0b9df16cf6f183 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 3 Feb 2023 17:18:35 +1000 Subject: lazy tlb: allow lazy tlb mm refcounting to be configurable Add CONFIG_MMU_TLB_REFCOUNT which enables refcounting of the lazy tlb mm when it is context switched. This can be disabled by architectures that don't require this refcounting if they clean up lazy tlb mms when the last refcount is dropped. Currently this is always enabled, so the patch introduces no functional change. Link: https://lkml.kernel.org/r/20230203071837.1136453-4-npiggin@gmail.com Signed-off-by: Nicholas Piggin Acked-by: Linus Torvalds Cc: Andy Lutomirski Cc: Catalin Marinas Cc: Christophe Leroy Cc: Dave Hansen Cc: Michael Ellerman Cc: Nadav Amit Cc: Peter Zijlstra Cc: Rik van Riel Cc: Will Deacon Signed-off-by: Andrew Morton --- Documentation/mm/active_mm.rst | 6 ++++++ arch/Kconfig | 17 +++++++++++++++++ include/linux/sched/mm.h | 18 +++++++++++++++--- 3 files changed, 38 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/Documentation/mm/active_mm.rst b/Documentation/mm/active_mm.rst index 45d89f8fb3a8..d096fc091e23 100644 --- a/Documentation/mm/active_mm.rst +++ b/Documentation/mm/active_mm.rst @@ -2,6 +2,12 @@ Active MM ========= +Note, the mm_count refcount may no longer include the "lazy" users +(running tasks with ->active_mm == mm && ->mm == NULL) on kernels +with CONFIG_MMU_LAZY_TLB_REFCOUNT=n. Taking and releasing these lazy +references must be done with mmgrab_lazy_tlb() and mmdrop_lazy_tlb() +helpers, which abstract this config option. + :: List: linux-kernel diff --git a/arch/Kconfig b/arch/Kconfig index e3511afbb7f2..643f8a04141e 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -465,6 +465,23 @@ config ARCH_WANT_IRQS_OFF_ACTIVATE_MM irqs disabled over activate_mm. Architectures that do IPI based TLB shootdowns should enable this. +# Use normal mm refcounting for MMU_LAZY_TLB kernel thread references. +# MMU_LAZY_TLB_REFCOUNT=n can improve the scalability of context switching +# to/from kernel threads when the same mm is running on a lot of CPUs (a large +# multi-threaded application), by reducing contention on the mm refcount. +# +# This can be disabled if the architecture ensures no CPUs are using an mm as a +# "lazy tlb" beyond its final refcount (i.e., by the time __mmdrop frees the mm +# or its kernel page tables). This could be arranged by arch_exit_mmap(), or +# final exit(2) TLB flush, for example. +# +# To implement this, an arch *must*: +# Ensure the _lazy_tlb variants of mmgrab/mmdrop are used when manipulating +# the lazy tlb reference of a kthread's ->active_mm (non-arch code has been +# converted already). +config MMU_LAZY_TLB_REFCOUNT + def_bool y + config ARCH_HAVE_NMI_SAFE_CMPXCHG bool diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 5376caf6fcf3..689dbe812563 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -82,17 +82,29 @@ static inline void mmdrop_sched(struct mm_struct *mm) /* Helpers for lazy TLB mm refcounting */ static inline void mmgrab_lazy_tlb(struct mm_struct *mm) { - mmgrab(mm); + if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT)) + mmgrab(mm); } static inline void mmdrop_lazy_tlb(struct mm_struct *mm) { - mmdrop(mm); + if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT)) { + mmdrop(mm); + } else { + /* + * mmdrop_lazy_tlb must provide a full memory barrier, see the + * membarrier comment finish_task_switch which relies on this. + */ + smp_mb(); + } } static inline void mmdrop_lazy_tlb_sched(struct mm_struct *mm) { - mmdrop_sched(mm); + if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT)) + mmdrop_sched(mm); + else + smp_mb(); /* see mmdrop_lazy_tlb() above */ } /** -- cgit v1.2.3 From 4c85c0be3d7a9a7ffe48bfe0954eacc0ba9d3c75 Mon Sep 17 00:00:00 2001 From: Hyeonggon Yoo <42.hyeyoo@gmail.com> Date: Mon, 30 Jan 2023 13:25:13 +0900 Subject: mm, printk: introduce new format %pGt for page_type %pGp format is used to display 'flags' field of a struct page. However, some page flags (i.e. PG_buddy, see page-flags.h for more details) are stored in page_type field. To display human-readable output of page_type, introduce %pGt format. It is important to note the meaning of bits are different in page_type. if page_type is 0xffffffff, no flags are set. Setting PG_buddy (0x00000080) flag results in a page_type of 0xffffff7f. Clearing a bit actually means setting a flag. Bits in page_type are inverted when displaying type names. Only values for which page_type_has_type() returns true are considered as page_type, to avoid confusion with mapcount values. if it returns false, only raw values are displayed and not page type names. Link: https://lkml.kernel.org/r/20230130042514.2418-3-42.hyeyoo@gmail.com Signed-off-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Reviewed-by: Petr Mladek [vsprintf part] Cc: Andy Shevchenko Cc: David Hildenbrand Cc: Joe Perches Cc: John Ogness Cc: Matthew Wilcox Cc: Sergey Senozhatsky Cc: Steven Rostedt (Google) Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- Documentation/core-api/printk-formats.rst | 16 +++++++++++----- include/linux/page-flags.h | 7 ++++++- include/trace/events/mmflags.h | 8 ++++++++ lib/test_printf.c | 26 ++++++++++++++++++++++++++ lib/vsprintf.c | 21 +++++++++++++++++++++ mm/debug.c | 5 +++++ mm/internal.h | 1 + 7 files changed, 78 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/Documentation/core-api/printk-formats.rst b/Documentation/core-api/printk-formats.rst index dbe1aacc79d0..dfe7e75a71de 100644 --- a/Documentation/core-api/printk-formats.rst +++ b/Documentation/core-api/printk-formats.rst @@ -575,20 +575,26 @@ The field width is passed by value, the bitmap is passed by reference. Helper macros cpumask_pr_args() and nodemask_pr_args() are available to ease printing cpumask and nodemask. -Flags bitfields such as page flags, gfp_flags ---------------------------------------------- +Flags bitfields such as page flags, page_type, gfp_flags +-------------------------------------------------------- :: %pGp 0x17ffffc0002036(referenced|uptodate|lru|active|private|node=0|zone=2|lastcpupid=0x1fffff) + %pGt 0xffffff7f(buddy) %pGg GFP_USER|GFP_DMA32|GFP_NOWARN %pGv read|exec|mayread|maywrite|mayexec|denywrite For printing flags bitfields as a collection of symbolic constants that would construct the value. The type of flags is given by the third -character. Currently supported are [p]age flags, [v]ma_flags (both -expect ``unsigned long *``) and [g]fp_flags (expects ``gfp_t *``). The flag -names and print order depends on the particular type. +character. Currently supported are: + + - p - [p]age flags, expects value of type (``unsigned long *``) + - t - page [t]ype, expects value of type (``unsigned int *``) + - v - [v]ma_flags, expects value of type (``unsigned long *``) + - g - [g]fp_flags, expects value of type (``gfp_t *``) + +The flag names and print order depends on the particular type. Note that this format should not be used directly in the :c:func:`TP_printk()` part of a tracepoint. Instead, use the show_*_flags() diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index a7e3a3405520..57287102c5bd 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -926,9 +926,14 @@ static inline bool is_page_hwpoison(struct page *page) #define PageType(page, flag) \ ((page->page_type & (PAGE_TYPE_BASE | flag)) == PAGE_TYPE_BASE) +static inline int page_type_has_type(unsigned int page_type) +{ + return (int)page_type < PAGE_MAPCOUNT_RESERVE; +} + static inline int page_has_type(struct page *page) { - return (int)page->page_type < PAGE_MAPCOUNT_RESERVE; + return page_type_has_type(page->page_type); } #define PAGE_TYPE_OPS(uname, lname) \ diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index f453babe29b3..b28218b7998e 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -141,6 +141,14 @@ IF_HAVE_PG_SKIP_KASAN_POISON(skip_kasan_poison) __def_pageflag_names \ ) : "none" +#define DEF_PAGETYPE_NAME(_name) { PG_##_name, __stringify(_name) } + +#define __def_pagetype_names \ + DEF_PAGETYPE_NAME(offline), \ + DEF_PAGETYPE_NAME(guard), \ + DEF_PAGETYPE_NAME(table), \ + DEF_PAGETYPE_NAME(buddy) + #if defined(CONFIG_X86) #define __VM_ARCH_SPECIFIC_1 {VM_PAT, "pat" } #elif defined(CONFIG_PPC) diff --git a/lib/test_printf.c b/lib/test_printf.c index 46b4e6c414a3..7677ebccf3c3 100644 --- a/lib/test_printf.c +++ b/lib/test_printf.c @@ -642,12 +642,26 @@ page_flags_test(int section, int node, int zone, int last_cpupid, test(cmp_buf, "%pGp", &flags); } +static void __init page_type_test(unsigned int page_type, const char *name, + char *cmp_buf) +{ + unsigned long size; + + size = scnprintf(cmp_buf, BUF_SIZE, "%#x(", page_type); + if (page_type_has_type(page_type)) + size += scnprintf(cmp_buf + size, BUF_SIZE - size, "%s", name); + + snprintf(cmp_buf + size, BUF_SIZE - size, ")"); + test(cmp_buf, "%pGt", &page_type); +} + static void __init flags(void) { unsigned long flags; char *cmp_buffer; gfp_t gfp; + unsigned int page_type; cmp_buffer = kmalloc(BUF_SIZE, GFP_KERNEL); if (!cmp_buffer) @@ -687,6 +701,18 @@ flags(void) gfp |= __GFP_HIGH; test(cmp_buffer, "%pGg", &gfp); + page_type = ~0; + page_type_test(page_type, "", cmp_buffer); + + page_type = 10; + page_type_test(page_type, "", cmp_buffer); + + page_type = ~PG_buddy; + page_type_test(page_type, "buddy", cmp_buffer); + + page_type = ~(PG_table | PG_buddy); + page_type_test(page_type, "table|buddy", cmp_buffer); + kfree(cmp_buffer); } diff --git a/lib/vsprintf.c b/lib/vsprintf.c index be71a03c936a..fbe320b5e89f 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -2052,6 +2052,25 @@ char *format_page_flags(char *buf, char *end, unsigned long flags) return buf; } +static +char *format_page_type(char *buf, char *end, unsigned int page_type) +{ + buf = number(buf, end, page_type, default_flag_spec); + + if (buf < end) + *buf = '('; + buf++; + + if (page_type_has_type(page_type)) + buf = format_flags(buf, end, ~page_type, pagetype_names); + + if (buf < end) + *buf = ')'; + buf++; + + return buf; +} + static noinline_for_stack char *flags_string(char *buf, char *end, void *flags_ptr, struct printf_spec spec, const char *fmt) @@ -2065,6 +2084,8 @@ char *flags_string(char *buf, char *end, void *flags_ptr, switch (fmt[1]) { case 'p': return format_page_flags(buf, end, *(unsigned long *)flags_ptr); + case 't': + return format_page_type(buf, end, *(unsigned int *)flags_ptr); case 'v': flags = *(unsigned long *)flags_ptr; names = vmaflag_names; diff --git a/mm/debug.c b/mm/debug.c index 96d594e16292..01cf0435723b 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -36,6 +36,11 @@ const struct trace_print_flags pageflag_names[] = { {0, NULL} }; +const struct trace_print_flags pagetype_names[] = { + __def_pagetype_names, + {0, NULL} +}; + const struct trace_print_flags gfpflag_names[] = { __def_gfpflag_names, {0, NULL} diff --git a/mm/internal.h b/mm/internal.h index 7920a8b7982e..2a7ffd9962c4 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -802,6 +802,7 @@ static inline void flush_tlb_batched_pending(struct mm_struct *mm) #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */ extern const struct trace_print_flags pageflag_names[]; +extern const struct trace_print_flags pagetype_names[]; extern const struct trace_print_flags vmaflag_names[]; extern const struct trace_print_flags gfpflag_names[]; -- cgit v1.2.3 From 16d91faf09be148d9c1cf4999f4d15fb24a7cd72 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Thu, 16 Feb 2023 11:59:24 -0800 Subject: kasan: call clear_page with a match-all tag instead of changing page tag Instead of changing the page's tag solely in order to obtain a pointer with a match-all tag and then changing it back again, just convert the pointer that we get from kmap_atomic() into one with a match-all tag before passing it to clear_page(). On a certain microarchitecture, this has been observed to cause a measurable improvement in microbenchmark performance, presumably as a result of being able to avoid the atomic operations on the page tag. Link: https://lkml.kernel.org/r/20230216195924.3287772-1-pcc@google.com Signed-off-by: Peter Collingbourne Link: https://linux-review.googlesource.com/id/I0249822cc29097ca7a04ad48e8eb14871f80e711 Reviewed-by: Andrey Konovalov Reviewed-by: Catalin Marinas Cc: Andrey Ryabinin Cc: Evgenii Stepanov Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/highmem.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 8fc10089e19e..9c7cdaa3de8c 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -243,12 +243,10 @@ static inline void clear_highpage(struct page *page) static inline void clear_highpage_kasan_tagged(struct page *page) { - u8 tag; + void *kaddr = kmap_local_page(page); - tag = page_kasan_tag(page); - page_kasan_tag_reset(page); - clear_highpage(page); - page_kasan_tag_set(page, tag); + clear_page(kasan_reset_tag(kaddr)); + kunmap_local(kaddr); } #ifndef __HAVE_ARCH_TAG_CLEAR_HIGHPAGE -- cgit v1.2.3 From 3e4fb13ac34bad94cf390415b1e6bc3ca9520d65 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Thu, 2 Mar 2023 19:58:35 +0800 Subject: mm: swap: remove unneeded cgroup_throttle_swaprate() All the callers of cgroup_throttle_swaprate() are converted to folio_throttle_swaprate(), so make __cgroup_throttle_swaprate() to take a folio, and rename it to __folio_throttle_swaprate(), also rename gfp_mask to gfp and drop redundant extern keyword. finally, drop unused cgroup_throttle_swaprate(). Link: https://lkml.kernel.org/r/20230302115835.105364-8-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviewed-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/swap.h | 12 ++++-------- mm/swapfile.c | 6 +++--- 2 files changed, 7 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index 209a425739a9..d5d0b54e90e8 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -641,22 +641,18 @@ extern atomic_t zswap_stored_pages; #endif #if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) -extern void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask); -static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) +void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp); +static inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp) { if (mem_cgroup_disabled()) return; - __cgroup_throttle_swaprate(page, gfp_mask); + __folio_throttle_swaprate(folio, gfp); } #else -static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) -{ -} -#endif static inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp) { - cgroup_throttle_swaprate(&folio->page, gfp); } +#endif #if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP) void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry); diff --git a/mm/swapfile.c b/mm/swapfile.c index 62ba2bf577d7..c1b97436f811 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -3635,12 +3635,12 @@ static void free_swap_count_continuations(struct swap_info_struct *si) } #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) -void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) +void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp) { struct swap_info_struct *si, *next; - int nid = page_to_nid(page); + int nid = folio_nid(folio); - if (!(gfp_mask & __GFP_IO)) + if (!(gfp & __GFP_IO)) return; if (!blk_cgroup_congested()) -- cgit v1.2.3 From 99c29133639a29fa803ea27ec79bf9e732efd062 Mon Sep 17 00:00:00 2001 From: Gerald Schaefer Date: Mon, 6 Mar 2023 17:15:48 +0100 Subject: mm: add PTE pointer parameter to flush_tlb_fix_spurious_fault() s390 can do more fine-grained handling of spurious TLB protection faults, when there also is the PTE pointer available. Therefore, pass on the PTE pointer to flush_tlb_fix_spurious_fault() as an additional parameter. This will add no functional change to other architectures, but those with private flush_tlb_fix_spurious_fault() implementations need to be made aware of the new parameter. Link: https://lkml.kernel.org/r/20230306161548.661740-1-gerald.schaefer@linux.ibm.com Signed-off-by: Gerald Schaefer Reviewed-by: Alexander Gordeev Acked-by: Catalin Marinas [arm64] Acked-by: Michael Ellerman [powerpc] Acked-by: David Hildenbrand Cc: Anshuman Khandual Cc: Borislav Petkov (AMD) Cc: Christophe Leroy Cc: Dave Hansen Cc: Ingo Molnar Cc: Matthew Wilcox (Oracle) Cc: Nicholas Piggin Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Will Deacon Signed-off-by: Andrew Morton --- arch/arm64/include/asm/pgtable.h | 2 +- arch/mips/include/asm/pgtable.h | 3 ++- arch/powerpc/include/asm/book3s/64/tlbflush.h | 3 ++- arch/s390/include/asm/pgtable.h | 12 +++++++----- arch/x86/include/asm/pgtable.h | 2 +- include/linux/pgtable.h | 2 +- mm/memory.c | 3 ++- mm/pgtable-generic.c | 2 +- 8 files changed, 17 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index b6ba466e2e8a..0bd18de9fd97 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -57,7 +57,7 @@ static inline bool arch_thp_swp_supported(void) * fault on one CPU which has been handled concurrently by another CPU * does not need to perform additional invalidation. */ -#define flush_tlb_fix_spurious_fault(vma, address) do { } while (0) +#define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0) /* * ZERO_PAGE is a global shared page that is always zero: used diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index 791389bf3c12..574fa14ac8b2 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -469,7 +469,8 @@ static inline pgprot_t pgprot_writecombine(pgprot_t _prot) } static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, - unsigned long address) + unsigned long address, + pte_t *ptep) { } diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h index 2bbc0fcce04a..ff7f0ee179e5 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h @@ -121,7 +121,8 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, - unsigned long address) + unsigned long address, + pte_t *ptep) { /* * Book3S 64 does not require spurious fault flushes because the PTE diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 2c70b4d1263d..c1f6b46ec555 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1239,7 +1239,8 @@ static inline int pte_allow_rdp(pte_t old, pte_t new) } static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, - unsigned long address) + unsigned long address, + pte_t *ptep) { /* * RDP might not have propagated the PTE protection reset to all CPUs, @@ -1247,11 +1248,12 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, * NOTE: This will also be called when a racing pagetable update on * another thread already installed the correct PTE. Both cases cannot * really be distinguished. - * Therefore, only do the local TLB flush when RDP can be used, to avoid - * unnecessary overhead. + * Therefore, only do the local TLB flush when RDP can be used, and the + * PTE does not have _PAGE_PROTECT set, to avoid unnecessary overhead. + * A local RDP can be used to do the flush. */ - if (MACHINE_HAS_RDP) - asm volatile("ptlb" : : : "memory"); + if (MACHINE_HAS_RDP && !(pte_val(*ptep) & _PAGE_PROTECT)) + __ptep_rdp(address, ptep, 0, 0, 1); } #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 7425f32e5293..15ae4d6ba476 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -1097,7 +1097,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte); } -#define flush_tlb_fix_spurious_fault(vma, address) do { } while (0) +#define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0) #define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot)) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 9dc936bc77d1..c5a51481bbb9 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -817,7 +817,7 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) #endif #ifndef flush_tlb_fix_spurious_fault -#define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address) +#define flush_tlb_fix_spurious_fault(vma, address, ptep) flush_tlb_page(vma, address) #endif /* diff --git a/mm/memory.c b/mm/memory.c index fdc903825981..6285cad1f4fb 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4944,7 +4944,8 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) * with threads. */ if (vmf->flags & FAULT_FLAG_WRITE) - flush_tlb_fix_spurious_fault(vmf->vma, vmf->address); + flush_tlb_fix_spurious_fault(vmf->vma, vmf->address, + vmf->pte); } unlock: pte_unmap_unlock(vmf->pte, vmf->ptl); diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index 90ab721a12a8..d2fc52bffafc 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -69,7 +69,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma, int changed = !pte_same(*ptep, entry); if (changed) { set_pte_at(vma->vm_mm, address, ptep, entry); - flush_tlb_fix_spurious_fault(vma, address); + flush_tlb_fix_spurious_fault(vma, address, ptep); } return changed; } -- cgit v1.2.3 From 82b3aa2681ca92421c4b508e7c390000273c53fe Mon Sep 17 00:00:00 2001 From: Yue Zhao Date: Mon, 6 Mar 2023 23:41:36 +0800 Subject: mm, memcg: Prevent memory.swappiness load/store tearing The knob for cgroup v1 memory controller: memory.swappiness is not protected by any locking so it can be modified while it is used. This is not an actual problem because races are unlikely. But it is better to use [READ|WRITE]_ONCE to prevent compiler from doing anything funky. The access of memcg->swappiness and vm_swappiness is lockless, so both of them can be concurrently set at the same time as we are trying to read them. All occurrences of memcg->swappiness and vm_swappiness are updated with [READ|WRITE]_ONCE. [findns94@gmail.com: v3] Link: https://lkml.kernel.org/r/20230308162555.14195-3-findns94@gmail.com Link: https://lkml.kernel.org/r/20230306154138.3775-3-findns94@gmail.com Signed-off-by: Yue Zhao Acked-by: Michal Hocko Acked-by: Shakeel Butt Acked-by: Roman Gushchin Cc: Johannes Weiner Cc: Matthew Wilcox (Oracle) Cc: Muchun Song Cc: Tang Yizhou Signed-off-by: Andrew Morton --- include/linux/swap.h | 8 ++++---- mm/memcontrol.c | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index d5d0b54e90e8..bfc3b06b5f8f 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -620,18 +620,18 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) { /* Cgroup2 doesn't have per-cgroup swappiness */ if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) - return vm_swappiness; + return READ_ONCE(vm_swappiness); /* root ? */ if (mem_cgroup_disabled() || mem_cgroup_is_root(memcg)) - return vm_swappiness; + return READ_ONCE(vm_swappiness); - return memcg->swappiness; + return READ_ONCE(memcg->swappiness); } #else static inline int mem_cgroup_swappiness(struct mem_cgroup *mem) { - return vm_swappiness; + return READ_ONCE(vm_swappiness); } #endif diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 06821e5f7604..1b0112afcad3 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4179,9 +4179,9 @@ static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css, return -EINVAL; if (!mem_cgroup_is_root(memcg)) - memcg->swappiness = val; + WRITE_ONCE(memcg->swappiness, val); else - vm_swappiness = val; + WRITE_ONCE(vm_swappiness, val); return 0; } @@ -5353,7 +5353,7 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) #endif page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX); if (parent) { - memcg->swappiness = mem_cgroup_swappiness(parent); + WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent)); memcg->oom_kill_disable = parent->oom_kill_disable; page_counter_init(&memcg->memory, &parent->memory); -- cgit v1.2.3 From 452a8f40728065800b5a5b81f1152e9a16d39656 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 7 Mar 2023 15:31:25 +0100 Subject: mm,jfs: move write_one_page/folio_write_one to jfs The last remaining user of folio_write_one through the write_one_page wrapper is jfs, so move the functionality there and hard code the call to metapage_writepage. Note that the use of the pagecache by the JFS 'metapage' buffer cache is a bit odd, and we could probably do without VM-level dirty tracking at all, but that's a change for another time. Link: https://lkml.kernel.org/r/20230307143125.27778-4-hch@lst.de Signed-off-by: Christoph Hellwig Acked-by: Dave Kleikamp Cc: Changwei Ge Cc: Evgeniy Dushistov Cc: Gang He Cc: Jan Kara Cc: Jan Kara via Ocfs2-devel Cc: Joel Becker Cc: Joseph Qi Cc: Joseph Qi Cc: Jun Piao Cc: Junxiao Bi Cc: Mark Fasheh Cc: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- fs/jfs/jfs_metapage.c | 39 ++++++++++++++++++++++++++++++++++----- include/linux/pagemap.h | 6 ------ mm/page-writeback.c | 40 ---------------------------------------- 3 files changed, 34 insertions(+), 51 deletions(-) (limited to 'include/linux') diff --git a/fs/jfs/jfs_metapage.c b/fs/jfs/jfs_metapage.c index 2e8461ce74de..961569c11159 100644 --- a/fs/jfs/jfs_metapage.c +++ b/fs/jfs/jfs_metapage.c @@ -691,6 +691,35 @@ void grab_metapage(struct metapage * mp) unlock_page(mp->page); } +static int metapage_write_one(struct page *page) +{ + struct folio *folio = page_folio(page); + struct address_space *mapping = folio->mapping; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = folio_nr_pages(folio), + }; + int ret = 0; + + BUG_ON(!folio_test_locked(folio)); + + folio_wait_writeback(folio); + + if (folio_clear_dirty_for_io(folio)) { + folio_get(folio); + ret = metapage_writepage(page, &wbc); + if (ret == 0) + folio_wait_writeback(folio); + folio_put(folio); + } else { + folio_unlock(folio); + } + + if (!ret) + ret = filemap_check_errors(mapping); + return ret; +} + void force_metapage(struct metapage *mp) { struct page *page = mp->page; @@ -700,8 +729,8 @@ void force_metapage(struct metapage *mp) get_page(page); lock_page(page); set_page_dirty(page); - if (write_one_page(page)) - jfs_error(mp->sb, "write_one_page() failed\n"); + if (metapage_write_one(page)) + jfs_error(mp->sb, "metapage_write_one() failed\n"); clear_bit(META_forcewrite, &mp->flag); put_page(page); } @@ -746,9 +775,9 @@ void release_metapage(struct metapage * mp) set_page_dirty(page); if (test_bit(META_sync, &mp->flag)) { clear_bit(META_sync, &mp->flag); - if (write_one_page(page)) - jfs_error(mp->sb, "write_one_page() failed\n"); - lock_page(page); /* write_one_page unlocks the page */ + if (metapage_write_one(page)) + jfs_error(mp->sb, "metapage_write_one() failed\n"); + lock_page(page); } } else if (mp->lsn) /* discard_metapage doesn't remove it */ remove_from_logsync(mp); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 0acb8e1fb7af..853184a46411 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1066,12 +1066,6 @@ static inline void folio_cancel_dirty(struct folio *folio) bool folio_clear_dirty_for_io(struct folio *folio); bool clear_page_dirty_for_io(struct page *page); void folio_invalidate(struct folio *folio, size_t offset, size_t length); -int __must_check folio_write_one(struct folio *folio); -static inline int __must_check write_one_page(struct page *page) -{ - return folio_write_one(page_folio(page)); -} - int __set_page_dirty_nobuffers(struct page *page); bool noop_dirty_folio(struct address_space *mapping, struct folio *folio); diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 516b1aa247e8..db7943999007 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -2583,46 +2583,6 @@ int do_writepages(struct address_space *mapping, struct writeback_control *wbc) return ret; } -/** - * folio_write_one - write out a single folio and wait on I/O. - * @folio: The folio to write. - * - * The folio must be locked by the caller and will be unlocked upon return. - * - * Note that the mapping's AS_EIO/AS_ENOSPC flags will be cleared when this - * function returns. - * - * Return: %0 on success, negative error code otherwise - */ -int folio_write_one(struct folio *folio) -{ - struct address_space *mapping = folio->mapping; - int ret = 0; - struct writeback_control wbc = { - .sync_mode = WB_SYNC_ALL, - .nr_to_write = folio_nr_pages(folio), - }; - - BUG_ON(!folio_test_locked(folio)); - - folio_wait_writeback(folio); - - if (folio_clear_dirty_for_io(folio)) { - folio_get(folio); - ret = mapping->a_ops->writepage(&folio->page, &wbc); - if (ret == 0) - folio_wait_writeback(folio); - folio_put(folio); - } else { - folio_unlock(folio); - } - - if (!ret) - ret = filemap_check_errors(mapping); - return ret; -} -EXPORT_SYMBOL(folio_write_one); - /* * For address_spaces which do not use buffers nor write back. */ -- cgit v1.2.3 From 2c6efe9cf2d7841b75fe38ed1adbd41a90f51ba0 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Thu, 9 Mar 2023 15:05:45 -0800 Subject: shmem: add support to ignore swap In doing experimentations with shmem having the option to avoid swap becomes a useful mechanism. One of the *raves* about brd over shmem is you can avoid swap, but that's not really a good reason to use brd if we can instead use shmem. Using brd has its own good reasons to exist, but just because "tmpfs" doesn't let you do that is not a great reason to avoid it if we can easily add support for it. I don't add support for reconfiguring incompatible options, but if we really wanted to we can add support for that. To avoid swap we use mapping_set_unevictable() upon inode creation, and put a WARN_ON_ONCE() stop-gap on writepages() for reclaim. Link: https://lkml.kernel.org/r/20230309230545.2930737-7-mcgrof@kernel.org Signed-off-by: Luis Chamberlain Acked-by: Christian Brauner Tested-by: Xin Hao Reviewed-by: Davidlohr Bueso Cc: Adam Manzanares Cc: David Hildenbrand Cc: Davidlohr Bueso Cc: Hugh Dickins Cc: Kees Cook Cc: Matthew Wilcox Cc: Pankaj Raghav Cc: Yosry Ahmed Signed-off-by: Andrew Morton --- Documentation/filesystems/tmpfs.rst | 9 ++++++--- Documentation/mm/unevictable-lru.rst | 2 ++ include/linux/shmem_fs.h | 1 + mm/shmem.c | 28 +++++++++++++++++++++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/Documentation/filesystems/tmpfs.rst b/Documentation/filesystems/tmpfs.rst index 1ec9a9f8196b..f18f46be5c0c 100644 --- a/Documentation/filesystems/tmpfs.rst +++ b/Documentation/filesystems/tmpfs.rst @@ -13,7 +13,8 @@ everything stored therein is lost. tmpfs puts everything into the kernel internal caches and grows and shrinks to accommodate the files it contains and is able to swap -unneeded pages out to swap space, and supports THP. +unneeded pages out to swap space, if swap was enabled for the tmpfs +mount. tmpfs also supports THP. tmpfs extends ramfs with a few userspace configurable options listed and explained further below, some of which can be reconfigured dynamically on the @@ -33,8 +34,8 @@ configured in size at initialization and you cannot dynamically resize them. Contrary to brd ramdisks, tmpfs has its own filesystem, it does not rely on the block layer at all. -Since tmpfs lives completely in the page cache and on swap, all tmpfs -pages will be shown as "Shmem" in /proc/meminfo and "Shared" in +Since tmpfs lives completely in the page cache and optionally on swap, +all tmpfs pages will be shown as "Shmem" in /proc/meminfo and "Shared" in free(1). Notice that these counters also include shared memory (shmem, see ipcs(1)). The most reliable way to get the count is using df(1) and du(1). @@ -83,6 +84,8 @@ nr_inodes The maximum number of inodes for this instance. The default is half of the number of your physical RAM pages, or (on a machine with highmem) the number of lowmem RAM pages, whichever is the lower. +noswap Disables swap. Remounts must respect the original settings. + By default swap is enabled. ========= ============================================================ These parameters accept a suffix k, m or g for kilo, mega and giga and diff --git a/Documentation/mm/unevictable-lru.rst b/Documentation/mm/unevictable-lru.rst index 92ac5dca420c..d5ac8511eb67 100644 --- a/Documentation/mm/unevictable-lru.rst +++ b/Documentation/mm/unevictable-lru.rst @@ -42,6 +42,8 @@ The unevictable list addresses the following classes of unevictable pages: * Those owned by ramfs. + * Those owned by tmpfs with the noswap mount option. + * Those mapped into SHM_LOCK'd shared memory regions. * Those mapped into VM_LOCKED [mlock()ed] VMAs. diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 103d1000a5a2..50bf82b36995 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -45,6 +45,7 @@ struct shmem_sb_info { kuid_t uid; /* Mount uid for root directory */ kgid_t gid; /* Mount gid for root directory */ bool full_inums; /* If i_ino should be uint or ino_t */ + bool noswap; /* ignores VM reclaim / swap requests */ ino_t next_ino; /* The next per-sb inode number to use */ ino_t __percpu *ino_batch; /* The next per-cpu inode number to use */ struct mempolicy *mpol; /* default memory policy for mappings */ diff --git a/mm/shmem.c b/mm/shmem.c index 7751c136eadb..787e83791eb5 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -116,10 +116,12 @@ struct shmem_options { bool full_inums; int huge; int seen; + bool noswap; #define SHMEM_SEEN_BLOCKS 1 #define SHMEM_SEEN_INODES 2 #define SHMEM_SEEN_HUGE 4 #define SHMEM_SEEN_INUMS 8 +#define SHMEM_SEEN_NOSWAP 16 }; #ifdef CONFIG_TMPFS @@ -1334,6 +1336,7 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) struct address_space *mapping = folio->mapping; struct inode *inode = mapping->host; struct shmem_inode_info *info = SHMEM_I(inode); + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); swp_entry_t swap; pgoff_t index; @@ -1347,7 +1350,7 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) if (WARN_ON_ONCE(!wbc->for_reclaim)) goto redirty; - if (WARN_ON_ONCE(info->flags & VM_LOCKED)) + if (WARN_ON_ONCE((info->flags & VM_LOCKED) || sbinfo->noswap)) goto redirty; if (!total_swap_pages) @@ -2372,6 +2375,8 @@ static struct inode *shmem_get_inode(struct mnt_idmap *idmap, struct super_block shmem_set_inode_flags(inode, info->fsflags); INIT_LIST_HEAD(&info->shrinklist); INIT_LIST_HEAD(&info->swaplist); + if (sbinfo->noswap) + mapping_set_unevictable(inode->i_mapping); simple_xattrs_init(&info->xattrs); cache_no_acl(inode); mapping_set_large_folios(inode->i_mapping); @@ -3459,6 +3464,7 @@ enum shmem_param { Opt_uid, Opt_inode32, Opt_inode64, + Opt_noswap, }; static const struct constant_table shmem_param_enums_huge[] = { @@ -3480,6 +3486,7 @@ const struct fs_parameter_spec shmem_fs_parameters[] = { fsparam_u32 ("uid", Opt_uid), fsparam_flag ("inode32", Opt_inode32), fsparam_flag ("inode64", Opt_inode64), + fsparam_flag ("noswap", Opt_noswap), {} }; @@ -3563,6 +3570,10 @@ static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param) ctx->full_inums = true; ctx->seen |= SHMEM_SEEN_INUMS; break; + case Opt_noswap: + ctx->noswap = true; + ctx->seen |= SHMEM_SEEN_NOSWAP; + break; } return 0; @@ -3661,6 +3672,14 @@ static int shmem_reconfigure(struct fs_context *fc) err = "Current inum too high to switch to 32-bit inums"; goto out; } + if ((ctx->seen & SHMEM_SEEN_NOSWAP) && ctx->noswap && !sbinfo->noswap) { + err = "Cannot disable swap on remount"; + goto out; + } + if (!(ctx->seen & SHMEM_SEEN_NOSWAP) && !ctx->noswap && sbinfo->noswap) { + err = "Cannot enable swap on remount if it was disabled on first mount"; + goto out; + } if (ctx->seen & SHMEM_SEEN_HUGE) sbinfo->huge = ctx->huge; @@ -3681,6 +3700,10 @@ static int shmem_reconfigure(struct fs_context *fc) sbinfo->mpol = ctx->mpol; /* transfers initial ref */ ctx->mpol = NULL; } + + if (ctx->noswap) + sbinfo->noswap = true; + raw_spin_unlock(&sbinfo->stat_lock); mpol_put(mpol); return 0; @@ -3735,6 +3758,8 @@ static int shmem_show_options(struct seq_file *seq, struct dentry *root) seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); #endif shmem_show_mpol(seq, sbinfo->mpol); + if (sbinfo->noswap) + seq_printf(seq, ",noswap"); return 0; } @@ -3778,6 +3803,7 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) ctx->inodes = shmem_default_max_inodes(); if (!(ctx->seen & SHMEM_SEEN_INUMS)) ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64); + sbinfo->noswap = ctx->noswap; } else { sb->s_flags |= SB_NOUSER; } -- cgit v1.2.3 From 7eb16f23b9a415f062db22739e59bb144e0b24ab Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 10 Mar 2023 17:29:05 +0100 Subject: io-mapping: don't disable preempt on RT in io_mapping_map_atomic_wc(). io_mapping_map_atomic_wc() disables preemption and pagefaults for historical reasons. The conversion to io_mapping_map_local_wc(), which only disables migration, cannot be done wholesale because quite some call sites need to be updated to accommodate with the changed semantics. On PREEMPT_RT enabled kernels the io_mapping_map_atomic_wc() semantics are problematic due to the implicit disabling of preemption which makes it impossible to acquire 'sleeping' spinlocks within the mapped atomic sections. PREEMPT_RT replaces the preempt_disable() with a migrate_disable() for more than a decade. It could be argued that this is a justification to do this unconditionally, but PREEMPT_RT covers only a limited number of architectures and it disables some functionality which limits the coverage further. Limit the replacement to PREEMPT_RT for now. This is also done kmap_atomic(). Link: https://lkml.kernel.org/r/20230310162905.O57Pj7hh@linutronix.de Signed-off-by: Sebastian Andrzej Siewior Reported-by: Richard Weinberger Link: https://lore.kernel.org/CAFLxGvw0WMxaMqYqJ5WgvVSbKHq2D2xcXTOgMCpgq9nDC-MWTQ@mail.gmail.com Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- include/linux/io-mapping.h | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io-mapping.h b/include/linux/io-mapping.h index 09d4f17c8d3b..7376c1df9c90 100644 --- a/include/linux/io-mapping.h +++ b/include/linux/io-mapping.h @@ -69,7 +69,10 @@ io_mapping_map_atomic_wc(struct io_mapping *mapping, BUG_ON(offset >= mapping->size); phys_addr = mapping->base + offset; - preempt_disable(); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_disable(); + else + migrate_disable(); pagefault_disable(); return __iomap_local_pfn_prot(PHYS_PFN(phys_addr), mapping->prot); } @@ -79,7 +82,10 @@ io_mapping_unmap_atomic(void __iomem *vaddr) { kunmap_local_indexed((void __force *)vaddr); pagefault_enable(); - preempt_enable(); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_enable(); + else + migrate_enable(); } static inline void __iomem * @@ -162,7 +168,10 @@ static inline void __iomem * io_mapping_map_atomic_wc(struct io_mapping *mapping, unsigned long offset) { - preempt_disable(); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_disable(); + else + migrate_disable(); pagefault_disable(); return io_mapping_map_wc(mapping, offset, PAGE_SIZE); } @@ -172,7 +181,10 @@ io_mapping_unmap_atomic(void __iomem *vaddr) { io_mapping_unmap(vaddr); pagefault_enable(); - preempt_enable(); + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_enable(); + else + migrate_enable(); } static inline void __iomem * -- cgit v1.2.3 From 0a54864f8dfb64b64c84c9db6ff70e0e93690a33 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Thu, 9 Mar 2023 20:29:14 -0800 Subject: kasan: remove PG_skip_kasan_poison flag Code inspection reveals that PG_skip_kasan_poison is redundant with kasantag, because the former is intended to be set iff the latter is the match-all tag. It can also be observed that it's basically pointless to poison pages which have kasantag=0, because any pages with this tag would have been pointed to by pointers with match-all tags, so poisoning the pages would have little to no effect in terms of bug detection. Therefore, change the condition in should_skip_kasan_poison() to check kasantag instead, and remove PG_skip_kasan_poison and associated flags. Link: https://lkml.kernel.org/r/20230310042914.3805818-3-pcc@google.com Link: https://linux-review.googlesource.com/id/I57f825f2eaeaf7e8389d6cf4597c8a5821359838 Signed-off-by: Peter Collingbourne Reviewed-by: Andrey Konovalov Cc: Andrey Ryabinin Cc: Catalin Marinas Cc: Evgenii Stepanov Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/gfp_types.h | 30 +++++++--------- include/linux/page-flags.h | 9 ----- include/trace/events/mmflags.h | 13 ++----- mm/kasan/hw_tags.c | 2 +- mm/page_alloc.c | 81 ++++++++++++++++-------------------------- mm/vmalloc.c | 2 +- 6 files changed, 47 insertions(+), 90 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 5088637fe5c2..6583a58670c5 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -47,16 +47,14 @@ typedef unsigned int __bitwise gfp_t; #define ___GFP_ACCOUNT 0x400000u #define ___GFP_ZEROTAGS 0x800000u #ifdef CONFIG_KASAN_HW_TAGS -#define ___GFP_SKIP_ZERO 0x1000000u -#define ___GFP_SKIP_KASAN_UNPOISON 0x2000000u -#define ___GFP_SKIP_KASAN_POISON 0x4000000u +#define ___GFP_SKIP_ZERO 0x1000000u +#define ___GFP_SKIP_KASAN 0x2000000u #else -#define ___GFP_SKIP_ZERO 0 -#define ___GFP_SKIP_KASAN_UNPOISON 0 -#define ___GFP_SKIP_KASAN_POISON 0 +#define ___GFP_SKIP_ZERO 0 +#define ___GFP_SKIP_KASAN 0 #endif #ifdef CONFIG_LOCKDEP -#define ___GFP_NOLOCKDEP 0x8000000u +#define ___GFP_NOLOCKDEP 0x4000000u #else #define ___GFP_NOLOCKDEP 0 #endif @@ -234,25 +232,24 @@ typedef unsigned int __bitwise gfp_t; * memory tags at the same time as zeroing memory has minimal additional * performace impact. * - * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation. - * Only effective in HW_TAGS mode. - * - * %__GFP_SKIP_KASAN_POISON makes KASAN skip poisoning on page deallocation. - * Typically, used for userspace pages. Only effective in HW_TAGS mode. + * %__GFP_SKIP_KASAN makes KASAN skip unpoisoning on page allocation. + * Used for userspace and vmalloc pages; the latter are unpoisoned by + * kasan_unpoison_vmalloc instead. For userspace pages, results in + * poisoning being skipped as well, see should_skip_kasan_poison for + * details. Only effective in HW_TAGS mode. */ #define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN) #define __GFP_COMP ((__force gfp_t)___GFP_COMP) #define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) #define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS) #define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO) -#define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON) -#define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON) +#define __GFP_SKIP_KASAN ((__force gfp_t)___GFP_SKIP_KASAN) /* Disable lockdep for GFP context tracking */ #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) /* Room for N __GFP_FOO bits */ -#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP)) +#define __GFP_BITS_SHIFT (26 + IS_ENABLED(CONFIG_LOCKDEP)) #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /** @@ -335,8 +332,7 @@ typedef unsigned int __bitwise gfp_t; #define GFP_DMA __GFP_DMA #define GFP_DMA32 __GFP_DMA32 #define GFP_HIGHUSER (GFP_USER | __GFP_HIGHMEM) -#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE | \ - __GFP_SKIP_KASAN_POISON | __GFP_SKIP_KASAN_UNPOISON) +#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE | __GFP_SKIP_KASAN) #define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \ __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM) #define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 57287102c5bd..dcda20c47b8f 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -135,9 +135,6 @@ enum pageflags { #ifdef CONFIG_ARCH_USES_PG_ARCH_X PG_arch_2, PG_arch_3, -#endif -#ifdef CONFIG_KASAN_HW_TAGS - PG_skip_kasan_poison, #endif __NR_PAGEFLAGS, @@ -594,12 +591,6 @@ TESTCLEARFLAG(Young, young, PF_ANY) PAGEFLAG(Idle, idle, PF_ANY) #endif -#ifdef CONFIG_KASAN_HW_TAGS -PAGEFLAG(SkipKASanPoison, skip_kasan_poison, PF_HEAD) -#else -PAGEFLAG_FALSE(SkipKASanPoison, skip_kasan_poison) -#endif - /* * PageReported() is used to track reported free pages within the Buddy * allocator. We can use the non-atomic version of the test and set diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index b28218b7998e..b63e7c0fbbe5 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -55,8 +55,7 @@ #ifdef CONFIG_KASAN_HW_TAGS #define __def_gfpflag_names_kasan , \ gfpflag_string(__GFP_SKIP_ZERO), \ - gfpflag_string(__GFP_SKIP_KASAN_POISON), \ - gfpflag_string(__GFP_SKIP_KASAN_UNPOISON) + gfpflag_string(__GFP_SKIP_KASAN) #else #define __def_gfpflag_names_kasan #endif @@ -96,13 +95,6 @@ #define IF_HAVE_PG_ARCH_X(_name) #endif -#ifdef CONFIG_KASAN_HW_TAGS -#define IF_HAVE_PG_SKIP_KASAN_POISON(_name) \ - ,{1UL << PG_##_name, __stringify(_name)} -#else -#define IF_HAVE_PG_SKIP_KASAN_POISON(_name) -#endif - #define DEF_PAGEFLAG_NAME(_name) { 1UL << PG_##_name, __stringify(_name) } #define __def_pageflag_names \ @@ -133,8 +125,7 @@ IF_HAVE_PG_HWPOISON(hwpoison) \ IF_HAVE_PG_IDLE(idle) \ IF_HAVE_PG_IDLE(young) \ IF_HAVE_PG_ARCH_X(arch_2) \ -IF_HAVE_PG_ARCH_X(arch_3) \ -IF_HAVE_PG_SKIP_KASAN_POISON(skip_kasan_poison) +IF_HAVE_PG_ARCH_X(arch_3) #define show_page_flags(flags) \ (flags) ? __print_flags(flags, "|", \ diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index d1bcb0205327..bb4f56e5bdec 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -318,7 +318,7 @@ void *__kasan_unpoison_vmalloc(const void *start, unsigned long size, * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags * the first virtual mapping, which is created by vmalloc(). * Tagging the page_alloc memory backing that vmalloc() allocation is - * skipped, see ___GFP_SKIP_KASAN_UNPOISON. + * skipped, see ___GFP_SKIP_KASAN. * * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual. */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index a109444e9f44..3737f9d58f5f 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -112,17 +112,6 @@ typedef int __bitwise fpi_t; */ #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) -/* - * Don't poison memory with KASAN (only for the tag-based modes). - * During boot, all non-reserved memblock memory is exposed to page_alloc. - * Poisoning all that memory lengthens boot time, especially on systems with - * large amount of RAM. This flag is used to skip that poisoning. - * This is only done for the tag-based KASAN modes, as those are able to - * detect memory corruptions with the memory tags assigned by default. - * All memory allocated normally after boot gets poisoned as usual. - */ -#define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2)) - /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ static DEFINE_MUTEX(pcp_batch_high_lock); #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8) @@ -1370,13 +1359,19 @@ out: /* * Skip KASAN memory poisoning when either: * - * 1. Deferred memory initialization has not yet completed, - * see the explanation below. - * 2. Skipping poisoning is requested via FPI_SKIP_KASAN_POISON, - * see the comment next to it. - * 3. Skipping poisoning is requested via __GFP_SKIP_KASAN_POISON, - * see the comment next to it. - * 4. The allocation is excluded from being checked due to sampling, + * 1. For generic KASAN: deferred memory initialization has not yet completed. + * Tag-based KASAN modes skip pages freed via deferred memory initialization + * using page tags instead (see below). + * 2. For tag-based KASAN modes: the page has a match-all KASAN tag, indicating + * that error detection is disabled for accesses via the page address. + * + * Pages will have match-all tags in the following circumstances: + * + * 1. Pages are being initialized for the first time, including during deferred + * memory init; see the call to page_kasan_tag_reset in __init_single_page. + * 2. The allocation was not unpoisoned due to __GFP_SKIP_KASAN, with the + * exception of pages unpoisoned by kasan_unpoison_vmalloc. + * 3. The allocation was excluded from being checked due to sampling, * see the call to kasan_unpoison_pages. * * Poisoning pages during deferred memory init will greatly lengthen the @@ -1392,10 +1387,10 @@ out: */ static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) { - return deferred_pages_enabled() || - (!IS_ENABLED(CONFIG_KASAN_GENERIC) && - (fpi_flags & FPI_SKIP_KASAN_POISON)) || - PageSkipKASanPoison(page); + if (IS_ENABLED(CONFIG_KASAN_GENERIC)) + return deferred_pages_enabled(); + + return page_kasan_tag(page) == 0xff; } static void kernel_init_pages(struct page *page, int numpages) @@ -1730,7 +1725,7 @@ void __free_pages_core(struct page *page, unsigned int order) * Bypass PCP and place fresh pages right to the tail, primarily * relevant for memory onlining. */ - __free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON); + __free_pages_ok(page, order, FPI_TO_TAIL); } #ifdef CONFIG_NUMA @@ -2396,9 +2391,9 @@ static inline bool should_skip_kasan_unpoison(gfp_t flags) /* * With hardware tag-based KASAN enabled, skip if this has been - * requested via __GFP_SKIP_KASAN_UNPOISON. + * requested via __GFP_SKIP_KASAN. */ - return flags & __GFP_SKIP_KASAN_UNPOISON; + return flags & __GFP_SKIP_KASAN; } static inline bool should_skip_init(gfp_t flags) @@ -2417,7 +2412,6 @@ inline void post_alloc_hook(struct page *page, unsigned int order, bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) && !should_skip_init(gfp_flags); bool zero_tags = init && (gfp_flags & __GFP_ZEROTAGS); - bool reset_tags = true; int i; set_page_private(page, 0); @@ -2451,37 +2445,22 @@ inline void post_alloc_hook(struct page *page, unsigned int order, /* Take note that memory was initialized by the loop above. */ init = false; } - if (!should_skip_kasan_unpoison(gfp_flags)) { - /* Try unpoisoning (or setting tags) and initializing memory. */ - if (kasan_unpoison_pages(page, order, init)) { - /* Take note that memory was initialized by KASAN. */ - if (kasan_has_integrated_init()) - init = false; - /* Take note that memory tags were set by KASAN. */ - reset_tags = false; - } else { - /* - * KASAN decided to exclude this allocation from being - * (un)poisoned due to sampling. Make KASAN skip - * poisoning when the allocation is freed. - */ - SetPageSkipKASanPoison(page); - } - } - /* - * If memory tags have not been set by KASAN, reset the page tags to - * ensure page_address() dereferencing does not fault. - */ - if (reset_tags) { + if (!should_skip_kasan_unpoison(gfp_flags) && + kasan_unpoison_pages(page, order, init)) { + /* Take note that memory was initialized by KASAN. */ + if (kasan_has_integrated_init()) + init = false; + } else { + /* + * If memory tags have not been set by KASAN, reset the page + * tags to ensure page_address() dereferencing does not fault. + */ for (i = 0; i != 1 << order; ++i) page_kasan_tag_reset(page + i); } /* If memory is still not initialized, initialize it now. */ if (init) kernel_init_pages(page, 1 << order); - /* Propagate __GFP_SKIP_KASAN_POISON to page flags. */ - if (kasan_hw_tags_enabled() && (gfp_flags & __GFP_SKIP_KASAN_POISON)) - SetPageSkipKASanPoison(page); set_page_owner(page, order, gfp_flags); page_table_check_alloc(page, order); diff --git a/mm/vmalloc.c b/mm/vmalloc.c index bef6cf2b4d46..5e60e9792cbf 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3188,7 +3188,7 @@ again: * pages backing VM_ALLOC mapping. Memory is instead * poisoned and zeroed by kasan_unpoison_vmalloc(). */ - gfp_mask |= __GFP_SKIP_KASAN_UNPOISON | __GFP_SKIP_ZERO; + gfp_mask |= __GFP_SKIP_KASAN | __GFP_SKIP_ZERO; } /* Take note that the mapping is PAGE_KERNEL. */ -- cgit v1.2.3 From 42c9db39704839eeb77b27db4c1d57bfa2a54a5b Mon Sep 17 00:00:00 2001 From: Qi Zheng Date: Mon, 13 Mar 2023 19:28:12 +0800 Subject: mm: vmscan: add a map_nr_max field to shrinker_info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "make slab shrink lockless", v5. This patch series aims to make slab shrink lockless. 1. Background ============= On our servers, we often find the following system cpu hotspots: 52.22% [kernel] [k] down_read_trylock 19.60% [kernel] [k] up_read 8.86% [kernel] [k] shrink_slab 2.44% [kernel] [k] idr_find 1.25% [kernel] [k] count_shadow_nodes 1.18% [kernel] [k] shrink lruvec 0.71% [kernel] [k] mem_cgroup_iter 0.71% [kernel] [k] shrink_node 0.55% [kernel] [k] find_next_bit And we used bpftrace to capture its calltrace as follows: @[ down_read_trylock+1 shrink_slab+128 shrink_node+371 do_try_to_free_pages+232 try_to_free_pages+243 _alloc_pages_slowpath+771 _alloc_pages_nodemask+702 pagecache_get_page+255 filemap_fault+1361 ext4_filemap_fault+44 __do_fault+76 handle_mm_fault+3543 do_user_addr_fault+442 do_page_fault+48 page_fault+62 ]: 1161690 @[ down_read_trylock+1 shrink_slab+128 shrink_node+371 balance_pgdat+690 kswapd+389 kthread+246 ret_from_fork+31 ]: 8424884 @[ down_read_trylock+1 shrink_slab+128 shrink_node+371 do_try_to_free_pages+232 try_to_free_pages+243 __alloc_pages_slowpath+771 __alloc_pages_nodemask+702 __do_page_cache_readahead+244 filemap_fault+1674 ext4_filemap_fault+44 __do_fault+76 handle_mm_fault+3543 do_user_addr_fault+442 do_page_fault+48 page_fault+62 ]: 20917631 We can see that down_read_trylock() of shrinker_rwsem is being called with high frequency at that time. Because of the poor multicore scalability of atomic operations, this can lead to a significant drop in IPC (instructions per cycle). And more, the shrinker_rwsem is a global read-write lock in shrinkers subsystem, which protects most operations such as slab shrink, registration and unregistration of shrinkers, etc. This can easily cause problems in the following cases. 1) When the memory pressure is high and there are many filesystems mounted or unmounted at the same time, slab shrink will be affected (down_read_trylock() failed). Such as the real workload mentioned by Kirill Tkhai: ``` One of the real workloads from my experience is start of an overcommitted node containing many starting containers after node crash (or many resuming containers after reboot for kernel update). In these cases memory pressure is huge, and the node goes round in long reclaim. ``` 2) If a shrinker is blocked (such as the case mentioned in [1]) and a writer comes in (such as mount a fs), then this writer will be blocked and cause all subsequent shrinker-related operations to be blocked. [1]. https://lore.kernel.org/lkml/20191129214541.3110-1-ptikhomirov@virtuozzo.com/ All the above cases can be solved by replacing the shrinker_rwsem trylocks with SRCU. 2. Survey ========= Before doing the code implementation, I found that there were many similar submissions in the community: a. Davidlohr Bueso submitted a patch in 2015. Subject: [PATCH -next v2] mm: srcu-ify shrinkers Link: https://lore.kernel.org/all/1437080113.3596.2.camel@stgolabs.net/ Result: It was finally merged into the linux-next branch, but failed on arm allnoconfig (without CONFIG_SRCU) b. Tetsuo Handa submitted a patchset in 2017. Subject: [PATCH 1/2] mm,vmscan: Kill global shrinker lock. Link: https://lore.kernel.org/lkml/1510609063-3327-1-git-send-email-penguin-kernel@I-love.SAKURA.ne.jp/ Result: Finally chose to use the current simple way (break when rwsem_is_contended()). And Christoph Hellwig suggested to using SRCU, but SRCU was not unconditionally enabled at the time. c. Kirill Tkhai submitted a patchset in 2018. Subject: [PATCH RFC 00/10] Introduce lockless shrink_slab() Link: https://lore.kernel.org/lkml/153365347929.19074.12509495712735843805.stgit@localhost.localdomain/ Result: At that time, SRCU was not unconditionally enabled, and there were some objections to enabling SRCU. Later, because Kirill's focus was moved to other things, this patchset was not continued to be updated. d. Sultan Alsawaf submitted a patch in 2021. Subject: [PATCH] mm: vmscan: Replace shrinker_rwsem trylocks with SRCU protection Link: https://lore.kernel.org/lkml/20210927074823.5825-1-sultan@kerneltoast.com/ Result: Rejected because SRCU was not unconditionally enabled. We can find that almost all these historical commits were abandoned because SRCU was not unconditionally enabled. But now SRCU has been unconditionally enable by Paul E. McKenney in 2023 [2], so it's time to replace shrinker_rwsem trylocks with SRCU. [2] https://lore.kernel.org/lkml/20230105003759.GA1769545@paulmck-ThinkPad-P17-Gen-1/ 3. Reproduction and testing =========================== We can reproduce the down_read_trylock() hotspot through the following script: ``` #!/bin/bash DIR="/root/shrinker/memcg/mnt" do_create() { mkdir -p /sys/fs/cgroup/memory/test mkdir -p /sys/fs/cgroup/perf_event/test echo 4G > /sys/fs/cgroup/memory/test/memory.limit_in_bytes for i in `seq 0 $1`; do mkdir -p /sys/fs/cgroup/memory/test/$i; echo $$ > /sys/fs/cgroup/memory/test/$i/cgroup.procs; echo $$ > /sys/fs/cgroup/perf_event/test/cgroup.procs; mkdir -p $DIR/$i; done } do_mount() { for i in `seq $1 $2`; do mount -t tmpfs $i $DIR/$i; done } do_touch() { for i in `seq $1 $2`; do echo $$ > /sys/fs/cgroup/memory/test/$i/cgroup.procs; echo $$ > /sys/fs/cgroup/perf_event/test/cgroup.procs; dd if=/dev/zero of=$DIR/$i/file$i bs=1M count=1 & done } case "$1" in touch) do_touch $2 $3 ;; test) do_create 4000 do_mount 0 4000 do_touch 0 3000 ;; *) exit 1 ;; esac ``` Save the above script, then run test and touch commands. Then we can use the following perf command to view hotspots: perf top -U -F 999 1) Before applying this patchset: 32.31% [kernel] [k] down_read_trylock 19.40% [kernel] [k] pv_native_safe_halt 16.24% [kernel] [k] up_read 15.70% [kernel] [k] shrink_slab 4.69% [kernel] [k] _find_next_bit 2.62% [kernel] [k] shrink_node 1.78% [kernel] [k] shrink_lruvec 0.76% [kernel] [k] do_shrink_slab 2) After applying this patchset: 27.83% [kernel] [k] _find_next_bit 16.97% [kernel] [k] shrink_slab 15.82% [kernel] [k] pv_native_safe_halt 9.58% [kernel] [k] shrink_node 8.31% [kernel] [k] shrink_lruvec 5.64% [kernel] [k] do_shrink_slab 3.88% [kernel] [k] mem_cgroup_iter At the same time, we use the following perf command to capture IPC information: perf stat -e cycles,instructions -G test -a --repeat 5 -- sleep 10 1) Before applying this patchset: Performance counter stats for 'system wide' (5 runs): 454187219766 cycles test ( +- 1.84% ) 78896433101 instructions test # 0.17 insn per cycle ( +- 0.44% ) 10.0020430 +- 0.0000366 seconds time elapsed ( +- 0.00% ) 2) After applying this patchset: Performance counter stats for 'system wide' (5 runs): 841954709443 cycles test ( +- 15.80% ) (98.69%) 527258677936 instructions test # 0.63 insn per cycle ( +- 15.11% ) (98.68%) 10.01064 +- 0.00831 seconds time elapsed ( +- 0.08% ) We can see that IPC drops very seriously when calling down_read_trylock() at high frequency. After using SRCU, the IPC is at a normal level. This patch (of 8): To prepare for the subsequent lockless memcg slab shrink, add a map_nr_max field to struct shrinker_info to records its own real shrinker_nr_max. Link: https://lkml.kernel.org/r/20230313112819.38938-1-zhengqi.arch@bytedance.com Link: https://lkml.kernel.org/r/20230313112819.38938-2-zhengqi.arch@bytedance.com Signed-off-by: Qi Zheng Suggested-by: Kirill Tkhai Acked-by: Vlastimil Babka Acked-by: Kirill Tkhai Acked-by: Roman Gushchin Cc: Christian König Cc: David Hildenbrand Cc: Davidlohr Bueso Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Paul E. McKenney Cc: Qi Zheng Cc: Shakeel Butt Cc: Sultan Alsawaf Cc: Tetsuo Handa Cc: Yang Shi Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 1 + mm/vmscan.c | 35 ++++++++++++++++++----------------- 2 files changed, 19 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index b6eda2ab205d..aa69ea98e2d8 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -97,6 +97,7 @@ struct shrinker_info { struct rcu_head rcu; atomic_long_t *nr_deferred; unsigned long *map; + int map_nr_max; }; struct lruvec_stats_percpu { diff --git a/mm/vmscan.c b/mm/vmscan.c index 9414226218f0..9a2a6301052c 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -226,7 +226,8 @@ static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg, static int expand_one_shrinker_info(struct mem_cgroup *memcg, int map_size, int defer_size, - int old_map_size, int old_defer_size) + int old_map_size, int old_defer_size, + int new_nr_max) { struct shrinker_info *new, *old; struct mem_cgroup_per_node *pn; @@ -240,12 +241,17 @@ static int expand_one_shrinker_info(struct mem_cgroup *memcg, if (!old) return 0; + /* Already expanded this shrinker_info */ + if (new_nr_max <= old->map_nr_max) + continue; + new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid); if (!new) return -ENOMEM; new->nr_deferred = (atomic_long_t *)(new + 1); new->map = (void *)new->nr_deferred + defer_size; + new->map_nr_max = new_nr_max; /* map: set all old bits, clear all new bits */ memset(new->map, (int)0xff, old_map_size); @@ -295,6 +301,7 @@ int alloc_shrinker_info(struct mem_cgroup *memcg) } info->nr_deferred = (atomic_long_t *)(info + 1); info->map = (void *)info->nr_deferred + defer_size; + info->map_nr_max = shrinker_nr_max; rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info); } up_write(&shrinker_rwsem); @@ -302,23 +309,14 @@ int alloc_shrinker_info(struct mem_cgroup *memcg) return ret; } -static inline bool need_expand(int nr_max) -{ - return round_up(nr_max, BITS_PER_LONG) > - round_up(shrinker_nr_max, BITS_PER_LONG); -} - static int expand_shrinker_info(int new_id) { int ret = 0; - int new_nr_max = new_id + 1; + int new_nr_max = round_up(new_id + 1, BITS_PER_LONG); int map_size, defer_size = 0; int old_map_size, old_defer_size = 0; struct mem_cgroup *memcg; - if (!need_expand(new_nr_max)) - goto out; - if (!root_mem_cgroup) goto out; @@ -332,7 +330,8 @@ static int expand_shrinker_info(int new_id) memcg = mem_cgroup_iter(NULL, NULL, NULL); do { ret = expand_one_shrinker_info(memcg, map_size, defer_size, - old_map_size, old_defer_size); + old_map_size, old_defer_size, + new_nr_max); if (ret) { mem_cgroup_iter_break(NULL, memcg); goto out; @@ -352,9 +351,11 @@ void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id) rcu_read_lock(); info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info); - /* Pairs with smp mb in shrink_slab() */ - smp_mb__before_atomic(); - set_bit(shrinker_id, info->map); + if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) { + /* Pairs with smp mb in shrink_slab() */ + smp_mb__before_atomic(); + set_bit(shrinker_id, info->map); + } rcu_read_unlock(); } } @@ -432,7 +433,7 @@ void reparent_shrinker_deferred(struct mem_cgroup *memcg) for_each_node(nid) { child_info = shrinker_info_protected(memcg, nid); parent_info = shrinker_info_protected(parent, nid); - for (i = 0; i < shrinker_nr_max; i++) { + for (i = 0; i < child_info->map_nr_max; i++) { nr = atomic_long_read(&child_info->nr_deferred[i]); atomic_long_add(nr, &parent_info->nr_deferred[i]); } @@ -899,7 +900,7 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, if (unlikely(!info)) goto unlock; - for_each_set_bit(i, info->map, shrinker_nr_max) { + for_each_set_bit(i, info->map, info->map_nr_max) { struct shrink_control sc = { .gfp_mask = gfp_mask, .nid = nid, -- cgit v1.2.3 From 3c556d2425b04054e22045d4ef7d34f163b7a71a Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 15 Mar 2023 13:16:42 -0400 Subject: mm/thp: rename TRANSPARENT_HUGEPAGE_NEVER_DAX to _UNSUPPORTED TRANSPARENT_HUGEPAGE_NEVER_DAX has nothing to do with DAX. It's set when has_transparent_hugepage() returns false, checked in hugepage_vma_check() and will disable THP completely if false. Rename it to TRANSPARENT_HUGEPAGE_UNSUPPORTED to reflect its real purpose. [peterx@redhat.com: fix comment, per David] Link: https://lkml.kernel.org/r/ZBMzQW674oHQJV7F@x1n Link: https://lkml.kernel.org/r/20230315171642.1244625-1-peterx@redhat.com Signed-off-by: Peter Xu Acked-by: David Hildenbrand Cc: Aneesh Kumar K.V Cc: Yang Shi Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 2 +- mm/huge_memory.c | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 70bd867eba94..9a3a3af2dd80 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -79,7 +79,7 @@ static inline vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, } enum transparent_hugepage_flag { - TRANSPARENT_HUGEPAGE_NEVER_DAX, + TRANSPARENT_HUGEPAGE_UNSUPPORTED, TRANSPARENT_HUGEPAGE_FLAG, TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 0c1df321ad64..cbf095e7f059 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -88,7 +88,7 @@ bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, /* * If the hardware/firmware marked hugepage support disabled. */ - if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_NEVER_DAX)) + if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED)) return false; /* khugepaged doesn't collapse DAX vma, but page fault is fine. */ @@ -460,11 +460,7 @@ static int __init hugepage_init(void) struct kobject *hugepage_kobj; if (!has_transparent_hugepage()) { - /* - * Hardware doesn't support hugepages, hence disable - * DAX PMD support. - */ - transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_NEVER_DAX; + transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED; return -EINVAL; } -- cgit v1.2.3 From 75a2d4226b53710380d1017b3f4c88f937ddba78 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Sat, 25 Mar 2023 09:45:37 +0100 Subject: driver core: class: mark the struct class for sysfs callbacks as constant struct class should never be modified in a sysfs callback as there is nothing in the structure to modify, and frankly, the structure is almost never used in a sysfs callback, so mark it as constant to allow struct class to be moved to read-only memory. While we are touching all class sysfs callbacks also mark the attribute as constant as it can not be modified. The bonding code still uses this structure so it can not be removed from the function callbacks. Cc: "David S. Miller" Cc: "Rafael J. Wysocki" Cc: Bartosz Golaszewski Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Jens Axboe Cc: Johannes Berg Cc: Linus Walleij Cc: Minchan Kim Cc: Miquel Raynal Cc: Namjae Jeon Cc: Paolo Abeni Cc: Russ Weight Cc: Sergey Senozhatsky Cc: Steve French Cc: Vignesh Raghavendra Cc: linux-cifs@vger.kernel.org Cc: linux-gpio@vger.kernel.org Cc: linux-mtd@lists.infradead.org Cc: linux-rdma@vger.kernel.org Cc: linux-s390@vger.kernel.org Cc: linuxppc-dev@lists.ozlabs.org Cc: netdev@vger.kernel.org Reviewed-by: Luis Chamberlain Link: https://lore.kernel.org/r/20230325084537.3622280-1-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- arch/powerpc/platforms/pseries/dlpar.c | 4 ++-- arch/powerpc/platforms/pseries/mobility.c | 4 ++-- drivers/base/class.c | 4 ++-- drivers/base/devcoredump.c | 4 ++-- drivers/base/firmware_loader/sysfs.c | 4 ++-- drivers/block/pktcdvd.c | 6 +++--- drivers/block/zram/zram_drv.c | 11 +++++------ drivers/gpio/gpiolib-sysfs.c | 8 ++++---- drivers/infiniband/core/user_mad.c | 4 ++-- drivers/mtd/ubi/build.c | 2 +- drivers/net/bonding/bond_sysfs.c | 18 +++++++++--------- drivers/s390/crypto/zcrypt_api.c | 8 ++++---- fs/ksmbd/server.c | 10 +++++----- include/linux/device/class.h | 9 +++++---- 14 files changed, 48 insertions(+), 48 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/platforms/pseries/dlpar.c b/arch/powerpc/platforms/pseries/dlpar.c index 75ffdbcd2865..719c97a155ed 100644 --- a/arch/powerpc/platforms/pseries/dlpar.c +++ b/arch/powerpc/platforms/pseries/dlpar.c @@ -512,7 +512,7 @@ static int dlpar_parse_id_type(char **cmd, struct pseries_hp_errorlog *hp_elog) return 0; } -static ssize_t dlpar_store(struct class *class, struct class_attribute *attr, +static ssize_t dlpar_store(const struct class *class, const struct class_attribute *attr, const char *buf, size_t count) { struct pseries_hp_errorlog hp_elog; @@ -551,7 +551,7 @@ dlpar_store_out: return rc ? rc : count; } -static ssize_t dlpar_show(struct class *class, struct class_attribute *attr, +static ssize_t dlpar_show(const struct class *class, const struct class_attribute *attr, char *buf) { return sprintf(buf, "%s\n", "memory,cpu"); diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c index 643d309d1bd0..6b25642adfa0 100644 --- a/arch/powerpc/platforms/pseries/mobility.c +++ b/arch/powerpc/platforms/pseries/mobility.c @@ -787,8 +787,8 @@ int rtas_syscall_dispatch_ibm_suspend_me(u64 handle) return pseries_migrate_partition(handle); } -static ssize_t migration_store(struct class *class, - struct class_attribute *attr, const char *buf, +static ssize_t migration_store(const struct class *class, + const struct class_attribute *attr, const char *buf, size_t count) { u64 streamid; diff --git a/drivers/base/class.c b/drivers/base/class.c index 8ae91e118827..04de20e0dba8 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -482,8 +482,8 @@ void class_interface_unregister(struct class_interface *class_intf) } EXPORT_SYMBOL_GPL(class_interface_unregister); -ssize_t show_class_attr_string(struct class *class, - struct class_attribute *attr, char *buf) +ssize_t show_class_attr_string(const struct class *class, + const struct class_attribute *attr, char *buf) { struct class_attribute_string *cs; diff --git a/drivers/base/devcoredump.c b/drivers/base/devcoredump.c index 59aaf2e1375a..91536ee05f14 100644 --- a/drivers/base/devcoredump.c +++ b/drivers/base/devcoredump.c @@ -167,7 +167,7 @@ static int devcd_free(struct device *dev, void *data) return 0; } -static ssize_t disabled_show(struct class *class, struct class_attribute *attr, +static ssize_t disabled_show(const struct class *class, const struct class_attribute *attr, char *buf) { return sysfs_emit(buf, "%d\n", devcd_disabled); @@ -197,7 +197,7 @@ static ssize_t disabled_show(struct class *class, struct class_attribute *attr, * so, above situation would not occur. */ -static ssize_t disabled_store(struct class *class, struct class_attribute *attr, +static ssize_t disabled_store(const struct class *class, const struct class_attribute *attr, const char *buf, size_t count) { long tmp = simple_strtol(buf, NULL, 10); diff --git a/drivers/base/firmware_loader/sysfs.c b/drivers/base/firmware_loader/sysfs.c index 56911d75b90a..c9c93b47d9a5 100644 --- a/drivers/base/firmware_loader/sysfs.c +++ b/drivers/base/firmware_loader/sysfs.c @@ -25,7 +25,7 @@ void __fw_load_abort(struct fw_priv *fw_priv) } #ifdef CONFIG_FW_LOADER_USER_HELPER -static ssize_t timeout_show(struct class *class, struct class_attribute *attr, +static ssize_t timeout_show(const struct class *class, const struct class_attribute *attr, char *buf) { return sysfs_emit(buf, "%d\n", __firmware_loading_timeout()); @@ -44,7 +44,7 @@ static ssize_t timeout_show(struct class *class, struct class_attribute *attr, * * Note: zero means 'wait forever'. **/ -static ssize_t timeout_store(struct class *class, struct class_attribute *attr, +static ssize_t timeout_store(const struct class *class, const struct class_attribute *attr, const char *buf, size_t count) { int tmp_loading_timeout = simple_strtol(buf, NULL, 10); diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 642e3377441a..ba9bbdef9ef5 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -343,7 +343,7 @@ static void class_pktcdvd_release(struct class *cls) kfree(cls); } -static ssize_t device_map_show(struct class *c, struct class_attribute *attr, +static ssize_t device_map_show(const struct class *c, const struct class_attribute *attr, char *data) { int n = 0; @@ -364,7 +364,7 @@ static ssize_t device_map_show(struct class *c, struct class_attribute *attr, } static CLASS_ATTR_RO(device_map); -static ssize_t add_store(struct class *c, struct class_attribute *attr, +static ssize_t add_store(const struct class *c, const struct class_attribute *attr, const char *buf, size_t count) { unsigned int major, minor; @@ -385,7 +385,7 @@ static ssize_t add_store(struct class *c, struct class_attribute *attr, } static CLASS_ATTR_WO(add); -static ssize_t remove_store(struct class *c, struct class_attribute *attr, +static ssize_t remove_store(const struct class *c, const struct class_attribute *attr, const char *buf, size_t count) { unsigned int major, minor; diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index b7bb52f8dfbd..3feadfb96114 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -2424,8 +2424,8 @@ static int zram_remove(struct zram *zram) * creates a new un-initialized zram device and returns back this device's * device_id (or an error code if it fails to create a new device). */ -static ssize_t hot_add_show(struct class *class, - struct class_attribute *attr, +static ssize_t hot_add_show(const struct class *class, + const struct class_attribute *attr, char *buf) { int ret; @@ -2438,11 +2438,10 @@ static ssize_t hot_add_show(struct class *class, return ret; return scnprintf(buf, PAGE_SIZE, "%d\n", ret); } -static struct class_attribute class_attr_hot_add = - __ATTR(hot_add, 0400, hot_add_show, NULL); +static CLASS_ATTR_RO(hot_add); -static ssize_t hot_remove_store(struct class *class, - struct class_attribute *attr, +static ssize_t hot_remove_store(const struct class *class, + const struct class_attribute *attr, const char *buf, size_t count) { diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 774755052618..a895915affa5 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -426,8 +426,8 @@ ATTRIBUTE_GROUPS(gpiochip); * /sys/class/gpio/unexport ... write-only * integer N ... number of GPIO to unexport */ -static ssize_t export_store(struct class *class, - struct class_attribute *attr, +static ssize_t export_store(const struct class *class, + const struct class_attribute *attr, const char *buf, size_t len) { long gpio; @@ -478,8 +478,8 @@ done: } static CLASS_ATTR_WO(export); -static ssize_t unexport_store(struct class *class, - struct class_attribute *attr, +static ssize_t unexport_store(const struct class *class, + const struct class_attribute *attr, const char *buf, size_t len) { long gpio; diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c index f83954180a33..0e9e04f8c685 100644 --- a/drivers/infiniband/core/user_mad.c +++ b/drivers/infiniband/core/user_mad.c @@ -1229,8 +1229,8 @@ static char *umad_devnode(const struct device *dev, umode_t *mode) return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev)); } -static ssize_t abi_version_show(struct class *class, - struct class_attribute *attr, char *buf) +static ssize_t abi_version_show(const struct class *class, + const struct class_attribute *attr, char *buf) { return sysfs_emit(buf, "%d\n", IB_USER_MAD_ABI_VERSION); } diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c index ae6d35e3da9c..32105bd35831 100644 --- a/drivers/mtd/ubi/build.c +++ b/drivers/mtd/ubi/build.c @@ -95,7 +95,7 @@ static DEFINE_SPINLOCK(ubi_devices_lock); /* "Show" method for files in '//class/ubi/' */ /* UBI version attribute ('//class/ubi/version') */ -static ssize_t version_show(struct class *class, struct class_attribute *attr, +static ssize_t version_show(const struct class *class, const struct class_attribute *attr, char *buf) { return sprintf(buf, "%d\n", UBI_VERSION); diff --git a/drivers/net/bonding/bond_sysfs.c b/drivers/net/bonding/bond_sysfs.c index 8996bd0a194a..0bb59da24922 100644 --- a/drivers/net/bonding/bond_sysfs.c +++ b/drivers/net/bonding/bond_sysfs.c @@ -31,12 +31,12 @@ /* "show" function for the bond_masters attribute. * The class parameter is ignored. */ -static ssize_t bonding_show_bonds(struct class *cls, - struct class_attribute *attr, +static ssize_t bonding_show_bonds(const struct class *cls, + const struct class_attribute *attr, char *buf) { - struct bond_net *bn = - container_of(attr, struct bond_net, class_attr_bonding_masters); + const struct bond_net *bn = + container_of_const(attr, struct bond_net, class_attr_bonding_masters); int res = 0; struct bonding *bond; @@ -59,7 +59,7 @@ static ssize_t bonding_show_bonds(struct class *cls, return res; } -static struct net_device *bond_get_by_name(struct bond_net *bn, const char *ifname) +static struct net_device *bond_get_by_name(const struct bond_net *bn, const char *ifname) { struct bonding *bond; @@ -75,12 +75,12 @@ static struct net_device *bond_get_by_name(struct bond_net *bn, const char *ifna * * The class parameter is ignored. */ -static ssize_t bonding_store_bonds(struct class *cls, - struct class_attribute *attr, +static ssize_t bonding_store_bonds(const struct class *cls, + const struct class_attribute *attr, const char *buffer, size_t count) { - struct bond_net *bn = - container_of(attr, struct bond_net, class_attr_bonding_masters); + const struct bond_net *bn = + container_of_const(attr, struct bond_net, class_attr_bonding_masters); char command[IFNAMSIZ + 1] = {0, }; char *ifname; int rv, res = count; diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 582ac301d315..cff2eea88f98 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -340,8 +340,8 @@ static const struct attribute_group *zcdn_dev_attr_groups[] = { NULL }; -static ssize_t zcdn_create_store(struct class *class, - struct class_attribute *attr, +static ssize_t zcdn_create_store(const struct class *class, + const struct class_attribute *attr, const char *buf, size_t count) { int rc; @@ -357,8 +357,8 @@ static ssize_t zcdn_create_store(struct class *class, static const struct class_attribute class_attr_zcdn_create = __ATTR(create, 0600, NULL, zcdn_create_store); -static ssize_t zcdn_destroy_store(struct class *class, - struct class_attribute *attr, +static ssize_t zcdn_destroy_store(const struct class *class, + const struct class_attribute *attr, const char *buf, size_t count) { int rc; diff --git a/fs/ksmbd/server.c b/fs/ksmbd/server.c index b5af3e43e677..c2c958a5423e 100644 --- a/fs/ksmbd/server.c +++ b/fs/ksmbd/server.c @@ -418,7 +418,7 @@ int server_queue_ctrl_reset_work(void) return __queue_ctrl_work(SERVER_CTRL_TYPE_RESET); } -static ssize_t stats_show(struct class *class, struct class_attribute *attr, +static ssize_t stats_show(const struct class *class, const struct class_attribute *attr, char *buf) { /* @@ -437,8 +437,8 @@ static ssize_t stats_show(struct class *class, struct class_attribute *attr, server_conf.ipc_last_active / HZ); } -static ssize_t kill_server_store(struct class *class, - struct class_attribute *attr, const char *buf, +static ssize_t kill_server_store(const struct class *class, + const struct class_attribute *attr, const char *buf, size_t len) { if (!sysfs_streq(buf, "hard")) @@ -458,7 +458,7 @@ static const char * const debug_type_strings[] = {"smb", "auth", "vfs", "oplock", "ipc", "conn", "rdma"}; -static ssize_t debug_show(struct class *class, struct class_attribute *attr, +static ssize_t debug_show(const struct class *class, const struct class_attribute *attr, char *buf) { ssize_t sz = 0; @@ -476,7 +476,7 @@ static ssize_t debug_show(struct class *class, struct class_attribute *attr, return sz; } -static ssize_t debug_store(struct class *class, struct class_attribute *attr, +static ssize_t debug_store(const struct class *class, const struct class_attribute *attr, const char *buf, size_t len) { int i; diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 73436c578d37..b53728ca56fb 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -174,10 +174,10 @@ static inline struct device *class_find_device_by_acpi_dev(const struct class *c struct class_attribute { struct attribute attr; - ssize_t (*show)(struct class *class, struct class_attribute *attr, + ssize_t (*show)(const struct class *class, const struct class_attribute *attr, char *buf); - ssize_t (*store)(struct class *class, struct class_attribute *attr, - const char *buf, size_t count); + ssize_t (*store)(const struct class *class, const struct class_attribute *attr, + const char *buf, size_t count); }; #define CLASS_ATTR_RW(_name) \ @@ -217,7 +217,8 @@ struct class_attribute_string { struct class_attribute_string class_attr_##_name = \ _CLASS_ATTR_STRING(_name, _mode, _str) -ssize_t show_class_attr_string(struct class *class, struct class_attribute *attr, char *buf); +ssize_t show_class_attr_string(const struct class *class, const struct class_attribute *attr, + char *buf); struct class_interface { struct list_head node; -- cgit v1.2.3 From 130eac4170859fb368681e00d390f20f44bbf27b Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 28 Mar 2023 15:10:43 +0200 Subject: xhci: use pm_ptr() instead of #ifdef for CONFIG_PM conditionals A recent patch caused an unused-function warning in builds with CONFIG_PM disabled, after the function became marked 'static': drivers/usb/host/xhci-pci.c:91:13: error: 'xhci_msix_sync_irqs' defined but not used [-Werror=unused-function] 91 | static void xhci_msix_sync_irqs(struct xhci_hcd *xhci) | ^~~~~~~~~~~~~~~~~~~ This could be solved by adding another #ifdef, but as there is a trend towards removing CONFIG_PM checks in favor of helper macros, do the same conversion here and use pm_ptr() to get either a function pointer or NULL but avoid the warning. As the hidden functions reference some other symbols, make sure those are visible at compile time, at the minimal cost of a few extra bytes for 'struct usb_device'. Fixes: 9abe15d55dcc ("xhci: Move xhci MSI sync function to to xhci-pci") Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20230328131114.1296430-1-arnd@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/usb/host/xhci-pci.c | 16 +++++----------- include/linux/usb.h | 3 +-- include/linux/usb/hcd.h | 2 -- 3 files changed, 6 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index a53ecc8ff8c5..bbbb01282038 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -728,7 +728,6 @@ static void xhci_pci_remove(struct pci_dev *dev) usb_hcd_pci_remove(dev); } -#ifdef CONFIG_PM /* * In some Intel xHCI controllers, in order to get D3 working, * through a vendor specific SSIC CONFIG register at offset 0x883c, @@ -927,7 +926,6 @@ static void xhci_pci_shutdown(struct usb_hcd *hcd) if (xhci->quirks & XHCI_SPURIOUS_WAKEUP) pci_set_power_state(pdev, PCI_D3hot); } -#endif /* CONFIG_PM */ /*-------------------------------------------------------------------------*/ @@ -970,9 +968,7 @@ static struct pci_driver xhci_pci_driver = { .shutdown = usb_hcd_pci_shutdown, .driver = { -#ifdef CONFIG_PM - .pm = &usb_hcd_pci_pm_ops, -#endif + .pm = pm_ptr(&usb_hcd_pci_pm_ops), .probe_type = PROBE_PREFER_ASYNCHRONOUS, }, }; @@ -980,12 +976,10 @@ static struct pci_driver xhci_pci_driver = { static int __init xhci_pci_init(void) { xhci_init_driver(&xhci_pci_hc_driver, &xhci_pci_overrides); -#ifdef CONFIG_PM - xhci_pci_hc_driver.pci_suspend = xhci_pci_suspend; - xhci_pci_hc_driver.pci_resume = xhci_pci_resume; - xhci_pci_hc_driver.pci_poweroff_late = xhci_pci_poweroff_late; - xhci_pci_hc_driver.shutdown = xhci_pci_shutdown; -#endif + xhci_pci_hc_driver.pci_suspend = pm_ptr(xhci_pci_suspend); + xhci_pci_hc_driver.pci_resume = pm_ptr(xhci_pci_resume); + xhci_pci_hc_driver.pci_poweroff_late = pm_ptr(xhci_pci_poweroff_late); + xhci_pci_hc_driver.shutdown = pm_ptr(xhci_pci_shutdown); xhci_pci_hc_driver.stop = xhci_pci_stop; return pci_register_driver(&xhci_pci_driver); } diff --git a/include/linux/usb.h b/include/linux/usb.h index 9642ee02d713..d510fabcafa2 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -704,13 +704,12 @@ struct usb_device { unsigned long active_duration; -#ifdef CONFIG_PM unsigned long connect_time; unsigned do_remote_wakeup:1; unsigned reset_resume:1; unsigned port_is_suspended:1; -#endif + struct wusb_dev *wusb_dev; int slot_id; struct usb2_lpm_parameters l1_params; diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index b51c07111729..094c77eaf455 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -488,9 +488,7 @@ extern void usb_hcd_pci_shutdown(struct pci_dev *dev); extern int usb_hcd_amd_remote_wakeup_quirk(struct pci_dev *dev); -#ifdef CONFIG_PM extern const struct dev_pm_ops usb_hcd_pci_pm_ops; -#endif #endif /* CONFIG_USB_PCI */ /* pci-ish (pdev null is ok) buffer alloc/mapping support */ -- cgit v1.2.3 From 77f7eb9f3416aace703971156133926e44e2195b Mon Sep 17 00:00:00 2001 From: Patrisious Haddad Date: Thu, 23 Mar 2023 12:13:51 +0200 Subject: net/mlx5: Introduce other vport query for Q-counters These new fields in QUERY_Q_COUNTER command allow us to access another vport counters during the query command, which is specially useful to query representor vports. In addition also add the required caps to check if this capability is actually supported. Signed-off-by: Patrisious Haddad Reviewed-by: Michael Guralnik Link: https://lore.kernel.org/r/75c73a4a0e60f18c37b35a4a11ca2e2415e4a6f3.1679566038.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- include/linux/mlx5/mlx5_ifc.h | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 62039b147432..e4306cd87cd7 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1729,7 +1729,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 log_max_transport_domain[0x5]; u8 reserved_at_328[0x3]; u8 log_max_pd[0x5]; - u8 reserved_at_330[0xb]; + u8 reserved_at_330[0x9]; + u8 q_counter_aggregation[0x1]; + u8 q_counter_other_vport[0x1]; u8 log_max_xrcd[0x5]; u8 nic_receive_steering_discard[0x1]; @@ -5603,10 +5605,15 @@ struct mlx5_ifc_query_q_counter_in_bits { u8 reserved_at_20[0x10]; u8 op_mod[0x10]; - u8 reserved_at_40[0x80]; + u8 other_vport[0x1]; + u8 reserved_at_41[0xf]; + u8 vport_number[0x10]; + + u8 reserved_at_60[0x60]; u8 clear[0x1]; - u8 reserved_at_c1[0x1f]; + u8 aggregate[0x1]; + u8 reserved_at_c2[0x1e]; u8 reserved_at_e0[0x18]; u8 counter_set_id[0x8]; -- cgit v1.2.3 From b93c2a68f3d9dc98ec30dcb342ae47c1c8d09d18 Mon Sep 17 00:00:00 2001 From: Elson Roy Serrao Date: Fri, 24 Mar 2023 14:47:57 -0700 Subject: usb: gadget: Properly configure the device for remote wakeup The wakeup bit in the bmAttributes field indicates whether the device is configured for remote wakeup. But this field should be allowed to set only if the UDC supports such wakeup mechanism. So configure this field based on UDC capability. Also inform the UDC whether the device is configured for remote wakeup by implementing a gadget op. Reviewed-by: Thinh Nguyen Signed-off-by: Elson Roy Serrao Link: https://lore.kernel.org/r/1679694482-16430-2-git-send-email-quic_eserrao@quicinc.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/composite.c | 18 ++++++++++++++++++ drivers/usb/gadget/configfs.c | 3 +++ drivers/usb/gadget/udc/core.c | 27 +++++++++++++++++++++++++++ drivers/usb/gadget/udc/trace.h | 5 +++++ include/linux/usb/composite.h | 2 ++ include/linux/usb/gadget.h | 8 ++++++++ 6 files changed, 63 insertions(+) (limited to 'include/linux') diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c index f1ecd12b62c3..00c89571de2a 100644 --- a/drivers/usb/gadget/composite.c +++ b/drivers/usb/gadget/composite.c @@ -513,6 +513,19 @@ static u8 encode_bMaxPower(enum usb_device_speed speed, return min(val, 900U) / 8; } +void check_remote_wakeup_config(struct usb_gadget *g, + struct usb_configuration *c) +{ + if (USB_CONFIG_ATT_WAKEUP & c->bmAttributes) { + /* Reset the rw bit if gadget is not capable of it */ + if (!g->wakeup_capable && g->ops->set_remote_wakeup) { + WARN(c->cdev, "Clearing wakeup bit for config c.%d\n", + c->bConfigurationValue); + c->bmAttributes &= ~USB_CONFIG_ATT_WAKEUP; + } + } +} + static int config_buf(struct usb_configuration *config, enum usb_device_speed speed, void *buf, u8 type) { @@ -994,6 +1007,11 @@ static int set_config(struct usb_composite_dev *cdev, power = min(power, 500U); else power = min(power, 900U); + + if (USB_CONFIG_ATT_WAKEUP & c->bmAttributes) + usb_gadget_set_remote_wakeup(gadget, 1); + else + usb_gadget_set_remote_wakeup(gadget, 0); done: if (power <= USB_SELF_POWER_VBUS_MAX_DRAW) usb_gadget_set_selfpowered(gadget); diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index b9f1136aa0a2..4c639e9ddedc 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -1761,6 +1761,9 @@ static int configfs_composite_bind(struct usb_gadget *gadget, if (gadget_is_otg(gadget)) c->descriptors = otg_desc; + /* Properly configure the bmAttributes wakeup bit */ + check_remote_wakeup_config(gadget, c); + cfg = container_of(c, struct config_usb_cfg, c); if (!list_empty(&cfg->string_list)) { i = 0; diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index 23b0629a8774..3dcbba739db6 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -513,6 +513,33 @@ out: } EXPORT_SYMBOL_GPL(usb_gadget_wakeup); +/** + * usb_gadget_set_remote_wakeup - configures the device remote wakeup feature. + * @gadget:the device being configured for remote wakeup + * @set:value to be configured. + * + * set to one to enable remote wakeup feature and zero to disable it. + * + * returns zero on success, else negative errno. + */ +int usb_gadget_set_remote_wakeup(struct usb_gadget *gadget, int set) +{ + int ret = 0; + + if (!gadget->ops->set_remote_wakeup) { + ret = -EOPNOTSUPP; + goto out; + } + + ret = gadget->ops->set_remote_wakeup(gadget, set); + +out: + trace_usb_gadget_set_remote_wakeup(gadget, ret); + + return ret; +} +EXPORT_SYMBOL_GPL(usb_gadget_set_remote_wakeup); + /** * usb_gadget_set_selfpowered - sets the device selfpowered feature. * @gadget:the device being declared as self-powered diff --git a/drivers/usb/gadget/udc/trace.h b/drivers/usb/gadget/udc/trace.h index abdbcb1bacb0..a5ed26fbc2da 100644 --- a/drivers/usb/gadget/udc/trace.h +++ b/drivers/usb/gadget/udc/trace.h @@ -91,6 +91,11 @@ DEFINE_EVENT(udc_log_gadget, usb_gadget_wakeup, TP_ARGS(g, ret) ); +DEFINE_EVENT(udc_log_gadget, usb_gadget_set_remote_wakeup, + TP_PROTO(struct usb_gadget *g, int ret), + TP_ARGS(g, ret) +); + DEFINE_EVENT(udc_log_gadget, usb_gadget_set_selfpowered, TP_PROTO(struct usb_gadget *g, int ret), TP_ARGS(g, ret) diff --git a/include/linux/usb/composite.h b/include/linux/usb/composite.h index 608dc962748b..d949e91ceb48 100644 --- a/include/linux/usb/composite.h +++ b/include/linux/usb/composite.h @@ -413,6 +413,8 @@ extern int composite_dev_prepare(struct usb_composite_driver *composite, extern int composite_os_desc_req_prepare(struct usb_composite_dev *cdev, struct usb_ep *ep0); void composite_dev_cleanup(struct usb_composite_dev *cdev); +void check_remote_wakeup_config(struct usb_gadget *g, + struct usb_configuration *c); static inline struct usb_composite_driver *to_cdriver( struct usb_gadget_driver *gdrv) diff --git a/include/linux/usb/gadget.h b/include/linux/usb/gadget.h index 00750f7020f3..1d796128030b 100644 --- a/include/linux/usb/gadget.h +++ b/include/linux/usb/gadget.h @@ -310,6 +310,7 @@ struct usb_udc; struct usb_gadget_ops { int (*get_frame)(struct usb_gadget *); int (*wakeup)(struct usb_gadget *); + int (*set_remote_wakeup)(struct usb_gadget *, int set); int (*set_selfpowered) (struct usb_gadget *, int is_selfpowered); int (*vbus_session) (struct usb_gadget *, int is_active); int (*vbus_draw) (struct usb_gadget *, unsigned mA); @@ -384,6 +385,8 @@ struct usb_gadget_ops { * @connected: True if gadget is connected. * @lpm_capable: If the gadget max_speed is FULL or HIGH, this flag * indicates that it supports LPM as per the LPM ECN & errata. + * @wakeup_capable: True if gadget is capable of sending remote wakeup. + * @wakeup_armed: True if gadget is armed by the host for remote wakeup. * @irq: the interrupt number for device controller. * @id_number: a unique ID number for ensuring that gadget names are distinct * @@ -445,6 +448,8 @@ struct usb_gadget { unsigned deactivated:1; unsigned connected:1; unsigned lpm_capable:1; + unsigned wakeup_capable:1; + unsigned wakeup_armed:1; int irq; int id_number; }; @@ -601,6 +606,7 @@ static inline int gadget_is_otg(struct usb_gadget *g) #if IS_ENABLED(CONFIG_USB_GADGET) int usb_gadget_frame_number(struct usb_gadget *gadget); int usb_gadget_wakeup(struct usb_gadget *gadget); +int usb_gadget_set_remote_wakeup(struct usb_gadget *gadget, int set); int usb_gadget_set_selfpowered(struct usb_gadget *gadget); int usb_gadget_clear_selfpowered(struct usb_gadget *gadget); int usb_gadget_vbus_connect(struct usb_gadget *gadget); @@ -616,6 +622,8 @@ static inline int usb_gadget_frame_number(struct usb_gadget *gadget) { return 0; } static inline int usb_gadget_wakeup(struct usb_gadget *gadget) { return 0; } +static inline int usb_gadget_set_remote_wakeup(struct usb_gadget *gadget, int set) +{ return 0; } static inline int usb_gadget_set_selfpowered(struct usb_gadget *gadget) { return 0; } static inline int usb_gadget_clear_selfpowered(struct usb_gadget *gadget) -- cgit v1.2.3 From f0db885fb05d35befa81896db6b19eb3ee9ccdfe Mon Sep 17 00:00:00 2001 From: Elson Roy Serrao Date: Fri, 24 Mar 2023 14:47:59 -0700 Subject: usb: gadget: Add function wakeup support USB3.2 spec section 9.2.5.4 quotes that a function may signal that it wants to exit from Function Suspend by sending a Function Wake Notification to the host if it is enabled for function remote wakeup. Add an api in composite layer that can be used by the function drivers to support this feature. Also expose a gadget op so that composite layer can trigger a wakeup request to the UDC driver. Reviewed-by: Thinh Nguyen Signed-off-by: Elson Roy Serrao Link: https://lore.kernel.org/r/1679694482-16430-4-git-send-email-quic_eserrao@quicinc.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/composite.c | 40 ++++++++++++++++++++++++++++++++++++++++ include/linux/usb/composite.h | 6 ++++++ include/linux/usb/gadget.h | 1 + 3 files changed, 47 insertions(+) (limited to 'include/linux') diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c index 00c89571de2a..d3fd2d1d8096 100644 --- a/drivers/usb/gadget/composite.c +++ b/drivers/usb/gadget/composite.c @@ -492,6 +492,46 @@ int usb_interface_id(struct usb_configuration *config, } EXPORT_SYMBOL_GPL(usb_interface_id); +/** + * usb_func_wakeup - sends function wake notification to the host. + * @func: function that sends the remote wakeup notification. + * + * Applicable to devices operating at enhanced superspeed when usb + * functions are put in function suspend state and armed for function + * remote wakeup. On completion, function wake notification is sent. If + * the device is in low power state it tries to bring the device to active + * state before sending the wake notification. Since it is a synchronous + * call, caller must take care of not calling it in interrupt context. + * For devices operating at lower speeds returns negative errno. + * + * Returns zero on success, else negative errno. + */ +int usb_func_wakeup(struct usb_function *func) +{ + struct usb_gadget *gadget = func->config->cdev->gadget; + int id; + + if (!gadget->ops->func_wakeup) + return -EOPNOTSUPP; + + if (!func->func_wakeup_armed) { + ERROR(func->config->cdev, "not armed for func remote wakeup\n"); + return -EINVAL; + } + + for (id = 0; id < MAX_CONFIG_INTERFACES; id++) + if (func->config->interface[id] == func) + break; + + if (id == MAX_CONFIG_INTERFACES) { + ERROR(func->config->cdev, "Invalid function\n"); + return -EINVAL; + } + + return gadget->ops->func_wakeup(gadget, id); +} +EXPORT_SYMBOL_GPL(usb_func_wakeup); + static u8 encode_bMaxPower(enum usb_device_speed speed, struct usb_configuration *c) { diff --git a/include/linux/usb/composite.h b/include/linux/usb/composite.h index d949e91ceb48..a2448e98854f 100644 --- a/include/linux/usb/composite.h +++ b/include/linux/usb/composite.h @@ -150,6 +150,9 @@ struct usb_os_desc_table { * GetStatus() request when the recipient is Interface. * @func_suspend: callback to be called when * SetFeature(FUNCTION_SUSPEND) is reseived + * @func_suspended: Indicates whether the function is in function suspend state. + * @func_wakeup_armed: Indicates whether the function is armed by the host for + * wakeup signaling. * * A single USB function uses one or more interfaces, and should in most * cases support operation at both full and high speeds. Each function is @@ -220,6 +223,8 @@ struct usb_function { int (*get_status)(struct usb_function *); int (*func_suspend)(struct usb_function *, u8 suspend_opt); + bool func_suspended; + bool func_wakeup_armed; /* private: */ /* internals */ struct list_head list; @@ -241,6 +246,7 @@ int config_ep_by_speed_and_alt(struct usb_gadget *g, struct usb_function *f, int config_ep_by_speed(struct usb_gadget *g, struct usb_function *f, struct usb_ep *_ep); +int usb_func_wakeup(struct usb_function *func); #define MAX_CONFIG_INTERFACES 16 /* arbitrary; max 255 */ diff --git a/include/linux/usb/gadget.h b/include/linux/usb/gadget.h index 1d796128030b..75bda0783395 100644 --- a/include/linux/usb/gadget.h +++ b/include/linux/usb/gadget.h @@ -310,6 +310,7 @@ struct usb_udc; struct usb_gadget_ops { int (*get_frame)(struct usb_gadget *); int (*wakeup)(struct usb_gadget *); + int (*func_wakeup)(struct usb_gadget *gadget, int intf_id); int (*set_remote_wakeup)(struct usb_gadget *, int set); int (*set_selfpowered) (struct usb_gadget *, int is_selfpowered); int (*vbus_session) (struct usb_gadget *, int is_active); -- cgit v1.2.3 From 2b76ffe81e32afd6d318dc4547e2ba8c46207b77 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 28 Mar 2023 19:15:29 -0700 Subject: linux/vt_buffer.h: allow either builtin or modular for macros Fix build errors on ARCH=alpha when CONFIG_MDA_CONSOLE=m. This allows the ARCH macros to be the only ones defined. In file included from ../drivers/video/console/mdacon.c:37: ../arch/alpha/include/asm/vga.h:17:40: error: expected identifier or '(' before 'volatile' 17 | static inline void scr_writew(u16 val, volatile u16 *addr) | ^~~~~~~~ ../include/linux/vt_buffer.h:24:34: note: in definition of macro 'scr_writew' 24 | #define scr_writew(val, addr) (*(addr) = (val)) | ^~~~ ../include/linux/vt_buffer.h:24:40: error: expected ')' before '=' token 24 | #define scr_writew(val, addr) (*(addr) = (val)) | ^ ../arch/alpha/include/asm/vga.h:17:20: note: in expansion of macro 'scr_writew' 17 | static inline void scr_writew(u16 val, volatile u16 *addr) | ^~~~~~~~~~ ../arch/alpha/include/asm/vga.h:25:29: error: expected identifier or '(' before 'volatile' 25 | static inline u16 scr_readw(volatile const u16 *addr) | ^~~~~~~~ Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: Randy Dunlap Cc: Greg Kroah-Hartman Cc: Jiri Slaby Cc: dri-devel@lists.freedesktop.org Cc: linux-fbdev@vger.kernel.org Link: https://lore.kernel.org/r/20230329021529.16188-1-rdunlap@infradead.org Signed-off-by: Greg Kroah-Hartman --- include/linux/vt_buffer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/vt_buffer.h b/include/linux/vt_buffer.h index 848db1b1569f..919d999a8c1d 100644 --- a/include/linux/vt_buffer.h +++ b/include/linux/vt_buffer.h @@ -16,7 +16,7 @@ #include -#if defined(CONFIG_VGA_CONSOLE) || defined(CONFIG_MDA_CONSOLE) +#if IS_ENABLED(CONFIG_VGA_CONSOLE) || IS_ENABLED(CONFIG_MDA_CONSOLE) #include #endif -- cgit v1.2.3 From 2959ab247061e67485d83b6af8feb3761ec08cb9 Mon Sep 17 00:00:00 2001 From: Nipun Gupta Date: Mon, 13 Mar 2023 18:56:30 +0530 Subject: cdx: add the cdx bus driver Introduce AMD CDX bus, which provides a mechanism for scanning and probing CDX devices. These devices are memory mapped on system bus for Application Processors(APUs). CDX devices can be changed dynamically in the Fabric and CDX bus interacts with CDX controller to rescan the bus and rediscover the devices. Signed-off-by: Nipun Gupta Reviewed-by: Pieter Jansen van Vuuren Tested-by: Nikhil Agarwal Link: https://lore.kernel.org/r/20230313132636.31850-2-nipun.gupta@amd.com Signed-off-by: Greg Kroah-Hartman --- Documentation/ABI/testing/sysfs-bus-cdx | 12 + MAINTAINERS | 7 + drivers/Kconfig | 2 + drivers/Makefile | 1 + drivers/cdx/Kconfig | 17 ++ drivers/cdx/Makefile | 8 + drivers/cdx/cdx.c | 408 ++++++++++++++++++++++++++++++++ drivers/cdx/cdx.h | 62 +++++ include/linux/cdx/cdx_bus.h | 147 ++++++++++++ include/linux/mod_devicetable.h | 15 ++ scripts/mod/devicetable-offsets.c | 4 + scripts/mod/file2alias.c | 12 + 12 files changed, 695 insertions(+) create mode 100644 Documentation/ABI/testing/sysfs-bus-cdx create mode 100644 drivers/cdx/Kconfig create mode 100644 drivers/cdx/Makefile create mode 100644 drivers/cdx/cdx.c create mode 100644 drivers/cdx/cdx.h create mode 100644 include/linux/cdx/cdx_bus.h (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-bus-cdx b/Documentation/ABI/testing/sysfs-bus-cdx new file mode 100644 index 000000000000..43b4e161f226 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-cdx @@ -0,0 +1,12 @@ +What: /sys/bus/cdx/rescan +Date: March 2023 +Contact: nipun.gupta@amd.com +Description: + Writing y/1/on to this file will cause rescan of the bus + and devices on the CDX bus. Any new devices are scanned and + added to the list of Linux devices and any devices removed are + also deleted from Linux. + + For example:: + + # echo 1 > /sys/bus/cdx/rescan diff --git a/MAINTAINERS b/MAINTAINERS index 619c30ec36dc..973cfbd3e3e0 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -964,6 +964,13 @@ Q: https://patchwork.kernel.org/project/linux-rdma/list/ F: drivers/infiniband/hw/efa/ F: include/uapi/rdma/efa-abi.h +AMD CDX BUS DRIVER +M: Nipun Gupta +M: Nikhil Agarwal +S: Maintained +F: drivers/cdx/* +F: include/linux/cdx/* + AMD CRYPTOGRAPHIC COPROCESSOR (CCP) DRIVER M: Tom Lendacky M: John Allen diff --git a/drivers/Kconfig b/drivers/Kconfig index 968bd0a6fd78..514ae6b24cb2 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -241,4 +241,6 @@ source "drivers/peci/Kconfig" source "drivers/hte/Kconfig" +source "drivers/cdx/Kconfig" + endmenu diff --git a/drivers/Makefile b/drivers/Makefile index 20b118dca999..7241d80a7b29 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -194,3 +194,4 @@ obj-$(CONFIG_MOST) += most/ obj-$(CONFIG_PECI) += peci/ obj-$(CONFIG_HTE) += hte/ obj-$(CONFIG_DRM_ACCEL) += accel/ +obj-$(CONFIG_CDX_BUS) += cdx/ diff --git a/drivers/cdx/Kconfig b/drivers/cdx/Kconfig new file mode 100644 index 000000000000..1aaee0ec5bd9 --- /dev/null +++ b/drivers/cdx/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# CDX bus configuration +# +# Copyright (C) 2022-2023, Advanced Micro Devices, Inc. +# + +config CDX_BUS + bool "CDX Bus driver" + depends on OF && ARM64 + help + Driver to enable Composable DMA Transfer(CDX) Bus. CDX bus + exposes Fabric devices which uses composable DMA IP to the + APU. CDX bus provides a mechanism for scanning and probing + of CDX devices. CDX devices are memory mapped on system bus + for embedded CPUs. CDX bus uses CDX controller and firmware + to scan these CDX devices. diff --git a/drivers/cdx/Makefile b/drivers/cdx/Makefile new file mode 100644 index 000000000000..c5feb11fc718 --- /dev/null +++ b/drivers/cdx/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for CDX +# +# Copyright (C) 2022-2023, Advanced Micro Devices, Inc. +# + +obj-$(CONFIG_CDX_BUS) += cdx.o diff --git a/drivers/cdx/cdx.c b/drivers/cdx/cdx.c new file mode 100644 index 000000000000..6b162bbb9b25 --- /dev/null +++ b/drivers/cdx/cdx.c @@ -0,0 +1,408 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * CDX bus driver. + * + * Copyright (C) 2022-2023, Advanced Micro Devices, Inc. + */ + +/* + * Architecture Overview + * ===================== + * CDX is a Hardware Architecture designed for AMD FPGA devices. It + * consists of sophisticated mechanism for interaction between FPGA, + * Firmware and the APUs (Application CPUs). + * + * Firmware resides on RPU (Realtime CPUs) which interacts with + * the FPGA program manager and the APUs. The RPU provides memory-mapped + * interface (RPU if) which is used to communicate with APUs. + * + * The diagram below shows an overview of the CDX architecture: + * + * +--------------------------------------+ + * | Application CPUs (APU) | + * | | + * | CDX device drivers| + * | Linux OS | | + * | CDX bus | + * | | | + * | CDX controller | + * | | | + * +-----------------------------|--------+ + * | (discover, config, + * | reset, rescan) + * | + * +------------------------| RPU if |----+ + * | | | + * | V | + * | Realtime CPUs (RPU) | + * | | + * +--------------------------------------+ + * | + * +---------------------|----------------+ + * | FPGA | | + * | +-----------------------+ | + * | | | | | + * | +-------+ +-------+ +-------+ | + * | | dev 1 | | dev 2 | | dev 3 | | + * | +-------+ +-------+ +-------+ | + * +--------------------------------------+ + * + * The RPU firmware extracts the device information from the loaded FPGA + * image and implements a mechanism that allows the APU drivers to + * enumerate such devices (device personality and resource details) via + * a dedicated communication channel. RPU mediates operations such as + * discover, reset and rescan of the FPGA devices for the APU. This is + * done using memory mapped interface provided by the RPU to APU. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "cdx.h" + +/* Default DMA mask for devices on a CDX bus */ +#define CDX_DEFAULT_DMA_MASK (~0ULL) +#define MAX_CDX_CONTROLLERS 16 + +/* CDX controllers registered with the CDX bus */ +static DEFINE_XARRAY_ALLOC(cdx_controllers); + +/** + * cdx_unregister_device - Unregister a CDX device + * @dev: CDX device + * @data: This is always passed as NULL, and is not used in this API, + * but is required here as the bus_for_each_dev() API expects + * the passed function (cdx_unregister_device) to have this + * as an argument. + * + * Return: 0 on success. + */ +static int cdx_unregister_device(struct device *dev, + void *data) +{ + struct cdx_device *cdx_dev = to_cdx_device(dev); + + kfree(cdx_dev->driver_override); + cdx_dev->driver_override = NULL; + /* + * Do not free cdx_dev here as it would be freed in + * cdx_device_release() called from within put_device(). + */ + device_del(&cdx_dev->dev); + put_device(&cdx_dev->dev); + + return 0; +} + +static void cdx_unregister_devices(struct bus_type *bus) +{ + /* Reset all the devices attached to cdx bus */ + bus_for_each_dev(bus, NULL, NULL, cdx_unregister_device); +} + +/** + * cdx_match_one_device - Tell if a CDX device structure has a matching + * CDX device id structure + * @id: single CDX device id structure to match + * @dev: the CDX device structure to match against + * + * Return: matching cdx_device_id structure or NULL if there is no match. + */ +static inline const struct cdx_device_id * +cdx_match_one_device(const struct cdx_device_id *id, + const struct cdx_device *dev) +{ + /* Use vendor ID and device ID for matching */ + if ((id->vendor == CDX_ANY_ID || id->vendor == dev->vendor) && + (id->device == CDX_ANY_ID || id->device == dev->device)) + return id; + return NULL; +} + +/** + * cdx_match_id - See if a CDX device matches a given cdx_id table + * @ids: array of CDX device ID structures to search in + * @dev: the CDX device structure to match against. + * + * Used by a driver to check whether a CDX device is in its list of + * supported devices. Returns the matching cdx_device_id structure or + * NULL if there is no match. + * + * Return: matching cdx_device_id structure or NULL if there is no match. + */ +static inline const struct cdx_device_id * +cdx_match_id(const struct cdx_device_id *ids, struct cdx_device *dev) +{ + if (ids) { + while (ids->vendor || ids->device) { + if (cdx_match_one_device(ids, dev)) + return ids; + ids++; + } + } + return NULL; +} + +/** + * cdx_bus_match - device to driver matching callback + * @dev: the cdx device to match against + * @drv: the device driver to search for matching cdx device + * structures + * + * Return: true on success, false otherwise. + */ +static int cdx_bus_match(struct device *dev, struct device_driver *drv) +{ + struct cdx_device *cdx_dev = to_cdx_device(dev); + struct cdx_driver *cdx_drv = to_cdx_driver(drv); + const struct cdx_device_id *found_id = NULL; + const struct cdx_device_id *ids; + + ids = cdx_drv->match_id_table; + + /* When driver_override is set, only bind to the matching driver */ + if (cdx_dev->driver_override && strcmp(cdx_dev->driver_override, drv->name)) + return false; + + found_id = cdx_match_id(ids, cdx_dev); + if (!found_id) + return false; + + do { + /* + * In case override_only was set, enforce driver_override + * matching. + */ + if (!found_id->override_only) + return true; + if (cdx_dev->driver_override) + return true; + + ids = found_id + 1; + found_id = cdx_match_id(ids, cdx_dev); + } while (found_id); + + return false; +} + +static int cdx_probe(struct device *dev) +{ + struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); + struct cdx_device *cdx_dev = to_cdx_device(dev); + int error; + + error = cdx_drv->probe(cdx_dev); + if (error) { + dev_err_probe(dev, error, "%s failed\n", __func__); + return error; + } + + return 0; +} + +static void cdx_remove(struct device *dev) +{ + struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); + struct cdx_device *cdx_dev = to_cdx_device(dev); + + if (cdx_drv && cdx_drv->remove) + cdx_drv->remove(cdx_dev); +} + +static void cdx_shutdown(struct device *dev) +{ + struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); + struct cdx_device *cdx_dev = to_cdx_device(dev); + + if (cdx_drv && cdx_drv->shutdown) + cdx_drv->shutdown(cdx_dev); +} + +static int cdx_dma_configure(struct device *dev) +{ + struct cdx_device *cdx_dev = to_cdx_device(dev); + u32 input_id = cdx_dev->req_id; + int ret; + + ret = of_dma_configure_id(dev, dev->parent->of_node, 0, &input_id); + if (ret && ret != -EPROBE_DEFER) { + dev_err(dev, "of_dma_configure_id() failed\n"); + return ret; + } + + return 0; +} + +static ssize_t rescan_store(struct bus_type *bus, + const char *buf, size_t count) +{ + struct cdx_controller *cdx; + unsigned long index; + bool val; + + if (kstrtobool(buf, &val) < 0) + return -EINVAL; + + if (!val) + return -EINVAL; + + /* Unregister all the devices on the bus */ + cdx_unregister_devices(&cdx_bus_type); + + /* Rescan all the devices */ + xa_for_each(&cdx_controllers, index, cdx) { + int ret; + + ret = cdx->ops->scan(cdx); + if (ret) + dev_err(cdx->dev, "cdx bus scanning failed\n"); + } + + return count; +} +static BUS_ATTR_WO(rescan); + +static struct attribute *cdx_bus_attrs[] = { + &bus_attr_rescan.attr, + NULL, +}; +ATTRIBUTE_GROUPS(cdx_bus); + +struct bus_type cdx_bus_type = { + .name = "cdx", + .match = cdx_bus_match, + .probe = cdx_probe, + .remove = cdx_remove, + .shutdown = cdx_shutdown, + .dma_configure = cdx_dma_configure, + .bus_groups = cdx_bus_groups, +}; +EXPORT_SYMBOL_GPL(cdx_bus_type); + +int __cdx_driver_register(struct cdx_driver *cdx_driver, + struct module *owner) +{ + int error; + + cdx_driver->driver.owner = owner; + cdx_driver->driver.bus = &cdx_bus_type; + + error = driver_register(&cdx_driver->driver); + if (error) { + pr_err("driver_register() failed for %s: %d\n", + cdx_driver->driver.name, error); + return error; + } + + return 0; +} +EXPORT_SYMBOL_GPL(__cdx_driver_register); + +void cdx_driver_unregister(struct cdx_driver *cdx_driver) +{ + driver_unregister(&cdx_driver->driver); +} +EXPORT_SYMBOL_GPL(cdx_driver_unregister); + +static void cdx_device_release(struct device *dev) +{ + struct cdx_device *cdx_dev = to_cdx_device(dev); + + kfree(cdx_dev); +} + +int cdx_device_add(struct cdx_dev_params *dev_params) +{ + struct cdx_controller *cdx = dev_params->cdx; + struct device *parent = cdx->dev; + struct cdx_device *cdx_dev; + int ret; + + cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL); + if (!cdx_dev) + return -ENOMEM; + + /* Populate resource */ + memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) * + dev_params->res_count); + cdx_dev->res_count = dev_params->res_count; + + /* Populate CDX dev params */ + cdx_dev->req_id = dev_params->req_id; + cdx_dev->vendor = dev_params->vendor; + cdx_dev->device = dev_params->device; + cdx_dev->bus_num = dev_params->bus_num; + cdx_dev->dev_num = dev_params->dev_num; + cdx_dev->cdx = dev_params->cdx; + cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK; + + /* Initialize generic device */ + device_initialize(&cdx_dev->dev); + cdx_dev->dev.parent = parent; + cdx_dev->dev.bus = &cdx_bus_type; + cdx_dev->dev.dma_mask = &cdx_dev->dma_mask; + cdx_dev->dev.release = cdx_device_release; + + /* Set Name */ + dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", + ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (cdx_dev->bus_num & CDX_BUS_NUM_MASK)), + cdx_dev->dev_num); + + ret = device_add(&cdx_dev->dev); + if (ret) { + dev_err(&cdx_dev->dev, + "cdx device add failed: %d", ret); + goto fail; + } + + return 0; +fail: + /* + * Do not free cdx_dev here as it would be freed in + * cdx_device_release() called from put_device(). + */ + put_device(&cdx_dev->dev); + + return ret; +} +EXPORT_SYMBOL_GPL(cdx_device_add); + +int cdx_register_controller(struct cdx_controller *cdx) +{ + int ret; + + ret = xa_alloc(&cdx_controllers, &cdx->id, cdx, + XA_LIMIT(0, MAX_CDX_CONTROLLERS - 1), GFP_KERNEL); + if (ret) { + dev_err(cdx->dev, + "No free index available. Maximum controllers already registered\n"); + cdx->id = (u8)MAX_CDX_CONTROLLERS; + return ret; + } + + /* Scan all the devices */ + cdx->ops->scan(cdx); + + return 0; +} +EXPORT_SYMBOL_GPL(cdx_register_controller); + +void cdx_unregister_controller(struct cdx_controller *cdx) +{ + if (cdx->id >= MAX_CDX_CONTROLLERS) + return; + + device_for_each_child(cdx->dev, NULL, cdx_unregister_device); + xa_erase(&cdx_controllers, cdx->id); +} +EXPORT_SYMBOL_GPL(cdx_unregister_controller); + +static int __init cdx_bus_init(void) +{ + return bus_register(&cdx_bus_type); +} +postcore_initcall(cdx_bus_init); diff --git a/drivers/cdx/cdx.h b/drivers/cdx/cdx.h new file mode 100644 index 000000000000..c436ac7ac86f --- /dev/null +++ b/drivers/cdx/cdx.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Header file for the CDX Bus + * + * Copyright (C) 2022-2023, Advanced Micro Devices, Inc. + */ + +#ifndef _CDX_H_ +#define _CDX_H_ + +#include + +/** + * struct cdx_dev_params - CDX device parameters + * @cdx: CDX controller associated with the device + * @parent: Associated CDX controller + * @vendor: Vendor ID for CDX device + * @device: Device ID for CDX device + * @bus_num: Bus number for this CDX device + * @dev_num: Device number for this device + * @res: array of MMIO region entries + * @res_count: number of valid MMIO regions + * @req_id: Requestor ID associated with CDX device + */ +struct cdx_dev_params { + struct cdx_controller *cdx; + u16 vendor; + u16 device; + u8 bus_num; + u8 dev_num; + struct resource res[MAX_CDX_DEV_RESOURCES]; + u8 res_count; + u32 req_id; +}; + +/** + * cdx_register_controller - Register a CDX controller and its ports + * on the CDX bus. + * @cdx: The CDX controller to register + * + * Return: -errno on failure, 0 on success. + */ +int cdx_register_controller(struct cdx_controller *cdx); + +/** + * cdx_unregister_controller - Unregister a CDX controller + * @cdx: The CDX controller to unregister + */ +void cdx_unregister_controller(struct cdx_controller *cdx); + +/** + * cdx_device_add - Add a CDX device. This function adds a CDX device + * on the CDX bus as per the device parameters provided + * by caller. It also creates and registers an associated + * Linux generic device. + * @dev_params: device parameters associated with the device to be created. + * + * Return: -errno on failure, 0 on success. + */ +int cdx_device_add(struct cdx_dev_params *dev_params); + +#endif /* _CDX_H_ */ diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h new file mode 100644 index 000000000000..d134e0104724 --- /dev/null +++ b/include/linux/cdx/cdx_bus.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * CDX bus public interface + * + * Copyright (C) 2022-2023, Advanced Micro Devices, Inc. + * + */ + +#ifndef _CDX_BUS_H_ +#define _CDX_BUS_H_ + +#include +#include +#include + +#define MAX_CDX_DEV_RESOURCES 4 +#define CDX_ANY_ID (0xFFFF) +#define CDX_CONTROLLER_ID_SHIFT 4 +#define CDX_BUS_NUM_MASK 0xF + +/* Forward declaration for CDX controller */ +struct cdx_controller; + +typedef int (*cdx_scan_cb)(struct cdx_controller *cdx); + +/** + * CDX_DEVICE_DRIVER_OVERRIDE - macro used to describe a CDX device with + * override_only flags. + * @vend: the 16 bit CDX Vendor ID + * @dev: the 16 bit CDX Device ID + * @driver_override: the 32 bit CDX Device override_only + * + * This macro is used to create a struct cdx_device_id that matches only a + * driver_override device. + */ +#define CDX_DEVICE_DRIVER_OVERRIDE(vend, dev, driver_override) \ + .vendor = (vend), .device = (dev), .override_only = (driver_override) + +/** + * struct cdx_ops - Callbacks supported by CDX controller. + * @scan: scan the devices on the controller + */ +struct cdx_ops { + cdx_scan_cb scan; +}; + +/** + * struct cdx_controller: CDX controller object + * @dev: Linux device associated with the CDX controller. + * @priv: private data + * @id: Controller ID + * @ops: CDX controller ops + */ +struct cdx_controller { + struct device *dev; + void *priv; + u32 id; + struct cdx_ops *ops; +}; + +/** + * struct cdx_device - CDX device object + * @dev: Linux driver model device object + * @cdx: CDX controller associated with the device + * @vendor: Vendor ID for CDX device + * @device: Device ID for CDX device + * @bus_num: Bus number for this CDX device + * @dev_num: Device number for this device + * @res: array of MMIO region entries + * @res_attr: resource binary attribute + * @res_count: number of valid MMIO regions + * @dma_mask: Default DMA mask + * @flags: CDX device flags + * @req_id: Requestor ID associated with CDX device + * @driver_override: driver name to force a match; do not set directly, + * because core frees it; use driver_set_override() to + * set or clear it. + */ +struct cdx_device { + struct device dev; + struct cdx_controller *cdx; + u16 vendor; + u16 device; + u8 bus_num; + u8 dev_num; + struct resource res[MAX_CDX_DEV_RESOURCES]; + u8 res_count; + u64 dma_mask; + u16 flags; + u32 req_id; + const char *driver_override; +}; + +#define to_cdx_device(_dev) \ + container_of(_dev, struct cdx_device, dev) + +/** + * struct cdx_driver - CDX device driver + * @driver: Generic device driver + * @match_id_table: table of supported device matching Ids + * @probe: Function called when a device is added + * @remove: Function called when a device is removed + * @shutdown: Function called at shutdown time to quiesce the device + * @driver_managed_dma: Device driver doesn't use kernel DMA API for DMA. + * For most device drivers, no need to care about this flag + * as long as all DMAs are handled through the kernel DMA API. + * For some special ones, for example VFIO drivers, they know + * how to manage the DMA themselves and set this flag so that + * the IOMMU layer will allow them to setup and manage their + * own I/O address space. + */ +struct cdx_driver { + struct device_driver driver; + const struct cdx_device_id *match_id_table; + int (*probe)(struct cdx_device *dev); + int (*remove)(struct cdx_device *dev); + void (*shutdown)(struct cdx_device *dev); + bool driver_managed_dma; +}; + +#define to_cdx_driver(_drv) \ + container_of(_drv, struct cdx_driver, driver) + +/* Macro to avoid include chaining to get THIS_MODULE */ +#define cdx_driver_register(drv) \ + __cdx_driver_register(drv, THIS_MODULE) + +/** + * __cdx_driver_register - registers a CDX device driver + * @cdx_driver: CDX driver to register + * @owner: module owner + * + * Return: -errno on failure, 0 on success. + */ +int __must_check __cdx_driver_register(struct cdx_driver *cdx_driver, + struct module *owner); + +/** + * cdx_driver_unregister - unregisters a device driver from the + * CDX bus. + * @cdx_driver: CDX driver to register + */ +void cdx_driver_unregister(struct cdx_driver *cdx_driver); + +extern struct bus_type cdx_bus_type; + +#endif /* _CDX_BUS_H_ */ diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index 35203ca3fc37..ccaaeda792c0 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -912,4 +912,19 @@ struct ishtp_device_id { kernel_ulong_t driver_data; }; +/** + * struct cdx_device_id - CDX device identifier + * @vendor: Vendor ID + * @device: Device ID + * @override_only: Match only when dev->driver_override is this driver. + * + * Type of entries in the "device Id" table for CDX devices supported by + * a CDX device driver. + */ +struct cdx_device_id { + __u16 vendor; + __u16 device; + __u32 override_only; +}; + #endif /* LINUX_MOD_DEVICETABLE_H */ diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c index c0d3bcb99138..62dc988df84d 100644 --- a/scripts/mod/devicetable-offsets.c +++ b/scripts/mod/devicetable-offsets.c @@ -262,5 +262,9 @@ int main(void) DEVID(ishtp_device_id); DEVID_FIELD(ishtp_device_id, guid); + DEVID(cdx_device_id); + DEVID_FIELD(cdx_device_id, vendor); + DEVID_FIELD(cdx_device_id, device); + return 0; } diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index 91c2e7ba5e52..28da34ba4359 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -1452,6 +1452,17 @@ static int do_dfl_entry(const char *filename, void *symval, char *alias) return 1; } +/* Looks like: cdx:vNdN */ +static int do_cdx_entry(const char *filename, void *symval, + char *alias) +{ + DEF_FIELD(symval, cdx_device_id, vendor); + DEF_FIELD(symval, cdx_device_id, device); + + sprintf(alias, "cdx:v%08Xd%08Xd", vendor, device); + return 1; +} + /* Does namelen bytes of name exactly match the symbol? */ static bool sym_is(const char *name, unsigned namelen, const char *symbol) { @@ -1531,6 +1542,7 @@ static const struct devtable devtable[] = { {"ssam", SIZE_ssam_device_id, do_ssam_entry}, {"dfl", SIZE_dfl_device_id, do_dfl_entry}, {"ishtp", SIZE_ishtp_device_id, do_ishtp_entry}, + {"cdx", SIZE_cdx_device_id, do_cdx_entry}, }; /* Create MODULE_ALIAS() statements. -- cgit v1.2.3 From 48a6c7bced2a781c911497f073347bec5ab4d7a5 Mon Sep 17 00:00:00 2001 From: Nipun Gupta Date: Mon, 13 Mar 2023 18:56:36 +0530 Subject: cdx: add device attributes Create sysfs entry for CDX devices. Sysfs entries provided in each of the CDX device detected by the CDX controller - vendor id - device id - remove - reset of the device. - driver override Signed-off-by: Puneet Gupta Signed-off-by: Nipun Gupta Signed-off-by: Tarak Reddy Reviewed-by: Pieter Jansen van Vuuren Tested-by: Nikhil Agarwal Link: https://lore.kernel.org/r/20230313132636.31850-8-nipun.gupta@amd.com Signed-off-by: Greg Kroah-Hartman --- Documentation/ABI/testing/sysfs-bus-cdx | 44 +++++++++++ drivers/cdx/cdx.c | 127 ++++++++++++++++++++++++++++++++ drivers/cdx/controller/cdx_controller.c | 18 +++++ drivers/cdx/controller/mcdi_functions.c | 14 ++++ drivers/cdx/controller/mcdi_functions.h | 11 +++ include/linux/cdx/cdx_bus.h | 27 +++++++ 6 files changed, 241 insertions(+) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-bus-cdx b/Documentation/ABI/testing/sysfs-bus-cdx index 43b4e161f226..7af477f49998 100644 --- a/Documentation/ABI/testing/sysfs-bus-cdx +++ b/Documentation/ABI/testing/sysfs-bus-cdx @@ -10,3 +10,47 @@ Description: For example:: # echo 1 > /sys/bus/cdx/rescan + +What: /sys/bus/cdx/devices/.../vendor +Date: March 2023 +Contact: nipun.gupta@amd.com +Description: + Vendor ID for this CDX device, in hexadecimal. Vendor ID is + 16 bit identifier which is specific to the device manufacturer. + Combination of Vendor ID and Device ID identifies a device. + +What: /sys/bus/cdx/devices/.../device +Date: March 2023 +Contact: nipun.gupta@amd.com +Description: + Device ID for this CDX device, in hexadecimal. Device ID is + 16 bit identifier to identify a device type within the range + of a device manufacturer. + Combination of Vendor ID and Device ID identifies a device. + +What: /sys/bus/cdx/devices/.../reset +Date: March 2023 +Contact: nipun.gupta@amd.com +Description: + Writing y/1/on to this file resets the CDX device. + On resetting the device, the corresponding driver is notified + twice, once before the device is being reset, and again after + the reset has been complete. + + For example:: + + # echo 1 > /sys/bus/cdx/.../reset + +What: /sys/bus/cdx/devices/.../remove +Date: March 2023 +Contact: tarak.reddy@amd.com +Description: + Writing y/1/on to this file removes the corresponding + device from the CDX bus. If the device is to be reconfigured + reconfigured in the Hardware, the device can be removed, so + that the device driver does not access the device while it is + being reconfigured. + + For example:: + + # echo 1 > /sys/bus/cdx/devices/.../remove diff --git a/drivers/cdx/cdx.c b/drivers/cdx/cdx.c index 6b162bbb9b25..67c32cb2c006 100644 --- a/drivers/cdx/cdx.c +++ b/drivers/cdx/cdx.c @@ -71,6 +71,39 @@ /* CDX controllers registered with the CDX bus */ static DEFINE_XARRAY_ALLOC(cdx_controllers); +/** + * cdx_dev_reset - Reset a CDX device + * @dev: CDX device + * + * Return: -errno on failure, 0 on success. + */ +int cdx_dev_reset(struct device *dev) +{ + struct cdx_device *cdx_dev = to_cdx_device(dev); + struct cdx_controller *cdx = cdx_dev->cdx; + struct cdx_device_config dev_config = {0}; + struct cdx_driver *cdx_drv; + int ret; + + cdx_drv = to_cdx_driver(dev->driver); + /* Notify driver that device is being reset */ + if (cdx_drv && cdx_drv->reset_prepare) + cdx_drv->reset_prepare(cdx_dev); + + dev_config.type = CDX_DEV_RESET_CONF; + ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num, + cdx_dev->dev_num, &dev_config); + if (ret) + dev_err(dev, "cdx device reset failed\n"); + + /* Notify driver that device reset is complete */ + if (cdx_drv && cdx_drv->reset_done) + cdx_drv->reset_done(cdx_dev); + + return ret; +} +EXPORT_SYMBOL_GPL(cdx_dev_reset); + /** * cdx_unregister_device - Unregister a CDX device * @dev: CDX device @@ -237,6 +270,99 @@ static int cdx_dma_configure(struct device *dev) return 0; } +/* show configuration fields */ +#define cdx_config_attr(field, format_string) \ +static ssize_t \ +field##_show(struct device *dev, struct device_attribute *attr, char *buf) \ +{ \ + struct cdx_device *cdx_dev = to_cdx_device(dev); \ + return sysfs_emit(buf, format_string, cdx_dev->field); \ +} \ +static DEVICE_ATTR_RO(field) + +cdx_config_attr(vendor, "0x%04x\n"); +cdx_config_attr(device, "0x%04x\n"); + +static ssize_t remove_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + bool val; + + if (kstrtobool(buf, &val) < 0) + return -EINVAL; + + if (!val) + return -EINVAL; + + if (device_remove_file_self(dev, attr)) { + int ret; + + ret = cdx_unregister_device(dev, NULL); + if (ret) + return ret; + } + + return count; +} +static DEVICE_ATTR_WO(remove); + +static ssize_t reset_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + bool val; + int ret; + + if (kstrtobool(buf, &val) < 0) + return -EINVAL; + + if (!val) + return -EINVAL; + + ret = cdx_dev_reset(dev); + if (ret) + return ret; + + return count; +} +static DEVICE_ATTR_WO(reset); + +static ssize_t driver_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cdx_device *cdx_dev = to_cdx_device(dev); + int ret; + + if (WARN_ON(dev->bus != &cdx_bus_type)) + return -EINVAL; + + ret = driver_set_override(dev, &cdx_dev->driver_override, buf, count); + if (ret) + return ret; + + return count; +} + +static ssize_t driver_override_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cdx_device *cdx_dev = to_cdx_device(dev); + + return sysfs_emit(buf, "%s\n", cdx_dev->driver_override); +} +static DEVICE_ATTR_RW(driver_override); + +static struct attribute *cdx_dev_attrs[] = { + &dev_attr_remove.attr, + &dev_attr_reset.attr, + &dev_attr_vendor.attr, + &dev_attr_device.attr, + &dev_attr_driver_override.attr, + NULL, +}; +ATTRIBUTE_GROUPS(cdx_dev); + static ssize_t rescan_store(struct bus_type *bus, const char *buf, size_t count) { @@ -280,6 +406,7 @@ struct bus_type cdx_bus_type = { .shutdown = cdx_shutdown, .dma_configure = cdx_dma_configure, .bus_groups = cdx_bus_groups, + .dev_groups = cdx_dev_groups, }; EXPORT_SYMBOL_GPL(cdx_bus_type); diff --git a/drivers/cdx/controller/cdx_controller.c b/drivers/cdx/controller/cdx_controller.c index 0d1826980935..dc52f95f8978 100644 --- a/drivers/cdx/controller/cdx_controller.c +++ b/drivers/cdx/controller/cdx_controller.c @@ -45,6 +45,23 @@ void cdx_rpmsg_pre_remove(struct cdx_controller *cdx) cdx_mcdi_wait_for_quiescence(cdx->priv, MCDI_RPC_TIMEOUT); } +static int cdx_configure_device(struct cdx_controller *cdx, + u8 bus_num, u8 dev_num, + struct cdx_device_config *dev_config) +{ + int ret = 0; + + switch (dev_config->type) { + case CDX_DEV_RESET_CONF: + ret = cdx_mcdi_reset_device(cdx->priv, bus_num, dev_num); + break; + default: + ret = -EINVAL; + } + + return ret; +} + static int cdx_scan_devices(struct cdx_controller *cdx) { struct cdx_mcdi *cdx_mcdi = cdx->priv; @@ -104,6 +121,7 @@ static int cdx_scan_devices(struct cdx_controller *cdx) static struct cdx_ops cdx_ops = { .scan = cdx_scan_devices, + .dev_configure = cdx_configure_device, }; static int xlnx_cdx_probe(struct platform_device *pdev) diff --git a/drivers/cdx/controller/mcdi_functions.c b/drivers/cdx/controller/mcdi_functions.c index 012b52881dd5..0158f26533dd 100644 --- a/drivers/cdx/controller/mcdi_functions.c +++ b/drivers/cdx/controller/mcdi_functions.c @@ -123,3 +123,17 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx, return 0; } + +int cdx_mcdi_reset_device(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num) +{ + MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_DEVICE_RESET_IN_LEN); + int ret; + + MCDI_SET_DWORD(inbuf, CDX_DEVICE_RESET_IN_BUS, bus_num); + MCDI_SET_DWORD(inbuf, CDX_DEVICE_RESET_IN_DEVICE, dev_num); + + ret = cdx_mcdi_rpc(cdx, MC_CMD_CDX_DEVICE_RESET, inbuf, sizeof(inbuf), + NULL, 0, NULL); + + return ret; +} diff --git a/drivers/cdx/controller/mcdi_functions.h b/drivers/cdx/controller/mcdi_functions.h index 6bf5a4a0778f..7440ace5539a 100644 --- a/drivers/cdx/controller/mcdi_functions.h +++ b/drivers/cdx/controller/mcdi_functions.h @@ -47,4 +47,15 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num, struct cdx_dev_params *dev_params); +/** + * cdx_mcdi_reset_device - Reset cdx device represented by bus_num:dev_num + * @cdx: pointer to MCDI interface. + * @bus_num: Bus number. + * @dev_num: Device number. + * + * Return: 0 on success, <0 on failure + */ +int cdx_mcdi_reset_device(struct cdx_mcdi *cdx, + u8 bus_num, u8 dev_num); + #endif /* CDX_MCDI_FUNCTIONS_H */ diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h index d134e0104724..35ef41d8a61a 100644 --- a/include/linux/cdx/cdx_bus.h +++ b/include/linux/cdx/cdx_bus.h @@ -21,8 +21,20 @@ /* Forward declaration for CDX controller */ struct cdx_controller; +enum { + CDX_DEV_RESET_CONF, +}; + +struct cdx_device_config { + u8 type; +}; + typedef int (*cdx_scan_cb)(struct cdx_controller *cdx); +typedef int (*cdx_dev_configure_cb)(struct cdx_controller *cdx, + u8 bus_num, u8 dev_num, + struct cdx_device_config *dev_config); + /** * CDX_DEVICE_DRIVER_OVERRIDE - macro used to describe a CDX device with * override_only flags. @@ -39,9 +51,12 @@ typedef int (*cdx_scan_cb)(struct cdx_controller *cdx); /** * struct cdx_ops - Callbacks supported by CDX controller. * @scan: scan the devices on the controller + * @dev_configure: configuration like reset, master_enable, + * msi_config etc for a CDX device */ struct cdx_ops { cdx_scan_cb scan; + cdx_dev_configure_cb dev_configure; }; /** @@ -101,6 +116,8 @@ struct cdx_device { * @probe: Function called when a device is added * @remove: Function called when a device is removed * @shutdown: Function called at shutdown time to quiesce the device + * @reset_prepare: Function called before is reset to notify driver + * @reset_done: Function called after reset is complete to notify driver * @driver_managed_dma: Device driver doesn't use kernel DMA API for DMA. * For most device drivers, no need to care about this flag * as long as all DMAs are handled through the kernel DMA API. @@ -115,6 +132,8 @@ struct cdx_driver { int (*probe)(struct cdx_device *dev); int (*remove)(struct cdx_device *dev); void (*shutdown)(struct cdx_device *dev); + void (*reset_prepare)(struct cdx_device *dev); + void (*reset_done)(struct cdx_device *dev); bool driver_managed_dma; }; @@ -144,4 +163,12 @@ void cdx_driver_unregister(struct cdx_driver *cdx_driver); extern struct bus_type cdx_bus_type; +/** + * cdx_dev_reset - Reset CDX device + * @dev: device pointer + * + * Return: 0 for success, -errno on failure + */ +int cdx_dev_reset(struct device *dev); + #endif /* _CDX_BUS_H_ */ -- cgit v1.2.3 From e5a26a4048eeb9558e5c84f340a989c78db4adf4 Mon Sep 17 00:00:00 2001 From: Beau Belgrave Date: Tue, 28 Mar 2023 16:52:08 -0700 Subject: tracing/user_events: Split header into uapi and kernel The UAPI parts need to be split out from the kernel parts of user_events now that other parts of the kernel will reference it. Do so by moving the existing include/linux/user_events.h into include/uapi/linux/user_events.h. Link: https://lkml.kernel.org/r/20230328235219.203-2-beaub@linux.microsoft.com Signed-off-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- include/linux/user_events.h | 52 +++++----------------------------------- include/uapi/linux/user_events.h | 48 +++++++++++++++++++++++++++++++++++++ kernel/trace/trace_events_user.c | 5 ---- 3 files changed, 54 insertions(+), 51 deletions(-) create mode 100644 include/uapi/linux/user_events.h (limited to 'include/linux') diff --git a/include/linux/user_events.h b/include/linux/user_events.h index 592a3fbed98e..13689589d36e 100644 --- a/include/linux/user_events.h +++ b/include/linux/user_events.h @@ -1,54 +1,14 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2021, Microsoft Corporation. + * Copyright (c) 2022, Microsoft Corporation. * * Authors: * Beau Belgrave */ -#ifndef _UAPI_LINUX_USER_EVENTS_H -#define _UAPI_LINUX_USER_EVENTS_H -#include -#include +#ifndef _LINUX_USER_EVENTS_H +#define _LINUX_USER_EVENTS_H -#ifdef __KERNEL__ -#include -#else -#include -#endif +#include -#define USER_EVENTS_SYSTEM "user_events" -#define USER_EVENTS_PREFIX "u:" - -/* Create dynamic location entry within a 32-bit value */ -#define DYN_LOC(offset, size) ((size) << 16 | (offset)) - -/* - * Describes an event registration and stores the results of the registration. - * This structure is passed to the DIAG_IOCSREG ioctl, callers at a minimum - * must set the size and name_args before invocation. - */ -struct user_reg { - - /* Input: Size of the user_reg structure being used */ - __u32 size; - - /* Input: Pointer to string with event name, description and flags */ - __u64 name_args; - - /* Output: Bitwise index of the event within the status page */ - __u32 status_bit; - - /* Output: Index of the event to use when writing data */ - __u32 write_index; -} __attribute__((__packed__)); - -#define DIAG_IOC_MAGIC '*' - -/* Requests to register a user_event */ -#define DIAG_IOCSREG _IOWR(DIAG_IOC_MAGIC, 0, struct user_reg*) - -/* Requests to delete a user_event */ -#define DIAG_IOCSDEL _IOW(DIAG_IOC_MAGIC, 1, char*) - -#endif /* _UAPI_LINUX_USER_EVENTS_H */ +#endif /* _LINUX_USER_EVENTS_H */ diff --git a/include/uapi/linux/user_events.h b/include/uapi/linux/user_events.h new file mode 100644 index 000000000000..03f92366068d --- /dev/null +++ b/include/uapi/linux/user_events.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright (c) 2021-2022, Microsoft Corporation. + * + * Authors: + * Beau Belgrave + */ +#ifndef _UAPI_LINUX_USER_EVENTS_H +#define _UAPI_LINUX_USER_EVENTS_H + +#include +#include + +#define USER_EVENTS_SYSTEM "user_events" +#define USER_EVENTS_PREFIX "u:" + +/* Create dynamic location entry within a 32-bit value */ +#define DYN_LOC(offset, size) ((size) << 16 | (offset)) + +/* + * Describes an event registration and stores the results of the registration. + * This structure is passed to the DIAG_IOCSREG ioctl, callers at a minimum + * must set the size and name_args before invocation. + */ +struct user_reg { + + /* Input: Size of the user_reg structure being used */ + __u32 size; + + /* Input: Pointer to string with event name, description and flags */ + __u64 name_args; + + /* Output: Bitwise index of the event within the status page */ + __u32 status_bit; + + /* Output: Index of the event to use when writing data */ + __u32 write_index; +} __attribute__((__packed__)); + +#define DIAG_IOC_MAGIC '*' + +/* Request to register a user_event */ +#define DIAG_IOCSREG _IOWR(DIAG_IOC_MAGIC, 0, struct user_reg *) + +/* Request to delete a user_event */ +#define DIAG_IOCSDEL _IOW(DIAG_IOC_MAGIC, 1, char *) + +#endif /* _UAPI_LINUX_USER_EVENTS_H */ diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 908e8a13c675..070551480747 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -19,12 +19,7 @@ #include #include #include -/* Reminder to move to uapi when everything works */ -#ifdef CONFIG_COMPILE_TEST #include -#else -#include -#endif #include "trace.h" #include "trace_dynevent.h" -- cgit v1.2.3 From fd593511cdfc0b0e38af2eb21c99f5154a1d7acf Mon Sep 17 00:00:00 2001 From: Beau Belgrave Date: Tue, 28 Mar 2023 16:52:09 -0700 Subject: tracing/user_events: Track fork/exec/exit for mm lifetime During tracefs discussions it was decided instead of requiring a mapping within a user-process to track the lifetime of memory descriptors we should hook the appropriate calls. Do this by adding the minimal stubs required for task fork, exec, and exit. Currently this is just a NOP. Future patches will implement these calls fully. Link: https://lkml.kernel.org/r/20230328235219.203-3-beaub@linux.microsoft.com Suggested-by: Mathieu Desnoyers Signed-off-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- fs/exec.c | 2 ++ include/linux/sched.h | 5 +++++ include/linux/user_events.h | 18 ++++++++++++++++++ kernel/exit.c | 2 ++ kernel/fork.c | 2 ++ 5 files changed, 29 insertions(+) (limited to 'include/linux') diff --git a/fs/exec.c b/fs/exec.c index 7c44d0c65b1b..2b0042f8deec 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -65,6 +65,7 @@ #include #include #include +#include #include #include @@ -1859,6 +1860,7 @@ static int bprm_execve(struct linux_binprm *bprm, current->fs->in_exec = 0; current->in_execve = 0; rseq_execve(current); + user_events_execve(current); acct_update_integrals(current); task_numa_free(current, false); return retval; diff --git a/include/linux/sched.h b/include/linux/sched.h index 63d242164b1a..bf37846e90c2 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -69,6 +69,7 @@ struct sighand_struct; struct signal_struct; struct task_delay_info; struct task_group; +struct user_event_mm; /* * Task state bitmask. NOTE! These bits are also @@ -1528,6 +1529,10 @@ struct task_struct { union rv_task_monitor rv[RV_PER_TASK_MONITORS]; #endif +#ifdef CONFIG_USER_EVENTS + struct user_event_mm *user_event_mm; +#endif + /* * New fields for task_struct should be added above here, so that * they are included in the randomized portion of task_struct. diff --git a/include/linux/user_events.h b/include/linux/user_events.h index 13689589d36e..3d747c45d2fa 100644 --- a/include/linux/user_events.h +++ b/include/linux/user_events.h @@ -11,4 +11,22 @@ #include +#ifdef CONFIG_USER_EVENTS +struct user_event_mm { +}; +#endif + +static inline void user_events_fork(struct task_struct *t, + unsigned long clone_flags) +{ +} + +static inline void user_events_execve(struct task_struct *t) +{ +} + +static inline void user_events_exit(struct task_struct *t) +{ +} + #endif /* _LINUX_USER_EVENTS_H */ diff --git a/kernel/exit.c b/kernel/exit.c index f2afdb0add7c..875d6a134df8 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -68,6 +68,7 @@ #include #include #include +#include #include #include @@ -818,6 +819,7 @@ void __noreturn do_exit(long code) coredump_task_exit(tsk); ptrace_event(PTRACE_EVENT_EXIT, code); + user_events_exit(tsk); validate_creds_for_do_exit(tsk); diff --git a/kernel/fork.c b/kernel/fork.c index d8cda4c6de6c..efb1f2257772 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -97,6 +97,7 @@ #include #include #include +#include #include #include @@ -2505,6 +2506,7 @@ static __latent_entropy struct task_struct *copy_process( trace_task_newtask(p, clone_flags); uprobe_copy_process(p, clone_flags); + user_events_fork(p, clone_flags); copy_oom_score_adj(clone_flags, p); -- cgit v1.2.3 From 7235759084a4f8524a46bd2638885ff3b34ce279 Mon Sep 17 00:00:00 2001 From: Beau Belgrave Date: Tue, 28 Mar 2023 16:52:10 -0700 Subject: tracing/user_events: Use remote writes for event enablement As part of the discussions for user_events aligned with user space tracers, it was determined that user programs should register a aligned value to set or clear a bit when an event becomes enabled. Currently a shared page is being used that requires mmap(). Remove the shared page implementation and move to a user registered address implementation. In this new model during the event registration from user programs 3 new values are specified. The first is the address to update when the event is either enabled or disabled. The second is the bit to set/clear to reflect the event being enabled. The third is the size of the value at the specified address. This allows for a local 32/64-bit value in user programs to support both kernel and user tracers. As an example, setting bit 31 for kernel tracers when the event becomes enabled allows for user tracers to use the other bits for ref counts or other flags. The kernel side updates the bit atomically, user programs need to also update these values atomically. User provided addresses must be aligned on a natural boundary, this allows for single page checking and prevents odd behaviors such as a enable value straddling 2 pages instead of a single page. Currently page faults are only logged, future patches will handle these. Link: https://lkml.kernel.org/r/20230328235219.203-4-beaub@linux.microsoft.com Suggested-by: Mathieu Desnoyers Signed-off-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- include/linux/user_events.h | 53 +++- include/uapi/linux/user_events.h | 15 +- kernel/trace/Kconfig | 5 +- kernel/trace/trace_events_user.c | 586 ++++++++++++++++++++++++++++++--------- 4 files changed, 517 insertions(+), 142 deletions(-) (limited to 'include/linux') diff --git a/include/linux/user_events.h b/include/linux/user_events.h index 3d747c45d2fa..0120b3dd5b03 100644 --- a/include/linux/user_events.h +++ b/include/linux/user_events.h @@ -9,13 +9,63 @@ #ifndef _LINUX_USER_EVENTS_H #define _LINUX_USER_EVENTS_H +#include +#include +#include +#include #include #ifdef CONFIG_USER_EVENTS struct user_event_mm { + struct list_head link; + struct list_head enablers; + struct mm_struct *mm; + struct user_event_mm *next; + refcount_t refcnt; + refcount_t tasks; + struct rcu_work put_rwork; }; -#endif +extern void user_event_mm_dup(struct task_struct *t, + struct user_event_mm *old_mm); + +extern void user_event_mm_remove(struct task_struct *t); + +static inline void user_events_fork(struct task_struct *t, + unsigned long clone_flags) +{ + struct user_event_mm *old_mm; + + if (!t || !current->user_event_mm) + return; + + old_mm = current->user_event_mm; + + if (clone_flags & CLONE_VM) { + t->user_event_mm = old_mm; + refcount_inc(&old_mm->tasks); + return; + } + + user_event_mm_dup(t, old_mm); +} + +static inline void user_events_execve(struct task_struct *t) +{ + if (!t || !t->user_event_mm) + return; + + user_event_mm_remove(t); +} + +static inline void user_events_exit(struct task_struct *t) +{ + if (!t || !t->user_event_mm) + return; + + user_event_mm_remove(t); +} +#else static inline void user_events_fork(struct task_struct *t, unsigned long clone_flags) { @@ -28,5 +78,6 @@ static inline void user_events_execve(struct task_struct *t) static inline void user_events_exit(struct task_struct *t) { } +#endif /* CONFIG_USER_EVENTS */ #endif /* _LINUX_USER_EVENTS_H */ diff --git a/include/uapi/linux/user_events.h b/include/uapi/linux/user_events.h index 03f92366068d..22521bc622db 100644 --- a/include/uapi/linux/user_events.h +++ b/include/uapi/linux/user_events.h @@ -27,12 +27,21 @@ struct user_reg { /* Input: Size of the user_reg structure being used */ __u32 size; + /* Input: Bit in enable address to use */ + __u8 enable_bit; + + /* Input: Enable size in bytes at address */ + __u8 enable_size; + + /* Input: Flags for future use, set to 0 */ + __u16 flags; + + /* Input: Address to update when enabled */ + __u64 enable_addr; + /* Input: Pointer to string with event name, description and flags */ __u64 name_args; - /* Output: Bitwise index of the event within the status page */ - __u32 status_bit; - /* Output: Index of the event to use when writing data */ __u32 write_index; } __attribute__((__packed__)); diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 5b1e7fa41ca8..c7020e071bf9 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -798,9 +798,10 @@ config USER_EVENTS can be used like an existing kernel trace event. User trace events are generated by writing to a tracefs file. User processes can determine if their tracing events should be - generated by memory mapping a tracefs file and checking for - an associated byte being non-zero. + generated by registering a value and bit with the kernel + that reflects when it is enabled or not. + See Documentation/trace/user_events.rst. If in doubt, say N. config HIST_TRIGGERS diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 070551480747..553a82ee7aeb 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "trace.h" #include "trace_dynevent.h" @@ -29,34 +30,11 @@ #define FIELD_DEPTH_NAME 1 #define FIELD_DEPTH_SIZE 2 -/* - * Limits how many trace_event calls user processes can create: - * Must be a power of two of PAGE_SIZE. - */ -#define MAX_PAGE_ORDER 0 -#define MAX_PAGES (1 << MAX_PAGE_ORDER) -#define MAX_BYTES (MAX_PAGES * PAGE_SIZE) -#define MAX_EVENTS (MAX_BYTES * 8) - /* Limit how long of an event name plus args within the subsystem. */ #define MAX_EVENT_DESC 512 #define EVENT_NAME(user_event) ((user_event)->tracepoint.name) #define MAX_FIELD_ARRAY_SIZE 1024 -/* - * The MAP_STATUS_* macros are used for taking a index and determining the - * appropriate byte and the bit in the byte to set/reset for an event. - * - * The lower 3 bits of the index decide which bit to set. - * The remaining upper bits of the index decide which byte to use for the bit. - * - * This is used when an event has a probe attached/removed to reflect live - * status of the event wanting tracing or not to user-programs via shared - * memory maps. - */ -#define MAP_STATUS_BYTE(index) ((index) >> 3) -#define MAP_STATUS_MASK(index) BIT((index) & 7) - /* * Internal bits (kernel side only) to keep track of connected probes: * These are used when status is requested in text form about an event. These @@ -70,20 +48,14 @@ #define EVENT_STATUS_OTHER BIT(7) /* - * Stores the pages, tables, and locks for a group of events. - * Each logical grouping of events has its own group, with a - * matching page for status checks within user programs. This - * allows for isolation of events to user programs by various - * means. + * Stores the system name, tables, and locks for a group of events. This + * allows isolation for events by various means. */ struct user_event_group { - struct page *pages; - char *register_page_data; char *system_name; struct hlist_node node; struct mutex reg_mutex; DECLARE_HASHTABLE(register_table, 8); - DECLARE_BITMAP(page_bitmap, MAX_EVENTS); }; /* Group for init_user_ns mapping, top-most group */ @@ -106,12 +78,34 @@ struct user_event { struct list_head fields; struct list_head validators; refcount_t refcnt; - int index; - int flags; int min_size; char status; }; +/* + * Stores per-mm/event properties that enable an address to be + * updated properly for each task. As tasks are forked, we use + * these to track enablement sites that are tied to an event. + */ +struct user_event_enabler { + struct list_head link; + struct user_event *event; + unsigned long addr; + + /* Track enable bit, flags, etc. Aligned for bitops. */ + unsigned int values; +}; + +/* Bits 0-5 are for the bit to update upon enable/disable (0-63 allowed) */ +#define ENABLE_VAL_BIT_MASK 0x3F + +/* Only duplicate the bit value */ +#define ENABLE_VAL_DUP_MASK ENABLE_VAL_BIT_MASK + +/* Global list of memory descriptors using user_events */ +static LIST_HEAD(user_event_mms); +static DEFINE_SPINLOCK(user_event_mms_lock); + /* * Stores per-file events references, as users register events * within a file this structure is modified and freed via RCU. @@ -145,33 +139,17 @@ static int user_event_parse(struct user_event_group *group, char *name, char *args, char *flags, struct user_event **newuser); +static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm); +static struct user_event_mm *user_event_mm_get_all(struct user_event *user); +static void user_event_mm_put(struct user_event_mm *mm); + static u32 user_event_key(char *name) { return jhash(name, strlen(name), 0); } -static void set_page_reservations(char *pages, bool set) -{ - int page; - - for (page = 0; page < MAX_PAGES; ++page) { - void *addr = pages + (PAGE_SIZE * page); - - if (set) - SetPageReserved(virt_to_page(addr)); - else - ClearPageReserved(virt_to_page(addr)); - } -} - static void user_event_group_destroy(struct user_event_group *group) { - if (group->register_page_data) - set_page_reservations(group->register_page_data, false); - - if (group->pages) - __free_pages(group->pages, MAX_PAGE_ORDER); - kfree(group->system_name); kfree(group); } @@ -242,19 +220,6 @@ static struct user_event_group if (!group->system_name) goto error; - group->pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, MAX_PAGE_ORDER); - - if (!group->pages) - goto error; - - group->register_page_data = page_address(group->pages); - - set_page_reservations(group->register_page_data, true); - - /* Zero all bits beside 0 (which is reserved for failures) */ - bitmap_zero(group->page_bitmap, MAX_EVENTS); - set_bit(0, group->page_bitmap); - mutex_init(&group->reg_mutex); hash_init(group->register_table); @@ -266,20 +231,367 @@ error: return NULL; }; -static __always_inline -void user_event_register_set(struct user_event *user) +static void user_event_enabler_destroy(struct user_event_enabler *enabler) +{ + list_del_rcu(&enabler->link); + + /* No longer tracking the event via the enabler */ + refcount_dec(&enabler->event->refcnt); + + kfree(enabler); +} + +static int user_event_mm_fault_in(struct user_event_mm *mm, unsigned long uaddr) +{ + bool unlocked; + int ret; + + mmap_read_lock(mm->mm); + + /* Ensure MM has tasks, cannot use after exit_mm() */ + if (refcount_read(&mm->tasks) == 0) { + ret = -ENOENT; + goto out; + } + + ret = fixup_user_fault(mm->mm, uaddr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, + &unlocked); +out: + mmap_read_unlock(mm->mm); + + return ret; +} + +static int user_event_enabler_write(struct user_event_mm *mm, + struct user_event_enabler *enabler) +{ + unsigned long uaddr = enabler->addr; + unsigned long *ptr; + struct page *page; + void *kaddr; + int ret; + + lockdep_assert_held(&event_mutex); + mmap_assert_locked(mm->mm); + + /* Ensure MM has tasks, cannot use after exit_mm() */ + if (refcount_read(&mm->tasks) == 0) + return -ENOENT; + + ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT, + &page, NULL, NULL); + + if (ret <= 0) { + pr_warn("user_events: Enable write failed\n"); + return -EFAULT; + } + + kaddr = kmap_local_page(page); + ptr = kaddr + (uaddr & ~PAGE_MASK); + + /* Update bit atomically, user tracers must be atomic as well */ + if (enabler->event && enabler->event->status) + set_bit(enabler->values & ENABLE_VAL_BIT_MASK, ptr); + else + clear_bit(enabler->values & ENABLE_VAL_BIT_MASK, ptr); + + kunmap_local(kaddr); + unpin_user_pages_dirty_lock(&page, 1, true); + + return 0; +} + +static void user_event_enabler_update(struct user_event *user) +{ + struct user_event_enabler *enabler; + struct user_event_mm *mm = user_event_mm_get_all(user); + struct user_event_mm *next; + + while (mm) { + next = mm->next; + mmap_read_lock(mm->mm); + rcu_read_lock(); + + list_for_each_entry_rcu(enabler, &mm->enablers, link) + if (enabler->event == user) + user_event_enabler_write(mm, enabler); + + rcu_read_unlock(); + mmap_read_unlock(mm->mm); + user_event_mm_put(mm); + mm = next; + } +} + +static bool user_event_enabler_dup(struct user_event_enabler *orig, + struct user_event_mm *mm) +{ + struct user_event_enabler *enabler; + + enabler = kzalloc(sizeof(*enabler), GFP_NOWAIT); + + if (!enabler) + return false; + + enabler->event = orig->event; + enabler->addr = orig->addr; + + /* Only dup part of value (ignore future flags, etc) */ + enabler->values = orig->values & ENABLE_VAL_DUP_MASK; + + refcount_inc(&enabler->event->refcnt); + list_add_rcu(&enabler->link, &mm->enablers); + + return true; +} + +static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm) +{ + refcount_inc(&mm->refcnt); + + return mm; +} + +static struct user_event_mm *user_event_mm_get_all(struct user_event *user) +{ + struct user_event_mm *found = NULL; + struct user_event_enabler *enabler; + struct user_event_mm *mm; + + /* + * We do not want to block fork/exec while enablements are being + * updated, so we use RCU to walk the current tasks that have used + * user_events ABI for 1 or more events. Each enabler found in each + * task that matches the event being updated has a write to reflect + * the kernel state back into the process. Waits/faults must not occur + * during this. So we scan the list under RCU for all the mm that have + * the event within it. This is needed because mm_read_lock() can wait. + * Each user mm returned has a ref inc to handle remove RCU races. + */ + rcu_read_lock(); + + list_for_each_entry_rcu(mm, &user_event_mms, link) + list_for_each_entry_rcu(enabler, &mm->enablers, link) + if (enabler->event == user) { + mm->next = found; + found = user_event_mm_get(mm); + break; + } + + rcu_read_unlock(); + + return found; +} + +static struct user_event_mm *user_event_mm_create(struct task_struct *t) +{ + struct user_event_mm *user_mm; + unsigned long flags; + + user_mm = kzalloc(sizeof(*user_mm), GFP_KERNEL); + + if (!user_mm) + return NULL; + + user_mm->mm = t->mm; + INIT_LIST_HEAD(&user_mm->enablers); + refcount_set(&user_mm->refcnt, 1); + refcount_set(&user_mm->tasks, 1); + + spin_lock_irqsave(&user_event_mms_lock, flags); + list_add_rcu(&user_mm->link, &user_event_mms); + spin_unlock_irqrestore(&user_event_mms_lock, flags); + + t->user_event_mm = user_mm; + + /* + * The lifetime of the memory descriptor can slightly outlast + * the task lifetime if a ref to the user_event_mm is taken + * between list_del_rcu() and call_rcu(). Therefore we need + * to take a reference to it to ensure it can live this long + * under this corner case. This can also occur in clones that + * outlast the parent. + */ + mmgrab(user_mm->mm); + + return user_mm; +} + +static struct user_event_mm *current_user_event_mm(void) +{ + struct user_event_mm *user_mm = current->user_event_mm; + + if (user_mm) + goto inc; + + user_mm = user_event_mm_create(current); + + if (!user_mm) + goto error; +inc: + refcount_inc(&user_mm->refcnt); +error: + return user_mm; +} + +static void user_event_mm_destroy(struct user_event_mm *mm) +{ + struct user_event_enabler *enabler, *next; + + list_for_each_entry_safe(enabler, next, &mm->enablers, link) + user_event_enabler_destroy(enabler); + + mmdrop(mm->mm); + kfree(mm); +} + +static void user_event_mm_put(struct user_event_mm *mm) +{ + if (mm && refcount_dec_and_test(&mm->refcnt)) + user_event_mm_destroy(mm); +} + +static void delayed_user_event_mm_put(struct work_struct *work) +{ + struct user_event_mm *mm; + + mm = container_of(to_rcu_work(work), struct user_event_mm, put_rwork); + user_event_mm_put(mm); +} + +void user_event_mm_remove(struct task_struct *t) { - int i = user->index; + struct user_event_mm *mm; + unsigned long flags; + + might_sleep(); + + mm = t->user_event_mm; + t->user_event_mm = NULL; + + /* Clone will increment the tasks, only remove if last clone */ + if (!refcount_dec_and_test(&mm->tasks)) + return; + + /* Remove the mm from the list, so it can no longer be enabled */ + spin_lock_irqsave(&user_event_mms_lock, flags); + list_del_rcu(&mm->link); + spin_unlock_irqrestore(&user_event_mms_lock, flags); + + /* + * We need to wait for currently occurring writes to stop within + * the mm. This is required since exit_mm() snaps the current rss + * stats and clears them. On the final mmdrop(), check_mm() will + * report a bug if these increment. + * + * All writes/pins are done under mmap_read lock, take the write + * lock to ensure in-progress faults have completed. Faults that + * are pending but yet to run will check the task count and skip + * the fault since the mm is going away. + */ + mmap_write_lock(mm->mm); + mmap_write_unlock(mm->mm); - user->group->register_page_data[MAP_STATUS_BYTE(i)] |= MAP_STATUS_MASK(i); + /* + * Put for mm must be done after RCU delay to handle new refs in + * between the list_del_rcu() and now. This ensures any get refs + * during rcu_read_lock() are accounted for during list removal. + * + * CPU A | CPU B + * --------------------------------------------------------------- + * user_event_mm_remove() | rcu_read_lock(); + * list_del_rcu() | list_for_each_entry_rcu(); + * call_rcu() | refcount_inc(); + * . | rcu_read_unlock(); + * schedule_work() | . + * user_event_mm_put() | . + * + * mmdrop() cannot be called in the softirq context of call_rcu() + * so we use a work queue after call_rcu() to run within. + */ + INIT_RCU_WORK(&mm->put_rwork, delayed_user_event_mm_put); + queue_rcu_work(system_wq, &mm->put_rwork); } -static __always_inline -void user_event_register_clear(struct user_event *user) +void user_event_mm_dup(struct task_struct *t, struct user_event_mm *old_mm) { - int i = user->index; + struct user_event_mm *mm = user_event_mm_create(t); + struct user_event_enabler *enabler; - user->group->register_page_data[MAP_STATUS_BYTE(i)] &= ~MAP_STATUS_MASK(i); + if (!mm) + return; + + rcu_read_lock(); + + list_for_each_entry_rcu(enabler, &old_mm->enablers, link) + if (!user_event_enabler_dup(enabler, mm)) + goto error; + + rcu_read_unlock(); + + return; +error: + rcu_read_unlock(); + user_event_mm_remove(t); +} + +static struct user_event_enabler +*user_event_enabler_create(struct user_reg *reg, struct user_event *user, + int *write_result) +{ + struct user_event_enabler *enabler; + struct user_event_mm *user_mm; + unsigned long uaddr = (unsigned long)reg->enable_addr; + + user_mm = current_user_event_mm(); + + if (!user_mm) + return NULL; + + enabler = kzalloc(sizeof(*enabler), GFP_KERNEL); + + if (!enabler) + goto out; + + enabler->event = user; + enabler->addr = uaddr; + enabler->values = reg->enable_bit; +retry: + /* Prevents state changes from racing with new enablers */ + mutex_lock(&event_mutex); + + /* Attempt to reflect the current state within the process */ + mmap_read_lock(user_mm->mm); + *write_result = user_event_enabler_write(user_mm, enabler); + mmap_read_unlock(user_mm->mm); + + /* + * If the write works, then we will track the enabler. A ref to the + * underlying user_event is held by the enabler to prevent it going + * away while the enabler is still in use by a process. The ref is + * removed when the enabler is destroyed. This means a event cannot + * be forcefully deleted from the system until all tasks using it + * exit or run exec(), which includes forks and clones. + */ + if (!*write_result) { + refcount_inc(&enabler->event->refcnt); + list_add_rcu(&enabler->link, &user_mm->enablers); + } + + mutex_unlock(&event_mutex); + + if (*write_result) { + /* Attempt to fault-in and retry if it worked */ + if (!user_event_mm_fault_in(user_mm, uaddr)) + goto retry; + + kfree(enabler); + enabler = NULL; + } +out: + user_event_mm_put(user_mm); + + return enabler; } static __always_inline __must_check @@ -824,9 +1136,6 @@ static int destroy_user_event(struct user_event *user) return ret; dyn_event_remove(&user->devent); - - user_event_register_clear(user); - clear_bit(user->index, user->group->page_bitmap); hash_del(&user->node); user_event_destroy_validators(user); @@ -972,9 +1281,9 @@ discard: #endif /* - * Update the register page that is shared between user processes. + * Update the enabled bit among all user processes. */ -static void update_reg_page_for(struct user_event *user) +static void update_enable_bit_for(struct user_event *user) { struct tracepoint *tp = &user->tracepoint; char status = 0; @@ -1005,12 +1314,9 @@ static void update_reg_page_for(struct user_event *user) rcu_read_unlock_sched(); } - if (status) - user_event_register_set(user); - else - user_event_register_clear(user); - user->status = status; + + user_event_enabler_update(user); } /* @@ -1067,10 +1373,10 @@ static int user_event_reg(struct trace_event_call *call, return ret; inc: refcount_inc(&user->refcnt); - update_reg_page_for(user); + update_enable_bit_for(user); return 0; dec: - update_reg_page_for(user); + update_enable_bit_for(user); refcount_dec(&user->refcnt); return 0; } @@ -1266,7 +1572,6 @@ static int user_event_parse(struct user_event_group *group, char *name, struct user_event **newuser) { int ret; - int index; u32 key; struct user_event *user; @@ -1285,11 +1590,6 @@ static int user_event_parse(struct user_event_group *group, char *name, return 0; } - index = find_first_zero_bit(group->page_bitmap, MAX_EVENTS); - - if (index == MAX_EVENTS) - return -EMFILE; - user = kzalloc(sizeof(*user), GFP_KERNEL); if (!user) @@ -1335,14 +1635,11 @@ static int user_event_parse(struct user_event_group *group, char *name, if (ret) goto put_user_lock; - user->index = index; - /* Ensure we track self ref and caller ref (2) */ refcount_set(&user->refcnt, 2); dyn_event_init(&user->devent, &user_event_dops); dyn_event_add(&user->devent, &user->call); - set_bit(user->index, group->page_bitmap); hash_add(group->register_table, &user->node, key); mutex_unlock(&event_mutex); @@ -1559,6 +1856,37 @@ static long user_reg_get(struct user_reg __user *ureg, struct user_reg *kreg) if (ret) return ret; + /* Ensure no flags, since we don't support any yet */ + if (kreg->flags != 0) + return -EINVAL; + + /* Ensure supported size */ + switch (kreg->enable_size) { + case 4: + /* 32-bit */ + break; +#if BITS_PER_LONG >= 64 + case 8: + /* 64-bit */ + break; +#endif + default: + return -EINVAL; + } + + /* Ensure natural alignment */ + if (kreg->enable_addr % kreg->enable_size) + return -EINVAL; + + /* Ensure bit range for size */ + if (kreg->enable_bit > (kreg->enable_size * BITS_PER_BYTE) - 1) + return -EINVAL; + + /* Ensure accessible */ + if (!access_ok((const void __user *)(uintptr_t)kreg->enable_addr, + kreg->enable_size)) + return -EFAULT; + kreg->size = size; return 0; @@ -1573,8 +1901,10 @@ static long user_events_ioctl_reg(struct user_event_file_info *info, struct user_reg __user *ureg = (struct user_reg __user *)uarg; struct user_reg reg; struct user_event *user; + struct user_event_enabler *enabler; char *name; long ret; + int write_result; ret = user_reg_get(ureg, ®); @@ -1605,8 +1935,28 @@ static long user_events_ioctl_reg(struct user_event_file_info *info, if (ret < 0) return ret; + /* + * user_events_ref_add succeeded: + * At this point we have a user_event, it's lifetime is bound by the + * reference count, not this file. If anything fails, the user_event + * still has a reference until the file is released. During release + * any remaining references (from user_events_ref_add) are decremented. + * + * Attempt to create an enabler, which too has a lifetime tied in the + * same way for the event. Once the task that caused the enabler to be + * created exits or issues exec() then the enablers it has created + * will be destroyed and the ref to the event will be decremented. + */ + enabler = user_event_enabler_create(®, user, &write_result); + + if (!enabler) + return -ENOMEM; + + /* Write failed/faulted, give error back to caller */ + if (write_result) + return write_result; + put_user((u32)ret, &ureg->write_index); - put_user(user->index, &ureg->status_bit); return 0; } @@ -1720,38 +2070,6 @@ static const struct file_operations user_data_fops = { .release = user_events_release, }; -static struct user_event_group *user_status_group(struct file *file) -{ - struct seq_file *m = file->private_data; - - if (!m) - return NULL; - - return m->private; -} - -/* - * Maps the shared page into the user process for checking if event is enabled. - */ -static int user_status_mmap(struct file *file, struct vm_area_struct *vma) -{ - char *pages; - struct user_event_group *group = user_status_group(file); - unsigned long size = vma->vm_end - vma->vm_start; - - if (size != MAX_BYTES) - return -EINVAL; - - if (!group) - return -EINVAL; - - pages = group->register_page_data; - - return remap_pfn_range(vma, vma->vm_start, - virt_to_phys(pages) >> PAGE_SHIFT, - size, vm_get_page_prot(VM_READ)); -} - static void *user_seq_start(struct seq_file *m, loff_t *pos) { if (*pos) @@ -1775,7 +2093,7 @@ static int user_seq_show(struct seq_file *m, void *p) struct user_event_group *group = m->private; struct user_event *user; char status; - int i, active = 0, busy = 0, flags; + int i, active = 0, busy = 0; if (!group) return -EINVAL; @@ -1784,11 +2102,10 @@ static int user_seq_show(struct seq_file *m, void *p) hash_for_each(group->register_table, i, user, node) { status = user->status; - flags = user->flags; - seq_printf(m, "%d:%s", user->index, EVENT_NAME(user)); + seq_printf(m, "%s", EVENT_NAME(user)); - if (flags != 0 || status != 0) + if (status != 0) seq_puts(m, " #"); if (status != 0) { @@ -1811,7 +2128,6 @@ static int user_seq_show(struct seq_file *m, void *p) seq_puts(m, "\n"); seq_printf(m, "Active: %d\n", active); seq_printf(m, "Busy: %d\n", busy); - seq_printf(m, "Max: %ld\n", MAX_EVENTS); return 0; } @@ -1847,7 +2163,6 @@ static int user_status_open(struct inode *node, struct file *file) static const struct file_operations user_status_fops = { .open = user_status_open, - .mmap = user_status_mmap, .read = seq_read, .llseek = seq_lseek, .release = seq_release, @@ -1868,8 +2183,7 @@ static int create_user_tracefs(void) goto err; } - /* mmap with MAP_SHARED requires writable fd */ - emmap = tracefs_create_file("user_events_status", TRACE_MODE_WRITE, + emmap = tracefs_create_file("user_events_status", TRACE_MODE_READ, NULL, NULL, &user_status_fops); if (!emmap) { -- cgit v1.2.3 From a4c40c1349e32f9510707ed09e0961626980d8cb Mon Sep 17 00:00:00 2001 From: Beau Belgrave Date: Tue, 28 Mar 2023 16:52:19 -0700 Subject: tracing/user_events: Align structs with tabs for readability Add tabs to make struct members easier to read and unify the style of the code. Link: https://lkml.kernel.org/r/20230328235219.203-13-beaub@linux.microsoft.com Signed-off-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- include/linux/user_events.h | 14 +++---- include/uapi/linux/user_events.h | 24 ++++++------ kernel/trace/trace_events_user.c | 82 ++++++++++++++++++++-------------------- 3 files changed, 60 insertions(+), 60 deletions(-) (limited to 'include/linux') diff --git a/include/linux/user_events.h b/include/linux/user_events.h index 0120b3dd5b03..2847f5a18a86 100644 --- a/include/linux/user_events.h +++ b/include/linux/user_events.h @@ -17,13 +17,13 @@ #ifdef CONFIG_USER_EVENTS struct user_event_mm { - struct list_head link; - struct list_head enablers; - struct mm_struct *mm; - struct user_event_mm *next; - refcount_t refcnt; - refcount_t tasks; - struct rcu_work put_rwork; + struct list_head link; + struct list_head enablers; + struct mm_struct *mm; + struct user_event_mm *next; + refcount_t refcnt; + refcount_t tasks; + struct rcu_work put_rwork; }; extern void user_event_mm_dup(struct task_struct *t, diff --git a/include/uapi/linux/user_events.h b/include/uapi/linux/user_events.h index 3e7275e3234a..2984aae4a2b4 100644 --- a/include/uapi/linux/user_events.h +++ b/include/uapi/linux/user_events.h @@ -25,25 +25,25 @@ struct user_reg { /* Input: Size of the user_reg structure being used */ - __u32 size; + __u32 size; /* Input: Bit in enable address to use */ - __u8 enable_bit; + __u8 enable_bit; /* Input: Enable size in bytes at address */ - __u8 enable_size; + __u8 enable_size; /* Input: Flags for future use, set to 0 */ - __u16 flags; + __u16 flags; /* Input: Address to update when enabled */ - __u64 enable_addr; + __u64 enable_addr; /* Input: Pointer to string with event name, description and flags */ - __u64 name_args; + __u64 name_args; /* Output: Index of the event to use when writing data */ - __u32 write_index; + __u32 write_index; } __attribute__((__packed__)); /* @@ -52,19 +52,19 @@ struct user_reg { */ struct user_unreg { /* Input: Size of the user_unreg structure being used */ - __u32 size; + __u32 size; /* Input: Bit to unregister */ - __u8 disable_bit; + __u8 disable_bit; /* Input: Reserved, set to 0 */ - __u8 __reserved; + __u8 __reserved; /* Input: Reserved, set to 0 */ - __u16 __reserved2; + __u16 __reserved2; /* Input: Address to unregister */ - __u64 disable_addr; + __u64 disable_addr; } __attribute__((__packed__)); #define DIAG_IOC_MAGIC '*' diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 9b43a02e1597..67cb7b53caf6 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -53,9 +53,9 @@ * allows isolation for events by various means. */ struct user_event_group { - char *system_name; - struct hlist_node node; - struct mutex reg_mutex; + char *system_name; + struct hlist_node node; + struct mutex reg_mutex; DECLARE_HASHTABLE(register_table, 8); }; @@ -76,17 +76,17 @@ static unsigned int current_user_events; * refcnt reaches one. */ struct user_event { - struct user_event_group *group; - struct tracepoint tracepoint; - struct trace_event_call call; - struct trace_event_class class; - struct dyn_event devent; - struct hlist_node node; - struct list_head fields; - struct list_head validators; - refcount_t refcnt; - int min_size; - char status; + struct user_event_group *group; + struct tracepoint tracepoint; + struct trace_event_call call; + struct trace_event_class class; + struct dyn_event devent; + struct hlist_node node; + struct list_head fields; + struct list_head validators; + refcount_t refcnt; + int min_size; + char status; }; /* @@ -95,12 +95,12 @@ struct user_event { * these to track enablement sites that are tied to an event. */ struct user_event_enabler { - struct list_head link; - struct user_event *event; - unsigned long addr; + struct list_head link; + struct user_event *event; + unsigned long addr; /* Track enable bit, flags, etc. Aligned for bitops. */ - unsigned int values; + unsigned int values; }; /* Bits 0-5 are for the bit to update upon enable/disable (0-63 allowed) */ @@ -119,9 +119,9 @@ struct user_event_enabler { /* Used for asynchronous faulting in of pages */ struct user_event_enabler_fault { - struct work_struct work; - struct user_event_mm *mm; - struct user_event_enabler *enabler; + struct work_struct work; + struct user_event_mm *mm; + struct user_event_enabler *enabler; }; static struct kmem_cache *fault_cache; @@ -137,23 +137,23 @@ static DEFINE_SPINLOCK(user_event_mms_lock); * These are not shared and only accessible by the file that created it. */ struct user_event_refs { - struct rcu_head rcu; - int count; - struct user_event *events[]; + struct rcu_head rcu; + int count; + struct user_event *events[]; }; struct user_event_file_info { - struct user_event_group *group; - struct user_event_refs *refs; + struct user_event_group *group; + struct user_event_refs *refs; }; #define VALIDATOR_ENSURE_NULL (1 << 0) #define VALIDATOR_REL (1 << 1) struct user_event_validator { - struct list_head link; - int offset; - int flags; + struct list_head link; + int offset; + int flags; }; typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i, @@ -2276,11 +2276,11 @@ out: } static const struct file_operations user_data_fops = { - .open = user_events_open, - .write = user_events_write, - .write_iter = user_events_write_iter, + .open = user_events_open, + .write = user_events_write, + .write_iter = user_events_write_iter, .unlocked_ioctl = user_events_ioctl, - .release = user_events_release, + .release = user_events_release, }; static void *user_seq_start(struct seq_file *m, loff_t *pos) @@ -2346,10 +2346,10 @@ static int user_seq_show(struct seq_file *m, void *p) } static const struct seq_operations user_seq_ops = { - .start = user_seq_start, - .next = user_seq_next, - .stop = user_seq_stop, - .show = user_seq_show, + .start = user_seq_start, + .next = user_seq_next, + .stop = user_seq_stop, + .show = user_seq_show, }; static int user_status_open(struct inode *node, struct file *file) @@ -2375,10 +2375,10 @@ static int user_status_open(struct inode *node, struct file *file) } static const struct file_operations user_status_fops = { - .open = user_status_open, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release, + .open = user_status_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, }; /* -- cgit v1.2.3 From 27a2195efa8d26447c40dd4a6299ea0247786d75 Mon Sep 17 00:00:00 2001 From: Sebastian Reichel Date: Fri, 17 Mar 2023 23:56:55 +0100 Subject: power: supply: core: auto-exposure of simple-battery data Automatically expose data from the simple-battery firmware node for all battery drivers. Reviewed-by: Linus Walleij Reviewed-by: Matti Vaittinen Signed-off-by: Sebastian Reichel --- drivers/power/supply/power_supply_core.c | 179 +++++++++++++++++++++++++++--- drivers/power/supply/power_supply_sysfs.c | 23 +++- include/linux/power_supply.h | 8 ++ 3 files changed, 191 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c index f3d7c1da299f..b2504d205035 100644 --- a/drivers/power/supply/power_supply_core.c +++ b/drivers/power/supply/power_supply_core.c @@ -388,7 +388,7 @@ static int __power_supply_get_supplier_property(struct device *dev, void *_data) struct psy_get_supplier_prop_data *data = _data; if (__power_supply_is_supplied_by(epsy, data->psy)) - if (!epsy->desc->get_property(epsy, data->psp, data->val)) + if (!power_supply_get_property(epsy, data->psp, data->val)) return 1; /* Success */ return 0; /* Continue iterating */ @@ -832,6 +832,133 @@ void power_supply_put_battery_info(struct power_supply *psy, } EXPORT_SYMBOL_GPL(power_supply_put_battery_info); +const enum power_supply_property power_supply_battery_info_properties[] = { + POWER_SUPPLY_PROP_TECHNOLOGY, + POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN, + POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, + POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN, + POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN, + POWER_SUPPLY_PROP_PRECHARGE_CURRENT, + POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX, + POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MIN, + POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MAX, + POWER_SUPPLY_PROP_TEMP_ALERT_MIN, + POWER_SUPPLY_PROP_TEMP_ALERT_MAX, + POWER_SUPPLY_PROP_TEMP_MIN, + POWER_SUPPLY_PROP_TEMP_MAX, +}; +EXPORT_SYMBOL_GPL(power_supply_battery_info_properties); + +const size_t power_supply_battery_info_properties_size = ARRAY_SIZE(power_supply_battery_info_properties); +EXPORT_SYMBOL_GPL(power_supply_battery_info_properties_size); + +bool power_supply_battery_info_has_prop(struct power_supply_battery_info *info, + enum power_supply_property psp) +{ + if (!info) + return false; + + switch (psp) { + case POWER_SUPPLY_PROP_TECHNOLOGY: + return info->technology != POWER_SUPPLY_TECHNOLOGY_UNKNOWN; + case POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN: + return info->energy_full_design_uwh >= 0; + case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: + return info->charge_full_design_uah >= 0; + case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: + return info->voltage_min_design_uv >= 0; + case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN: + return info->voltage_max_design_uv >= 0; + case POWER_SUPPLY_PROP_PRECHARGE_CURRENT: + return info->precharge_current_ua >= 0; + case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT: + return info->charge_term_current_ua >= 0; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + return info->constant_charge_current_max_ua >= 0; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX: + return info->constant_charge_voltage_max_uv >= 0; + case POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MIN: + return info->temp_ambient_alert_min > INT_MIN; + case POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MAX: + return info->temp_ambient_alert_max < INT_MAX; + case POWER_SUPPLY_PROP_TEMP_ALERT_MIN: + return info->temp_alert_min > INT_MIN; + case POWER_SUPPLY_PROP_TEMP_ALERT_MAX: + return info->temp_alert_max < INT_MAX; + case POWER_SUPPLY_PROP_TEMP_MIN: + return info->temp_min > INT_MIN; + case POWER_SUPPLY_PROP_TEMP_MAX: + return info->temp_max < INT_MAX; + default: + return false; + } +} +EXPORT_SYMBOL_GPL(power_supply_battery_info_has_prop); + +int power_supply_battery_info_get_prop(struct power_supply_battery_info *info, + enum power_supply_property psp, + union power_supply_propval *val) +{ + if (!info) + return -EINVAL; + + if (!power_supply_battery_info_has_prop(info, psp)) + return -EINVAL; + + switch (psp) { + case POWER_SUPPLY_PROP_TECHNOLOGY: + val->intval = info->technology; + return 0; + case POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN: + val->intval = info->energy_full_design_uwh; + return 0; + case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: + val->intval = info->charge_full_design_uah; + return 0; + case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: + val->intval = info->voltage_min_design_uv; + return 0; + case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN: + val->intval = info->voltage_max_design_uv; + return 0; + case POWER_SUPPLY_PROP_PRECHARGE_CURRENT: + val->intval = info->precharge_current_ua; + return 0; + case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT: + val->intval = info->charge_term_current_ua; + return 0; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + val->intval = info->constant_charge_current_max_ua; + return 0; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX: + val->intval = info->constant_charge_voltage_max_uv; + return 0; + case POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MIN: + val->intval = info->temp_ambient_alert_min; + return 0; + case POWER_SUPPLY_PROP_TEMP_AMBIENT_ALERT_MAX: + val->intval = info->temp_ambient_alert_max; + return 0; + case POWER_SUPPLY_PROP_TEMP_ALERT_MIN: + val->intval = info->temp_alert_min; + return 0; + case POWER_SUPPLY_PROP_TEMP_ALERT_MAX: + val->intval = info->temp_alert_max; + return 0; + case POWER_SUPPLY_PROP_TEMP_MIN: + val->intval = info->temp_min; + return 0; + case POWER_SUPPLY_PROP_TEMP_MAX: + val->intval = info->temp_max; + return 0; + default: + return -EINVAL; + } +} +EXPORT_SYMBOL_GPL(power_supply_battery_info_get_prop); + /** * power_supply_temp2resist_simple() - find the battery internal resistance * percent from temperature @@ -1046,6 +1173,22 @@ bool power_supply_battery_bti_in_range(struct power_supply_battery_info *info, } EXPORT_SYMBOL_GPL(power_supply_battery_bti_in_range); +static bool psy_has_property(const struct power_supply_desc *psy_desc, + enum power_supply_property psp) +{ + bool found = false; + int i; + + for (i = 0; i < psy_desc->num_properties; i++) { + if (psy_desc->properties[i] == psp) { + found = true; + break; + } + } + + return found; +} + int power_supply_get_property(struct power_supply *psy, enum power_supply_property psp, union power_supply_propval *val) @@ -1056,7 +1199,12 @@ int power_supply_get_property(struct power_supply *psy, return -ENODEV; } - return psy->desc->get_property(psy, psp, val); + if (psy_has_property(psy->desc, psp)) + return psy->desc->get_property(psy, psp, val); + else if (power_supply_battery_info_has_prop(psy->battery_info, psp)) + return power_supply_battery_info_get_prop(psy->battery_info, psp, val); + else + return -EINVAL; } EXPORT_SYMBOL_GPL(power_supply_get_property); @@ -1117,22 +1265,6 @@ void power_supply_unreg_notifier(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(power_supply_unreg_notifier); -static bool psy_has_property(const struct power_supply_desc *psy_desc, - enum power_supply_property psp) -{ - bool found = false; - int i; - - for (i = 0; i < psy_desc->num_properties; i++) { - if (psy_desc->properties[i] == psp) { - found = true; - break; - } - } - - return found; -} - #ifdef CONFIG_THERMAL static int power_supply_read_temp(struct thermal_zone_device *tzd, int *temp) @@ -1255,6 +1387,17 @@ __power_supply_register(struct device *parent, goto check_supplies_failed; } + /* + * Expose constant battery info, if it is available. While there are + * some chargers accessing constant battery data, we only want to + * expose battery data to userspace for battery devices. + */ + if (desc->type == POWER_SUPPLY_TYPE_BATTERY) { + rc = power_supply_get_battery_info(psy, &psy->battery_info); + if (rc && rc != -ENODEV && rc != -ENOENT) + goto check_supplies_failed; + } + spin_lock_init(&psy->changed_lock); rc = device_add(dev); if (rc) diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c index c228205e0953..ba3b125cd66e 100644 --- a/drivers/power/supply/power_supply_sysfs.c +++ b/drivers/power/supply/power_supply_sysfs.c @@ -221,9 +221,10 @@ static struct power_supply_attr power_supply_attrs[] = { POWER_SUPPLY_ATTR(MANUFACTURER), POWER_SUPPLY_ATTR(SERIAL_NUMBER), }; +#define POWER_SUPPLY_ATTR_CNT ARRAY_SIZE(power_supply_attrs) static struct attribute * -__power_supply_attrs[ARRAY_SIZE(power_supply_attrs) + 1]; +__power_supply_attrs[POWER_SUPPLY_ATTR_CNT + 1]; static struct power_supply_attr *to_ps_attr(struct device_attribute *attr) { @@ -380,6 +381,9 @@ static umode_t power_supply_attr_is_visible(struct kobject *kobj, } } + if (power_supply_battery_info_has_prop(psy->battery_info, attrno)) + return mode; + return 0; } @@ -461,6 +465,10 @@ static int add_prop_uevent(const struct device *dev, struct kobj_uevent_env *env int power_supply_uevent(const struct device *dev, struct kobj_uevent_env *env) { const struct power_supply *psy = dev_get_drvdata(dev); + const enum power_supply_property *battery_props = + power_supply_battery_info_properties; + unsigned long psy_drv_properties[POWER_SUPPLY_ATTR_CNT / + sizeof(unsigned long) + 1] = {0}; int ret = 0, j; char *prop_buf; @@ -482,12 +490,25 @@ int power_supply_uevent(const struct device *dev, struct kobj_uevent_env *env) goto out; for (j = 0; j < psy->desc->num_properties; j++) { + set_bit(psy->desc->properties[j], psy_drv_properties); ret = add_prop_uevent(dev, env, psy->desc->properties[j], prop_buf); if (ret) goto out; } + for (j = 0; j < power_supply_battery_info_properties_size; j++) { + if (test_bit(battery_props[j], psy_drv_properties)) + continue; + if (!power_supply_battery_info_has_prop(psy->battery_info, + battery_props[j])) + continue; + ret = add_prop_uevent(dev, env, battery_props[j], + prop_buf); + if (ret) + goto out; + } + out: free_page((unsigned long)prop_buf); diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index aa2c4a7c4826..a427f13c757f 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -301,6 +301,7 @@ struct power_supply { bool initialized; bool removing; atomic_t use_cnt; + struct power_supply_battery_info *battery_info; #ifdef CONFIG_THERMAL struct thermal_zone_device *tzd; struct thermal_cooling_device *tcd; @@ -791,10 +792,17 @@ devm_power_supply_get_by_phandle(struct device *dev, const char *property) { return NULL; } #endif /* CONFIG_OF */ +extern const enum power_supply_property power_supply_battery_info_properties[]; +extern const size_t power_supply_battery_info_properties_size; extern int power_supply_get_battery_info(struct power_supply *psy, struct power_supply_battery_info **info_out); extern void power_supply_put_battery_info(struct power_supply *psy, struct power_supply_battery_info *info); +extern bool power_supply_battery_info_has_prop(struct power_supply_battery_info *info, + enum power_supply_property psp); +extern int power_supply_battery_info_get_prop(struct power_supply_battery_info *info, + enum power_supply_property psp, + union power_supply_propval *val); extern int power_supply_ocv2cap_simple(struct power_supply_battery_ocv_table *table, int table_len, int ocv); extern struct power_supply_battery_ocv_table * -- cgit v1.2.3 From c8f573f312f36861db8e40d9953b6d4b84f1321b Mon Sep 17 00:00:00 2001 From: Sebastian Reichel Date: Fri, 17 Mar 2023 23:56:58 +0100 Subject: power: supply: generic-adc-battery: drop jitter delay support Drop support for configuring IRQ jitter delay by using big enough fixed value. Reviewed-by: Linus Walleij Signed-off-by: Sebastian Reichel --- drivers/power/supply/generic-adc-battery.c | 13 ++++--------- include/linux/power/generic-adc-battery.h | 3 --- 2 files changed, 4 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/power/supply/generic-adc-battery.c b/drivers/power/supply/generic-adc-battery.c index 535972a332b3..e20894460d7f 100644 --- a/drivers/power/supply/generic-adc-battery.c +++ b/drivers/power/supply/generic-adc-battery.c @@ -227,12 +227,10 @@ static void gab_work(struct work_struct *work) static irqreturn_t gab_charged(int irq, void *dev_id) { struct gab *adc_bat = dev_id; - struct gab_platform_data *pdata = adc_bat->pdata; - int delay; - delay = pdata->jitter_delay ? pdata->jitter_delay : JITTER_DEFAULT; schedule_delayed_work(&adc_bat->bat_work, - msecs_to_jiffies(delay)); + msecs_to_jiffies(JITTER_DEFAULT)); + return IRQ_HANDLED; } @@ -358,14 +356,11 @@ static int __maybe_unused gab_suspend(struct device *dev) static int __maybe_unused gab_resume(struct device *dev) { struct gab *adc_bat = dev_get_drvdata(dev); - struct gab_platform_data *pdata = adc_bat->pdata; - int delay; - - delay = pdata->jitter_delay ? pdata->jitter_delay : JITTER_DEFAULT; /* Schedule timer to check current status */ schedule_delayed_work(&adc_bat->bat_work, - msecs_to_jiffies(delay)); + msecs_to_jiffies(JITTER_DEFAULT)); + return 0; } diff --git a/include/linux/power/generic-adc-battery.h b/include/linux/power/generic-adc-battery.h index c68cbf34cd34..50eb4bf28286 100644 --- a/include/linux/power/generic-adc-battery.h +++ b/include/linux/power/generic-adc-battery.h @@ -11,13 +11,10 @@ * @battery_info: recommended structure to specify static power supply * parameters * @cal_charge: calculate charge level. - * @jitter_delay: delay required after the interrupt to check battery - * status.Default set is 10ms. */ struct gab_platform_data { struct power_supply_info battery_info; int (*cal_charge)(long value); - int jitter_delay; }; #endif /* GENERIC_ADC_BATTERY_H */ -- cgit v1.2.3 From 3b6fd262bfcd696aa98af99d71dcf952f289cbee Mon Sep 17 00:00:00 2001 From: Sebastian Reichel Date: Fri, 17 Mar 2023 23:56:59 +0100 Subject: power: supply: generic-adc-battery: drop charge now support Drop CHARGE_NOW support, which requires a platform specific calculation method. Reviewed-by: Linus Walleij Reviewed-by: Matti Vaittinen Signed-off-by: Sebastian Reichel --- drivers/power/supply/generic-adc-battery.c | 4 ---- include/linux/power/generic-adc-battery.h | 2 -- 2 files changed, 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/power/supply/generic-adc-battery.c b/drivers/power/supply/generic-adc-battery.c index e20894460d7f..d07eeb7d46d3 100644 --- a/drivers/power/supply/generic-adc-battery.c +++ b/drivers/power/supply/generic-adc-battery.c @@ -72,7 +72,6 @@ static const enum power_supply_property gab_props[] = { POWER_SUPPLY_PROP_STATUS, POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, POWER_SUPPLY_PROP_CHARGE_EMPTY_DESIGN, - POWER_SUPPLY_PROP_CHARGE_NOW, POWER_SUPPLY_PROP_VOLTAGE_NOW, POWER_SUPPLY_PROP_CURRENT_NOW, POWER_SUPPLY_PROP_TECHNOLOGY, @@ -166,9 +165,6 @@ static int gab_get_property(struct power_supply *psy, case POWER_SUPPLY_PROP_CHARGE_EMPTY_DESIGN: val->intval = 0; break; - case POWER_SUPPLY_PROP_CHARGE_NOW: - val->intval = pdata->cal_charge(result); - break; case POWER_SUPPLY_PROP_VOLTAGE_NOW: case POWER_SUPPLY_PROP_CURRENT_NOW: case POWER_SUPPLY_PROP_POWER_NOW: diff --git a/include/linux/power/generic-adc-battery.h b/include/linux/power/generic-adc-battery.h index 50eb4bf28286..54434e4304d3 100644 --- a/include/linux/power/generic-adc-battery.h +++ b/include/linux/power/generic-adc-battery.h @@ -10,11 +10,9 @@ * struct gab_platform_data - platform_data for generic adc iio battery driver. * @battery_info: recommended structure to specify static power supply * parameters - * @cal_charge: calculate charge level. */ struct gab_platform_data { struct power_supply_info battery_info; - int (*cal_charge)(long value); }; #endif /* GENERIC_ADC_BATTERY_H */ -- cgit v1.2.3 From 1b27bf793fd46219c882b8e545ec736cfadc0841 Mon Sep 17 00:00:00 2001 From: Sebastian Reichel Date: Fri, 17 Mar 2023 23:57:01 +0100 Subject: power: supply: generic-adc-battery: use simple-battery API Constant battery data is available through power-supply's simple-battery API. This works automatically, so the manual handling can be removed without loosing any feature :) Note, that the POWER_SUPPLY_STATUS_FULL check for the level variable can be dropped, since the variable is never written. It can be re-introduced properly once the driver gets functionality to calculate the current charge level. Apart from that the check must be done fuzzy anyways, since charge estimation usually is not precise enough to always return exactly the full charge capacity for a full battery. Reviewed-by: Linus Walleij Reviewed-by: Matti Vaittinen Signed-off-by: Sebastian Reichel --- drivers/power/supply/generic-adc-battery.c | 60 ++---------------------------- include/linux/power/generic-adc-battery.h | 18 --------- 2 files changed, 4 insertions(+), 74 deletions(-) delete mode 100644 include/linux/power/generic-adc-battery.h (limited to 'include/linux') diff --git a/drivers/power/supply/generic-adc-battery.c b/drivers/power/supply/generic-adc-battery.c index 771e5cfc49c3..42765cbf7518 100644 --- a/drivers/power/supply/generic-adc-battery.c +++ b/drivers/power/supply/generic-adc-battery.c @@ -22,7 +22,6 @@ #include #include #include -#include #include #define JITTER_DEFAULT 10 /* hope 10ms is enough */ @@ -48,9 +47,7 @@ struct gab { struct power_supply *psy; struct power_supply_desc psy_desc; struct iio_channel *channel[GAB_MAX_CHAN_TYPE]; - struct gab_platform_data *pdata; struct delayed_work bat_work; - int level; int status; bool cable_plugged; struct gpio_desc *charge_finished; @@ -70,14 +67,6 @@ static void gab_ext_power_changed(struct power_supply *psy) static const enum power_supply_property gab_props[] = { POWER_SUPPLY_PROP_STATUS, - POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, - POWER_SUPPLY_PROP_CHARGE_EMPTY_DESIGN, - POWER_SUPPLY_PROP_VOLTAGE_NOW, - POWER_SUPPLY_PROP_CURRENT_NOW, - POWER_SUPPLY_PROP_TECHNOLOGY, - POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN, - POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN, - POWER_SUPPLY_PROP_MODEL_NAME, }; /* @@ -97,17 +86,6 @@ static bool gab_charge_finished(struct gab *adc_bat) return gpiod_get_value(adc_bat->charge_finished); } -static int gab_get_status(struct gab *adc_bat) -{ - struct gab_platform_data *pdata = adc_bat->pdata; - struct power_supply_info *bat_info; - - bat_info = &pdata->battery_info; - if (adc_bat->level == bat_info->charge_full_design) - return POWER_SUPPLY_STATUS_FULL; - return adc_bat->status; -} - static enum gab_chan_type gab_prop_to_chan(enum power_supply_property psp) { switch (psp) { @@ -144,27 +122,14 @@ static int read_channel(struct gab *adc_bat, enum power_supply_property psp, static int gab_get_property(struct power_supply *psy, enum power_supply_property psp, union power_supply_propval *val) { - struct gab *adc_bat; - struct gab_platform_data *pdata; - struct power_supply_info *bat_info; + struct gab *adc_bat = to_generic_bat(psy); int result = 0; int ret = 0; - adc_bat = to_generic_bat(psy); - if (!adc_bat) { - dev_err(&psy->dev, "no battery infos ?!\n"); - return -EINVAL; - } - pdata = adc_bat->pdata; - bat_info = &pdata->battery_info; - switch (psp) { case POWER_SUPPLY_PROP_STATUS: - val->intval = gab_get_status(adc_bat); - break; - case POWER_SUPPLY_PROP_CHARGE_EMPTY_DESIGN: - val->intval = 0; - break; + val->intval = adc_bat->status; + return 0; case POWER_SUPPLY_PROP_VOLTAGE_NOW: case POWER_SUPPLY_PROP_CURRENT_NOW: case POWER_SUPPLY_PROP_POWER_NOW: @@ -173,21 +138,6 @@ static int gab_get_property(struct power_supply *psy, goto err; val->intval = result; break; - case POWER_SUPPLY_PROP_TECHNOLOGY: - val->intval = bat_info->technology; - break; - case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: - val->intval = bat_info->voltage_min_design; - break; - case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN: - val->intval = bat_info->voltage_max_design; - break; - case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: - val->intval = bat_info->charge_full_design; - break; - case POWER_SUPPLY_PROP_MODEL_NAME: - val->strval = bat_info->name; - break; default: return -EINVAL; } @@ -235,7 +185,6 @@ static int gab_probe(struct platform_device *pdev) struct gab *adc_bat; struct power_supply_desc *psy_desc; struct power_supply_config psy_cfg = {}; - struct gab_platform_data *pdata = pdev->dev.platform_data; enum power_supply_property *properties; int ret = 0; int chan; @@ -248,7 +197,7 @@ static int gab_probe(struct platform_device *pdev) psy_cfg.drv_data = adc_bat; psy_desc = &adc_bat->psy_desc; - psy_desc->name = pdata->battery_info.name; + psy_desc->name = dev_name(&pdev->dev); /* bootup default values for the battery */ adc_bat->cable_plugged = false; @@ -256,7 +205,6 @@ static int gab_probe(struct platform_device *pdev) psy_desc->type = POWER_SUPPLY_TYPE_BATTERY; psy_desc->get_property = gab_get_property; psy_desc->external_power_changed = gab_ext_power_changed; - adc_bat->pdata = pdata; /* * copying the static properties and allocating extra memory for holding diff --git a/include/linux/power/generic-adc-battery.h b/include/linux/power/generic-adc-battery.h deleted file mode 100644 index 54434e4304d3..000000000000 --- a/include/linux/power/generic-adc-battery.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012, Anish Kumar - */ - -#ifndef GENERIC_ADC_BATTERY_H -#define GENERIC_ADC_BATTERY_H - -/** - * struct gab_platform_data - platform_data for generic adc iio battery driver. - * @battery_info: recommended structure to specify static power supply - * parameters - */ -struct gab_platform_data { - struct power_supply_info battery_info; -}; - -#endif /* GENERIC_ADC_BATTERY_H */ -- cgit v1.2.3 From cd3891158a77685aee6129f7374a018d13540b2c Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Wed, 22 Mar 2023 13:07:58 -0700 Subject: iommu/sva: Move PASID helpers to sva code Preparing to remove IOASID infrastructure, PASID management will be under SVA code. Decouple mm code from IOASID. Reviewed-by: Jason Gunthorpe Signed-off-by: Jacob Pan Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20230322200803.869130-3-jacob.jun.pan@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu-sva.c | 10 +++++++++- include/linux/ioasid.h | 11 +---------- include/linux/iommu.h | 14 +++++++++++++- include/linux/sched/mm.h | 26 -------------------------- kernel/fork.c | 1 + 5 files changed, 24 insertions(+), 38 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 24bf9b2b58aa..fcfdc80a3939 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -44,7 +44,7 @@ int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max) if (!pasid_valid(pasid)) ret = -ENOMEM; else - mm_pasid_set(mm, pasid); + mm->pasid = pasid; out: mutex_unlock(&iommu_sva_lock); return ret; @@ -238,3 +238,11 @@ out_put_mm: return status; } + +void mm_pasid_drop(struct mm_struct *mm) +{ + if (pasid_valid(mm->pasid)) { + ioasid_free(mm->pasid); + mm->pasid = INVALID_IOASID; + } +} diff --git a/include/linux/ioasid.h b/include/linux/ioasid.h index af1c9d62e642..734bf0e036ee 100644 --- a/include/linux/ioasid.h +++ b/include/linux/ioasid.h @@ -4,8 +4,8 @@ #include #include +#include -#define INVALID_IOASID ((ioasid_t)-1) typedef unsigned int ioasid_t; typedef ioasid_t (*ioasid_alloc_fn_t)(ioasid_t min, ioasid_t max, void *data); typedef void (*ioasid_free_fn_t)(ioasid_t ioasid, void *data); @@ -40,10 +40,6 @@ void *ioasid_find(struct ioasid_set *set, ioasid_t ioasid, int ioasid_register_allocator(struct ioasid_allocator_ops *allocator); void ioasid_unregister_allocator(struct ioasid_allocator_ops *allocator); int ioasid_set_data(ioasid_t ioasid, void *data); -static inline bool pasid_valid(ioasid_t ioasid) -{ - return ioasid != INVALID_IOASID; -} #else /* !CONFIG_IOASID */ static inline ioasid_t ioasid_alloc(struct ioasid_set *set, ioasid_t min, @@ -74,10 +70,5 @@ static inline int ioasid_set_data(ioasid_t ioasid, void *data) return -ENOTSUPP; } -static inline bool pasid_valid(ioasid_t ioasid) -{ - return false; -} - #endif /* CONFIG_IOASID */ #endif /* __LINUX_IOASID_H */ diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 6595454d4f48..d3f81dc6e4dd 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -13,7 +13,6 @@ #include #include #include -#include #include #define IOMMU_READ (1 << 0) @@ -192,6 +191,8 @@ enum iommu_dev_features { }; #define IOMMU_PASID_INVALID (-1U) +typedef unsigned int ioasid_t; +#define INVALID_IOASID ((ioasid_t)-1) #ifdef CONFIG_IOMMU_API @@ -1172,7 +1173,16 @@ static inline bool tegra_dev_iommu_get_stream_id(struct device *dev, u32 *stream return false; } +static inline bool pasid_valid(ioasid_t ioasid) +{ + return ioasid != INVALID_IOASID; +} #ifdef CONFIG_IOMMU_SVA +static inline void mm_pasid_init(struct mm_struct *mm) +{ + mm->pasid = INVALID_IOASID; +} +void mm_pasid_drop(struct mm_struct *mm); struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm); void iommu_sva_unbind_device(struct iommu_sva *handle); @@ -1192,6 +1202,8 @@ static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle) { return IOMMU_PASID_INVALID; } +static inline void mm_pasid_init(struct mm_struct *mm) {} +static inline void mm_pasid_drop(struct mm_struct *mm) {} #endif /* CONFIG_IOMMU_SVA */ #endif /* __LINUX_IOMMU_H */ diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 2a243616f222..da9712a3ba73 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -8,7 +8,6 @@ #include #include #include -#include /* * Routines for handling mm_structs @@ -451,29 +450,4 @@ static inline void membarrier_update_current_mm(struct mm_struct *next_mm) } #endif -#ifdef CONFIG_IOMMU_SVA -static inline void mm_pasid_init(struct mm_struct *mm) -{ - mm->pasid = INVALID_IOASID; -} - -/* Associate a PASID with an mm_struct: */ -static inline void mm_pasid_set(struct mm_struct *mm, u32 pasid) -{ - mm->pasid = pasid; -} - -static inline void mm_pasid_drop(struct mm_struct *mm) -{ - if (pasid_valid(mm->pasid)) { - ioasid_free(mm->pasid); - mm->pasid = INVALID_IOASID; - } -} -#else -static inline void mm_pasid_init(struct mm_struct *mm) {} -static inline void mm_pasid_set(struct mm_struct *mm, u32 pasid) {} -static inline void mm_pasid_drop(struct mm_struct *mm) {} -#endif - #endif /* _LINUX_SCHED_MM_H */ diff --git a/kernel/fork.c b/kernel/fork.c index d8cda4c6de6c..9f14e4084fc0 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -97,6 +97,7 @@ #include #include #include +#include #include #include -- cgit v1.2.3 From fffaed1e24b8d114e958d180cb4a8aed3febbb5a Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Wed, 22 Mar 2023 13:08:02 -0700 Subject: iommu/ioasid: Rename INVALID_IOASID INVALID_IOASID and IOMMU_PASID_INVALID are duplicated. Rename INVALID_IOASID and consolidate since we are moving away from IOASID infrastructure. Reviewed-by: Dave Jiang Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Lu Baolu Signed-off-by: Jacob Pan Link: https://lore.kernel.org/r/20230322200803.869130-7-jacob.jun.pan@linux.intel.com Signed-off-by: Joerg Roedel --- Documentation/x86/sva.rst | 2 +- arch/x86/kernel/traps.c | 2 +- drivers/dma/idxd/device.c | 8 ++++---- drivers/dma/idxd/idxd.h | 1 + drivers/dma/idxd/init.c | 2 +- drivers/dma/idxd/irq.c | 2 +- drivers/iommu/intel/dmar.c | 4 ++-- drivers/iommu/intel/iommu.c | 2 +- drivers/iommu/intel/svm.c | 2 +- include/linux/ioasid.h | 1 + include/linux/iommu.h | 6 +++--- mm/init-mm.c | 4 ++-- 12 files changed, 19 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/Documentation/x86/sva.rst b/Documentation/x86/sva.rst index 2e9b8b0f9a0f..33cb05005982 100644 --- a/Documentation/x86/sva.rst +++ b/Documentation/x86/sva.rst @@ -107,7 +107,7 @@ process share the same page tables, thus the same MSR value. PASID Life Cycle Management =========================== -PASID is initialized as INVALID_IOASID (-1) when a process is created. +PASID is initialized as IOMMU_PASID_INVALID (-1) when a process is created. Only processes that access SVA-capable devices need to have a PASID allocated. This allocation happens when a process opens/binds an SVA-capable diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index d317dc3d06a3..492a60febb11 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -40,7 +40,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 5f321f3b4242..6fca8fa8d3a8 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -1194,7 +1194,7 @@ static void idxd_device_set_perm_entry(struct idxd_device *idxd, { union msix_perm mperm; - if (ie->pasid == INVALID_IOASID) + if (ie->pasid == IOMMU_PASID_INVALID) return; mperm.bits = 0; @@ -1224,7 +1224,7 @@ void idxd_wq_free_irq(struct idxd_wq *wq) idxd_device_clear_perm_entry(idxd, ie); ie->vector = -1; ie->int_handle = INVALID_INT_HANDLE; - ie->pasid = INVALID_IOASID; + ie->pasid = IOMMU_PASID_INVALID; } int idxd_wq_request_irq(struct idxd_wq *wq) @@ -1240,7 +1240,7 @@ int idxd_wq_request_irq(struct idxd_wq *wq) ie = &wq->ie; ie->vector = pci_irq_vector(pdev, ie->id); - ie->pasid = device_pasid_enabled(idxd) ? idxd->pasid : INVALID_IOASID; + ie->pasid = device_pasid_enabled(idxd) ? idxd->pasid : IOMMU_PASID_INVALID; idxd_device_set_perm_entry(idxd, ie); rc = request_threaded_irq(ie->vector, NULL, idxd_wq_thread, 0, "idxd-portal", ie); @@ -1265,7 +1265,7 @@ err_int_handle: free_irq(ie->vector, ie); err_irq: idxd_device_clear_perm_entry(idxd, ie); - ie->pasid = INVALID_IOASID; + ie->pasid = IOMMU_PASID_INVALID; return rc; } diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 7ced8d283d98..417e602a46b6 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "registers.h" diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 640d3048368e..e6ee267da0ff 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -105,7 +105,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) ie = idxd_get_ie(idxd, msix_idx); ie->id = msix_idx; ie->int_handle = INVALID_INT_HANDLE; - ie->pasid = INVALID_IOASID; + ie->pasid = IOMMU_PASID_INVALID; spin_lock_init(&ie->list_lock); init_llist_head(&ie->pending_llist); diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index aa314ebec587..242f1f0b9f09 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -80,7 +80,7 @@ static void idxd_int_handle_revoke_drain(struct idxd_irq_entry *ie) desc.opcode = DSA_OPCODE_DRAIN; desc.priv = 1; - if (ie->pasid != INVALID_IOASID) + if (ie->pasid != IOMMU_PASID_INVALID) desc.pasid = ie->pasid; desc.int_handle = ie->int_handle; portal = idxd_wq_portal_addr(wq); diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 0f348439ef0e..10bb20ff50fd 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1958,7 +1958,7 @@ static int dmar_fault_do_one(struct intel_iommu *iommu, int type, return 0; } - if (pasid == INVALID_IOASID) + if (pasid == IOMMU_PASID_INVALID) pr_err("[%s NO_PASID] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n", type ? "DMA Read" : "DMA Write", source_id >> 8, PCI_SLOT(source_id & 0xFF), @@ -2039,7 +2039,7 @@ irqreturn_t dmar_fault(int irq, void *dev_id) if (!ratelimited) /* Using pasid -1 if pasid is not present */ dmar_fault_do_one(iommu, type, fault_reason, - pasid_present ? pasid : INVALID_IOASID, + pasid_present ? pasid : IOMMU_PASID_INVALID, source_id, guest_addr); fault_index++; diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 6c2d1ffb5b0a..350c33605fd3 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -876,7 +876,7 @@ void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, return; } /* For request-without-pasid, get the pasid from context entry */ - if (intel_iommu_sm && pasid == INVALID_IOASID) + if (intel_iommu_sm && pasid == IOMMU_PASID_INVALID) pasid = PASID_RID2PASID; dir_index = pasid >> PASID_PDE_SHIFT; diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 7367f56c3bad..3848a1b0800c 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -273,7 +273,7 @@ static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid, if (WARN_ON(!mutex_is_locked(&pasid_mutex))) return -EINVAL; - if (pasid == INVALID_IOASID || pasid >= PASID_MAX) + if (pasid == IOMMU_PASID_INVALID || pasid >= PASID_MAX) return -EINVAL; svm = pasid_private_find(pasid); diff --git a/include/linux/ioasid.h b/include/linux/ioasid.h index 734bf0e036ee..bbc9c6fd3310 100644 --- a/include/linux/ioasid.h +++ b/include/linux/ioasid.h @@ -7,6 +7,7 @@ #include typedef unsigned int ioasid_t; +#define INVALID_IOASID ((ioasid_t)-1) typedef ioasid_t (*ioasid_alloc_fn_t)(ioasid_t min, ioasid_t max, void *data); typedef void (*ioasid_free_fn_t)(ioasid_t ioasid, void *data); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index d3f81dc6e4dd..54f535ff9868 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -192,7 +192,6 @@ enum iommu_dev_features { #define IOMMU_PASID_INVALID (-1U) typedef unsigned int ioasid_t; -#define INVALID_IOASID ((ioasid_t)-1) #ifdef CONFIG_IOMMU_API @@ -1175,12 +1174,13 @@ static inline bool tegra_dev_iommu_get_stream_id(struct device *dev, u32 *stream static inline bool pasid_valid(ioasid_t ioasid) { - return ioasid != INVALID_IOASID; + return ioasid != IOMMU_PASID_INVALID; } + #ifdef CONFIG_IOMMU_SVA static inline void mm_pasid_init(struct mm_struct *mm) { - mm->pasid = INVALID_IOASID; + mm->pasid = IOMMU_PASID_INVALID; } void mm_pasid_drop(struct mm_struct *mm); struct iommu_sva *iommu_sva_bind_device(struct device *dev, diff --git a/mm/init-mm.c b/mm/init-mm.c index c9327abb771c..a084039f55d8 100644 --- a/mm/init-mm.c +++ b/mm/init-mm.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #ifndef INIT_MM_CONTEXT @@ -40,7 +40,7 @@ struct mm_struct init_mm = { .user_ns = &init_user_ns, .cpu_bitmap = CPU_BITS_NONE, #ifdef CONFIG_IOMMU_SVA - .pasid = INVALID_IOASID, + .pasid = IOMMU_PASID_INVALID, #endif INIT_MM_CONTEXT(init_mm) }; -- cgit v1.2.3 From 99b5726b44230329f35b4c4d7fe1577d4f4edb31 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 22 Mar 2023 13:08:03 -0700 Subject: iommu: Remove ioasid infrastructure This has no use anymore, delete it all. Reviewed-by: Kevin Tian Reviewed-by: Lu Baolu Signed-off-by: Jason Gunthorpe Signed-off-by: Jacob Pan Link: https://lore.kernel.org/r/20230322200803.869130-8-jacob.jun.pan@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/dma/idxd/idxd.h | 1 - drivers/iommu/Kconfig | 5 - drivers/iommu/Makefile | 1 - drivers/iommu/intel/iommu.h | 1 - drivers/iommu/intel/svm.c | 1 - drivers/iommu/ioasid.c | 422 -------------------------------------------- drivers/iommu/iommu-sva.h | 1 - include/linux/ioasid.h | 75 -------- 8 files changed, 507 deletions(-) delete mode 100644 drivers/iommu/ioasid.c delete mode 100644 include/linux/ioasid.h (limited to 'include/linux') diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 417e602a46b6..dd2a6ed8949b 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index c4928514e5e2..db98c3f86e8c 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -3,10 +3,6 @@ config IOMMU_IOVA tristate -# The IOASID library may also be used by non-IOMMU_API users -config IOASID - tristate - # IOMMU_API always gets selected by whoever wants it. config IOMMU_API bool @@ -160,7 +156,6 @@ config IOMMU_DMA # Shared Virtual Addressing config IOMMU_SVA bool - select IOASID config FSL_PAMU bool "Freescale IOMMU support" diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index f461d0651385..769e43d780ce 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -9,7 +9,6 @@ obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o obj-$(CONFIG_IOMMU_IO_PGTABLE_DART) += io-pgtable-dart.o -obj-$(CONFIG_IOASID) += ioasid.o obj-$(CONFIG_IOMMU_IOVA) += iova.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index a2010fb120e2..65b15be72878 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 3848a1b0800c..e95b339e9cdc 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include diff --git a/drivers/iommu/ioasid.c b/drivers/iommu/ioasid.c deleted file mode 100644 index a786c034907c..000000000000 --- a/drivers/iommu/ioasid.c +++ /dev/null @@ -1,422 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * I/O Address Space ID allocator. There is one global IOASID space, split into - * subsets. Users create a subset with DECLARE_IOASID_SET, then allocate and - * free IOASIDs with ioasid_alloc() and ioasid_free(). - */ -#include -#include -#include -#include -#include - -struct ioasid_data { - ioasid_t id; - struct ioasid_set *set; - void *private; - struct rcu_head rcu; -}; - -/* - * struct ioasid_allocator_data - Internal data structure to hold information - * about an allocator. There are two types of allocators: - * - * - Default allocator always has its own XArray to track the IOASIDs allocated. - * - Custom allocators may share allocation helpers with different private data. - * Custom allocators that share the same helper functions also share the same - * XArray. - * Rules: - * 1. Default allocator is always available, not dynamically registered. This is - * to prevent race conditions with early boot code that want to register - * custom allocators or allocate IOASIDs. - * 2. Custom allocators take precedence over the default allocator. - * 3. When all custom allocators sharing the same helper functions are - * unregistered (e.g. due to hotplug), all outstanding IOASIDs must be - * freed. Otherwise, outstanding IOASIDs will be lost and orphaned. - * 4. When switching between custom allocators sharing the same helper - * functions, outstanding IOASIDs are preserved. - * 5. When switching between custom allocator and default allocator, all IOASIDs - * must be freed to ensure unadulterated space for the new allocator. - * - * @ops: allocator helper functions and its data - * @list: registered custom allocators - * @slist: allocators share the same ops but different data - * @flags: attributes of the allocator - * @xa: xarray holds the IOASID space - * @rcu: used for kfree_rcu when unregistering allocator - */ -struct ioasid_allocator_data { - struct ioasid_allocator_ops *ops; - struct list_head list; - struct list_head slist; -#define IOASID_ALLOCATOR_CUSTOM BIT(0) /* Needs framework to track results */ - unsigned long flags; - struct xarray xa; - struct rcu_head rcu; -}; - -static DEFINE_SPINLOCK(ioasid_allocator_lock); -static LIST_HEAD(allocators_list); - -static ioasid_t default_alloc(ioasid_t min, ioasid_t max, void *opaque); -static void default_free(ioasid_t ioasid, void *opaque); - -static struct ioasid_allocator_ops default_ops = { - .alloc = default_alloc, - .free = default_free, -}; - -static struct ioasid_allocator_data default_allocator = { - .ops = &default_ops, - .flags = 0, - .xa = XARRAY_INIT(ioasid_xa, XA_FLAGS_ALLOC), -}; - -static struct ioasid_allocator_data *active_allocator = &default_allocator; - -static ioasid_t default_alloc(ioasid_t min, ioasid_t max, void *opaque) -{ - ioasid_t id; - - if (xa_alloc(&default_allocator.xa, &id, opaque, XA_LIMIT(min, max), GFP_ATOMIC)) { - pr_err("Failed to alloc ioasid from %d to %d\n", min, max); - return INVALID_IOASID; - } - - return id; -} - -static void default_free(ioasid_t ioasid, void *opaque) -{ - struct ioasid_data *ioasid_data; - - ioasid_data = xa_erase(&default_allocator.xa, ioasid); - kfree_rcu(ioasid_data, rcu); -} - -/* Allocate and initialize a new custom allocator with its helper functions */ -static struct ioasid_allocator_data *ioasid_alloc_allocator(struct ioasid_allocator_ops *ops) -{ - struct ioasid_allocator_data *ia_data; - - ia_data = kzalloc(sizeof(*ia_data), GFP_ATOMIC); - if (!ia_data) - return NULL; - - xa_init_flags(&ia_data->xa, XA_FLAGS_ALLOC); - INIT_LIST_HEAD(&ia_data->slist); - ia_data->flags |= IOASID_ALLOCATOR_CUSTOM; - ia_data->ops = ops; - - /* For tracking custom allocators that share the same ops */ - list_add_tail(&ops->list, &ia_data->slist); - - return ia_data; -} - -static bool use_same_ops(struct ioasid_allocator_ops *a, struct ioasid_allocator_ops *b) -{ - return (a->free == b->free) && (a->alloc == b->alloc); -} - -/** - * ioasid_register_allocator - register a custom allocator - * @ops: the custom allocator ops to be registered - * - * Custom allocators take precedence over the default xarray based allocator. - * Private data associated with the IOASID allocated by the custom allocators - * are managed by IOASID framework similar to data stored in xa by default - * allocator. - * - * There can be multiple allocators registered but only one is active. In case - * of runtime removal of a custom allocator, the next one is activated based - * on the registration ordering. - * - * Multiple allocators can share the same alloc() function, in this case the - * IOASID space is shared. - */ -int ioasid_register_allocator(struct ioasid_allocator_ops *ops) -{ - struct ioasid_allocator_data *ia_data; - struct ioasid_allocator_data *pallocator; - int ret = 0; - - spin_lock(&ioasid_allocator_lock); - - ia_data = ioasid_alloc_allocator(ops); - if (!ia_data) { - ret = -ENOMEM; - goto out_unlock; - } - - /* - * No particular preference, we activate the first one and keep - * the later registered allocators in a list in case the first one gets - * removed due to hotplug. - */ - if (list_empty(&allocators_list)) { - WARN_ON(active_allocator != &default_allocator); - /* Use this new allocator if default is not active */ - if (xa_empty(&active_allocator->xa)) { - rcu_assign_pointer(active_allocator, ia_data); - list_add_tail(&ia_data->list, &allocators_list); - goto out_unlock; - } - pr_warn("Default allocator active with outstanding IOASID\n"); - ret = -EAGAIN; - goto out_free; - } - - /* Check if the allocator is already registered */ - list_for_each_entry(pallocator, &allocators_list, list) { - if (pallocator->ops == ops) { - pr_err("IOASID allocator already registered\n"); - ret = -EEXIST; - goto out_free; - } else if (use_same_ops(pallocator->ops, ops)) { - /* - * If the new allocator shares the same ops, - * then they will share the same IOASID space. - * We should put them under the same xarray. - */ - list_add_tail(&ops->list, &pallocator->slist); - goto out_free; - } - } - list_add_tail(&ia_data->list, &allocators_list); - - spin_unlock(&ioasid_allocator_lock); - return 0; -out_free: - kfree(ia_data); -out_unlock: - spin_unlock(&ioasid_allocator_lock); - return ret; -} -EXPORT_SYMBOL_GPL(ioasid_register_allocator); - -/** - * ioasid_unregister_allocator - Remove a custom IOASID allocator ops - * @ops: the custom allocator to be removed - * - * Remove an allocator from the list, activate the next allocator in - * the order it was registered. Or revert to default allocator if all - * custom allocators are unregistered without outstanding IOASIDs. - */ -void ioasid_unregister_allocator(struct ioasid_allocator_ops *ops) -{ - struct ioasid_allocator_data *pallocator; - struct ioasid_allocator_ops *sops; - - spin_lock(&ioasid_allocator_lock); - if (list_empty(&allocators_list)) { - pr_warn("No custom IOASID allocators active!\n"); - goto exit_unlock; - } - - list_for_each_entry(pallocator, &allocators_list, list) { - if (!use_same_ops(pallocator->ops, ops)) - continue; - - if (list_is_singular(&pallocator->slist)) { - /* No shared helper functions */ - list_del(&pallocator->list); - /* - * All IOASIDs should have been freed before - * the last allocator that shares the same ops - * is unregistered. - */ - WARN_ON(!xa_empty(&pallocator->xa)); - if (list_empty(&allocators_list)) { - pr_info("No custom IOASID allocators, switch to default.\n"); - rcu_assign_pointer(active_allocator, &default_allocator); - } else if (pallocator == active_allocator) { - rcu_assign_pointer(active_allocator, - list_first_entry(&allocators_list, - struct ioasid_allocator_data, list)); - pr_info("IOASID allocator changed"); - } - kfree_rcu(pallocator, rcu); - break; - } - /* - * Find the matching shared ops to delete, - * but keep outstanding IOASIDs - */ - list_for_each_entry(sops, &pallocator->slist, list) { - if (sops == ops) { - list_del(&ops->list); - break; - } - } - break; - } - -exit_unlock: - spin_unlock(&ioasid_allocator_lock); -} -EXPORT_SYMBOL_GPL(ioasid_unregister_allocator); - -/** - * ioasid_set_data - Set private data for an allocated ioasid - * @ioasid: the ID to set data - * @data: the private data - * - * For IOASID that is already allocated, private data can be set - * via this API. Future lookup can be done via ioasid_find. - */ -int ioasid_set_data(ioasid_t ioasid, void *data) -{ - struct ioasid_data *ioasid_data; - int ret = 0; - - spin_lock(&ioasid_allocator_lock); - ioasid_data = xa_load(&active_allocator->xa, ioasid); - if (ioasid_data) - rcu_assign_pointer(ioasid_data->private, data); - else - ret = -ENOENT; - spin_unlock(&ioasid_allocator_lock); - - /* - * Wait for readers to stop accessing the old private data, so the - * caller can free it. - */ - if (!ret) - synchronize_rcu(); - - return ret; -} -EXPORT_SYMBOL_GPL(ioasid_set_data); - -/** - * ioasid_alloc - Allocate an IOASID - * @set: the IOASID set - * @min: the minimum ID (inclusive) - * @max: the maximum ID (inclusive) - * @private: data private to the caller - * - * Allocate an ID between @min and @max. The @private pointer is stored - * internally and can be retrieved with ioasid_find(). - * - * Return: the allocated ID on success, or %INVALID_IOASID on failure. - */ -ioasid_t ioasid_alloc(struct ioasid_set *set, ioasid_t min, ioasid_t max, - void *private) -{ - struct ioasid_data *data; - void *adata; - ioasid_t id; - - data = kzalloc(sizeof(*data), GFP_ATOMIC); - if (!data) - return INVALID_IOASID; - - data->set = set; - data->private = private; - - /* - * Custom allocator needs allocator data to perform platform specific - * operations. - */ - spin_lock(&ioasid_allocator_lock); - adata = active_allocator->flags & IOASID_ALLOCATOR_CUSTOM ? active_allocator->ops->pdata : data; - id = active_allocator->ops->alloc(min, max, adata); - if (id == INVALID_IOASID) { - pr_err("Failed ASID allocation %lu\n", active_allocator->flags); - goto exit_free; - } - - if ((active_allocator->flags & IOASID_ALLOCATOR_CUSTOM) && - xa_alloc(&active_allocator->xa, &id, data, XA_LIMIT(id, id), GFP_ATOMIC)) { - /* Custom allocator needs framework to store and track allocation results */ - pr_err("Failed to alloc ioasid from %d\n", id); - active_allocator->ops->free(id, active_allocator->ops->pdata); - goto exit_free; - } - data->id = id; - - spin_unlock(&ioasid_allocator_lock); - return id; -exit_free: - spin_unlock(&ioasid_allocator_lock); - kfree(data); - return INVALID_IOASID; -} -EXPORT_SYMBOL_GPL(ioasid_alloc); - -/** - * ioasid_free - Free an ioasid - * @ioasid: the ID to remove - */ -void ioasid_free(ioasid_t ioasid) -{ - struct ioasid_data *ioasid_data; - - spin_lock(&ioasid_allocator_lock); - ioasid_data = xa_load(&active_allocator->xa, ioasid); - if (!ioasid_data) { - pr_err("Trying to free unknown IOASID %u\n", ioasid); - goto exit_unlock; - } - - active_allocator->ops->free(ioasid, active_allocator->ops->pdata); - /* Custom allocator needs additional steps to free the xa element */ - if (active_allocator->flags & IOASID_ALLOCATOR_CUSTOM) { - ioasid_data = xa_erase(&active_allocator->xa, ioasid); - kfree_rcu(ioasid_data, rcu); - } - -exit_unlock: - spin_unlock(&ioasid_allocator_lock); -} -EXPORT_SYMBOL_GPL(ioasid_free); - -/** - * ioasid_find - Find IOASID data - * @set: the IOASID set - * @ioasid: the IOASID to find - * @getter: function to call on the found object - * - * The optional getter function allows to take a reference to the found object - * under the rcu lock. The function can also check if the object is still valid: - * if @getter returns false, then the object is invalid and NULL is returned. - * - * If the IOASID exists, return the private pointer passed to ioasid_alloc. - * Private data can be NULL if not set. Return an error if the IOASID is not - * found, or if @set is not NULL and the IOASID does not belong to the set. - */ -void *ioasid_find(struct ioasid_set *set, ioasid_t ioasid, - bool (*getter)(void *)) -{ - void *priv; - struct ioasid_data *ioasid_data; - struct ioasid_allocator_data *idata; - - rcu_read_lock(); - idata = rcu_dereference(active_allocator); - ioasid_data = xa_load(&idata->xa, ioasid); - if (!ioasid_data) { - priv = ERR_PTR(-ENOENT); - goto unlock; - } - if (set && ioasid_data->set != set) { - /* data found but does not belong to the set */ - priv = ERR_PTR(-EACCES); - goto unlock; - } - /* Now IOASID and its set is verified, we can return the private data */ - priv = rcu_dereference(ioasid_data->private); - if (getter && !getter(priv)) - priv = NULL; -unlock: - rcu_read_unlock(); - - return priv; -} -EXPORT_SYMBOL_GPL(ioasid_find); - -MODULE_AUTHOR("Jean-Philippe Brucker "); -MODULE_AUTHOR("Jacob Pan "); -MODULE_DESCRIPTION("IO Address Space ID (IOASID) allocator"); -MODULE_LICENSE("GPL"); diff --git a/drivers/iommu/iommu-sva.h b/drivers/iommu/iommu-sva.h index c22d0174ad61..54946b5a7caf 100644 --- a/drivers/iommu/iommu-sva.h +++ b/drivers/iommu/iommu-sva.h @@ -5,7 +5,6 @@ #ifndef _IOMMU_SVA_H #define _IOMMU_SVA_H -#include #include /* I/O Page fault */ diff --git a/include/linux/ioasid.h b/include/linux/ioasid.h deleted file mode 100644 index bbc9c6fd3310..000000000000 --- a/include/linux/ioasid.h +++ /dev/null @@ -1,75 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __LINUX_IOASID_H -#define __LINUX_IOASID_H - -#include -#include -#include - -typedef unsigned int ioasid_t; -#define INVALID_IOASID ((ioasid_t)-1) -typedef ioasid_t (*ioasid_alloc_fn_t)(ioasid_t min, ioasid_t max, void *data); -typedef void (*ioasid_free_fn_t)(ioasid_t ioasid, void *data); - -struct ioasid_set { - int dummy; -}; - -/** - * struct ioasid_allocator_ops - IOASID allocator helper functions and data - * - * @alloc: helper function to allocate IOASID - * @free: helper function to free IOASID - * @list: for tracking ops that share helper functions but not data - * @pdata: data belong to the allocator, provided when calling alloc() - */ -struct ioasid_allocator_ops { - ioasid_alloc_fn_t alloc; - ioasid_free_fn_t free; - struct list_head list; - void *pdata; -}; - -#define DECLARE_IOASID_SET(name) struct ioasid_set name = { 0 } - -#if IS_ENABLED(CONFIG_IOASID) -ioasid_t ioasid_alloc(struct ioasid_set *set, ioasid_t min, ioasid_t max, - void *private); -void ioasid_free(ioasid_t ioasid); -void *ioasid_find(struct ioasid_set *set, ioasid_t ioasid, - bool (*getter)(void *)); -int ioasid_register_allocator(struct ioasid_allocator_ops *allocator); -void ioasid_unregister_allocator(struct ioasid_allocator_ops *allocator); -int ioasid_set_data(ioasid_t ioasid, void *data); - -#else /* !CONFIG_IOASID */ -static inline ioasid_t ioasid_alloc(struct ioasid_set *set, ioasid_t min, - ioasid_t max, void *private) -{ - return INVALID_IOASID; -} - -static inline void ioasid_free(ioasid_t ioasid) { } - -static inline void *ioasid_find(struct ioasid_set *set, ioasid_t ioasid, - bool (*getter)(void *)) -{ - return NULL; -} - -static inline int ioasid_register_allocator(struct ioasid_allocator_ops *allocator) -{ - return -ENOTSUPP; -} - -static inline void ioasid_unregister_allocator(struct ioasid_allocator_ops *allocator) -{ -} - -static inline int ioasid_set_data(ioasid_t ioasid, void *data) -{ - return -ENOTSUPP; -} - -#endif /* CONFIG_IOASID */ -#endif /* __LINUX_IOASID_H */ -- cgit v1.2.3 From 27fc5ec673b527dbc2f44787246a39c5ecc01de5 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Wed, 1 Mar 2023 17:32:53 +0100 Subject: clk: Introduce devm_clk_hw_register_gate_parent_data() Add an API for clock gate that uses parent_data for the parent instead of a string parent_name. Reviewed-by: Peng Fan Reviewed-by: Fabio Estevam Tested-by: Adam Ford #imx8mp-beacon-kit Tested-by: Alexander Stein Signed-off-by: Marek Vasut Tested-by: Richard Leitner Reviewed-by: Stephen Boyd Signed-off-by: Abel Vesa Link: https://lore.kernel.org/r/20230301163257.49005-1-marex@denx.de --- include/linux/clk-provider.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'include/linux') diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index 842e72a5348f..92b7c794c627 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -608,6 +608,25 @@ struct clk *clk_register_gate(struct device *dev, const char *name, __devm_clk_hw_register_gate((dev), NULL, (name), (parent_name), NULL, \ NULL, (flags), (reg), (bit_idx), \ (clk_gate_flags), (lock)) + +/** + * devm_clk_hw_register_gate - register a gate clock with the clock framework + * @dev: device that is registering this clock + * @name: name of this clock + * @parent_data: parent clk data + * @flags: framework-specific flags for this clock + * @reg: register address to control gating of this clock + * @bit_idx: which bit in the register controls gating of this clock + * @clk_gate_flags: gate-specific flags for this clock + * @lock: shared register lock for this clock + */ +#define devm_clk_hw_register_gate_parent_data(dev, name, parent_data, flags, \ + reg, bit_idx, clk_gate_flags, \ + lock) \ + __devm_clk_hw_register_gate((dev), NULL, (name), NULL, NULL, \ + (parent_data), (flags), (reg), (bit_idx), \ + (clk_gate_flags), (lock)) + void clk_unregister_gate(struct clk *clk); void clk_hw_unregister_gate(struct clk_hw *hw); int clk_gate_is_enabled(struct clk_hw *hw); -- cgit v1.2.3 From f806bea3093cb3568e01b4375d5e1d7c8c47c1d4 Mon Sep 17 00:00:00 2001 From: Vignesh Raghavendra Date: Thu, 23 Mar 2023 17:31:07 +0530 Subject: dmaengine: ti: k3-udma: Workaround errata i2234 Per [1], UDMA TR15 transactions may hang if ICNT0 is less than 64B Work around is to set EOL flag is to 1 for ICNT0. Since, there is no performance penalty / side effects of setting EOL flag event ICNTO > 64B, just set the flag for all UDMAP TR15 descriptors. [1] https://www.ti.com/lit/er/sprz455a/sprz455a.pdf Errata doc for J721E DRA829/TDA4VM Processors Silicon Revision 1.1/1.0 (Rev. A) Signed-off-by: Vignesh Raghavendra [j-choudhary@ti.com: minor cleanups] Signed-off-by: Jayesh Choudhary Link: https://lore.kernel.org/r/20230323120107.27638-1-j-choudhary@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma.c | 20 +++++++++++--------- include/linux/dma/ti-cppi5.h | 1 + 2 files changed, 12 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index f652a217be76..c07feaff69c0 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -2966,6 +2966,7 @@ udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, struct scatterlist *sgent; struct cppi5_tr_type15_t *tr_req = NULL; enum dma_slave_buswidth dev_width; + u32 csf = CPPI5_TR_CSF_SUPR_EVT; u16 tr_cnt0, tr_cnt1; dma_addr_t dev_addr; struct udma_desc *d; @@ -3036,6 +3037,7 @@ udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, if (uc->ud->match_data->type == DMA_TYPE_UDMA) { asel = 0; + csf |= CPPI5_TR_CSF_EOL_ICNT0; } else { asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; dev_addr |= asel; @@ -3059,7 +3061,7 @@ udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false, true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); - cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT); + cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf); cppi5_tr_set_trigger(&tr_req[tr_idx].flags, uc->config.tr_trigger_type, CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 0, 0); @@ -3105,8 +3107,7 @@ udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false, true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); - cppi5_tr_csf_set(&tr_req[tr_idx].flags, - CPPI5_TR_CSF_SUPR_EVT); + cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf); cppi5_tr_set_trigger(&tr_req[tr_idx].flags, uc->config.tr_trigger_type, CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, @@ -3150,8 +3151,7 @@ udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, d->residue += sg_len; } - cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, - CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); + cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, csf | CPPI5_TR_CSF_EOP); return d; } @@ -3680,6 +3680,7 @@ udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, int num_tr; size_t tr_size = sizeof(struct cppi5_tr_type15_t); u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; + u32 csf = CPPI5_TR_CSF_SUPR_EVT; if (uc->config.dir != DMA_MEM_TO_MEM) { dev_err(chan->device->dev, @@ -3710,13 +3711,15 @@ udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, if (uc->ud->match_data->type != DMA_TYPE_UDMA) { src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT; dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT; + } else { + csf |= CPPI5_TR_CSF_EOL_ICNT0; } tr_req = d->hwdesc[0].tr_req_base; cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); - cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT); + cppi5_tr_csf_set(&tr_req[0].flags, csf); tr_req[0].addr = src; tr_req[0].icnt0 = tr0_cnt0; @@ -3735,7 +3738,7 @@ udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, if (num_tr == 2) { cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); - cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT); + cppi5_tr_csf_set(&tr_req[1].flags, csf); tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0; tr_req[1].icnt0 = tr1_cnt0; @@ -3750,8 +3753,7 @@ udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, tr_req[1].dicnt3 = 1; } - cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, - CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); + cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, csf | CPPI5_TR_CSF_EOP); if (uc->config.metadata_size) d->vd.tx.metadata_ops = &metadata_ops; diff --git a/include/linux/dma/ti-cppi5.h b/include/linux/dma/ti-cppi5.h index efa2f0309f00..c53c0f6e3b1a 100644 --- a/include/linux/dma/ti-cppi5.h +++ b/include/linux/dma/ti-cppi5.h @@ -616,6 +616,7 @@ static inline void *cppi5_hdesc_get_swdata(struct cppi5_host_desc_t *desc) #define CPPI5_TR_CSF_SUPR_EVT BIT(2) #define CPPI5_TR_CSF_EOL_ADV_SHIFT (4U) #define CPPI5_TR_CSF_EOL_ADV_MASK GENMASK(6, 4) +#define CPPI5_TR_CSF_EOL_ICNT0 BIT(4) #define CPPI5_TR_CSF_EOP BIT(7) /** -- cgit v1.2.3 From 6f14c02220c791d5c46b0f965b9340c58f3d503d Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 31 Mar 2023 11:33:13 +0200 Subject: driver core: create class_is_registered() Some classes (i.e. gpio), want to know if they have been registered or not, and poke around in the class's internal structures to try to figure this out. Because this is not really a good idea, provide a function for classes to call to try to figure this out. Note, this is racy as the state of the class could change at any moment in time after the call is made, but as usually a class only wants to know if it has been registered yet or not, it should be fairly safe to use, and is just as safe as the previous "poke at the class internals" check was. Move the gpiolib code to use this function as proof that it works properly. Cc: Bartosz Golaszewski Cc: Sebastian Reichel Cc: Benjamin Tissoires Cc: linux-gpio@vger.kernel.org Reviewed-by: Linus Walleij Reviewed-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230331093318.82288-2-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 25 +++++++++++++++++++++++++ drivers/gpio/gpiolib-sysfs.c | 4 ++-- include/linux/device/class.h | 1 + 3 files changed, 28 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index 68a6f9b56d19..a8a1bf976290 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -634,6 +634,31 @@ void class_compat_remove_link(struct class_compat *cls, struct device *dev, } EXPORT_SYMBOL_GPL(class_compat_remove_link); +/** + * class_is_registered - determine if at this moment in time, a class is + * registered in the driver core or not. + * @class: the class to check + * + * Returns a boolean to state if the class is registered in the driver core + * or not. Note that the value could switch right after this call is made, + * so only use this in places where you "know" it is safe to do so (usually + * to determine if the specific class has been registered yet or not). + * + * Be careful in using this. + */ +bool class_is_registered(const struct class *class) +{ + struct subsys_private *sp = class_to_subsys(class); + bool is_initialized = false; + + if (sp) { + is_initialized = true; + subsys_put(sp); + } + return is_initialized; +} +EXPORT_SYMBOL_GPL(class_is_registered); + int __init classes_init(void) { class_kset = kset_create_and_add("class", NULL, NULL); diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index a895915affa5..1a9b21731cc9 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -554,7 +554,7 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) int offset; /* can't export until sysfs is available ... */ - if (!gpio_class.p) { + if (!class_is_registered(&gpio_class)) { pr_debug("%s: called too early!\n", __func__); return -ENOENT; } @@ -728,7 +728,7 @@ int gpiochip_sysfs_register(struct gpio_device *gdev) * register later, in gpiolib_sysfs_init() ... here we just * verify that _some_ field of gpio_class got initialized. */ - if (!gpio_class.p) + if (!class_is_registered(&gpio_class)) return 0; /* diff --git a/include/linux/device/class.h b/include/linux/device/class.h index b53728ca56fb..9cb5db0588c8 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -84,6 +84,7 @@ extern struct kobject *sysfs_dev_block_kobj; int __must_check class_register(struct class *class); void class_unregister(const struct class *class); +bool class_is_registered(const struct class *class); struct class_compat; struct class_compat *class_compat_register(const char *name); -- cgit v1.2.3 From 2df418cf4b720fe3a0db4b4aab67be43d26af9dd Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 31 Mar 2023 11:33:14 +0200 Subject: driver core: class: remove subsystem private pointer from struct class Now that the last users of the subsystem private pointer in struct class are gone, the pointer can be removed, as no one is using it. One step closer to allowing struct class to be const and moved into read-only memory. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230331093318.82288-3-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 4 ---- include/linux/device/class.h | 2 -- 2 files changed, 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index a8a1bf976290..fcfb295363cc 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -97,8 +97,6 @@ static void class_release(struct kobject *kobj) pr_debug("class '%s': release.\n", class->name); - class->p = NULL; - if (class->class_release) class->class_release(class); else @@ -206,7 +204,6 @@ int class_register(struct class *cls) cp->subsys.kobj.kset = class_kset; cp->subsys.kobj.ktype = &class_ktype; cp->class = cls; - cls->p = cp; error = kset_register(&cp->subsys); if (error) @@ -222,7 +219,6 @@ int class_register(struct class *cls) err_out: kfree(cp); - cls->p = NULL; return error; } EXPORT_SYMBOL_GPL(class_register); diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 9cb5db0588c8..f7aad64e256a 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -71,8 +71,6 @@ struct class { void (*get_ownership)(const struct device *dev, kuid_t *uid, kgid_t *gid); const struct dev_pm_ops *pm; - - struct subsys_private *p; }; struct class_dev_iter { -- cgit v1.2.3 From e78195d52981fc634a4e1e66610a316088bd7458 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 31 Mar 2023 11:33:16 +0200 Subject: driver core: class: remove dev_kobj from struct class The dev_kobj field in struct class is now only written to, but never read from, so it can be removed as it is useless. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230331093318.82288-5-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- block/genhd.c | 1 - drivers/base/class.c | 4 ---- include/linux/device/class.h | 2 -- 3 files changed, 7 deletions(-) (limited to 'include/linux') diff --git a/block/genhd.c b/block/genhd.c index e1e1230b1b9f..af7208a37c53 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -899,7 +899,6 @@ static int __init genhd_device_init(void) { int error; - block_class.dev_kobj = sysfs_dev_block_kobj; error = class_register(&block_class); if (unlikely(error)) return error; diff --git a/drivers/base/class.c b/drivers/base/class.c index fcfb295363cc..06b96d6faa19 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -197,10 +197,6 @@ int class_register(struct class *cls) return error; } - /* set the default /sys/dev directory for devices of this class */ - if (!cls->dev_kobj) - cls->dev_kobj = sysfs_dev_char_kobj; - cp->subsys.kobj.kset = class_kset; cp->subsys.kobj.ktype = &class_ktype; cp->class = cls; diff --git a/include/linux/device/class.h b/include/linux/device/class.h index f7aad64e256a..e946642c314e 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -27,7 +27,6 @@ struct fwnode_handle; * @name: Name of the class. * @class_groups: Default attributes of this class. * @dev_groups: Default attributes of the devices that belong to the class. - * @dev_kobj: The kobject that represents this class and links it into the hierarchy. * @dev_uevent: Called when a device is added, removed from this class, or a * few other things that generate uevents to add the environment * variables. @@ -55,7 +54,6 @@ struct class { const struct attribute_group **class_groups; const struct attribute_group **dev_groups; - struct kobject *dev_kobj; int (*dev_uevent)(const struct device *dev, struct kobj_uevent_env *env); char *(*devnode)(const struct device *dev, umode_t *mode); -- cgit v1.2.3 From 575ab414c90a317158db48475a88de42f37e0afa Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 31 Mar 2023 11:33:17 +0200 Subject: driver core: make sysfs_dev_block_kobj static Nothing outside of drivers/base/core.c uses sysfs_dev_block_kobj, so make it static and document what it is used for so we remember it the next time we touch it 15 years from now. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20230331093318.82288-6-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- drivers/base/core.c | 4 +++- include/linux/device/class.h | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/core.c b/drivers/base/core.c index dbc2ba6dfffc..cf6f41c2060c 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -2256,7 +2256,9 @@ int (*platform_notify)(struct device *dev) = NULL; int (*platform_notify_remove)(struct device *dev) = NULL; static struct kobject *dev_kobj; struct kobject *sysfs_dev_char_kobj; -struct kobject *sysfs_dev_block_kobj; + +/* /sys/dev/block */ +static struct kobject *sysfs_dev_block_kobj; static DEFINE_MUTEX(device_hotplug_lock); diff --git a/include/linux/device/class.h b/include/linux/device/class.h index e946642c314e..7e4a1a6329f4 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -76,8 +76,6 @@ struct class_dev_iter { const struct device_type *type; }; -extern struct kobject *sysfs_dev_block_kobj; - int __must_check class_register(struct class *class); void class_unregister(const struct class *class); bool class_is_registered(const struct class *class); -- cgit v1.2.3 From 979207cac517833c828133ffb2633bcdf6edce00 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Sun, 2 Apr 2023 19:58:46 +0200 Subject: driver core: class: mark class_release() as taking a const * The struct class callback, class_release(), is only called in 2 places, the pcmcia cardservices code, and in the class driver core code. Both places it is safe to mark the structure as a const *, to allow us to in the future mark all struct class usages as constant and move into read-only memory. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/2023040248-outrage-obsolete-5a9a@gregkh Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 2 +- drivers/pcmcia/cs.c | 2 +- include/linux/device/class.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index 65502bd7d5c5..53fc7052340c 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -235,7 +235,7 @@ void class_unregister(const struct class *cls) } EXPORT_SYMBOL_GPL(class_unregister); -static void class_create_release(struct class *cls) +static void class_create_release(const struct class *cls) { pr_debug("%s called for %s\n", __func__, cls->name); kfree(cls); diff --git a/drivers/pcmcia/cs.c b/drivers/pcmcia/cs.c index e3224e49c43f..5658745c398f 100644 --- a/drivers/pcmcia/cs.c +++ b/drivers/pcmcia/cs.c @@ -824,7 +824,7 @@ static int pcmcia_socket_uevent(const struct device *dev, static struct completion pcmcia_unload; -static void pcmcia_release_socket_class(struct class *data) +static void pcmcia_release_socket_class(const struct class *data) { complete(&pcmcia_unload); } diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 7e4a1a6329f4..f3c418fa129a 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -58,7 +58,7 @@ struct class { int (*dev_uevent)(const struct device *dev, struct kobj_uevent_env *env); char *(*devnode)(const struct device *dev, umode_t *mode); - void (*class_release)(struct class *class); + void (*class_release)(const struct class *class); void (*dev_release)(struct device *dev); int (*shutdown_pre)(struct device *dev); -- cgit v1.2.3 From 43a7206b0963c2153c95d6985624d1dc1b3abd4d Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Sun, 2 Apr 2023 19:58:47 +0200 Subject: driver core: class: make class_register() take a const * Now that the class code is cleaned up to not modify the class pointer registered with it, change class_register() to take a const * to allow the structure to be placed into read-only memory. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/2023040248-customary-release-4aec@gregkh Signed-off-by: Greg Kroah-Hartman --- drivers/base/base.h | 2 +- drivers/base/class.c | 6 +++--- include/linux/device/class.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/base.h b/drivers/base/base.h index e96f3343fd7c..eb4c0ace9242 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -54,7 +54,7 @@ struct subsys_private { struct device *dev_root; struct kset glue_dirs; - struct class *class; + const struct class *class; struct lock_class_key lock_key; }; diff --git a/drivers/base/class.c b/drivers/base/class.c index 53fc7052340c..05bce79d3d19 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -93,7 +93,7 @@ static ssize_t class_attr_store(struct kobject *kobj, struct attribute *attr, static void class_release(struct kobject *kobj) { struct subsys_private *cp = to_subsys_private(kobj); - struct class *class = cp->class; + const struct class *class = cp->class; pr_debug("class '%s': release.\n", class->name); @@ -110,7 +110,7 @@ static void class_release(struct kobject *kobj) static const struct kobj_ns_type_operations *class_child_ns_type(const struct kobject *kobj) { const struct subsys_private *cp = to_subsys_private(kobj); - struct class *class = cp->class; + const struct class *class = cp->class; return class->ns_type; } @@ -175,7 +175,7 @@ static void klist_class_dev_put(struct klist_node *n) put_device(dev); } -int class_register(struct class *cls) +int class_register(const struct class *cls) { struct subsys_private *cp; struct lock_class_key *key; diff --git a/include/linux/device/class.h b/include/linux/device/class.h index f3c418fa129a..4bf46f9bbb56 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -76,7 +76,7 @@ struct class_dev_iter { const struct device_type *type; }; -int __must_check class_register(struct class *class); +int __must_check class_register(const struct class *class); void class_unregister(const struct class *class); bool class_is_registered(const struct class *class); -- cgit v1.2.3 From 6b0d49be81cf4f1cf30ebd079cbf4643cab0a01d Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Sun, 2 Apr 2023 19:58:48 +0200 Subject: driver core: class: mark the struct class in struct class_interface constant The struct class pointer in struct class_interface is never modified, so mark it as const so that no one accidentally tries to modify it in the future. Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/2023040249-handball-gruffly-5da7@gregkh Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 2 +- include/linux/device/class.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index 05bce79d3d19..ad8b9f163fd2 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -498,7 +498,7 @@ EXPORT_SYMBOL_GPL(class_interface_register); void class_interface_unregister(struct class_interface *class_intf) { struct subsys_private *sp; - struct class *parent = class_intf->class; + const struct class *parent = class_intf->class; struct class_dev_iter iter; struct device *dev; diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 4bf46f9bbb56..53287aa105b8 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -217,7 +217,7 @@ ssize_t show_class_attr_string(const struct class *class, const struct class_att struct class_interface { struct list_head node; - struct class *class; + const struct class *class; int (*add_dev) (struct device *, struct class_interface *); void (*remove_dev) (struct device *, struct class_interface *); -- cgit v1.2.3 From 2243acd50ac4026e93ac4f024109be6dbd7f9098 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Sun, 2 Apr 2023 19:58:49 +0200 Subject: driver core: class: remove struct class_interface * from callbacks The add_dev and remove_dev callbacks in struct class_interface currently pass in a pointer back to the class_interface structure that is calling them, but none of the callback implementations actually use this pointer as it is pointless (the structure is known, the driver passed it in in the first place if it is really needed again.) So clean this up and just remove the pointer from the callbacks and fix up all callback functions. Cc: Jean Delvare Cc: Guenter Roeck Cc: "David S. Miller" Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Paolo Abeni Cc: Kurt Schwemmer Cc: Jon Mason Cc: Dave Jiang Cc: Allen Hubbe Cc: Dominik Brodowski Cc: Matt Porter Cc: Alexandre Bounine Cc: "James E.J. Bottomley" Cc: "Martin K. Petersen" Cc: Doug Gilbert Cc: John Stultz Cc: Thomas Gleixner Cc: Stephen Boyd Cc: Hans de Goede Cc: Andrew Morton Cc: Wang Weiyang Cc: Yang Yingliang Cc: Jakob Koschel Cc: Cai Xinchen Acked-by: Rafael J. Wysocki Acked-by: Logan Gunthorpe Link: https://lore.kernel.org/r/2023040250-pushover-platter-509c@gregkh Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 4 ++-- drivers/base/core.c | 10 ++++------ drivers/hwmon/drivetemp.c | 4 ++-- drivers/net/rionet.c | 3 +-- drivers/ntb/hw/mscc/ntb_hw_switchtec.c | 6 ++---- drivers/pcmcia/ds.c | 6 ++---- drivers/pcmcia/rsrc_nonstatic.c | 6 ++---- drivers/rapidio/devices/rio_mport_cdev.c | 7 ++----- drivers/rapidio/rio_cm.c | 8 ++------ drivers/scsi/ses.c | 6 ++---- drivers/scsi/sg.c | 8 ++++---- include/linux/device/class.h | 4 ++-- kernel/time/alarmtimer.c | 3 +-- 13 files changed, 28 insertions(+), 47 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index ad8b9f163fd2..ac1808d1a2e8 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -486,7 +486,7 @@ int class_interface_register(struct class_interface *class_intf) if (class_intf->add_dev) { class_dev_iter_init(&iter, parent, NULL, NULL); while ((dev = class_dev_iter_next(&iter))) - class_intf->add_dev(dev, class_intf); + class_intf->add_dev(dev); class_dev_iter_exit(&iter); } mutex_unlock(&sp->mutex); @@ -514,7 +514,7 @@ void class_interface_unregister(struct class_interface *class_intf) if (class_intf->remove_dev) { class_dev_iter_init(&iter, parent, NULL, NULL); while ((dev = class_dev_iter_next(&iter))) - class_intf->remove_dev(dev, class_intf); + class_intf->remove_dev(dev); class_dev_iter_exit(&iter); } mutex_unlock(&sp->mutex); diff --git a/drivers/base/core.c b/drivers/base/core.c index 64d188be4df9..7a42d1b6b721 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -541,8 +541,7 @@ static struct class devlink_class = { .dev_release = devlink_dev_release, }; -static int devlink_add_symlinks(struct device *dev, - struct class_interface *class_intf) +static int devlink_add_symlinks(struct device *dev) { int ret; size_t len; @@ -591,8 +590,7 @@ out: return ret; } -static void devlink_remove_symlinks(struct device *dev, - struct class_interface *class_intf) +static void devlink_remove_symlinks(struct device *dev) { struct device_link *link = to_devlink(dev); size_t len; @@ -3647,7 +3645,7 @@ int device_add(struct device *dev) /* notify any interfaces that the device is here */ list_for_each_entry(class_intf, &sp->interfaces, node) if (class_intf->add_dev) - class_intf->add_dev(dev, class_intf); + class_intf->add_dev(dev); mutex_unlock(&sp->mutex); subsys_put(sp); } @@ -3805,7 +3803,7 @@ void device_del(struct device *dev) /* notify any interfaces that the device is now gone */ list_for_each_entry(class_intf, &sp->interfaces, node) if (class_intf->remove_dev) - class_intf->remove_dev(dev, class_intf); + class_intf->remove_dev(dev); /* remove the device from the class list */ klist_del(&dev->p->knode_class); mutex_unlock(&sp->mutex); diff --git a/drivers/hwmon/drivetemp.c b/drivers/hwmon/drivetemp.c index 8e5759b42390..86171031ddc5 100644 --- a/drivers/hwmon/drivetemp.c +++ b/drivers/hwmon/drivetemp.c @@ -550,7 +550,7 @@ static const struct hwmon_chip_info drivetemp_chip_info = { * The device argument points to sdev->sdev_dev. Its parent is * sdev->sdev_gendev, which we can use to get the scsi_device pointer. */ -static int drivetemp_add(struct device *dev, struct class_interface *intf) +static int drivetemp_add(struct device *dev) { struct scsi_device *sdev = to_scsi_device(dev->parent); struct drivetemp_data *st; @@ -585,7 +585,7 @@ abort: return err; } -static void drivetemp_remove(struct device *dev, struct class_interface *intf) +static void drivetemp_remove(struct device *dev) { struct drivetemp_data *st, *tmp; diff --git a/drivers/net/rionet.c b/drivers/net/rionet.c index fbcb9d05da64..4eececc94513 100644 --- a/drivers/net/rionet.c +++ b/drivers/net/rionet.c @@ -662,8 +662,7 @@ static int rionet_shutdown(struct notifier_block *nb, unsigned long code, return NOTIFY_DONE; } -static void rionet_remove_mport(struct device *dev, - struct class_interface *class_intf) +static void rionet_remove_mport(struct device *dev) { struct rio_mport *mport = to_rio_mport(dev); struct net_device *ndev; diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c index 88ae18b0efa8..d6bbcc7b5b90 100644 --- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c +++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c @@ -1470,8 +1470,7 @@ static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev) return rc; } -static int switchtec_ntb_add(struct device *dev, - struct class_interface *class_intf) +static int switchtec_ntb_add(struct device *dev) { struct switchtec_dev *stdev = to_stdev(dev); struct switchtec_ntb *sndev; @@ -1541,8 +1540,7 @@ free_and_exit: return rc; } -static void switchtec_ntb_remove(struct device *dev, - struct class_interface *class_intf) +static void switchtec_ntb_remove(struct device *dev) { struct switchtec_dev *stdev = to_stdev(dev); struct switchtec_ntb *sndev = stdev->sndev; diff --git a/drivers/pcmcia/ds.c b/drivers/pcmcia/ds.c index c8087efa5e4a..d500e5dbbc3f 100644 --- a/drivers/pcmcia/ds.c +++ b/drivers/pcmcia/ds.c @@ -1335,8 +1335,7 @@ static struct pcmcia_callback pcmcia_bus_callback = { .resume = pcmcia_bus_resume, }; -static int pcmcia_bus_add_socket(struct device *dev, - struct class_interface *class_intf) +static int pcmcia_bus_add_socket(struct device *dev) { struct pcmcia_socket *socket = dev_get_drvdata(dev); int ret; @@ -1369,8 +1368,7 @@ static int pcmcia_bus_add_socket(struct device *dev, return 0; } -static void pcmcia_bus_remove_socket(struct device *dev, - struct class_interface *class_intf) +static void pcmcia_bus_remove_socket(struct device *dev) { struct pcmcia_socket *socket = dev_get_drvdata(dev); diff --git a/drivers/pcmcia/rsrc_nonstatic.c b/drivers/pcmcia/rsrc_nonstatic.c index ad1141fddb4c..471e0c5815f3 100644 --- a/drivers/pcmcia/rsrc_nonstatic.c +++ b/drivers/pcmcia/rsrc_nonstatic.c @@ -1200,8 +1200,7 @@ static const struct attribute_group rsrc_attributes = { .attrs = pccard_rsrc_attributes, }; -static int pccard_sysfs_add_rsrc(struct device *dev, - struct class_interface *class_intf) +static int pccard_sysfs_add_rsrc(struct device *dev) { struct pcmcia_socket *s = dev_get_drvdata(dev); @@ -1210,8 +1209,7 @@ static int pccard_sysfs_add_rsrc(struct device *dev, return sysfs_create_group(&dev->kobj, &rsrc_attributes); } -static void pccard_sysfs_remove_rsrc(struct device *dev, - struct class_interface *class_intf) +static void pccard_sysfs_remove_rsrc(struct device *dev) { struct pcmcia_socket *s = dev_get_drvdata(dev); diff --git a/drivers/rapidio/devices/rio_mport_cdev.c b/drivers/rapidio/devices/rio_mport_cdev.c index deb96c3160a7..a115730ebf14 100644 --- a/drivers/rapidio/devices/rio_mport_cdev.c +++ b/drivers/rapidio/devices/rio_mport_cdev.c @@ -2536,10 +2536,8 @@ static void mport_cdev_remove(struct mport_dev *md) /* * mport_add_mport() - Add rio_mport from LDM device struct * @dev: Linux device model struct - * @class_intf: Linux class_interface */ -static int mport_add_mport(struct device *dev, - struct class_interface *class_intf) +static int mport_add_mport(struct device *dev) { struct rio_mport *mport = NULL; struct mport_dev *chdev = NULL; @@ -2559,8 +2557,7 @@ static int mport_add_mport(struct device *dev, * mport_remove_mport() - Remove rio_mport from global list * TODO remove device from global mport_dev list */ -static void mport_remove_mport(struct device *dev, - struct class_interface *class_intf) +static void mport_remove_mport(struct device *dev) { struct rio_mport *mport = NULL; struct mport_dev *chdev; diff --git a/drivers/rapidio/rio_cm.c b/drivers/rapidio/rio_cm.c index acaf9cda014c..49f8d111e546 100644 --- a/drivers/rapidio/rio_cm.c +++ b/drivers/rapidio/rio_cm.c @@ -2087,13 +2087,11 @@ static int riocm_cdev_add(dev_t devno) /* * riocm_add_mport - add new local mport device into channel management core * @dev: device object associated with mport - * @class_intf: class interface * * When a new mport device is added, CM immediately reserves inbound and * outbound RapidIO mailboxes that will be used. */ -static int riocm_add_mport(struct device *dev, - struct class_interface *class_intf) +static int riocm_add_mport(struct device *dev) { int rc; int i; @@ -2166,14 +2164,12 @@ static int riocm_add_mport(struct device *dev, /* * riocm_remove_mport - remove local mport device from channel management core * @dev: device object associated with mport - * @class_intf: class interface * * Removes a local mport device from the list of registered devices that provide * channel management services. Returns an error if the specified mport is not * registered with the CM core. */ -static void riocm_remove_mport(struct device *dev, - struct class_interface *class_intf) +static void riocm_remove_mport(struct device *dev) { struct rio_mport *mport = to_rio_mport(dev); struct cm_dev *cm; diff --git a/drivers/scsi/ses.c b/drivers/scsi/ses.c index b11a9162e73a..57feb0cae896 100644 --- a/drivers/scsi/ses.c +++ b/drivers/scsi/ses.c @@ -663,8 +663,7 @@ static void ses_match_to_enclosure(struct enclosure_device *edev, } } -static int ses_intf_add(struct device *cdev, - struct class_interface *intf) +static int ses_intf_add(struct device *cdev) { struct scsi_device *sdev = to_scsi_device(cdev->parent); struct scsi_device *tmp_sdev; @@ -869,8 +868,7 @@ static void ses_intf_remove_enclosure(struct scsi_device *sdev) enclosure_unregister(edev); } -static void ses_intf_remove(struct device *cdev, - struct class_interface *intf) +static void ses_intf_remove(struct device *cdev) { struct scsi_device *sdev = to_scsi_device(cdev->parent); diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index 4997f880d4a4..037f8c98a6d3 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -96,8 +96,8 @@ static int scatter_elem_sz_prev = SG_SCATTER_SZ; #define SG_SECTOR_SZ 512 -static int sg_add_device(struct device *, struct class_interface *); -static void sg_remove_device(struct device *, struct class_interface *); +static int sg_add_device(struct device *); +static void sg_remove_device(struct device *); static DEFINE_IDR(sg_index_idr); static DEFINE_RWLOCK(sg_index_lock); /* Also used to lock @@ -1488,7 +1488,7 @@ out_unlock: } static int -sg_add_device(struct device *cl_dev, struct class_interface *cl_intf) +sg_add_device(struct device *cl_dev) { struct scsi_device *scsidp = to_scsi_device(cl_dev->parent); Sg_device *sdp = NULL; @@ -1578,7 +1578,7 @@ sg_device_destroy(struct kref *kref) } static void -sg_remove_device(struct device *cl_dev, struct class_interface *cl_intf) +sg_remove_device(struct device *cl_dev) { struct scsi_device *scsidp = to_scsi_device(cl_dev->parent); Sg_device *sdp = dev_get_drvdata(cl_dev); diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 53287aa105b8..9deeaeb457bb 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -219,8 +219,8 @@ struct class_interface { struct list_head node; const struct class *class; - int (*add_dev) (struct device *, struct class_interface *); - void (*remove_dev) (struct device *, struct class_interface *); + int (*add_dev) (struct device *dev); + void (*remove_dev) (struct device *dev); }; int __must_check class_interface_register(struct class_interface *); diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 7e5dff602585..82b28ab0f328 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -81,8 +81,7 @@ struct rtc_device *alarmtimer_get_rtcdev(void) } EXPORT_SYMBOL_GPL(alarmtimer_get_rtcdev); -static int alarmtimer_rtc_add_device(struct device *dev, - struct class_interface *class_intf) +static int alarmtimer_rtc_add_device(struct device *dev) { unsigned long flags; struct rtc_device *rtc = to_rtc_device(dev); -- cgit v1.2.3 From 862d8312eed994a8a9af7aa8e9e15456183b10a7 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Sun, 2 Apr 2023 19:58:50 +0200 Subject: tty: make tty_class a static const structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the driver core allows for struct class to be in read-only memory, move the tty_class structure to be declared at build time placing it into read-only memory, instead of having to be dynamically allocated at boot time. Cc: "Ilpo Järvinen" Reviewed-by: Jiri Slaby Link: https://lore.kernel.org/r/2023040250-landowner-unfitted-11f4@gregkh Signed-off-by: Greg Kroah-Hartman --- drivers/tty/pty.c | 2 +- drivers/tty/tty_io.c | 24 +++++++++++------------- drivers/tty/vt/vt.c | 2 +- include/linux/tty.h | 2 +- 4 files changed, 14 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/pty.c b/drivers/tty/pty.c index 07394fdaf522..2b1c8ab99dba 100644 --- a/drivers/tty/pty.c +++ b/drivers/tty/pty.c @@ -931,7 +931,7 @@ static void __init unix98_pty_init(void) if (cdev_add(&ptmx_cdev, MKDEV(TTYAUX_MAJOR, 2), 1) || register_chrdev_region(MKDEV(TTYAUX_MAJOR, 2), 1, "/dev/ptmx") < 0) panic("Couldn't register /dev/ptmx driver"); - device_create(tty_class, NULL, MKDEV(TTYAUX_MAJOR, 2), NULL, "ptmx"); + device_create(&tty_class, NULL, MKDEV(TTYAUX_MAJOR, 2), NULL, "ptmx"); } #else diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 1382d9050ce8..44e0d53aa0b8 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -3070,7 +3070,7 @@ static struct device *tty_get_device(struct tty_struct *tty) { dev_t devt = tty_devnum(tty); - return class_find_device_by_devt(tty_class, devt); + return class_find_device_by_devt(&tty_class, devt); } @@ -3143,8 +3143,6 @@ int tty_put_char(struct tty_struct *tty, unsigned char ch) } EXPORT_SYMBOL_GPL(tty_put_char); -struct class *tty_class; - static int tty_cdev_add(struct tty_driver *driver, dev_t dev, unsigned int index, unsigned int count) { @@ -3239,7 +3237,7 @@ struct device *tty_register_device_attr(struct tty_driver *driver, return ERR_PTR(-ENOMEM); dev->devt = devt; - dev->class = tty_class; + dev->class = &tty_class; dev->parent = device; dev->release = tty_device_create_release; dev_set_name(dev, "%s", name); @@ -3294,8 +3292,7 @@ EXPORT_SYMBOL_GPL(tty_register_device_attr); */ void tty_unregister_device(struct tty_driver *driver, unsigned index) { - device_destroy(tty_class, - MKDEV(driver->major, driver->minor_start) + index); + device_destroy(&tty_class, MKDEV(driver->major, driver->minor_start) + index); if (!(driver->flags & TTY_DRIVER_DYNAMIC_ALLOC)) { cdev_del(driver->cdevs[index]); driver->cdevs[index] = NULL; @@ -3510,13 +3507,14 @@ static char *tty_devnode(const struct device *dev, umode_t *mode) return NULL; } +const struct class tty_class = { + .name = "tty", + .devnode = tty_devnode, +}; + static int __init tty_class_init(void) { - tty_class = class_create("tty"); - if (IS_ERR(tty_class)) - return PTR_ERR(tty_class); - tty_class->devnode = tty_devnode; - return 0; + return class_register(&tty_class); } postcore_initcall(tty_class_init); @@ -3643,13 +3641,13 @@ int __init tty_init(void) if (cdev_add(&tty_cdev, MKDEV(TTYAUX_MAJOR, 0), 1) || register_chrdev_region(MKDEV(TTYAUX_MAJOR, 0), 1, "/dev/tty") < 0) panic("Couldn't register /dev/tty driver\n"); - device_create(tty_class, NULL, MKDEV(TTYAUX_MAJOR, 0), NULL, "tty"); + device_create(&tty_class, NULL, MKDEV(TTYAUX_MAJOR, 0), NULL, "tty"); cdev_init(&console_cdev, &console_fops); if (cdev_add(&console_cdev, MKDEV(TTYAUX_MAJOR, 1), 1) || register_chrdev_region(MKDEV(TTYAUX_MAJOR, 1), 1, "/dev/console") < 0) panic("Couldn't register /dev/console driver\n"); - consdev = device_create_with_groups(tty_class, NULL, + consdev = device_create_with_groups(&tty_class, NULL, MKDEV(TTYAUX_MAJOR, 1), NULL, cons_dev_groups, "console"); if (IS_ERR(consdev)) diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 81c148064aa6..40e4928eddaf 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -3539,7 +3539,7 @@ int __init vty_init(const struct file_operations *console_fops) if (cdev_add(&vc0_cdev, MKDEV(TTY_MAJOR, 0), 1) || register_chrdev_region(MKDEV(TTY_MAJOR, 0), 1, "/dev/vc/0") < 0) panic("Couldn't register /dev/tty0 driver\n"); - tty0dev = device_create_with_groups(tty_class, NULL, + tty0dev = device_create_with_groups(&tty_class, NULL, MKDEV(TTY_MAJOR, 0), NULL, vt_dev_groups, "tty0"); if (IS_ERR(tty0dev)) diff --git a/include/linux/tty.h b/include/linux/tty.h index 093935e97f42..121eb4b0d325 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -387,7 +387,7 @@ extern struct ktermios tty_std_termios; int vcs_init(void); -extern struct class *tty_class; +extern const struct class tty_class; /** * tty_kref_get - get a tty reference -- cgit v1.2.3 From 65457b74aa9437418e552e8d52d7112d4f9901a6 Mon Sep 17 00:00:00 2001 From: Domenico Cerasuolo Date: Thu, 30 Mar 2023 12:54:16 +0200 Subject: sched/psi: Rename existing poll members in preparation Renaming in PSI implementation to make a clear distinction between privileged and unprivileged triggers code to be implemented in the next patch. Suggested-by: Johannes Weiner Signed-off-by: Domenico Cerasuolo Signed-off-by: Peter Zijlstra (Intel) Acked-by: Johannes Weiner Link: https://lore.kernel.org/r/20230330105418.77061-3-cerasuolodomenico@gmail.com --- include/linux/psi_types.h | 36 +++++----- kernel/sched/psi.c | 163 +++++++++++++++++++++++----------------------- 2 files changed, 100 insertions(+), 99 deletions(-) (limited to 'include/linux') diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h index 1e0a0d7ace3a..1819afa8b198 100644 --- a/include/linux/psi_types.h +++ b/include/linux/psi_types.h @@ -175,26 +175,26 @@ struct psi_group { u64 total[NR_PSI_AGGREGATORS][NR_PSI_STATES - 1]; unsigned long avg[NR_PSI_STATES - 1][3]; - /* Monitor work control */ - struct task_struct __rcu *poll_task; - struct timer_list poll_timer; - wait_queue_head_t poll_wait; - atomic_t poll_wakeup; - atomic_t poll_scheduled; + /* Monitor RT polling work control */ + struct task_struct __rcu *rtpoll_task; + struct timer_list rtpoll_timer; + wait_queue_head_t rtpoll_wait; + atomic_t rtpoll_wakeup; + atomic_t rtpoll_scheduled; /* Protects data used by the monitor */ - struct mutex trigger_lock; - - /* Configured polling triggers */ - struct list_head triggers; - u32 nr_triggers[NR_PSI_STATES - 1]; - u32 poll_states; - u64 poll_min_period; - - /* Total stall times at the start of monitor activation */ - u64 polling_total[NR_PSI_STATES - 1]; - u64 polling_next_update; - u64 polling_until; + struct mutex rtpoll_trigger_lock; + + /* Configured RT polling triggers */ + struct list_head rtpoll_triggers; + u32 rtpoll_nr_triggers[NR_PSI_STATES - 1]; + u32 rtpoll_states; + u64 rtpoll_min_period; + + /* Total stall times at the start of RT polling monitor activation */ + u64 rtpoll_total[NR_PSI_STATES - 1]; + u64 rtpoll_next_update; + u64 rtpoll_until; }; #else /* CONFIG_PSI */ diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index fe9269f1d2a4..a3d0b5cf797a 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -189,14 +189,14 @@ static void group_init(struct psi_group *group) INIT_DELAYED_WORK(&group->avgs_work, psi_avgs_work); mutex_init(&group->avgs_lock); /* Init trigger-related members */ - atomic_set(&group->poll_scheduled, 0); - mutex_init(&group->trigger_lock); - INIT_LIST_HEAD(&group->triggers); - group->poll_min_period = U32_MAX; - group->polling_next_update = ULLONG_MAX; - init_waitqueue_head(&group->poll_wait); - timer_setup(&group->poll_timer, poll_timer_fn, 0); - rcu_assign_pointer(group->poll_task, NULL); + atomic_set(&group->rtpoll_scheduled, 0); + mutex_init(&group->rtpoll_trigger_lock); + INIT_LIST_HEAD(&group->rtpoll_triggers); + group->rtpoll_min_period = U32_MAX; + group->rtpoll_next_update = ULLONG_MAX; + init_waitqueue_head(&group->rtpoll_wait); + timer_setup(&group->rtpoll_timer, poll_timer_fn, 0); + rcu_assign_pointer(group->rtpoll_task, NULL); } void __init psi_init(void) @@ -440,11 +440,11 @@ static u64 update_triggers(struct psi_group *group, u64 now) * On subsequent updates, calculate growth deltas and let * watchers know when their specified thresholds are exceeded. */ - list_for_each_entry(t, &group->triggers, node) { + list_for_each_entry(t, &group->rtpoll_triggers, node) { u64 growth; bool new_stall; - new_stall = group->polling_total[t->state] != total[t->state]; + new_stall = group->rtpoll_total[t->state] != total[t->state]; /* Check for stall activity or a previous threshold breach */ if (!new_stall && !t->pending_event) @@ -486,10 +486,10 @@ static u64 update_triggers(struct psi_group *group, u64 now) } if (update_total) - memcpy(group->polling_total, total, - sizeof(group->polling_total)); + memcpy(group->rtpoll_total, total, + sizeof(group->rtpoll_total)); - return now + group->poll_min_period; + return now + group->rtpoll_min_period; } static u64 update_averages(struct psi_group *group, u64 now) @@ -582,53 +582,53 @@ static void init_triggers(struct psi_group *group, u64 now) { struct psi_trigger *t; - list_for_each_entry(t, &group->triggers, node) + list_for_each_entry(t, &group->rtpoll_triggers, node) window_reset(&t->win, now, group->total[PSI_POLL][t->state], 0); - memcpy(group->polling_total, group->total[PSI_POLL], - sizeof(group->polling_total)); - group->polling_next_update = now + group->poll_min_period; + memcpy(group->rtpoll_total, group->total[PSI_POLL], + sizeof(group->rtpoll_total)); + group->rtpoll_next_update = now + group->rtpoll_min_period; } /* Schedule polling if it's not already scheduled or forced. */ -static void psi_schedule_poll_work(struct psi_group *group, unsigned long delay, +static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long delay, bool force) { struct task_struct *task; /* * atomic_xchg should be called even when !force to provide a - * full memory barrier (see the comment inside psi_poll_work). + * full memory barrier (see the comment inside psi_rtpoll_work). */ - if (atomic_xchg(&group->poll_scheduled, 1) && !force) + if (atomic_xchg(&group->rtpoll_scheduled, 1) && !force) return; rcu_read_lock(); - task = rcu_dereference(group->poll_task); + task = rcu_dereference(group->rtpoll_task); /* * kworker might be NULL in case psi_trigger_destroy races with * psi_task_change (hotpath) which can't use locks */ if (likely(task)) - mod_timer(&group->poll_timer, jiffies + delay); + mod_timer(&group->rtpoll_timer, jiffies + delay); else - atomic_set(&group->poll_scheduled, 0); + atomic_set(&group->rtpoll_scheduled, 0); rcu_read_unlock(); } -static void psi_poll_work(struct psi_group *group) +static void psi_rtpoll_work(struct psi_group *group) { bool force_reschedule = false; u32 changed_states; u64 now; - mutex_lock(&group->trigger_lock); + mutex_lock(&group->rtpoll_trigger_lock); now = sched_clock(); - if (now > group->polling_until) { + if (now > group->rtpoll_until) { /* * We are either about to start or might stop polling if no * state change was recorded. Resetting poll_scheduled leaves @@ -638,7 +638,7 @@ static void psi_poll_work(struct psi_group *group) * should be negligible and polling_next_update still keeps * updates correctly on schedule. */ - atomic_set(&group->poll_scheduled, 0); + atomic_set(&group->rtpoll_scheduled, 0); /* * A task change can race with the poll worker that is supposed to * report on it. To avoid missing events, ensure ordering between @@ -667,9 +667,9 @@ static void psi_poll_work(struct psi_group *group) collect_percpu_times(group, PSI_POLL, &changed_states); - if (changed_states & group->poll_states) { + if (changed_states & group->rtpoll_states) { /* Initialize trigger windows when entering polling mode */ - if (now > group->polling_until) + if (now > group->rtpoll_until) init_triggers(group, now); /* @@ -677,50 +677,50 @@ static void psi_poll_work(struct psi_group *group) * minimum tracking window as long as monitor states are * changing. */ - group->polling_until = now + - group->poll_min_period * UPDATES_PER_WINDOW; + group->rtpoll_until = now + + group->rtpoll_min_period * UPDATES_PER_WINDOW; } - if (now > group->polling_until) { - group->polling_next_update = ULLONG_MAX; + if (now > group->rtpoll_until) { + group->rtpoll_next_update = ULLONG_MAX; goto out; } - if (now >= group->polling_next_update) - group->polling_next_update = update_triggers(group, now); + if (now >= group->rtpoll_next_update) + group->rtpoll_next_update = update_triggers(group, now); - psi_schedule_poll_work(group, - nsecs_to_jiffies(group->polling_next_update - now) + 1, + psi_schedule_rtpoll_work(group, + nsecs_to_jiffies(group->rtpoll_next_update - now) + 1, force_reschedule); out: - mutex_unlock(&group->trigger_lock); + mutex_unlock(&group->rtpoll_trigger_lock); } -static int psi_poll_worker(void *data) +static int psi_rtpoll_worker(void *data) { struct psi_group *group = (struct psi_group *)data; sched_set_fifo_low(current); while (true) { - wait_event_interruptible(group->poll_wait, - atomic_cmpxchg(&group->poll_wakeup, 1, 0) || + wait_event_interruptible(group->rtpoll_wait, + atomic_cmpxchg(&group->rtpoll_wakeup, 1, 0) || kthread_should_stop()); if (kthread_should_stop()) break; - psi_poll_work(group); + psi_rtpoll_work(group); } return 0; } static void poll_timer_fn(struct timer_list *t) { - struct psi_group *group = from_timer(group, t, poll_timer); + struct psi_group *group = from_timer(group, t, rtpoll_timer); - atomic_set(&group->poll_wakeup, 1); - wake_up_interruptible(&group->poll_wait); + atomic_set(&group->rtpoll_wakeup, 1); + wake_up_interruptible(&group->rtpoll_wait); } static void record_times(struct psi_group_cpu *groupc, u64 now) @@ -851,8 +851,8 @@ static void psi_group_change(struct psi_group *group, int cpu, write_seqcount_end(&groupc->seq); - if (state_mask & group->poll_states) - psi_schedule_poll_work(group, 1, false); + if (state_mask & group->rtpoll_states) + psi_schedule_rtpoll_work(group, 1, false); if (wake_clock && !delayed_work_pending(&group->avgs_work)) schedule_delayed_work(&group->avgs_work, PSI_FREQ); @@ -1005,8 +1005,8 @@ void psi_account_irqtime(struct task_struct *task, u32 delta) write_seqcount_end(&groupc->seq); - if (group->poll_states & (1 << PSI_IRQ_FULL)) - psi_schedule_poll_work(group, 1, false); + if (group->rtpoll_states & (1 << PSI_IRQ_FULL)) + psi_schedule_rtpoll_work(group, 1, false); } while ((group = group->parent)); } #endif @@ -1101,7 +1101,7 @@ void psi_cgroup_free(struct cgroup *cgroup) cancel_delayed_work_sync(&cgroup->psi->avgs_work); free_percpu(cgroup->psi->pcpu); /* All triggers must be removed by now */ - WARN_ONCE(cgroup->psi->poll_states, "psi: trigger leak\n"); + WARN_ONCE(cgroup->psi->rtpoll_states, "psi: trigger leak\n"); kfree(cgroup->psi); } @@ -1302,29 +1302,29 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, init_waitqueue_head(&t->event_wait); t->pending_event = false; - mutex_lock(&group->trigger_lock); + mutex_lock(&group->rtpoll_trigger_lock); - if (!rcu_access_pointer(group->poll_task)) { + if (!rcu_access_pointer(group->rtpoll_task)) { struct task_struct *task; - task = kthread_create(psi_poll_worker, group, "psimon"); + task = kthread_create(psi_rtpoll_worker, group, "psimon"); if (IS_ERR(task)) { kfree(t); - mutex_unlock(&group->trigger_lock); + mutex_unlock(&group->rtpoll_trigger_lock); return ERR_CAST(task); } - atomic_set(&group->poll_wakeup, 0); + atomic_set(&group->rtpoll_wakeup, 0); wake_up_process(task); - rcu_assign_pointer(group->poll_task, task); + rcu_assign_pointer(group->rtpoll_task, task); } - list_add(&t->node, &group->triggers); - group->poll_min_period = min(group->poll_min_period, + list_add(&t->node, &group->rtpoll_triggers); + group->rtpoll_min_period = min(group->rtpoll_min_period, div_u64(t->win.size, UPDATES_PER_WINDOW)); - group->nr_triggers[t->state]++; - group->poll_states |= (1 << t->state); + group->rtpoll_nr_triggers[t->state]++; + group->rtpoll_states |= (1 << t->state); - mutex_unlock(&group->trigger_lock); + mutex_unlock(&group->rtpoll_trigger_lock); return t; } @@ -1349,51 +1349,52 @@ void psi_trigger_destroy(struct psi_trigger *t) */ wake_up_pollfree(&t->event_wait); - mutex_lock(&group->trigger_lock); + mutex_lock(&group->rtpoll_trigger_lock); if (!list_empty(&t->node)) { struct psi_trigger *tmp; u64 period = ULLONG_MAX; list_del(&t->node); - group->nr_triggers[t->state]--; - if (!group->nr_triggers[t->state]) - group->poll_states &= ~(1 << t->state); + group->rtpoll_nr_triggers[t->state]--; + if (!group->rtpoll_nr_triggers[t->state]) + group->rtpoll_states &= ~(1 << t->state); /* reset min update period for the remaining triggers */ - list_for_each_entry(tmp, &group->triggers, node) + list_for_each_entry(tmp, &group->rtpoll_triggers, node) period = min(period, div_u64(tmp->win.size, UPDATES_PER_WINDOW)); - group->poll_min_period = period; - /* Destroy poll_task when the last trigger is destroyed */ - if (group->poll_states == 0) { - group->polling_until = 0; + group->rtpoll_min_period = period; + /* Destroy rtpoll_task when the last trigger is destroyed */ + if (group->rtpoll_states == 0) { + group->rtpoll_until = 0; task_to_destroy = rcu_dereference_protected( - group->poll_task, - lockdep_is_held(&group->trigger_lock)); - rcu_assign_pointer(group->poll_task, NULL); - del_timer(&group->poll_timer); + group->rtpoll_task, + lockdep_is_held(&group->rtpoll_trigger_lock)); + rcu_assign_pointer(group->rtpoll_task, NULL); + del_timer(&group->rtpoll_timer); } } - mutex_unlock(&group->trigger_lock); + mutex_unlock(&group->rtpoll_trigger_lock); /* - * Wait for psi_schedule_poll_work RCU to complete its read-side + * Wait for psi_schedule_rtpoll_work RCU to complete its read-side * critical section before destroying the trigger and optionally the - * poll_task. + * rtpoll_task. */ synchronize_rcu(); /* - * Stop kthread 'psimon' after releasing trigger_lock to prevent a - * deadlock while waiting for psi_poll_work to acquire trigger_lock + * Stop kthread 'psimon' after releasing rtpoll_trigger_lock to prevent + * a deadlock while waiting for psi_rtpoll_work to acquire + * rtpoll_trigger_lock */ if (task_to_destroy) { /* * After the RCU grace period has expired, the worker - * can no longer be found through group->poll_task. + * can no longer be found through group->rtpoll_task. */ kthread_stop(task_to_destroy); - atomic_set(&group->poll_scheduled, 0); + atomic_set(&group->rtpoll_scheduled, 0); } kfree(t); } -- cgit v1.2.3 From d82caa273565b45fcf103148950549af76c314b0 Mon Sep 17 00:00:00 2001 From: Domenico Cerasuolo Date: Thu, 30 Mar 2023 12:54:18 +0200 Subject: sched/psi: Allow unprivileged polling of N*2s period PSI offers 2 mechanisms to get information about a specific resource pressure. One is reading from /proc/pressure/, which gives average pressures aggregated every 2s. The other is creating a pollable fd for a specific resource and cgroup. The trigger creation requires CAP_SYS_RESOURCE, and gives the possibility to pick specific time window and threshold, spawing an RT thread to aggregate the data. Systemd would like to provide containers the option to monitor pressure on their own cgroup and sub-cgroups. For example, if systemd launches a container that itself then launches services, the container should have the ability to poll() for pressure in individual services. But neither the container nor the services are privileged. This patch implements a mechanism to allow unprivileged users to create pressure triggers. The difference with privileged triggers creation is that unprivileged ones must have a time window that's a multiple of 2s. This is so that we can avoid unrestricted spawning of rt threads, and use instead the same aggregation mechanism done for the averages, which runs independently of any triggers. Suggested-by: Johannes Weiner Signed-off-by: Domenico Cerasuolo Signed-off-by: Peter Zijlstra (Intel) Acked-by: Johannes Weiner Link: https://lore.kernel.org/r/20230330105418.77061-5-cerasuolodomenico@gmail.com --- Documentation/accounting/psi.rst | 4 + include/linux/psi.h | 2 +- include/linux/psi_types.h | 7 ++ kernel/cgroup/cgroup.c | 2 +- kernel/sched/psi.c | 175 ++++++++++++++++++++++++--------------- 5 files changed, 121 insertions(+), 69 deletions(-) (limited to 'include/linux') diff --git a/Documentation/accounting/psi.rst b/Documentation/accounting/psi.rst index 5e40b3f437f9..df6062eb3abb 100644 --- a/Documentation/accounting/psi.rst +++ b/Documentation/accounting/psi.rst @@ -105,6 +105,10 @@ prevent overly frequent polling. Max limit is chosen as a high enough number after which monitors are most likely not needed and psi averages can be used instead. +Unprivileged users can also create monitors, with the only limitation that the +window size must be a multiple of 2s, in order to prevent excessive resource +usage. + When activated, psi monitor stays active for at least the duration of one tracking window to avoid repeated activations/deactivations when system is bouncing in and out of the stall state. diff --git a/include/linux/psi.h b/include/linux/psi.h index b029a847def1..ab26200c2803 100644 --- a/include/linux/psi.h +++ b/include/linux/psi.h @@ -24,7 +24,7 @@ void psi_memstall_leave(unsigned long *flags); int psi_show(struct seq_file *s, struct psi_group *group, enum psi_res res); struct psi_trigger *psi_trigger_create(struct psi_group *group, - char *buf, enum psi_res res); + char *buf, enum psi_res res, struct file *file); void psi_trigger_destroy(struct psi_trigger *t); __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h index 1819afa8b198..040c089581c6 100644 --- a/include/linux/psi_types.h +++ b/include/linux/psi_types.h @@ -151,6 +151,9 @@ struct psi_trigger { /* Deferred event(s) from previous ratelimit window */ bool pending_event; + + /* Trigger type - PSI_AVGS for unprivileged, PSI_POLL for RT */ + enum psi_aggregators aggregator; }; struct psi_group { @@ -171,6 +174,10 @@ struct psi_group { /* Aggregator work control */ struct delayed_work avgs_work; + /* Unprivileged triggers against N*PSI_FREQ windows */ + struct list_head avg_triggers; + u32 avg_nr_triggers[NR_PSI_STATES - 1]; + /* Total stall times and sampled pressure averages */ u64 total[NR_PSI_AGGREGATORS][NR_PSI_STATES - 1]; unsigned long avg[NR_PSI_STATES - 1][3]; diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 935e8121b21e..dead36969bba 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3761,7 +3761,7 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, } psi = cgroup_psi(cgrp); - new = psi_trigger_create(psi, buf, res); + new = psi_trigger_create(psi, buf, res, of->file); if (IS_ERR(new)) { cgroup_put(cgrp); return PTR_ERR(new); diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index f3df6a8ff493..e072f6b31bf3 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -186,9 +186,14 @@ static void group_init(struct psi_group *group) seqcount_init(&per_cpu_ptr(group->pcpu, cpu)->seq); group->avg_last_update = sched_clock(); group->avg_next_update = group->avg_last_update + psi_period; - INIT_DELAYED_WORK(&group->avgs_work, psi_avgs_work); mutex_init(&group->avgs_lock); - /* Init trigger-related members */ + + /* Init avg trigger-related members */ + INIT_LIST_HEAD(&group->avg_triggers); + memset(group->avg_nr_triggers, 0, sizeof(group->avg_nr_triggers)); + INIT_DELAYED_WORK(&group->avgs_work, psi_avgs_work); + + /* Init rtpoll trigger-related members */ atomic_set(&group->rtpoll_scheduled, 0); mutex_init(&group->rtpoll_trigger_lock); INIT_LIST_HEAD(&group->rtpoll_triggers); @@ -430,21 +435,32 @@ static u64 window_update(struct psi_window *win, u64 now, u64 value) return growth; } -static u64 update_triggers(struct psi_group *group, u64 now, bool *update_total) +static u64 update_triggers(struct psi_group *group, u64 now, bool *update_total, + enum psi_aggregators aggregator) { struct psi_trigger *t; - u64 *total = group->total[PSI_POLL]; + u64 *total = group->total[aggregator]; + struct list_head *triggers; + u64 *aggregator_total; *update_total = false; + if (aggregator == PSI_AVGS) { + triggers = &group->avg_triggers; + aggregator_total = group->avg_total; + } else { + triggers = &group->rtpoll_triggers; + aggregator_total = group->rtpoll_total; + } + /* * On subsequent updates, calculate growth deltas and let * watchers know when their specified thresholds are exceeded. */ - list_for_each_entry(t, &group->rtpoll_triggers, node) { + list_for_each_entry(t, triggers, node) { u64 growth; bool new_stall; - new_stall = group->rtpoll_total[t->state] != total[t->state]; + new_stall = aggregator_total[t->state] != total[t->state]; /* Check for stall activity or a previous threshold breach */ if (!new_stall && !t->pending_event) @@ -546,6 +562,7 @@ static void psi_avgs_work(struct work_struct *work) struct delayed_work *dwork; struct psi_group *group; u32 changed_states; + bool update_total; u64 now; dwork = to_delayed_work(work); @@ -563,8 +580,10 @@ static void psi_avgs_work(struct work_struct *work) * Once restarted, we'll catch up the running averages in one * go - see calc_avgs() and missed_periods. */ - if (now >= group->avg_next_update) + if (now >= group->avg_next_update) { + update_triggers(group, now, &update_total, PSI_AVGS); group->avg_next_update = update_averages(group, now); + } if (changed_states & PSI_STATE_RESCHEDULE) { schedule_delayed_work(dwork, nsecs_to_jiffies( @@ -574,7 +593,7 @@ static void psi_avgs_work(struct work_struct *work) mutex_unlock(&group->avgs_lock); } -static void init_triggers(struct psi_group *group, u64 now) +static void init_rtpoll_triggers(struct psi_group *group, u64 now) { struct psi_trigger *t; @@ -667,7 +686,7 @@ static void psi_rtpoll_work(struct psi_group *group) if (changed_states & group->rtpoll_states) { /* Initialize trigger windows when entering polling mode */ if (now > group->rtpoll_until) - init_triggers(group, now); + init_rtpoll_triggers(group, now); /* * Keep the monitor active for at least the duration of the @@ -684,7 +703,7 @@ static void psi_rtpoll_work(struct psi_group *group) } if (now >= group->rtpoll_next_update) { - group->rtpoll_next_update = update_triggers(group, now, &update_total); + group->rtpoll_next_update = update_triggers(group, now, &update_total, PSI_POLL); if (update_total) memcpy(group->rtpoll_total, group->total[PSI_POLL], sizeof(group->rtpoll_total)); @@ -1254,16 +1273,23 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) } struct psi_trigger *psi_trigger_create(struct psi_group *group, - char *buf, enum psi_res res) + char *buf, enum psi_res res, struct file *file) { struct psi_trigger *t; enum psi_states state; u32 threshold_us; + bool privileged; u32 window_us; if (static_branch_likely(&psi_disabled)) return ERR_PTR(-EOPNOTSUPP); + /* + * Checking the privilege here on file->f_cred implies that a privileged user + * could open the file and delegate the write to an unprivileged one. + */ + privileged = cap_raised(file->f_cred->cap_effective, CAP_SYS_RESOURCE); + if (sscanf(buf, "some %u %u", &threshold_us, &window_us) == 2) state = PSI_IO_SOME + res * 2; else if (sscanf(buf, "full %u %u", &threshold_us, &window_us) == 2) @@ -1283,6 +1309,13 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, window_us > WINDOW_MAX_US) return ERR_PTR(-EINVAL); + /* + * Unprivileged users can only use 2s windows so that averages aggregation + * work is used, and no RT threads need to be spawned. + */ + if (!privileged && window_us % 2000000) + return ERR_PTR(-EINVAL); + /* Check threshold */ if (threshold_us == 0 || threshold_us > window_us) return ERR_PTR(-EINVAL); @@ -1302,31 +1335,40 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, t->last_event_time = 0; init_waitqueue_head(&t->event_wait); t->pending_event = false; + t->aggregator = privileged ? PSI_POLL : PSI_AVGS; - mutex_lock(&group->rtpoll_trigger_lock); + if (privileged) { + mutex_lock(&group->rtpoll_trigger_lock); - if (!rcu_access_pointer(group->rtpoll_task)) { - struct task_struct *task; + if (!rcu_access_pointer(group->rtpoll_task)) { + struct task_struct *task; - task = kthread_create(psi_rtpoll_worker, group, "psimon"); - if (IS_ERR(task)) { - kfree(t); - mutex_unlock(&group->rtpoll_trigger_lock); - return ERR_CAST(task); + task = kthread_create(psi_rtpoll_worker, group, "psimon"); + if (IS_ERR(task)) { + kfree(t); + mutex_unlock(&group->rtpoll_trigger_lock); + return ERR_CAST(task); + } + atomic_set(&group->rtpoll_wakeup, 0); + wake_up_process(task); + rcu_assign_pointer(group->rtpoll_task, task); } - atomic_set(&group->rtpoll_wakeup, 0); - wake_up_process(task); - rcu_assign_pointer(group->rtpoll_task, task); - } - list_add(&t->node, &group->rtpoll_triggers); - group->rtpoll_min_period = min(group->rtpoll_min_period, - div_u64(t->win.size, UPDATES_PER_WINDOW)); - group->rtpoll_nr_triggers[t->state]++; - group->rtpoll_states |= (1 << t->state); + list_add(&t->node, &group->rtpoll_triggers); + group->rtpoll_min_period = min(group->rtpoll_min_period, + div_u64(t->win.size, UPDATES_PER_WINDOW)); + group->rtpoll_nr_triggers[t->state]++; + group->rtpoll_states |= (1 << t->state); - mutex_unlock(&group->rtpoll_trigger_lock); + mutex_unlock(&group->rtpoll_trigger_lock); + } else { + mutex_lock(&group->avgs_lock); + + list_add(&t->node, &group->avg_triggers); + group->avg_nr_triggers[t->state]++; + mutex_unlock(&group->avgs_lock); + } return t; } @@ -1350,34 +1392,41 @@ void psi_trigger_destroy(struct psi_trigger *t) */ wake_up_pollfree(&t->event_wait); - mutex_lock(&group->rtpoll_trigger_lock); - - if (!list_empty(&t->node)) { - struct psi_trigger *tmp; - u64 period = ULLONG_MAX; - - list_del(&t->node); - group->rtpoll_nr_triggers[t->state]--; - if (!group->rtpoll_nr_triggers[t->state]) - group->rtpoll_states &= ~(1 << t->state); - /* reset min update period for the remaining triggers */ - list_for_each_entry(tmp, &group->rtpoll_triggers, node) - period = min(period, div_u64(tmp->win.size, - UPDATES_PER_WINDOW)); - group->rtpoll_min_period = period; - /* Destroy rtpoll_task when the last trigger is destroyed */ - if (group->rtpoll_states == 0) { - group->rtpoll_until = 0; - task_to_destroy = rcu_dereference_protected( - group->rtpoll_task, - lockdep_is_held(&group->rtpoll_trigger_lock)); - rcu_assign_pointer(group->rtpoll_task, NULL); - del_timer(&group->rtpoll_timer); + if (t->aggregator == PSI_AVGS) { + mutex_lock(&group->avgs_lock); + if (!list_empty(&t->node)) { + list_del(&t->node); + group->avg_nr_triggers[t->state]--; } + mutex_unlock(&group->avgs_lock); + } else { + mutex_lock(&group->rtpoll_trigger_lock); + if (!list_empty(&t->node)) { + struct psi_trigger *tmp; + u64 period = ULLONG_MAX; + + list_del(&t->node); + group->rtpoll_nr_triggers[t->state]--; + if (!group->rtpoll_nr_triggers[t->state]) + group->rtpoll_states &= ~(1 << t->state); + /* reset min update period for the remaining triggers */ + list_for_each_entry(tmp, &group->rtpoll_triggers, node) + period = min(period, div_u64(tmp->win.size, + UPDATES_PER_WINDOW)); + group->rtpoll_min_period = period; + /* Destroy rtpoll_task when the last trigger is destroyed */ + if (group->rtpoll_states == 0) { + group->rtpoll_until = 0; + task_to_destroy = rcu_dereference_protected( + group->rtpoll_task, + lockdep_is_held(&group->rtpoll_trigger_lock)); + rcu_assign_pointer(group->rtpoll_task, NULL); + del_timer(&group->rtpoll_timer); + } + } + mutex_unlock(&group->rtpoll_trigger_lock); } - mutex_unlock(&group->rtpoll_trigger_lock); - /* * Wait for psi_schedule_rtpoll_work RCU to complete its read-side * critical section before destroying the trigger and optionally the @@ -1437,27 +1486,19 @@ static int psi_cpu_show(struct seq_file *m, void *v) return psi_show(m, &psi_system, PSI_CPU); } -static int psi_open(struct file *file, int (*psi_show)(struct seq_file *, void *)) -{ - if (file->f_mode & FMODE_WRITE && !capable(CAP_SYS_RESOURCE)) - return -EPERM; - - return single_open(file, psi_show, NULL); -} - static int psi_io_open(struct inode *inode, struct file *file) { - return psi_open(file, psi_io_show); + return single_open(file, psi_io_show, NULL); } static int psi_memory_open(struct inode *inode, struct file *file) { - return psi_open(file, psi_memory_show); + return single_open(file, psi_memory_show, NULL); } static int psi_cpu_open(struct inode *inode, struct file *file) { - return psi_open(file, psi_cpu_show); + return single_open(file, psi_cpu_show, NULL); } static ssize_t psi_write(struct file *file, const char __user *user_buf, @@ -1491,7 +1532,7 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, return -EBUSY; } - new = psi_trigger_create(&psi_system, buf, res); + new = psi_trigger_create(&psi_system, buf, res, file); if (IS_ERR(new)) { mutex_unlock(&seq->lock); return PTR_ERR(new); @@ -1571,7 +1612,7 @@ static int psi_irq_show(struct seq_file *m, void *v) static int psi_irq_open(struct inode *inode, struct file *file) { - return psi_open(file, psi_irq_show); + return single_open(file, psi_irq_show, NULL); } static ssize_t psi_irq_write(struct file *file, const char __user *user_buf, -- cgit v1.2.3 From b2d110cd5d28ded813f060b735cd68bfe2e5010f Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 6 Mar 2023 08:56:51 +0100 Subject: interconnect: drop unused icc_link_destroy() interface Now that the link array is deallocated when destroying nodes and the explicit link removal has been dropped from the exynos driver there are no further users of and no need for the icc_link_destroy() interface. Signed-off-by: Johan Hovold Link: https://lore.kernel.org/r/20230306075651.2449-24-johan+linaro@kernel.org Signed-off-by: Georgi Djakov --- drivers/interconnect/core.c | 46 ----------------------------------- include/linux/interconnect-provider.h | 6 ----- 2 files changed, 52 deletions(-) (limited to 'include/linux') diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c index da456329c524..0edf85ba055e 100644 --- a/drivers/interconnect/core.c +++ b/drivers/interconnect/core.c @@ -910,52 +910,6 @@ out: } EXPORT_SYMBOL_GPL(icc_link_create); -/** - * icc_link_destroy() - destroy a link between two nodes - * @src: pointer to source node - * @dst: pointer to destination node - * - * Return: 0 on success, or an error code otherwise - */ -int icc_link_destroy(struct icc_node *src, struct icc_node *dst) -{ - struct icc_node **new; - size_t slot; - int ret = 0; - - if (IS_ERR_OR_NULL(src)) - return -EINVAL; - - if (IS_ERR_OR_NULL(dst)) - return -EINVAL; - - mutex_lock(&icc_lock); - - for (slot = 0; slot < src->num_links; slot++) - if (src->links[slot] == dst) - break; - - if (WARN_ON(slot == src->num_links)) { - ret = -ENXIO; - goto out; - } - - src->links[slot] = src->links[--src->num_links]; - - new = krealloc(src->links, src->num_links * sizeof(*src->links), - GFP_KERNEL); - if (new) - src->links = new; - else - ret = -ENOMEM; - -out: - mutex_unlock(&icc_lock); - - return ret; -} -EXPORT_SYMBOL_GPL(icc_link_destroy); - /** * icc_node_add() - add interconnect node to interconnect provider * @node: pointer to the interconnect node diff --git a/include/linux/interconnect-provider.h b/include/linux/interconnect-provider.h index b9af9016a95e..e6d8aca6886d 100644 --- a/include/linux/interconnect-provider.h +++ b/include/linux/interconnect-provider.h @@ -118,7 +118,6 @@ int icc_std_aggregate(struct icc_node *node, u32 tag, u32 avg_bw, struct icc_node *icc_node_create(int id); void icc_node_destroy(int id); int icc_link_create(struct icc_node *node, const int dst_id); -int icc_link_destroy(struct icc_node *src, struct icc_node *dst); void icc_node_add(struct icc_node *node, struct icc_provider *provider); void icc_node_del(struct icc_node *node); int icc_nodes_remove(struct icc_provider *provider); @@ -150,11 +149,6 @@ static inline int icc_link_create(struct icc_node *node, const int dst_id) return -ENOTSUPP; } -static inline int icc_link_destroy(struct icc_node *src, struct icc_node *dst) -{ - return -ENOTSUPP; -} - static inline void icc_node_add(struct icc_node *node, struct icc_provider *provider) { } -- cgit v1.2.3 From 673aa1ed1c9b6710bf24e3f0957d85e2f46c77db Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Tue, 4 Apr 2023 18:21:16 +0100 Subject: of: Rename of_modalias_node() This helper does not produce a real modalias, but tries to get the "product" compatible part of the "vendor,product" compatibles only. It is far from creating a purely useful modalias string and does not seem to be used like that directly anyway, so let's try to give this helper a more meaningful name before moving there a real modalias helper (already existing under of/device.c). Also update the various documentations to refer to the strings as "aliases" rather than "modaliases" which has a real meaning in the Linux kernel. There is no functional change. Cc: Rafael J. Wysocki Cc: Len Brown Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Thomas Zimmermann Cc: Sebastian Reichel Cc: Wolfram Sang Cc: Mark Brown Signed-off-by: Miquel Raynal Reviewed-by: Rob Herring Acked-by: Mark Brown Signed-off-by: Srinivas Kandagatla Acked-by: Sebastian Reichel Link: https://lore.kernel.org/r/20230404172148.82422-9-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/acpi/bus.c | 7 ++++--- drivers/gpu/drm/drm_mipi_dsi.c | 2 +- drivers/hsi/hsi_core.c | 2 +- drivers/i2c/busses/i2c-powermac.c | 2 +- drivers/i2c/i2c-core-of.c | 2 +- drivers/of/base.c | 18 +++++++++++------- drivers/spi/spi.c | 4 ++-- include/linux/of.h | 3 ++- 8 files changed, 23 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index 9531dd0fef50..fc74c786a867 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -817,9 +817,10 @@ static bool acpi_of_modalias(struct acpi_device *adev, * @modalias: Pointer to buffer that modalias value will be copied into * @len: Length of modalias buffer * - * This is a counterpart of of_modalias_node() for struct acpi_device objects. - * If there is a compatible string for @adev, it will be copied to @modalias - * with the vendor prefix stripped; otherwise, @default_id will be used. + * This is a counterpart of of_alias_from_compatible() for struct acpi_device + * objects. If there is a compatible string for @adev, it will be copied to + * @modalias with the vendor prefix stripped; otherwise, @default_id will be + * used. */ void acpi_set_modalias(struct acpi_device *adev, const char *default_id, char *modalias, size_t len) diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c index b41aaf2bb9f1..b62f5e4425f4 100644 --- a/drivers/gpu/drm/drm_mipi_dsi.c +++ b/drivers/gpu/drm/drm_mipi_dsi.c @@ -160,7 +160,7 @@ of_mipi_dsi_device_add(struct mipi_dsi_host *host, struct device_node *node) int ret; u32 reg; - if (of_modalias_node(node, info.type, sizeof(info.type)) < 0) { + if (of_alias_from_compatible(node, info.type, sizeof(info.type)) < 0) { drm_err(host, "modalias failure on %pOF\n", node); return ERR_PTR(-EINVAL); } diff --git a/drivers/hsi/hsi_core.c b/drivers/hsi/hsi_core.c index 8fda8f1d064d..acbf82f755a8 100644 --- a/drivers/hsi/hsi_core.c +++ b/drivers/hsi/hsi_core.c @@ -207,7 +207,7 @@ static void hsi_add_client_from_dt(struct hsi_port *port, if (!cl) return; - err = of_modalias_node(client, name, sizeof(name)); + err = of_alias_from_compatible(client, name, sizeof(name)); if (err) goto err; diff --git a/drivers/i2c/busses/i2c-powermac.c b/drivers/i2c/busses/i2c-powermac.c index 2e74747eec9c..ec706a3aba26 100644 --- a/drivers/i2c/busses/i2c-powermac.c +++ b/drivers/i2c/busses/i2c-powermac.c @@ -284,7 +284,7 @@ static bool i2c_powermac_get_type(struct i2c_adapter *adap, */ /* First try proper modalias */ - if (of_modalias_node(node, tmp, sizeof(tmp)) >= 0) { + if (of_alias_from_compatible(node, tmp, sizeof(tmp)) >= 0) { snprintf(type, type_size, "MAC,%s", tmp); return true; } diff --git a/drivers/i2c/i2c-core-of.c b/drivers/i2c/i2c-core-of.c index bce6b796e04c..8941a30574e3 100644 --- a/drivers/i2c/i2c-core-of.c +++ b/drivers/i2c/i2c-core-of.c @@ -27,7 +27,7 @@ int of_i2c_get_board_info(struct device *dev, struct device_node *node, memset(info, 0, sizeof(*info)); - if (of_modalias_node(node, info->type, sizeof(info->type)) < 0) { + if (of_alias_from_compatible(node, info->type, sizeof(info->type)) < 0) { dev_err(dev, "of_i2c: modalias failure on %pOF\n", node); return -EINVAL; } diff --git a/drivers/of/base.c b/drivers/of/base.c index ac6fde53342f..161fe3192c46 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -1208,19 +1208,23 @@ struct device_node *of_find_matching_node_and_match(struct device_node *from, EXPORT_SYMBOL(of_find_matching_node_and_match); /** - * of_modalias_node - Lookup appropriate modalias for a device node + * of_alias_from_compatible - Lookup appropriate alias for a device node + * depending on compatible * @node: pointer to a device tree node - * @modalias: Pointer to buffer that modalias value will be copied into - * @len: Length of modalias value + * @alias: Pointer to buffer that alias value will be copied into + * @len: Length of alias value * * Based on the value of the compatible property, this routine will attempt - * to choose an appropriate modalias value for a particular device tree node. + * to choose an appropriate alias value for a particular device tree node. * It does this by stripping the manufacturer prefix (as delimited by a ',') * from the first entry in the compatible list property. * + * Note: The matching on just the "product" side of the compatible is a relic + * from I2C and SPI. Please do not add any new user. + * * Return: This routine returns 0 on success, <0 on failure. */ -int of_modalias_node(struct device_node *node, char *modalias, int len) +int of_alias_from_compatible(const struct device_node *node, char *alias, int len) { const char *compatible, *p; int cplen; @@ -1229,10 +1233,10 @@ int of_modalias_node(struct device_node *node, char *modalias, int len) if (!compatible || strlen(compatible) > cplen) return -ENODEV; p = strchr(compatible, ','); - strscpy(modalias, p ? p + 1 : compatible, len); + strscpy(alias, p ? p + 1 : compatible, len); return 0; } -EXPORT_SYMBOL_GPL(of_modalias_node); +EXPORT_SYMBOL_GPL(of_alias_from_compatible); /** * of_find_node_by_phandle - Find a node given a phandle diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 44b85a8d47f1..3bbdc5fe3b99 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -2354,8 +2354,8 @@ of_register_spi_device(struct spi_controller *ctlr, struct device_node *nc) } /* Select device driver */ - rc = of_modalias_node(nc, spi->modalias, - sizeof(spi->modalias)); + rc = of_alias_from_compatible(nc, spi->modalias, + sizeof(spi->modalias)); if (rc < 0) { dev_err(&ctlr->dev, "cannot find modalias for %pOF\n", nc); goto err_out; diff --git a/include/linux/of.h b/include/linux/of.h index 0af611307db2..b1eea8569043 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -373,7 +373,8 @@ extern int of_n_addr_cells(struct device_node *np); extern int of_n_size_cells(struct device_node *np); extern const struct of_device_id *of_match_node( const struct of_device_id *matches, const struct device_node *node); -extern int of_modalias_node(struct device_node *node, char *modalias, int len); +extern int of_alias_from_compatible(const struct device_node *node, char *alias, + int len); extern void of_print_phandle_args(const char *msg, const struct of_phandle_args *args); extern int __of_parse_phandle_with_args(const struct device_node *np, const char *list_name, const char *cells_name, int cell_count, -- cgit v1.2.3 From bd7a7ed774afd1a4174df34227626c95573be517 Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Tue, 4 Apr 2023 18:21:17 +0100 Subject: of: Move of_modalias() to module.c Create a specific .c file for OF related module handling. Move of_modalias() inside as a first step. The helper is exposed through of.h even though it is only used by core files because the users from device.c will soon be split into an OF-only helper in module.c as well as a device-oriented inline helper in of_device.h. Putting this helper in of_private.h would require to include of_private.h from of_device.h, which is not acceptable. Suggested-by: Rob Herring Signed-off-by: Miquel Raynal Reviewed-by: Rob Herring Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20230404172148.82422-10-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/of/Makefile | 2 +- drivers/of/device.c | 37 ------------------------------------- drivers/of/module.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/of.h | 9 +++++++++ 4 files changed, 54 insertions(+), 38 deletions(-) create mode 100644 drivers/of/module.c (limited to 'include/linux') diff --git a/drivers/of/Makefile b/drivers/of/Makefile index e0360a44306e..ae9923fd2940 100644 --- a/drivers/of/Makefile +++ b/drivers/of/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y = base.o device.o platform.o property.o +obj-y = base.o device.o module.o platform.o property.o obj-$(CONFIG_OF_KOBJ) += kobj.o obj-$(CONFIG_OF_DYNAMIC) += dynamic.o obj-$(CONFIG_OF_FLATTREE) += fdt.o diff --git a/drivers/of/device.c b/drivers/of/device.c index 351c505ecb50..7183cfd754db 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -#include #include #include #include @@ -248,42 +247,6 @@ const void *of_device_get_match_data(const struct device *dev) } EXPORT_SYMBOL(of_device_get_match_data); -static ssize_t of_modalias(const struct device_node *np, char *str, ssize_t len) -{ - const char *compat; - char *c; - struct property *p; - ssize_t csize; - ssize_t tsize; - - /* Name & Type */ - /* %p eats all alphanum characters, so %c must be used here */ - csize = snprintf(str, len, "of:N%pOFn%c%s", np, 'T', - of_node_get_device_type(np)); - tsize = csize; - len -= csize; - if (str) - str += csize; - - of_property_for_each_string(np, "compatible", p, compat) { - csize = strlen(compat) + 1; - tsize += csize; - if (csize > len) - continue; - - csize = snprintf(str, len, "C%s", compat); - for (c = str; c; ) { - c = strchr(c, ' '); - if (c) - *c++ = '_'; - } - len -= csize; - str += csize; - } - - return tsize; -} - int of_device_request_module(struct device *dev) { char *str; diff --git a/drivers/of/module.c b/drivers/of/module.c new file mode 100644 index 000000000000..4c59752bc8d6 --- /dev/null +++ b/drivers/of/module.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Linux kernel module helpers. + */ + +#include +#include +#include + +ssize_t of_modalias(const struct device_node *np, char *str, ssize_t len) +{ + const char *compat; + char *c; + struct property *p; + ssize_t csize; + ssize_t tsize; + + /* Name & Type */ + /* %p eats all alphanum characters, so %c must be used here */ + csize = snprintf(str, len, "of:N%pOFn%c%s", np, 'T', + of_node_get_device_type(np)); + tsize = csize; + len -= csize; + if (str) + str += csize; + + of_property_for_each_string(np, "compatible", p, compat) { + csize = strlen(compat) + 1; + tsize += csize; + if (csize > len) + continue; + + csize = snprintf(str, len, "C%s", compat); + for (c = str; c; ) { + c = strchr(c, ' '); + if (c) + *c++ = '_'; + } + len -= csize; + str += csize; + } + + return tsize; +} diff --git a/include/linux/of.h b/include/linux/of.h index b1eea8569043..be26c7e8ef9e 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -385,6 +385,9 @@ extern int of_parse_phandle_with_args_map(const struct device_node *np, extern int of_count_phandle_with_args(const struct device_node *np, const char *list_name, const char *cells_name); +/* module functions */ +extern ssize_t of_modalias(const struct device_node *np, char *str, ssize_t len); + /* phandle iterator functions */ extern int of_phandle_iterator_init(struct of_phandle_iterator *it, const struct device_node *np, @@ -742,6 +745,12 @@ static inline int of_count_phandle_with_args(const struct device_node *np, return -ENOSYS; } +static inline ssize_t of_modalias(const struct device_node *np, char *str, + ssize_t len) +{ + return -ENODEV; +} + static inline int of_phandle_iterator_init(struct of_phandle_iterator *it, const struct device_node *np, const char *list_name, -- cgit v1.2.3 From e6506f06d5e82765666902ccf9e9162f3e31d518 Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Tue, 4 Apr 2023 18:21:18 +0100 Subject: of: Move the request module helper logic to module.c Depending on device.c for pure OF handling is considered backwards. Let's extract the content of of_device_request_module() to have the real logic under module.c. The next step will be to convert users of of_device_request_module() to use the new helper. Signed-off-by: Miquel Raynal Reviewed-by: Rob Herring Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20230404172148.82422-11-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/of/device.c | 25 ++----------------------- drivers/of/module.c | 30 ++++++++++++++++++++++++++++++ include/linux/of.h | 6 ++++++ 3 files changed, 38 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/drivers/of/device.c b/drivers/of/device.c index 7183cfd754db..874a2e1f6308 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -8,7 +8,6 @@ #include /* for bus_dma_region */ #include #include -#include #include #include #include @@ -249,30 +248,10 @@ EXPORT_SYMBOL(of_device_get_match_data); int of_device_request_module(struct device *dev) { - char *str; - ssize_t size; - int ret; - - if (!dev || !dev->of_node) + if (!dev) return -ENODEV; - size = of_modalias(dev->of_node, NULL, 0); - if (size < 0) - return size; - - /* Reserve an additional byte for the trailing '\0' */ - size++; - - str = kmalloc(size, GFP_KERNEL); - if (!str) - return -ENOMEM; - - of_modalias(dev->of_node, str, size); - str[size - 1] = '\0'; - ret = request_module(str); - kfree(str); - - return ret; + return of_request_module(dev->of_node); } EXPORT_SYMBOL_GPL(of_device_request_module); diff --git a/drivers/of/module.c b/drivers/of/module.c index 4c59752bc8d6..0e8aa974f0f2 100644 --- a/drivers/of/module.c +++ b/drivers/of/module.c @@ -4,6 +4,7 @@ */ #include +#include #include #include @@ -42,3 +43,32 @@ ssize_t of_modalias(const struct device_node *np, char *str, ssize_t len) return tsize; } + +int of_request_module(const struct device_node *np) +{ + char *str; + ssize_t size; + int ret; + + if (!np) + return -ENODEV; + + size = of_modalias(np, NULL, 0); + if (size < 0) + return size; + + /* Reserve an additional byte for the trailing '\0' */ + size++; + + str = kmalloc(size, GFP_KERNEL); + if (!str) + return -ENOMEM; + + of_modalias(np, str, size); + str[size - 1] = '\0'; + ret = request_module(str); + kfree(str); + + return ret; +} +EXPORT_SYMBOL_GPL(of_request_module); diff --git a/include/linux/of.h b/include/linux/of.h index be26c7e8ef9e..9b7a99499ef3 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -387,6 +387,7 @@ extern int of_count_phandle_with_args(const struct device_node *np, /* module functions */ extern ssize_t of_modalias(const struct device_node *np, char *str, ssize_t len); +extern int of_request_module(const struct device_node *np); /* phandle iterator functions */ extern int of_phandle_iterator_init(struct of_phandle_iterator *it, @@ -751,6 +752,11 @@ static inline ssize_t of_modalias(const struct device_node *np, char *str, return -ENODEV; } +static inline int of_request_module(const struct device_node *np) +{ + return -ENODEV; +} + static inline int of_phandle_iterator_init(struct of_phandle_iterator *it, const struct device_node *np, const char *list_name, -- cgit v1.2.3 From 2f555f58f5ce33b201235e104823662d5573c4a5 Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Tue, 4 Apr 2023 18:21:20 +0100 Subject: of: device: Kill of_device_request_module() A new helper has been introduced, of_request_module(). Users have been converted, this helper can now be deleted. Signed-off-by: Miquel Raynal Reviewed-by: Rob Herring Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20230404172148.82422-13-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/of/device.c | 9 --------- include/linux/of_device.h | 6 ------ 2 files changed, 15 deletions(-) (limited to 'include/linux') diff --git a/drivers/of/device.c b/drivers/of/device.c index 874a2e1f6308..0f00f1b80708 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -246,15 +246,6 @@ const void *of_device_get_match_data(const struct device *dev) } EXPORT_SYMBOL(of_device_get_match_data); -int of_device_request_module(struct device *dev) -{ - if (!dev) - return -ENODEV; - - return of_request_module(dev->of_node); -} -EXPORT_SYMBOL_GPL(of_device_request_module); - /** * of_device_modalias - Fill buffer with newline terminated modalias string * @dev: Calling device diff --git a/include/linux/of_device.h b/include/linux/of_device.h index f4b57614979d..ce20d8b00b3e 100644 --- a/include/linux/of_device.h +++ b/include/linux/of_device.h @@ -33,7 +33,6 @@ extern void of_device_unregister(struct platform_device *ofdev); extern const void *of_device_get_match_data(const struct device *dev); extern ssize_t of_device_modalias(struct device *dev, char *str, ssize_t len); -extern int of_device_request_module(struct device *dev); extern void of_device_uevent(const struct device *dev, struct kobj_uevent_env *env); extern int of_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *env); @@ -78,11 +77,6 @@ static inline int of_device_modalias(struct device *dev, return -ENODEV; } -static inline int of_device_request_module(struct device *dev) -{ - return -ENODEV; -} - static inline int of_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *env) { -- cgit v1.2.3 From 266570f496b90dea8fda893c2cf7c28d63ae2bd9 Mon Sep 17 00:00:00 2001 From: Michael Walle Date: Tue, 4 Apr 2023 18:21:21 +0100 Subject: nvmem: core: introduce NVMEM layouts NVMEM layouts are used to generate NVMEM cells during runtime. Think of an EEPROM with a well-defined conent. For now, the content can be described by a device tree or a board file. But this only works if the offsets and lengths are static and don't change. One could also argue that putting the layout of the EEPROM in the device tree is the wrong place. Instead, the device tree should just have a specific compatible string. Right now there are two use cases: (1) The NVMEM cell needs special processing. E.g. if it only specifies a base MAC address offset and you need to add an offset, or it needs to parse a MAC from ASCII format or some proprietary format. (Post processing of cells is added in a later commit). (2) u-boot environment parsing. The cells don't have a particular offset but it needs parsing the content to determine the offsets and length. Co-developed-by: Miquel Raynal Signed-off-by: Miquel Raynal Signed-off-by: Michael Walle Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20230404172148.82422-14-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/nvmem.rst | 15 +++++ drivers/nvmem/Kconfig | 4 ++ drivers/nvmem/Makefile | 1 + drivers/nvmem/core.c | 120 +++++++++++++++++++++++++++++++++++++ drivers/nvmem/layouts/Kconfig | 5 ++ drivers/nvmem/layouts/Makefile | 4 ++ include/linux/nvmem-consumer.h | 7 +++ include/linux/nvmem-provider.h | 51 ++++++++++++++++ 8 files changed, 207 insertions(+) create mode 100644 drivers/nvmem/layouts/Kconfig create mode 100644 drivers/nvmem/layouts/Makefile (limited to 'include/linux') diff --git a/Documentation/driver-api/nvmem.rst b/Documentation/driver-api/nvmem.rst index e3366322d46c..de221e91c8e3 100644 --- a/Documentation/driver-api/nvmem.rst +++ b/Documentation/driver-api/nvmem.rst @@ -185,3 +185,18 @@ ex:: ===================== See Documentation/devicetree/bindings/nvmem/nvmem.txt + +8. NVMEM layouts +================ + +NVMEM layouts are yet another mechanism to create cells. With the device +tree binding it is possible to specify simple cells by using an offset +and a length. Sometimes, the cells doesn't have a static offset, but +the content is still well defined, e.g. tag-length-values. In this case, +the NVMEM device content has to be first parsed and the cells need to +be added accordingly. Layouts let you read the content of the NVMEM device +and let you add cells dynamically. + +Another use case for layouts is the post processing of cells. With layouts, +it is possible to associate a custom post processing hook to a cell. It +even possible to add this hook to cells not created by the layout itself. diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig index 3b3832f4dfad..a2afba11c890 100644 --- a/drivers/nvmem/Kconfig +++ b/drivers/nvmem/Kconfig @@ -21,6 +21,10 @@ config NVMEM_SYSFS This interface is mostly used by userspace applications to read/write directly into nvmem. +# Layouts + +source "drivers/nvmem/layouts/Kconfig" + # Devices config NVMEM_APPLE_EFUSES diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile index 6a1efffa88f0..f82431ec8aef 100644 --- a/drivers/nvmem/Makefile +++ b/drivers/nvmem/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_NVMEM) += nvmem_core.o nvmem_core-y := core.o +obj-y += layouts/ # Devices obj-$(CONFIG_NVMEM_APPLE_EFUSES) += nvmem-apple-efuses.o diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 22024b830788..b9be1faeb7be 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -40,6 +40,7 @@ struct nvmem_device { nvmem_reg_write_t reg_write; nvmem_cell_post_process_t cell_post_process; struct gpio_desc *wp_gpio; + struct nvmem_layout *layout; void *priv; }; @@ -74,6 +75,9 @@ static LIST_HEAD(nvmem_lookup_list); static BLOCKING_NOTIFIER_HEAD(nvmem_notifier); +static DEFINE_SPINLOCK(nvmem_layout_lock); +static LIST_HEAD(nvmem_layouts); + static int __nvmem_reg_read(struct nvmem_device *nvmem, unsigned int offset, void *val, size_t bytes) { @@ -728,6 +732,101 @@ static int nvmem_add_cells_from_of(struct nvmem_device *nvmem) return 0; } +int __nvmem_layout_register(struct nvmem_layout *layout, struct module *owner) +{ + layout->owner = owner; + + spin_lock(&nvmem_layout_lock); + list_add(&layout->node, &nvmem_layouts); + spin_unlock(&nvmem_layout_lock); + + return 0; +} +EXPORT_SYMBOL_GPL(__nvmem_layout_register); + +void nvmem_layout_unregister(struct nvmem_layout *layout) +{ + spin_lock(&nvmem_layout_lock); + list_del(&layout->node); + spin_unlock(&nvmem_layout_lock); +} +EXPORT_SYMBOL_GPL(nvmem_layout_unregister); + +static struct nvmem_layout *nvmem_layout_get(struct nvmem_device *nvmem) +{ + struct device_node *layout_np, *np = nvmem->dev.of_node; + struct nvmem_layout *l, *layout = NULL; + + layout_np = of_get_child_by_name(np, "nvmem-layout"); + if (!layout_np) + return NULL; + + spin_lock(&nvmem_layout_lock); + + list_for_each_entry(l, &nvmem_layouts, node) { + if (of_match_node(l->of_match_table, layout_np)) { + if (try_module_get(l->owner)) + layout = l; + + break; + } + } + + spin_unlock(&nvmem_layout_lock); + of_node_put(layout_np); + + return layout; +} + +static void nvmem_layout_put(struct nvmem_layout *layout) +{ + if (layout) + module_put(layout->owner); +} + +static int nvmem_add_cells_from_layout(struct nvmem_device *nvmem) +{ + struct nvmem_layout *layout = nvmem->layout; + int ret; + + if (layout && layout->add_cells) { + ret = layout->add_cells(&nvmem->dev, nvmem, layout); + if (ret) + return ret; + } + + return 0; +} + +#if IS_ENABLED(CONFIG_OF) +/** + * of_nvmem_layout_get_container() - Get OF node to layout container. + * + * @nvmem: nvmem device. + * + * Return: a node pointer with refcount incremented or NULL if no + * container exists. Use of_node_put() on it when done. + */ +struct device_node *of_nvmem_layout_get_container(struct nvmem_device *nvmem) +{ + return of_get_child_by_name(nvmem->dev.of_node, "nvmem-layout"); +} +EXPORT_SYMBOL_GPL(of_nvmem_layout_get_container); +#endif + +const void *nvmem_layout_get_match_data(struct nvmem_device *nvmem, + struct nvmem_layout *layout) +{ + struct device_node __maybe_unused *layout_np; + const struct of_device_id *match; + + layout_np = of_nvmem_layout_get_container(nvmem); + match = of_match_node(layout->of_match_table, layout_np); + + return match ? match->data : NULL; +} +EXPORT_SYMBOL_GPL(nvmem_layout_get_match_data); + /** * nvmem_register() - Register a nvmem device for given nvmem_config. * Also creates a binary entry in /sys/bus/nvmem/devices/dev-name/nvmem @@ -834,6 +933,12 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config) goto err_put_device; } + /* + * If the driver supplied a layout by config->layout, the module + * pointer will be NULL and nvmem_layout_put() will be a noop. + */ + nvmem->layout = config->layout ?: nvmem_layout_get(nvmem); + if (config->cells) { rval = nvmem_add_cells(nvmem, config->cells, config->ncells); if (rval) @@ -854,12 +959,17 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config) if (rval) goto err_remove_cells; + rval = nvmem_add_cells_from_layout(nvmem); + if (rval) + goto err_remove_cells; + blocking_notifier_call_chain(&nvmem_notifier, NVMEM_ADD, nvmem); return nvmem; err_remove_cells: nvmem_device_remove_all_cells(nvmem); + nvmem_layout_put(nvmem->layout); if (config->compat) nvmem_sysfs_remove_compat(nvmem, config); err_put_device: @@ -881,6 +991,7 @@ static void nvmem_device_release(struct kref *kref) device_remove_bin_file(nvmem->base_dev, &nvmem->eeprom); nvmem_device_remove_all_cells(nvmem); + nvmem_layout_put(nvmem->layout); device_unregister(&nvmem->dev); } @@ -1246,6 +1357,15 @@ struct nvmem_cell *of_nvmem_cell_get(struct device_node *np, const char *id) return ERR_PTR(-EINVAL); } + /* nvmem layouts produce cells within the nvmem-layout container */ + if (of_node_name_eq(nvmem_np, "nvmem-layout")) { + nvmem_np = of_get_next_parent(nvmem_np); + if (!nvmem_np) { + of_node_put(cell_np); + return ERR_PTR(-EINVAL); + } + } + nvmem = __nvmem_device_get(nvmem_np, device_match_of_node); of_node_put(nvmem_np); if (IS_ERR(nvmem)) { diff --git a/drivers/nvmem/layouts/Kconfig b/drivers/nvmem/layouts/Kconfig new file mode 100644 index 000000000000..9ad3911d1605 --- /dev/null +++ b/drivers/nvmem/layouts/Kconfig @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Layout Types" + +endmenu diff --git a/drivers/nvmem/layouts/Makefile b/drivers/nvmem/layouts/Makefile new file mode 100644 index 000000000000..6fdb3c60a4fa --- /dev/null +++ b/drivers/nvmem/layouts/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for nvmem layouts. +# diff --git a/include/linux/nvmem-consumer.h b/include/linux/nvmem-consumer.h index 1f62f7ba71ca..fa030d93b768 100644 --- a/include/linux/nvmem-consumer.h +++ b/include/linux/nvmem-consumer.h @@ -239,6 +239,7 @@ struct nvmem_cell *of_nvmem_cell_get(struct device_node *np, const char *id); struct nvmem_device *of_nvmem_device_get(struct device_node *np, const char *name); +struct device_node *of_nvmem_layout_get_container(struct nvmem_device *nvmem); #else static inline struct nvmem_cell *of_nvmem_cell_get(struct device_node *np, const char *id) @@ -251,6 +252,12 @@ static inline struct nvmem_device *of_nvmem_device_get(struct device_node *np, { return ERR_PTR(-EOPNOTSUPP); } + +static inline struct device_node * +of_nvmem_layout_get_container(struct nvmem_device *nvmem) +{ + return ERR_PTR(-EOPNOTSUPP); +} #endif /* CONFIG_NVMEM && CONFIG_OF */ #endif /* ifndef _LINUX_NVMEM_CONSUMER_H */ diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h index 0262b86194eb..535c5f9f3309 100644 --- a/include/linux/nvmem-provider.h +++ b/include/linux/nvmem-provider.h @@ -88,6 +88,7 @@ struct nvmem_cell_info { * @stride: Minimum read/write access stride. * @priv: User context passed to read/write callbacks. * @ignore_wp: Write Protect pin is managed by the provider. + * @layout: Fixed layout associated with this nvmem device. * * Note: A default "nvmem" name will be assigned to the device if * no name is specified in its configuration. In such case "" is @@ -109,6 +110,7 @@ struct nvmem_config { bool read_only; bool root_only; bool ignore_wp; + struct nvmem_layout *layout; struct device_node *of_node; bool no_of_node; nvmem_reg_read_t reg_read; @@ -142,6 +144,33 @@ struct nvmem_cell_table { struct list_head node; }; +/** + * struct nvmem_layout - NVMEM layout definitions + * + * @name: Layout name. + * @of_match_table: Open firmware match table. + * @add_cells: Will be called if a nvmem device is found which + * has this layout. The function will add layout + * specific cells with nvmem_add_one_cell(). + * @owner: Pointer to struct module. + * @node: List node. + * + * A nvmem device can hold a well defined structure which can just be + * evaluated during runtime. For example a TLV list, or a list of "name=val" + * pairs. A nvmem layout can parse the nvmem device and add appropriate + * cells. + */ +struct nvmem_layout { + const char *name; + const struct of_device_id *of_match_table; + int (*add_cells)(struct device *dev, struct nvmem_device *nvmem, + struct nvmem_layout *layout); + + /* private */ + struct module *owner; + struct list_head node; +}; + #if IS_ENABLED(CONFIG_NVMEM) struct nvmem_device *nvmem_register(const struct nvmem_config *cfg); @@ -156,6 +185,14 @@ void nvmem_del_cell_table(struct nvmem_cell_table *table); int nvmem_add_one_cell(struct nvmem_device *nvmem, const struct nvmem_cell_info *info); +int __nvmem_layout_register(struct nvmem_layout *layout, struct module *owner); +#define nvmem_layout_register(layout) \ + __nvmem_layout_register(layout, THIS_MODULE) +void nvmem_layout_unregister(struct nvmem_layout *layout); + +const void *nvmem_layout_get_match_data(struct nvmem_device *nvmem, + struct nvmem_layout *layout); + #else static inline struct nvmem_device *nvmem_register(const struct nvmem_config *c) @@ -179,5 +216,19 @@ static inline int nvmem_add_one_cell(struct nvmem_device *nvmem, return -EOPNOTSUPP; } +static inline int nvmem_layout_register(struct nvmem_layout *layout) +{ + return -EOPNOTSUPP; +} + +static inline void nvmem_layout_unregister(struct nvmem_layout *layout) {} + +static inline const void * +nvmem_layout_get_match_data(struct nvmem_device *nvmem, + struct nvmem_layout *layout) +{ + return NULL; +} + #endif /* CONFIG_NVMEM */ #endif /* ifndef _LINUX_NVMEM_PROVIDER_H */ -- cgit v1.2.3 From 345ec382cd4b736c36e01f155d08c913b225b736 Mon Sep 17 00:00:00 2001 From: Michael Walle Date: Tue, 4 Apr 2023 18:21:24 +0100 Subject: nvmem: core: add per-cell post processing Instead of relying on the name the consumer is using for the cell, like it is done for the nvmem .cell_post_process configuration parameter, provide a per-cell post processing hook. This can then be populated by the NVMEM provider (or the NVMEM layout) when adding the cell. Signed-off-by: Michael Walle Signed-off-by: Miquel Raynal Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20230404172148.82422-17-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/core.c | 17 +++++++++++++++++ include/linux/nvmem-provider.h | 3 +++ 2 files changed, 20 insertions(+) (limited to 'include/linux') diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 49b4bbaf59e8..0708f9f27898 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -54,6 +54,7 @@ struct nvmem_cell_entry { int bytes; int bit_offset; int nbits; + nvmem_cell_post_process_t read_post_process; struct device_node *np; struct nvmem_device *nvmem; struct list_head node; @@ -470,6 +471,7 @@ static int nvmem_cell_info_to_nvmem_cell_entry_nodup(struct nvmem_device *nvmem, cell->offset = info->offset; cell->bytes = info->bytes; cell->name = info->name; + cell->read_post_process = info->read_post_process; cell->bit_offset = info->bit_offset; cell->nbits = info->nbits; @@ -1563,6 +1565,13 @@ static int __nvmem_cell_read(struct nvmem_device *nvmem, if (cell->bit_offset || cell->nbits) nvmem_shift_read_buffer_in_place(cell, buf); + if (cell->read_post_process) { + rc = cell->read_post_process(nvmem->priv, id, index, + cell->offset, buf, cell->bytes); + if (rc) + return rc; + } + if (nvmem->cell_post_process) { rc = nvmem->cell_post_process(nvmem->priv, id, index, cell->offset, buf, cell->bytes); @@ -1671,6 +1680,14 @@ static int __nvmem_cell_entry_write(struct nvmem_cell_entry *cell, void *buf, si (cell->bit_offset == 0 && len != cell->bytes)) return -EINVAL; + /* + * Any cells which have a read_post_process hook are read-only because + * we cannot reverse the operation and it might affect other cells, + * too. + */ + if (cell->read_post_process) + return -EINVAL; + if (cell->bit_offset || cell->nbits) { buf = nvmem_cell_prepare_write_buffer(cell, buf, len); if (IS_ERR(buf)) diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h index 535c5f9f3309..3bfc23553a9e 100644 --- a/include/linux/nvmem-provider.h +++ b/include/linux/nvmem-provider.h @@ -54,6 +54,8 @@ struct nvmem_keepout { * @bit_offset: Bit offset if cell is smaller than a byte. * @nbits: Number of bits. * @np: Optional device_node pointer. + * @read_post_process: Callback for optional post processing of cell data + * on reads. */ struct nvmem_cell_info { const char *name; @@ -62,6 +64,7 @@ struct nvmem_cell_info { unsigned int bit_offset; unsigned int nbits; struct device_node *np; + nvmem_cell_post_process_t read_post_process; }; /** -- cgit v1.2.3 From de12c9691501ccba41a154c223869f82be4c12fd Mon Sep 17 00:00:00 2001 From: Michael Walle Date: Tue, 4 Apr 2023 18:21:25 +0100 Subject: nvmem: core: allow to modify a cell before adding it Provide a way to modify a cell before it will get added. This is useful to attach a custom post processing hook via a layout. Signed-off-by: Michael Walle Signed-off-by: Miquel Raynal Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20230404172148.82422-18-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/core.c | 4 ++++ include/linux/nvmem-provider.h | 5 +++++ 2 files changed, 9 insertions(+) (limited to 'include/linux') diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 0708f9f27898..f43025ad315b 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -695,6 +695,7 @@ static int nvmem_validate_keepouts(struct nvmem_device *nvmem) static int nvmem_add_cells_from_of(struct nvmem_device *nvmem) { + struct nvmem_layout *layout = nvmem->layout; struct device *dev = &nvmem->dev; struct device_node *child; const __be32 *addr; @@ -724,6 +725,9 @@ static int nvmem_add_cells_from_of(struct nvmem_device *nvmem) info.np = of_node_get(child); + if (layout && layout->fixup_cell_info) + layout->fixup_cell_info(nvmem, layout, &info); + ret = nvmem_add_one_cell(nvmem, &info); kfree(info.name); if (ret) { diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h index 3bfc23553a9e..be81cc88eabc 100644 --- a/include/linux/nvmem-provider.h +++ b/include/linux/nvmem-provider.h @@ -155,6 +155,8 @@ struct nvmem_cell_table { * @add_cells: Will be called if a nvmem device is found which * has this layout. The function will add layout * specific cells with nvmem_add_one_cell(). + * @fixup_cell_info: Will be called before a cell is added. Can be + * used to modify the nvmem_cell_info. * @owner: Pointer to struct module. * @node: List node. * @@ -168,6 +170,9 @@ struct nvmem_layout { const struct of_device_id *of_match_table; int (*add_cells)(struct device *dev, struct nvmem_device *nvmem, struct nvmem_layout *layout); + void (*fixup_cell_info)(struct nvmem_device *nvmem, + struct nvmem_layout *layout, + struct nvmem_cell_info *cell); /* private */ struct module *owner; -- cgit v1.2.3 From 011e40a166fdaa65fb9946b7cd91efec85b70dbb Mon Sep 17 00:00:00 2001 From: Michael Walle Date: Tue, 4 Apr 2023 18:21:27 +0100 Subject: nvmem: cell: drop global cell_post_process There are no users anymore for the global cell_post_process callback anymore. New users should use proper nvmem layouts. Signed-off-by: Michael Walle Signed-off-by: Miquel Raynal Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20230404172148.82422-20-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/core.c | 9 --------- include/linux/nvmem-provider.h | 2 -- 2 files changed, 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index f43025ad315b..fccb2728193a 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -39,7 +39,6 @@ struct nvmem_device { unsigned int nkeepout; nvmem_reg_read_t reg_read; nvmem_reg_write_t reg_write; - nvmem_cell_post_process_t cell_post_process; struct gpio_desc *wp_gpio; struct nvmem_layout *layout; void *priv; @@ -903,7 +902,6 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config) nvmem->type = config->type; nvmem->reg_read = config->reg_read; nvmem->reg_write = config->reg_write; - nvmem->cell_post_process = config->cell_post_process; nvmem->keepout = config->keepout; nvmem->nkeepout = config->nkeepout; if (config->of_node) @@ -1576,13 +1574,6 @@ static int __nvmem_cell_read(struct nvmem_device *nvmem, return rc; } - if (nvmem->cell_post_process) { - rc = nvmem->cell_post_process(nvmem->priv, id, index, - cell->offset, buf, cell->bytes); - if (rc) - return rc; - } - if (len) *len = cell->bytes; diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h index be81cc88eabc..d3d7af86a283 100644 --- a/include/linux/nvmem-provider.h +++ b/include/linux/nvmem-provider.h @@ -85,7 +85,6 @@ struct nvmem_cell_info { * @no_of_node: Device should not use the parent's of_node even if it's !NULL. * @reg_read: Callback to read data. * @reg_write: Callback to write data. - * @cell_post_process: Callback for vendor specific post processing of cell data * @size: Device size. * @word_size: Minimum read/write access granularity. * @stride: Minimum read/write access stride. @@ -118,7 +117,6 @@ struct nvmem_config { bool no_of_node; nvmem_reg_read_t reg_read; nvmem_reg_write_t reg_write; - nvmem_cell_post_process_t cell_post_process; int size; int word_size; int stride; -- cgit v1.2.3 From 8a134fd9f9323f4c39ec27055b3d3723cfb5c1e9 Mon Sep 17 00:00:00 2001 From: Michael Walle Date: Tue, 4 Apr 2023 18:21:28 +0100 Subject: nvmem: core: provide own priv pointer in post process callback It doesn't make any more sense to have a opaque pointer set up by the nvmem device. Usually, the layout isn't associated with a particular nvmem device. Instead, let the caller who set the post process callback provide the priv pointer. Signed-off-by: Michael Walle Signed-off-by: Miquel Raynal Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20230404172148.82422-21-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/core.c | 4 +++- include/linux/nvmem-provider.h | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index fccb2728193a..212c5ba5789f 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -54,6 +54,7 @@ struct nvmem_cell_entry { int bit_offset; int nbits; nvmem_cell_post_process_t read_post_process; + void *priv; struct device_node *np; struct nvmem_device *nvmem; struct list_head node; @@ -471,6 +472,7 @@ static int nvmem_cell_info_to_nvmem_cell_entry_nodup(struct nvmem_device *nvmem, cell->bytes = info->bytes; cell->name = info->name; cell->read_post_process = info->read_post_process; + cell->priv = info->priv; cell->bit_offset = info->bit_offset; cell->nbits = info->nbits; @@ -1568,7 +1570,7 @@ static int __nvmem_cell_read(struct nvmem_device *nvmem, nvmem_shift_read_buffer_in_place(cell, buf); if (cell->read_post_process) { - rc = cell->read_post_process(nvmem->priv, id, index, + rc = cell->read_post_process(cell->priv, id, index, cell->offset, buf, cell->bytes); if (rc) return rc; diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h index d3d7af86a283..0cf9f9490514 100644 --- a/include/linux/nvmem-provider.h +++ b/include/linux/nvmem-provider.h @@ -20,7 +20,8 @@ typedef int (*nvmem_reg_write_t)(void *priv, unsigned int offset, void *val, size_t bytes); /* used for vendor specific post processing of cell data */ typedef int (*nvmem_cell_post_process_t)(void *priv, const char *id, int index, - unsigned int offset, void *buf, size_t bytes); + unsigned int offset, void *buf, + size_t bytes); enum nvmem_type { NVMEM_TYPE_UNKNOWN = 0, @@ -56,6 +57,7 @@ struct nvmem_keepout { * @np: Optional device_node pointer. * @read_post_process: Callback for optional post processing of cell data * on reads. + * @priv: Opaque data passed to the read_post_process hook. */ struct nvmem_cell_info { const char *name; @@ -65,6 +67,7 @@ struct nvmem_cell_info { unsigned int nbits; struct device_node *np; nvmem_cell_post_process_t read_post_process; + void *priv; }; /** -- cgit v1.2.3 From 55d4980ce55b6bb4be66877de4dbec513911b988 Mon Sep 17 00:00:00 2001 From: Rafał Miłecki Date: Tue, 4 Apr 2023 18:21:42 +0100 Subject: nvmem: core: support specifying both: cell raw data & post read lengths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Callback .read_post_process() is designed to modify raw cell content before providing it to the consumer. So far we were dealing with modifications that didn't affect cell size (length). In some cases however cell content needs to be reformatted and resized. It's required e.g. to provide properly formatted MAC address in case it's stored in a non-binary format (e.g. using ASCII). There were few discussions how to optimally handle that. Following possible solutions were considered: 1. Allow .read_post_process() to realloc (resize) content buffer 2. Allow .read_post_process() to adjust (decrease) just buffer length 3. Register NVMEM cells using post-read sizes The preferred solution was the last one. The problem is that simply adjusting "bytes" in NVMEM providers would result in core code NOT passing whole raw data to .read_post_process() callbacks. It means callback functions couldn't do their job without somehow manually reading original cell content on their own. This patch deals with that by registering NVMEM cells with both lengths: raw content one and post read one. It allows: 1. Core code to read whole raw cell content 2. Callbacks to return content they want Signed-off-by: Rafał Miłecki Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20230404172148.82422-35-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/core.c | 11 +++++++---- include/linux/nvmem-provider.h | 2 ++ 2 files changed, 9 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 212c5ba5789f..a62973d010ff 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -50,6 +50,7 @@ struct nvmem_device { struct nvmem_cell_entry { const char *name; int offset; + size_t raw_len; int bytes; int bit_offset; int nbits; @@ -469,6 +470,7 @@ static int nvmem_cell_info_to_nvmem_cell_entry_nodup(struct nvmem_device *nvmem, { cell->nvmem = nvmem; cell->offset = info->offset; + cell->raw_len = info->raw_len ?: info->bytes; cell->bytes = info->bytes; cell->name = info->name; cell->read_post_process = info->read_post_process; @@ -1560,7 +1562,7 @@ static int __nvmem_cell_read(struct nvmem_device *nvmem, { int rc; - rc = nvmem_reg_read(nvmem, cell->offset, buf, cell->bytes); + rc = nvmem_reg_read(nvmem, cell->offset, buf, cell->raw_len); if (rc) return rc; @@ -1571,7 +1573,7 @@ static int __nvmem_cell_read(struct nvmem_device *nvmem, if (cell->read_post_process) { rc = cell->read_post_process(cell->priv, id, index, - cell->offset, buf, cell->bytes); + cell->offset, buf, cell->raw_len); if (rc) return rc; } @@ -1594,14 +1596,15 @@ static int __nvmem_cell_read(struct nvmem_device *nvmem, */ void *nvmem_cell_read(struct nvmem_cell *cell, size_t *len) { - struct nvmem_device *nvmem = cell->entry->nvmem; + struct nvmem_cell_entry *entry = cell->entry; + struct nvmem_device *nvmem = entry->nvmem; u8 *buf; int rc; if (!nvmem) return ERR_PTR(-EINVAL); - buf = kzalloc(cell->entry->bytes, GFP_KERNEL); + buf = kzalloc(max_t(size_t, entry->raw_len, entry->bytes), GFP_KERNEL); if (!buf) return ERR_PTR(-ENOMEM); diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h index 0cf9f9490514..8ffb42ba0f62 100644 --- a/include/linux/nvmem-provider.h +++ b/include/linux/nvmem-provider.h @@ -51,6 +51,7 @@ struct nvmem_keepout { * struct nvmem_cell_info - NVMEM cell description * @name: Name. * @offset: Offset within the NVMEM device. + * @raw_len: Length of raw data (without post processing). * @bytes: Length of the cell. * @bit_offset: Bit offset if cell is smaller than a byte. * @nbits: Number of bits. @@ -62,6 +63,7 @@ struct nvmem_keepout { struct nvmem_cell_info { const char *name; unsigned int offset; + size_t raw_len; unsigned int bytes; unsigned int bit_offset; unsigned int nbits; -- cgit v1.2.3 From 814c978f02db17f16e6aa2efa2a929372f06da09 Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Tue, 4 Apr 2023 18:21:44 +0100 Subject: nvmem: Add macro to register nvmem layout drivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provide a module_nvmem_layout_driver() macro at the end of the nvmem-provider.h header to reduce the boilerplate when registering nvmem layout drivers. Suggested-by: Srinivas Kandagatla Signed-off-by: Miquel Raynal Acked-by: Rafał Miłecki Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20230404172148.82422-37-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- include/linux/nvmem-provider.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h index 8ffb42ba0f62..dae26295e6be 100644 --- a/include/linux/nvmem-provider.h +++ b/include/linux/nvmem-provider.h @@ -9,6 +9,7 @@ #ifndef _LINUX_NVMEM_PROVIDER_H #define _LINUX_NVMEM_PROVIDER_H +#include #include #include #include @@ -242,4 +243,9 @@ nvmem_layout_get_match_data(struct nvmem_device *nvmem, } #endif /* CONFIG_NVMEM */ + +#define module_nvmem_layout_driver(__layout_driver) \ + module_driver(__layout_driver, nvmem_layout_register, \ + nvmem_layout_unregister) + #endif /* ifndef _LINUX_NVMEM_PROVIDER_H */ -- cgit v1.2.3 From 14e985482111a2c6bc75a0348701a4acdc5558d8 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 2 Apr 2023 11:42:07 +0200 Subject: clk: Remove mmask and nmask fields in struct clk_fractional_divider All users of these fields have been removed. They are now computed when needed with [mn]shift and [mn]width. This shrinks the size of struct clk_fractional_divider from 72 to 56 bytes. Signed-off-by: Christophe JAILLET Link: https://lore.kernel.org/r/680357e5acb338433bfc94114b65b4a4ce2c99e2.1680423909.git.christophe.jaillet@wanadoo.fr Reviewed-by: Heiko Stuebner Signed-off-by: Stephen Boyd --- include/linux/clk-provider.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index 842e72a5348f..3271a2bc352f 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -1135,10 +1135,8 @@ struct clk_fractional_divider { void __iomem *reg; u8 mshift; u8 mwidth; - u32 mmask; u8 nshift; u8 nwidth; - u32 nmask; u8 flags; void (*approximation)(struct clk_hw *hw, unsigned long rate, unsigned long *parent_rate, -- cgit v1.2.3 From 263e721e3ba1f3b42ad61aed3d504f3c8c9c0eb6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 7 Mar 2023 15:34:05 +0100 Subject: mm: make mapping_get_entry available outside of filemap.c mapping_get_entry is useful for page cache API users that need to know about xa_value internals. Rename it and make it available in pagemap.h. Link: https://lkml.kernel.org/r/20230307143410.28031-3-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Matthew Wilcox (Oracle) Cc: Andreas Gruenbacher Cc: Hugh Dickins Cc: Ryusuke Konishi Signed-off-by: Andrew Morton --- include/linux/pagemap.h | 1 + mm/filemap.c | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 853184a46411..bcfd798ce0b9 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -507,6 +507,7 @@ pgoff_t page_cache_prev_miss(struct address_space *mapping, #define FGP_ENTRY 0x00000080 #define FGP_STABLE 0x00000100 +void *filemap_get_entry(struct address_space *mapping, pgoff_t index); struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, int fgp_flags, gfp_t gfp); struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index, diff --git a/mm/filemap.c b/mm/filemap.c index 2723104cc06a..a674108a4d52 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1836,7 +1836,7 @@ EXPORT_SYMBOL(page_cache_prev_miss); */ /* - * mapping_get_entry - Get a page cache entry. + * filemap_get_entry - Get a page cache entry. * @mapping: the address_space to search * @index: The page cache index. * @@ -1847,7 +1847,7 @@ EXPORT_SYMBOL(page_cache_prev_miss); * * Return: The folio, swap or shadow entry, %NULL if nothing is found. */ -static void *mapping_get_entry(struct address_space *mapping, pgoff_t index) +void *filemap_get_entry(struct address_space *mapping, pgoff_t index) { XA_STATE(xas, &mapping->i_pages, index); struct folio *folio; @@ -1917,7 +1917,7 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, struct folio *folio; repeat: - folio = mapping_get_entry(mapping, index); + folio = filemap_get_entry(mapping, index); if (xa_is_value(folio)) { if (fgp_flags & FGP_ENTRY) return folio; -- cgit v1.2.3 From 48c9d11375fc66f1e59d0e9b27d121e015a50904 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 7 Mar 2023 15:34:09 +0100 Subject: mm: remove FGP_ENTRY FGP_ENTRY is unused now, so remove it. Link: https://lkml.kernel.org/r/20230307143410.28031-7-hch@lst.de Signed-off-by: Christoph Hellwig Cc: Andreas Gruenbacher Cc: Hugh Dickins Cc: Matthew Wilcox (Oracle) Cc: Ryusuke Konishi Signed-off-by: Andrew Morton --- include/linux/pagemap.h | 3 +-- mm/filemap.c | 7 +------ mm/folio-compat.c | 4 ++-- 3 files changed, 4 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index bcfd798ce0b9..306a0f63cea8 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -504,8 +504,7 @@ pgoff_t page_cache_prev_miss(struct address_space *mapping, #define FGP_NOFS 0x00000010 #define FGP_NOWAIT 0x00000020 #define FGP_FOR_MMAP 0x00000040 -#define FGP_ENTRY 0x00000080 -#define FGP_STABLE 0x00000100 +#define FGP_STABLE 0x00000080 void *filemap_get_entry(struct address_space *mapping, pgoff_t index); struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, diff --git a/mm/filemap.c b/mm/filemap.c index a674108a4d52..ac161b50f5bc 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1891,8 +1891,6 @@ out: * * * %FGP_ACCESSED - The folio will be marked accessed. * * %FGP_LOCK - The folio is returned locked. - * * %FGP_ENTRY - If there is a shadow / swap / DAX entry, return it - * instead of allocating a new folio to replace it. * * %FGP_CREAT - If no page is present then a new page is allocated using * @gfp and added to the page cache and the VM's LRU list. * The page is returned locked and with an increased refcount. @@ -1918,11 +1916,8 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, repeat: folio = filemap_get_entry(mapping, index); - if (xa_is_value(folio)) { - if (fgp_flags & FGP_ENTRY) - return folio; + if (xa_is_value(folio)) folio = NULL; - } if (!folio) goto no_page; diff --git a/mm/folio-compat.c b/mm/folio-compat.c index cabcd1de9ecb..1754daa85d35 100644 --- a/mm/folio-compat.c +++ b/mm/folio-compat.c @@ -97,8 +97,8 @@ struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index, struct folio *folio; folio = __filemap_get_folio(mapping, index, fgp_flags, gfp); - if (!folio || xa_is_value(folio)) - return &folio->page; + if (!folio) + return NULL; return folio_file_page(folio, index); } EXPORT_SYMBOL(pagecache_get_page); -- cgit v1.2.3 From 66dabbb65d673aef40dd17bf62c042be8f6d4a4b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 7 Mar 2023 15:34:10 +0100 Subject: mm: return an ERR_PTR from __filemap_get_folio Instead of returning NULL for all errors, distinguish between: - no entry found and not asked to allocated (-ENOENT) - failed to allocate memory (-ENOMEM) - would block (-EAGAIN) so that callers don't have to guess the error based on the passed in flags. Also pass through the error through the direct callers: filemap_get_folio, filemap_lock_folio filemap_grab_folio and filemap_get_incore_folio. [hch@lst.de: fix null-pointer deref] Link: https://lkml.kernel.org/r/20230310070023.GA13563@lst.de Link: https://lkml.kernel.org/r/20230310043137.GA1624890@u2004 Link: https://lkml.kernel.org/r/20230307143410.28031-8-hch@lst.de Signed-off-by: Christoph Hellwig Acked-by: Ryusuke Konishi [nilfs2] Cc: Andreas Gruenbacher Cc: Hugh Dickins Cc: Matthew Wilcox (Oracle) Cc: Naoya Horiguchi Signed-off-by: Andrew Morton --- fs/afs/dir.c | 10 +++++----- fs/afs/dir_edit.c | 2 +- fs/afs/write.c | 4 ++-- fs/ext4/inode.c | 2 +- fs/ext4/move_extent.c | 8 ++++---- fs/hugetlbfs/inode.c | 2 +- fs/iomap/buffered-io.c | 11 ++--------- fs/netfs/buffered_read.c | 4 ++-- fs/nfs/file.c | 4 ++-- fs/nilfs2/page.c | 6 +++--- include/linux/pagemap.h | 11 ++++++----- mm/filemap.c | 14 ++++++++------ mm/folio-compat.c | 2 +- mm/huge_memory.c | 2 +- mm/hugetlb.c | 6 ++++-- mm/memcontrol.c | 2 +- mm/mincore.c | 2 +- mm/shmem.c | 4 ++-- mm/swap_state.c | 17 ++++++++++------- mm/swapfile.c | 4 ++-- mm/truncate.c | 15 ++++++++------- 21 files changed, 67 insertions(+), 65 deletions(-) (limited to 'include/linux') diff --git a/fs/afs/dir.c b/fs/afs/dir.c index 82690d1dd49a..f92b9e62d567 100644 --- a/fs/afs/dir.c +++ b/fs/afs/dir.c @@ -319,16 +319,16 @@ expand: struct folio *folio; folio = filemap_get_folio(mapping, i); - if (!folio) { + if (IS_ERR(folio)) { if (test_and_clear_bit(AFS_VNODE_DIR_VALID, &dvnode->flags)) afs_stat_v(dvnode, n_inval); - - ret = -ENOMEM; folio = __filemap_get_folio(mapping, i, FGP_LOCK | FGP_CREAT, mapping->gfp_mask); - if (!folio) + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); goto error; + } folio_attach_private(folio, (void *)1); folio_unlock(folio); } @@ -524,7 +524,7 @@ static int afs_dir_iterate(struct inode *dir, struct dir_context *ctx, */ folio = __filemap_get_folio(dir->i_mapping, ctx->pos / PAGE_SIZE, FGP_ACCESSED, 0); - if (!folio) { + if (IS_ERR(folio)) { ret = afs_bad(dvnode, afs_file_error_dir_missing_page); break; } diff --git a/fs/afs/dir_edit.c b/fs/afs/dir_edit.c index 0ab7752d1b75..f0eddccbdd95 100644 --- a/fs/afs/dir_edit.c +++ b/fs/afs/dir_edit.c @@ -115,7 +115,7 @@ static struct folio *afs_dir_get_folio(struct afs_vnode *vnode, pgoff_t index) folio = __filemap_get_folio(mapping, index, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, mapping->gfp_mask); - if (!folio) + if (IS_ERR(folio)) clear_bit(AFS_VNODE_DIR_VALID, &vnode->flags); else if (folio && !folio_test_private(folio)) folio_attach_private(folio, (void *)1); diff --git a/fs/afs/write.c b/fs/afs/write.c index 571f3b9a417e..c822d6006033 100644 --- a/fs/afs/write.c +++ b/fs/afs/write.c @@ -232,7 +232,7 @@ static void afs_kill_pages(struct address_space *mapping, _debug("kill %lx (to %lx)", index, last); folio = filemap_get_folio(mapping, index); - if (!folio) { + if (IS_ERR(folio)) { next = index + 1; continue; } @@ -270,7 +270,7 @@ static void afs_redirty_pages(struct writeback_control *wbc, _debug("redirty %llx @%llx", len, start); folio = filemap_get_folio(mapping, index); - if (!folio) { + if (IS_ERR(folio)) { next = index + 1; continue; } diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index bf0b7dea4900..d7973743417b 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -5395,7 +5395,7 @@ static void ext4_wait_for_tail_page_commit(struct inode *inode) while (1) { struct folio *folio = filemap_lock_folio(inode->i_mapping, inode->i_size >> PAGE_SHIFT); - if (!folio) + if (IS_ERR(folio)) return; ret = __ext4_journalled_invalidate_folio(folio, offset, folio_size(folio) - offset); diff --git a/fs/ext4/move_extent.c b/fs/ext4/move_extent.c index 2de9829aed63..7bf6d069199c 100644 --- a/fs/ext4/move_extent.c +++ b/fs/ext4/move_extent.c @@ -141,18 +141,18 @@ mext_folio_double_lock(struct inode *inode1, struct inode *inode2, flags = memalloc_nofs_save(); folio[0] = __filemap_get_folio(mapping[0], index1, fgp_flags, mapping_gfp_mask(mapping[0])); - if (!folio[0]) { + if (IS_ERR(folio[0])) { memalloc_nofs_restore(flags); - return -ENOMEM; + return PTR_ERR(folio[0]); } folio[1] = __filemap_get_folio(mapping[1], index2, fgp_flags, mapping_gfp_mask(mapping[1])); memalloc_nofs_restore(flags); - if (!folio[1]) { + if (IS_ERR(folio[1])) { folio_unlock(folio[0]); folio_put(folio[0]); - return -ENOMEM; + return PTR_ERR(folio[1]); } /* * __filemap_get_folio() may not wait on folio's writeback if diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 9062da6da567..702d79639c0d 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -697,7 +697,7 @@ static void hugetlbfs_zero_partial_page(struct hstate *h, struct folio *folio; folio = filemap_lock_folio(mapping, idx); - if (!folio) + if (IS_ERR(folio)) return; start = start & ~huge_page_mask(h); diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 6f4c97a6d7e9..96bb56c203f4 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -468,19 +468,12 @@ EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate); struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos) { unsigned fgp = FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_STABLE | FGP_NOFS; - struct folio *folio; if (iter->flags & IOMAP_NOWAIT) fgp |= FGP_NOWAIT; - folio = __filemap_get_folio(iter->inode->i_mapping, pos >> PAGE_SHIFT, + return __filemap_get_folio(iter->inode->i_mapping, pos >> PAGE_SHIFT, fgp, mapping_gfp_mask(iter->inode->i_mapping)); - if (folio) - return folio; - - if (iter->flags & IOMAP_NOWAIT) - return ERR_PTR(-EAGAIN); - return ERR_PTR(-ENOMEM); } EXPORT_SYMBOL_GPL(iomap_get_folio); @@ -911,7 +904,7 @@ static int iomap_write_delalloc_scan(struct inode *inode, /* grab locked page */ folio = filemap_lock_folio(inode->i_mapping, start_byte >> PAGE_SHIFT); - if (!folio) { + if (IS_ERR(folio)) { start_byte = ALIGN_DOWN(start_byte, PAGE_SIZE) + PAGE_SIZE; continue; diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index 7679a68e8193..209726a9cfdb 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -350,8 +350,8 @@ int netfs_write_begin(struct netfs_inode *ctx, retry: folio = __filemap_get_folio(mapping, index, fgp_flags, mapping_gfp_mask(mapping)); - if (!folio) - return -ENOMEM; + if (IS_ERR(folio)) + return PTR_ERR(folio); if (ctx->ops->check_write_begin) { /* Allow the netfs (eg. ceph) to flush conflicts. */ diff --git a/fs/nfs/file.c b/fs/nfs/file.c index 893625eacab9..1d03406e6c03 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -336,8 +336,8 @@ static int nfs_write_begin(struct file *file, struct address_space *mapping, start: folio = nfs_folio_grab_cache_write_begin(mapping, pos >> PAGE_SHIFT); - if (!folio) - return -ENOMEM; + if (IS_ERR(folio)) + return PTR_ERR(folio); *pagep = &folio->page; ret = nfs_flush_incompatible(file, folio); diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c index 41ccd43cd979..5cf30827f244 100644 --- a/fs/nilfs2/page.c +++ b/fs/nilfs2/page.c @@ -259,10 +259,10 @@ repeat: NILFS_PAGE_BUG(&folio->page, "inconsistent dirty state"); dfolio = filemap_grab_folio(dmap, folio->index); - if (unlikely(!dfolio)) { + if (unlikely(IS_ERR(dfolio))) { /* No empty page is added to the page cache */ - err = -ENOMEM; folio_unlock(folio); + err = PTR_ERR(dfolio); break; } if (unlikely(!folio_buffers(folio))) @@ -311,7 +311,7 @@ repeat: folio_lock(folio); dfolio = filemap_lock_folio(dmap, index); - if (dfolio) { + if (!IS_ERR(dfolio)) { /* overwrite existing folio in the destination cache */ WARN_ON(folio_test_dirty(dfolio)); nilfs_copy_page(&dfolio->page, &folio->page, 0); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 306a0f63cea8..fdcd595d2294 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -520,7 +520,8 @@ struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index, * Looks up the page cache entry at @mapping & @index. If a folio is * present, it is returned with an increased refcount. * - * Otherwise, %NULL is returned. + * Return: A folio or ERR_PTR(-ENOENT) if there is no folio in the cache for + * this index. Will not return a shadow, swap or DAX entry. */ static inline struct folio *filemap_get_folio(struct address_space *mapping, pgoff_t index) @@ -537,8 +538,8 @@ static inline struct folio *filemap_get_folio(struct address_space *mapping, * present, it is returned locked with an increased refcount. * * Context: May sleep. - * Return: A folio or %NULL if there is no folio in the cache for this - * index. Will not return a shadow, swap or DAX entry. + * Return: A folio or ERR_PTR(-ENOENT) if there is no folio in the cache for + * this index. Will not return a shadow, swap or DAX entry. */ static inline struct folio *filemap_lock_folio(struct address_space *mapping, pgoff_t index) @@ -555,8 +556,8 @@ static inline struct folio *filemap_lock_folio(struct address_space *mapping, * a new folio is created. The folio is locked, marked as accessed, and * returned. * - * Return: A found or created folio. NULL if no folio is found and failed to - * create a folio. + * Return: A found or created folio. ERR_PTR(-ENOMEM) if no folio is found + * and failed to create a folio. */ static inline struct folio *filemap_grab_folio(struct address_space *mapping, pgoff_t index) diff --git a/mm/filemap.c b/mm/filemap.c index ac161b50f5bc..a34abfe8c654 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1907,7 +1907,7 @@ out: * * If there is a page cache page, it is returned with an increased refcount. * - * Return: The found folio or %NULL otherwise. + * Return: The found folio or an ERR_PTR() otherwise. */ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, int fgp_flags, gfp_t gfp) @@ -1925,7 +1925,7 @@ repeat: if (fgp_flags & FGP_NOWAIT) { if (!folio_trylock(folio)) { folio_put(folio); - return NULL; + return ERR_PTR(-EAGAIN); } } else { folio_lock(folio); @@ -1964,7 +1964,7 @@ no_page: folio = filemap_alloc_folio(gfp, 0); if (!folio) - return NULL; + return ERR_PTR(-ENOMEM); if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP)))) fgp_flags |= FGP_LOCK; @@ -1989,6 +1989,8 @@ no_page: folio_unlock(folio); } + if (!folio) + return ERR_PTR(-ENOENT); return folio; } EXPORT_SYMBOL(__filemap_get_folio); @@ -3258,7 +3260,7 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) * Do we have something in the page cache already? */ folio = filemap_get_folio(mapping, index); - if (likely(folio)) { + if (likely(!IS_ERR(folio))) { /* * We found the page, so try async readahead before waiting for * the lock. @@ -3287,7 +3289,7 @@ retry_find: folio = __filemap_get_folio(mapping, index, FGP_CREAT|FGP_FOR_MMAP, vmf->gfp_mask); - if (!folio) { + if (IS_ERR(folio)) { if (fpin) goto out_retry; filemap_invalidate_unlock_shared(mapping); @@ -3638,7 +3640,7 @@ static struct folio *do_read_cache_folio(struct address_space *mapping, filler = mapping->a_ops->read_folio; repeat: folio = filemap_get_folio(mapping, index); - if (!folio) { + if (IS_ERR(folio)) { folio = filemap_alloc_folio(gfp, 0); if (!folio) return ERR_PTR(-ENOMEM); diff --git a/mm/folio-compat.c b/mm/folio-compat.c index 1754daa85d35..2511c055a35f 100644 --- a/mm/folio-compat.c +++ b/mm/folio-compat.c @@ -97,7 +97,7 @@ struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index, struct folio *folio; folio = __filemap_get_folio(mapping, index, fgp_flags, gfp); - if (!folio) + if (IS_ERR(folio)) return NULL; return folio_file_page(folio, index); } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 70008dd7f215..2d860e70fe88 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3092,7 +3092,7 @@ static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start, struct folio *folio = filemap_get_folio(mapping, index); nr_pages = 1; - if (!folio) + if (IS_ERR(folio)) continue; if (!folio_test_large(folio)) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 07abcb6eb203..712e32b38295 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5780,7 +5780,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, */ new_folio = false; folio = filemap_lock_folio(mapping, idx); - if (!folio) { + if (IS_ERR(folio)) { size = i_size_read(mapping->host) >> huge_page_shift(h); if (idx >= size) goto out; @@ -6071,6 +6071,8 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, vma_end_reservation(h, vma, haddr); pagecache_folio = filemap_lock_folio(mapping, idx); + if (IS_ERR(pagecache_folio)) + pagecache_folio = NULL; } ptl = huge_pte_lock(h, mm, ptep); @@ -6182,7 +6184,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, if (is_continue) { ret = -EFAULT; folio = filemap_lock_folio(mapping, idx); - if (!folio) + if (IS_ERR(folio)) goto out; folio_in_pagecache = true; } else if (!*pagep) { diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 13ec89c45389..0524add35cae 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5705,7 +5705,7 @@ static struct page *mc_handle_file_pte(struct vm_area_struct *vma, /* shmem/tmpfs may report page out on swap: account for that too. */ index = linear_page_index(vma, addr); folio = filemap_get_incore_folio(vma->vm_file->f_mapping, index); - if (!folio) + if (IS_ERR(folio)) return NULL; return folio_file_page(folio, index); } diff --git a/mm/mincore.c b/mm/mincore.c index d359650b0f75..2d5be013a25a 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -61,7 +61,7 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) * tmpfs's .fault). So swapped out tmpfs mappings are tested here. */ folio = filemap_get_incore_folio(mapping, index); - if (folio) { + if (!IS_ERR(folio)) { present = folio_test_uptodate(folio); folio_put(folio); } diff --git a/mm/shmem.c b/mm/shmem.c index 93cb39852a16..fa6e38f2f55f 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -605,7 +605,7 @@ next: index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT; folio = filemap_get_folio(inode->i_mapping, index); - if (!folio) + if (IS_ERR(folio)) goto drop; /* No huge page at the end of the file: nothing to split */ @@ -3214,7 +3214,7 @@ static const char *shmem_get_link(struct dentry *dentry, if (!dentry) { folio = filemap_get_folio(inode->i_mapping, 0); - if (!folio) + if (IS_ERR(folio)) return ERR_PTR(-ECHILD); if (PageHWPoison(folio_page(folio, 0)) || !folio_test_uptodate(folio)) { diff --git a/mm/swap_state.c b/mm/swap_state.c index 92234f4b51d2..b76a65ac28b3 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -336,7 +336,7 @@ struct folio *swap_cache_get_folio(swp_entry_t entry, struct folio *folio; folio = filemap_get_folio(swap_address_space(entry), swp_offset(entry)); - if (folio) { + if (!IS_ERR(folio)) { bool vma_ra = swap_use_vma_readahead(); bool readahead; @@ -366,6 +366,8 @@ struct folio *swap_cache_get_folio(swp_entry_t entry, if (!vma || !vma_ra) atomic_inc(&swapin_readahead_hits); } + } else { + folio = NULL; } return folio; @@ -388,23 +390,24 @@ struct folio *filemap_get_incore_folio(struct address_space *mapping, struct swap_info_struct *si; struct folio *folio = filemap_get_entry(mapping, index); + if (!folio) + return ERR_PTR(-ENOENT); if (!xa_is_value(folio)) - goto out; + return folio; if (!shmem_mapping(mapping)) - return NULL; + return ERR_PTR(-ENOENT); swp = radix_to_swp_entry(folio); /* There might be swapin error entries in shmem mapping. */ if (non_swap_entry(swp)) - return NULL; + return ERR_PTR(-ENOENT); /* Prevent swapoff from happening to us */ si = get_swap_device(swp); if (!si) - return NULL; + return ERR_PTR(-ENOENT); index = swp_offset(swp); folio = filemap_get_folio(swap_address_space(swp), index); put_swap_device(si); -out: return folio; } @@ -431,7 +434,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, folio = filemap_get_folio(swap_address_space(entry), swp_offset(entry)); put_swap_device(si); - if (folio) + if (!IS_ERR(folio)) return folio_file_page(folio, swp_offset(entry)); /* diff --git a/mm/swapfile.c b/mm/swapfile.c index c1b97436f811..00b3e46becad 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -136,7 +136,7 @@ static int __try_to_reclaim_swap(struct swap_info_struct *si, int ret = 0; folio = filemap_get_folio(swap_address_space(entry), offset); - if (!folio) + if (IS_ERR(folio)) return 0; /* * When this function is called from scan_swap_map_slots() and it's @@ -2095,7 +2095,7 @@ retry: entry = swp_entry(type, i); folio = filemap_get_folio(swap_address_space(entry), i); - if (!folio) + if (IS_ERR(folio)) continue; /* diff --git a/mm/truncate.c b/mm/truncate.c index 7b4ea4c4a46b..86de31ed4d32 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -375,7 +375,7 @@ void truncate_inode_pages_range(struct address_space *mapping, same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT); folio = __filemap_get_folio(mapping, lstart >> PAGE_SHIFT, FGP_LOCK, 0); - if (folio) { + if (!IS_ERR(folio)) { same_folio = lend < folio_pos(folio) + folio_size(folio); if (!truncate_inode_partial_folio(folio, lstart, lend)) { start = folio->index + folio_nr_pages(folio); @@ -387,14 +387,15 @@ void truncate_inode_pages_range(struct address_space *mapping, folio = NULL; } - if (!same_folio) + if (!same_folio) { folio = __filemap_get_folio(mapping, lend >> PAGE_SHIFT, FGP_LOCK, 0); - if (folio) { - if (!truncate_inode_partial_folio(folio, lstart, lend)) - end = folio->index; - folio_unlock(folio); - folio_put(folio); + if (!IS_ERR(folio)) { + if (!truncate_inode_partial_folio(folio, lstart, lend)) + end = folio->index; + folio_unlock(folio); + folio_put(folio); + } } index = start; -- cgit v1.2.3 From 2bad466cc9d9b4c3b4b16eb9c03c919b59561316 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Thu, 9 Mar 2023 17:37:10 -0500 Subject: mm/uffd: UFFD_FEATURE_WP_UNPOPULATED Patch series "mm/uffd: Add feature bit UFFD_FEATURE_WP_UNPOPULATED", v4. The new feature bit makes anonymous memory acts the same as file memory on userfaultfd-wp in that it'll also wr-protect none ptes. It can be useful in two cases: (1) Uffd-wp app that needs to wr-protect none ptes like QEMU snapshot, so pre-fault can be replaced by enabling this flag and speed up protections (2) It helps to implement async uffd-wp mode that Muhammad is working on [1] It's debatable whether this is the most ideal solution because with the new feature bit set, wr-protect none pte needs to pre-populate the pgtables to the last level (PAGE_SIZE). But it seems fine so far to service either purpose above, so we can leave optimizations for later. The series brings pte markers to anonymous memory too. There's some change in the common mm code path in the 1st patch, great to have some eye looking at it, but hopefully they're still relatively straightforward. This patch (of 2): This is a new feature that controls how uffd-wp handles none ptes. When it's set, the kernel will handle anonymous memory the same way as file memory, by allowing the user to wr-protect unpopulated ptes. File memories handles none ptes consistently by allowing wr-protecting of none ptes because of the unawareness of page cache being exist or not. For anonymous it was not as persistent because we used to assume that we don't need protections on none ptes or known zero pages. One use case of such a feature bit was VM live snapshot, where if without wr-protecting empty ptes the snapshot can contain random rubbish in the holes of the anonymous memory, which can cause misbehave of the guest when the guest OS assumes the pages should be all zeros. QEMU worked it around by pre-populate the section with reads to fill in zero page entries before starting the whole snapshot process [1]. Recently there's another need raised on using userfaultfd wr-protect for detecting dirty pages (to replace soft-dirty in some cases) [2]. In that case if without being able to wr-protect none ptes by default, the dirty info can get lost, since we cannot treat every none pte to be dirty (the current design is identify a page dirty based on uffd-wp bit being cleared). In general, we want to be able to wr-protect empty ptes too even for anonymous. This patch implements UFFD_FEATURE_WP_UNPOPULATED so that it'll make uffd-wp handling on none ptes being consistent no matter what the memory type is underneath. It doesn't have any impact on file memories so far because we already have pte markers taking care of that. So it only affects anonymous. The feature bit is by default off, so the old behavior will be maintained. Sometimes it may be wanted because the wr-protect of none ptes will contain overheads not only during UFFDIO_WRITEPROTECT (by applying pte markers to anonymous), but also on creating the pgtables to store the pte markers. So there's potentially less chance of using thp on the first fault for a none pmd or larger than a pmd. The major implementation part is teaching the whole kernel to understand pte markers even for anonymously mapped ranges, meanwhile allowing the UFFDIO_WRITEPROTECT ioctl to apply pte markers for anonymous too when the new feature bit is set. Note that even if the patch subject starts with mm/uffd, there're a few small refactors to major mm path of handling anonymous page faults. But they should be straightforward. With WP_UNPOPUATED, application like QEMU can avoid pre-read faults all the memory before wr-protect during taking a live snapshot. Quotting from Muhammad's test result here [3] based on a simple program [4]: (1) With huge page disabled echo madvise > /sys/kernel/mm/transparent_hugepage/enabled ./uffd_wp_perf Test DEFAULT: 4 Test PRE-READ: 1111453 (pre-fault 1101011) Test MADVISE: 278276 (pre-fault 266378) Test WP-UNPOPULATE: 11712 (2) With Huge page enabled echo always > /sys/kernel/mm/transparent_hugepage/enabled ./uffd_wp_perf Test DEFAULT: 4 Test PRE-READ: 22521 (pre-fault 22348) Test MADVISE: 4909 (pre-fault 4743) Test WP-UNPOPULATE: 14448 There'll be a great perf boost for no-thp case, while for thp enabled with extreme case of all-thp-zero WP_UNPOPULATED can be slower than MADVISE, but that's low possibility in reality, also the overhead was not reduced but postponed until a follow up write on any huge zero thp, so potentially it is faster by making the follow up writes slower. [1] https://lore.kernel.org/all/20210401092226.102804-4-andrey.gruzdev@virtuozzo.com/ [2] https://lore.kernel.org/all/Y+v2HJ8+3i%2FKzDBu@x1n/ [3] https://lore.kernel.org/all/d0eb0a13-16dc-1ac1-653a-78b7273781e3@collabora.com/ [4] https://github.com/xzpeter/clibs/blob/master/uffd-test/uffd-wp-perf.c [peterx@redhat.com: comment changes, oneliner fix to khugepaged] Link: https://lkml.kernel.org/r/ZB2/8jPhD3fpx5U8@x1n Link: https://lkml.kernel.org/r/20230309223711.823547-1-peterx@redhat.com Link: https://lkml.kernel.org/r/20230309223711.823547-2-peterx@redhat.com Signed-off-by: Peter Xu Acked-by: David Hildenbrand Cc: Andrea Arcangeli Cc: Axel Rasmussen Cc: Mike Rapoport Cc: Muhammad Usama Anjum Cc: Nadav Amit Cc: Paul Gofman Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/userfaultfd.rst | 17 +++++++++ fs/userfaultfd.c | 16 ++++++++ include/linux/mm_inline.h | 6 +++ include/linux/userfaultfd_k.h | 23 ++++++++++++ include/uapi/linux/userfaultfd.h | 10 ++++- mm/khugepaged.c | 2 +- mm/memory.c | 56 +++++++++++++++++++++------- mm/mprotect.c | 51 ++++++++++++++++++++----- 8 files changed, 155 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst index 7dc823b56ca4..bd2226299583 100644 --- a/Documentation/admin-guide/mm/userfaultfd.rst +++ b/Documentation/admin-guide/mm/userfaultfd.rst @@ -219,6 +219,23 @@ former will have ``UFFD_PAGEFAULT_FLAG_WP`` set, the latter you still need to supply a page when ``UFFDIO_REGISTER_MODE_MISSING`` was used. +Userfaultfd write-protect mode currently behave differently on none ptes +(when e.g. page is missing) over different types of memories. + +For anonymous memory, ``ioctl(UFFDIO_WRITEPROTECT)`` will ignore none ptes +(e.g. when pages are missing and not populated). For file-backed memories +like shmem and hugetlbfs, none ptes will be write protected just like a +present pte. In other words, there will be a userfaultfd write fault +message generated when writing to a missing page on file typed memories, +as long as the page range was write-protected before. Such a message will +not be generated on anonymous memories by default. + +If the application wants to be able to write protect none ptes on anonymous +memory, one can pre-populate the memory with e.g. MADV_POPULATE_READ. On +newer kernels, one can also detect the feature UFFD_FEATURE_WP_UNPOPULATED +and set the feature bit in advance to make sure none ptes will also be +write protected even upon anonymous memory. + QEMU/KVM ======== diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 44d1ee429eb0..881e9c82b9d1 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -108,6 +108,21 @@ static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx) return ctx->features & UFFD_FEATURE_INITIALIZED; } +/* + * Whether WP_UNPOPULATED is enabled on the uffd context. It is only + * meaningful when userfaultfd_wp()==true on the vma and when it's + * anonymous. + */ +bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) +{ + struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; + + if (!ctx) + return false; + + return ctx->features & UFFD_FEATURE_WP_UNPOPULATED; +} + static void userfaultfd_set_vm_flags(struct vm_area_struct *vma, vm_flags_t flags) { @@ -1971,6 +1986,7 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, #endif #ifndef CONFIG_PTE_MARKER_UFFD_WP uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM; + uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED; #endif uffdio_api.ioctls = UFFD_API_IOCTLS; ret = -EFAULT; diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index de1e622dd366..0e1d239a882c 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -557,6 +557,12 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, /* The current status of the pte should be "cleared" before calling */ WARN_ON_ONCE(!pte_none(*pte)); + /* + * NOTE: userfaultfd_wp_unpopulated() doesn't need this whole + * thing, because when zapping either it means it's dropping the + * page, or in TTU where the present pte will be quickly replaced + * with a swap pte. There's no way of leaking the bit. + */ if (vma_is_anonymous(vma) || !userfaultfd_wp(vma)) return; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 3767f18114ef..0cf8880219da 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -179,6 +179,7 @@ extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, unsigned long end, struct list_head *uf); extern void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf); +extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma); #else /* CONFIG_USERFAULTFD */ @@ -274,8 +275,30 @@ static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) return false; } +static inline bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) +{ + return false; +} + #endif /* CONFIG_USERFAULTFD */ +static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) +{ + /* Only wr-protect mode uses pte markers */ + if (!userfaultfd_wp(vma)) + return false; + + /* File-based uffd-wp always need markers */ + if (!vma_is_anonymous(vma)) + return true; + + /* + * Anonymous uffd-wp only needs the markers if WP_UNPOPULATED + * enabled (to apply markers on zero pages). + */ + return userfaultfd_wp_unpopulated(vma); +} + static inline bool pte_marker_entry_uffd_wp(swp_entry_t entry) { #ifdef CONFIG_PTE_MARKER_UFFD_WP diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 005e5e306266..90c958952bfc 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -38,7 +38,8 @@ UFFD_FEATURE_MINOR_HUGETLBFS | \ UFFD_FEATURE_MINOR_SHMEM | \ UFFD_FEATURE_EXACT_ADDRESS | \ - UFFD_FEATURE_WP_HUGETLBFS_SHMEM) + UFFD_FEATURE_WP_HUGETLBFS_SHMEM | \ + UFFD_FEATURE_WP_UNPOPULATED) #define UFFD_API_IOCTLS \ ((__u64)1 << _UFFDIO_REGISTER | \ (__u64)1 << _UFFDIO_UNREGISTER | \ @@ -203,6 +204,12 @@ struct uffdio_api { * * UFFD_FEATURE_WP_HUGETLBFS_SHMEM indicates that userfaultfd * write-protection mode is supported on both shmem and hugetlbfs. + * + * UFFD_FEATURE_WP_UNPOPULATED indicates that userfaultfd + * write-protection mode will always apply to unpopulated pages + * (i.e. empty ptes). This will be the default behavior for shmem + * & hugetlbfs, so this flag only affects anonymous memory behavior + * when userfault write-protection mode is registered. */ #define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0) #define UFFD_FEATURE_EVENT_FORK (1<<1) @@ -217,6 +224,7 @@ struct uffdio_api { #define UFFD_FEATURE_MINOR_SHMEM (1<<10) #define UFFD_FEATURE_EXACT_ADDRESS (1<<11) #define UFFD_FEATURE_WP_HUGETLBFS_SHMEM (1<<12) +#define UFFD_FEATURE_WP_UNPOPULATED (1<<13) __u64 features; __u64 ioctls; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 074ea534f786..c7317678cb10 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1177,7 +1177,7 @@ static int hpage_collapse_scan_pmd(struct mm_struct *mm, * enabled swap entries. Please see * comment below for pte_uffd_wp(). */ - if (pte_swp_uffd_wp(pteval)) { + if (pte_swp_uffd_wp_any(pteval)) { result = SCAN_PTE_UFFD_WP; goto out_unmap; } diff --git a/mm/memory.c b/mm/memory.c index 6285cad1f4fb..a890b2951b53 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -104,6 +104,20 @@ EXPORT_SYMBOL(mem_map); #endif static vm_fault_t do_fault(struct vm_fault *vmf); +static vm_fault_t do_anonymous_page(struct vm_fault *vmf); +static bool vmf_pte_changed(struct vm_fault *vmf); + +/* + * Return true if the original pte was a uffd-wp pte marker (so the pte was + * wr-protected). + */ +static bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf) +{ + if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)) + return false; + + return pte_marker_uffd_wp(vmf->orig_pte); +} /* * A number of key systems in x86 including ioremap() rely on the assumption @@ -1346,6 +1360,10 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, pte_t *pte, struct zap_details *details, pte_t pteval) { + /* Zap on anonymous always means dropping everything */ + if (vma_is_anonymous(vma)) + return; + if (zap_drop_file_uffd_wp(details)) return; @@ -1452,8 +1470,12 @@ again: continue; rss[mm_counter(page)]--; } else if (pte_marker_entry_uffd_wp(entry)) { - /* Only drop the uffd-wp marker if explicitly requested */ - if (!zap_drop_file_uffd_wp(details)) + /* + * For anon: always drop the marker; for file: only + * drop the marker if explicitly requested. + */ + if (!vma_is_anonymous(vma) && + !zap_drop_file_uffd_wp(details)) continue; } else if (is_hwpoison_entry(entry) || is_swapin_error_entry(entry)) { @@ -3620,6 +3642,14 @@ static vm_fault_t pte_marker_clear(struct vm_fault *vmf) return 0; } +static vm_fault_t do_pte_missing(struct vm_fault *vmf) +{ + if (vma_is_anonymous(vmf->vma)) + return do_anonymous_page(vmf); + else + return do_fault(vmf); +} + /* * This is actually a page-missing access, but with uffd-wp special pte * installed. It means this pte was wr-protected before being unmapped. @@ -3630,11 +3660,10 @@ static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf) * Just in case there're leftover special ptes even after the region * got unregistered - we can simply clear them. */ - if (unlikely(!userfaultfd_wp(vmf->vma) || vma_is_anonymous(vmf->vma))) + if (unlikely(!userfaultfd_wp(vmf->vma))) return pte_marker_clear(vmf); - /* do_fault() can handle pte markers too like none pte */ - return do_fault(vmf); + return do_pte_missing(vmf); } static vm_fault_t handle_pte_marker(struct vm_fault *vmf) @@ -3999,6 +4028,7 @@ out_release: */ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) { + bool uffd_wp = vmf_orig_pte_uffd_wp(vmf); struct vm_area_struct *vma = vmf->vma; struct folio *folio; vm_fault_t ret = 0; @@ -4032,7 +4062,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) vma->vm_page_prot)); vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); - if (!pte_none(*vmf->pte)) { + if (vmf_pte_changed(vmf)) { update_mmu_tlb(vma, vmf->address, vmf->pte); goto unlock; } @@ -4072,7 +4102,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); - if (!pte_none(*vmf->pte)) { + if (vmf_pte_changed(vmf)) { update_mmu_tlb(vma, vmf->address, vmf->pte); goto release; } @@ -4092,6 +4122,8 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) folio_add_new_anon_rmap(folio, vma, vmf->address); folio_add_lru_vma(folio, vma); setpte: + if (uffd_wp) + entry = pte_mkuffd_wp(entry); set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry); /* No need to invalidate - it was non-present before */ @@ -4259,7 +4291,7 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr) { struct vm_area_struct *vma = vmf->vma; - bool uffd_wp = pte_marker_uffd_wp(vmf->orig_pte); + bool uffd_wp = vmf_orig_pte_uffd_wp(vmf); bool write = vmf->flags & FAULT_FLAG_WRITE; bool prefault = vmf->address != addr; pte_t entry; @@ -4903,12 +4935,8 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) } } - if (!vmf->pte) { - if (vma_is_anonymous(vmf->vma)) - return do_anonymous_page(vmf); - else - return do_fault(vmf); - } + if (!vmf->pte) + return do_pte_missing(vmf); if (!pte_present(vmf->orig_pte)) return do_swap_page(vmf); diff --git a/mm/mprotect.c b/mm/mprotect.c index 13e84d8c0797..b9da9a5f87fe 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -276,7 +276,15 @@ static long change_pte_range(struct mmu_gather *tlb, } else { /* It must be an none page, or what else?.. */ WARN_ON_ONCE(!pte_none(oldpte)); - if (unlikely(uffd_wp && !vma_is_anonymous(vma))) { + + /* + * Nobody plays with any none ptes besides + * userfaultfd when applying the protections. + */ + if (likely(!uffd_wp)) + continue; + + if (userfaultfd_wp_use_markers(vma)) { /* * For file-backed mem, we need to be able to * wr-protect a none pte, because even if the @@ -320,23 +328,46 @@ static inline int pmd_none_or_clear_bad_unless_trans_huge(pmd_t *pmd) return 0; } -/* Return true if we're uffd wr-protecting file-backed memory, or false */ +/* + * Return true if we want to split THPs into PTE mappings in change + * protection procedure, false otherwise. + */ static inline bool -uffd_wp_protect_file(struct vm_area_struct *vma, unsigned long cp_flags) +pgtable_split_needed(struct vm_area_struct *vma, unsigned long cp_flags) { + /* + * pte markers only resides in pte level, if we need pte markers, + * we need to split. We cannot wr-protect shmem thp because file + * thp is handled differently when split by erasing the pmd so far. + */ return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma); } /* - * If wr-protecting the range for file-backed, populate pgtable for the case - * when pgtable is empty but page cache exists. When {pte|pmd|...}_alloc() - * failed we treat it the same way as pgtable allocation failures during - * page faults by kicking OOM and returning error. + * Return true if we want to populate pgtables in change protection + * procedure, false otherwise + */ +static inline bool +pgtable_populate_needed(struct vm_area_struct *vma, unsigned long cp_flags) +{ + /* If not within ioctl(UFFDIO_WRITEPROTECT), then don't bother */ + if (!(cp_flags & MM_CP_UFFD_WP)) + return false; + + /* Populate if the userfaultfd mode requires pte markers */ + return userfaultfd_wp_use_markers(vma); +} + +/* + * Populate the pgtable underneath for whatever reason if requested. + * When {pte|pmd|...}_alloc() failed we treat it the same way as pgtable + * allocation failures during page faults by kicking OOM and returning + * error. */ #define change_pmd_prepare(vma, pmd, cp_flags) \ ({ \ long err = 0; \ - if (unlikely(uffd_wp_protect_file(vma, cp_flags))) { \ + if (unlikely(pgtable_populate_needed(vma, cp_flags))) { \ if (pte_alloc(vma->vm_mm, pmd)) \ err = -ENOMEM; \ } \ @@ -351,7 +382,7 @@ uffd_wp_protect_file(struct vm_area_struct *vma, unsigned long cp_flags) #define change_prepare(vma, high, low, addr, cp_flags) \ ({ \ long err = 0; \ - if (unlikely(uffd_wp_protect_file(vma, cp_flags))) { \ + if (unlikely(pgtable_populate_needed(vma, cp_flags))) { \ low##_t *p = low##_alloc(vma->vm_mm, high, addr); \ if (p == NULL) \ err = -ENOMEM; \ @@ -404,7 +435,7 @@ static inline long change_pmd_range(struct mmu_gather *tlb, if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { if ((next - addr != HPAGE_PMD_SIZE) || - uffd_wp_protect_file(vma, cp_flags)) { + pgtable_split_needed(vma, cp_flags)) { __split_huge_pmd(vma, pmd, addr, false, NULL); /* * For file-backed, the pmd could have been -- cgit v1.2.3 From 23baf831a32c04f9a968812511540b1b3e648bf5 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Wed, 15 Mar 2023 14:31:33 +0300 Subject: mm, treewide: redefine MAX_ORDER sanely MAX_ORDER currently defined as number of orders page allocator supports: user can ask buddy allocator for page order between 0 and MAX_ORDER-1. This definition is counter-intuitive and lead to number of bugs all over the kernel. Change the definition of MAX_ORDER to be inclusive: the range of orders user can ask from buddy allocator is 0..MAX_ORDER now. [kirill@shutemov.name: fix min() warning] Link: https://lkml.kernel.org/r/20230315153800.32wib3n5rickolvh@box [akpm@linux-foundation.org: fix another min_t warning] [kirill@shutemov.name: fixups per Zi Yan] Link: https://lkml.kernel.org/r/20230316232144.b7ic4cif4kjiabws@box.shutemov.name [akpm@linux-foundation.org: fix underlining in docs] Link: https://lore.kernel.org/oe-kbuild-all/202303191025.VRCTk6mP-lkp@intel.com/ Link: https://lkml.kernel.org/r/20230315113133.11326-11-kirill.shutemov@linux.intel.com Signed-off-by: Kirill A. Shutemov Reviewed-by: Michael Ellerman [powerpc] Cc: "Kirill A. Shutemov" Cc: Zi Yan Signed-off-by: Andrew Morton --- Documentation/admin-guide/kdump/vmcoreinfo.rst | 6 ++-- Documentation/admin-guide/kernel-parameters.txt | 2 +- arch/arc/Kconfig | 4 +-- arch/arm/Kconfig | 9 ++---- arch/arm/configs/imx_v6_v7_defconfig | 2 +- arch/arm/configs/milbeaut_m10v_defconfig | 2 +- arch/arm/configs/oxnas_v6_defconfig | 2 +- arch/arm/configs/pxa_defconfig | 2 +- arch/arm/configs/sama7_defconfig | 2 +- arch/arm/configs/sp7021_defconfig | 2 +- arch/arm64/Kconfig | 27 ++++++++---------- arch/arm64/include/asm/sparsemem.h | 2 +- arch/arm64/kvm/hyp/include/nvhe/gfp.h | 2 +- arch/arm64/kvm/hyp/nvhe/page_alloc.c | 10 +++---- arch/csky/Kconfig | 2 +- arch/ia64/Kconfig | 8 +++--- arch/ia64/include/asm/sparsemem.h | 4 +-- arch/ia64/mm/hugetlbpage.c | 2 +- arch/loongarch/Kconfig | 15 ++++------ arch/m68k/Kconfig.cpu | 5 +--- arch/mips/Kconfig | 19 ++++++------- arch/nios2/Kconfig | 7 ++--- arch/powerpc/Kconfig | 27 ++++++++---------- arch/powerpc/configs/85xx/ge_imp3a_defconfig | 2 +- arch/powerpc/configs/fsl-emb-nonhw.config | 2 +- arch/powerpc/mm/book3s64/iommu_api.c | 2 +- arch/powerpc/mm/hugetlbpage.c | 2 +- arch/powerpc/platforms/powernv/pci-ioda.c | 2 +- arch/sh/configs/ecovec24_defconfig | 2 +- arch/sh/mm/Kconfig | 17 +++++------ arch/sparc/Kconfig | 5 +--- arch/sparc/kernel/pci_sun4v.c | 2 +- arch/sparc/kernel/traps_64.c | 2 +- arch/sparc/mm/tsb.c | 4 +-- arch/um/kernel/um_arch.c | 4 +-- arch/xtensa/Kconfig | 5 +--- drivers/base/regmap/regmap-debugfs.c | 8 +++--- drivers/block/floppy.c | 2 +- drivers/crypto/ccp/sev-dev.c | 2 +- drivers/crypto/hisilicon/sgl.c | 6 ++-- drivers/gpu/drm/i915/gem/i915_gem_internal.c | 2 +- drivers/gpu/drm/i915/gem/selftests/huge_pages.c | 2 +- drivers/gpu/drm/ttm/ttm_pool.c | 22 +++++++------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 +- drivers/iommu/dma-iommu.c | 2 +- drivers/irqchip/irq-gic-v3-its.c | 4 +-- drivers/md/dm-bufio.c | 2 +- drivers/misc/genwqe/card_dev.c | 2 +- drivers/misc/genwqe/card_utils.c | 4 +-- drivers/net/ethernet/hisilicon/hns3/hns3_enet.c | 2 +- drivers/net/ethernet/ibm/ibmvnic.h | 2 +- drivers/video/fbdev/hyperv_fb.c | 4 +-- drivers/video/fbdev/vermilion/vermilion.c | 2 +- drivers/virtio/virtio_balloon.c | 2 +- drivers/virtio/virtio_mem.c | 12 ++++---- fs/ramfs/file-nommu.c | 2 +- include/drm/ttm/ttm_pool.h | 2 +- include/linux/hugetlb.h | 2 +- include/linux/mmzone.h | 10 +++---- include/linux/pageblock-flags.h | 4 +-- include/linux/slab.h | 6 ++-- kernel/crash_core.c | 2 +- kernel/dma/pool.c | 6 ++-- kernel/events/ring_buffer.c | 6 ++-- mm/Kconfig | 10 +++---- mm/compaction.c | 8 +++--- mm/debug_vm_pgtable.c | 4 +-- mm/huge_memory.c | 2 +- mm/hugetlb.c | 4 +-- mm/kmsan/init.c | 6 ++-- mm/memblock.c | 2 +- mm/memory_hotplug.c | 4 +-- mm/page_alloc.c | 38 ++++++++++++------------- mm/page_isolation.c | 12 ++++---- mm/page_owner.c | 6 ++-- mm/page_reporting.c | 6 ++-- mm/shuffle.h | 2 +- mm/slab.c | 2 +- mm/slub.c | 6 ++-- mm/vmscan.c | 2 +- mm/vmstat.c | 14 ++++----- net/smc/smc_ib.c | 2 +- security/integrity/ima/ima_crypto.c | 2 +- tools/testing/memblock/linux/mmzone.h | 6 ++-- 84 files changed, 223 insertions(+), 253 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/kdump/vmcoreinfo.rst b/Documentation/admin-guide/kdump/vmcoreinfo.rst index 86fd88492870..c18d94fa6470 100644 --- a/Documentation/admin-guide/kdump/vmcoreinfo.rst +++ b/Documentation/admin-guide/kdump/vmcoreinfo.rst @@ -172,7 +172,7 @@ variables. Offset of the free_list's member. This value is used to compute the number of free pages. -Each zone has a free_area structure array called free_area[MAX_ORDER]. +Each zone has a free_area structure array called free_area[MAX_ORDER + 1]. The free_list represents a linked list of free page blocks. (list_head, next|prev) @@ -189,8 +189,8 @@ Offsets of the vmap_area's members. They carry vmalloc-specific information. Makedumpfile gets the start address of the vmalloc region from this. -(zone.free_area, MAX_ORDER) ---------------------------- +(zone.free_area, MAX_ORDER + 1) +------------------------------- Free areas descriptor. User-space tools use this value to iterate the free_area ranges. MAX_ORDER is used by the zone buddy allocator. diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 6221a1d057dd..50da4f26fad5 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3969,7 +3969,7 @@ [KNL] Minimal page reporting order Format: Adjust the minimal page reporting order. The page - reporting is disabled when it exceeds (MAX_ORDER-1). + reporting is disabled when it exceeds MAX_ORDER. panic= [KNL] Kernel behaviour on panic: delay timeout > 0: seconds before rebooting diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index d9a13ccf89a3..ab6d701365bb 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -556,7 +556,7 @@ endmenu # "ARC Architecture Configuration" config ARCH_FORCE_MAX_ORDER int "Maximum zone order" - default "12" if ARC_HUGEPAGE_16M - default "11" + default "11" if ARC_HUGEPAGE_16M + default "10" source "kernel/power/Kconfig" diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e24a9820e12f..929e646e84b9 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1355,9 +1355,9 @@ config ARM_MODULE_PLTS config ARCH_FORCE_MAX_ORDER int "Maximum zone order" - default "12" if SOC_AM33XX - default "9" if SA1111 - default "11" + default "11" if SOC_AM33XX + default "8" if SA1111 + default "10" help The kernel memory allocator divides physically contiguous memory blocks into "zones", where each zone is a power of two number of @@ -1366,9 +1366,6 @@ config ARCH_FORCE_MAX_ORDER blocks of physically contiguous memory, then you may need to increase this value. - This config option is actually maximum order plus one. For example, - a value of 11 means that the largest free memory block is 2^10 pages. - config ALIGNMENT_TRAP def_bool CPU_CP15_MMU select HAVE_PROC_CPU if PROC_FS diff --git a/arch/arm/configs/imx_v6_v7_defconfig b/arch/arm/configs/imx_v6_v7_defconfig index 6dc6fed12af8..345a67e67dbd 100644 --- a/arch/arm/configs/imx_v6_v7_defconfig +++ b/arch/arm/configs/imx_v6_v7_defconfig @@ -31,7 +31,7 @@ CONFIG_SOC_VF610=y CONFIG_SMP=y CONFIG_ARM_PSCI=y CONFIG_HIGHMEM=y -CONFIG_ARCH_FORCE_MAX_ORDER=14 +CONFIG_ARCH_FORCE_MAX_ORDER=13 CONFIG_CMDLINE="noinitrd console=ttymxc0,115200" CONFIG_KEXEC=y CONFIG_CPU_FREQ=y diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig index bd29e5012cb0..385ad0f391a8 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -26,7 +26,7 @@ CONFIG_THUMB2_KERNEL=y # CONFIG_THUMB2_AVOID_R_ARM_THM_JUMP11 is not set # CONFIG_ARM_PATCH_IDIV is not set CONFIG_HIGHMEM=y -CONFIG_ARCH_FORCE_MAX_ORDER=12 +CONFIG_ARCH_FORCE_MAX_ORDER=11 CONFIG_SECCOMP=y CONFIG_KEXEC=y CONFIG_EFI=y diff --git a/arch/arm/configs/oxnas_v6_defconfig b/arch/arm/configs/oxnas_v6_defconfig index 70a67b3fc91b..90779812c6dd 100644 --- a/arch/arm/configs/oxnas_v6_defconfig +++ b/arch/arm/configs/oxnas_v6_defconfig @@ -12,7 +12,7 @@ CONFIG_ARCH_OXNAS=y CONFIG_MACH_OX820=y CONFIG_SMP=y CONFIG_NR_CPUS=16 -CONFIG_ARCH_FORCE_MAX_ORDER=12 +CONFIG_ARCH_FORCE_MAX_ORDER=11 CONFIG_SECCOMP=y CONFIG_ARM_APPENDED_DTB=y CONFIG_ARM_ATAG_DTB_COMPAT=y diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig index e656d3af2266..b46e39369dbb 100644 --- a/arch/arm/configs/pxa_defconfig +++ b/arch/arm/configs/pxa_defconfig @@ -20,7 +20,7 @@ CONFIG_PXA_SHARPSL=y CONFIG_MACH_AKITA=y CONFIG_MACH_BORZOI=y CONFIG_AEABI=y -CONFIG_ARCH_FORCE_MAX_ORDER=9 +CONFIG_ARCH_FORCE_MAX_ORDER=8 CONFIG_CMDLINE="root=/dev/ram0 ro" CONFIG_KEXEC=y CONFIG_CPU_FREQ=y diff --git a/arch/arm/configs/sama7_defconfig b/arch/arm/configs/sama7_defconfig index 0d964c613d71..954112041403 100644 --- a/arch/arm/configs/sama7_defconfig +++ b/arch/arm/configs/sama7_defconfig @@ -19,7 +19,7 @@ CONFIG_ATMEL_CLOCKSOURCE_TCB=y # CONFIG_CACHE_L2X0 is not set # CONFIG_ARM_PATCH_IDIV is not set # CONFIG_CPU_SW_DOMAIN_PAN is not set -CONFIG_ARCH_FORCE_MAX_ORDER=15 +CONFIG_ARCH_FORCE_MAX_ORDER=14 CONFIG_UACCESS_WITH_MEMCPY=y # CONFIG_ATAGS is not set CONFIG_CMDLINE="console=ttyS0,115200 earlyprintk ignore_loglevel" diff --git a/arch/arm/configs/sp7021_defconfig b/arch/arm/configs/sp7021_defconfig index 5bca2eb59b86..c6448ac860b6 100644 --- a/arch/arm/configs/sp7021_defconfig +++ b/arch/arm/configs/sp7021_defconfig @@ -17,7 +17,7 @@ CONFIG_ARCH_SUNPLUS=y # CONFIG_VDSO is not set CONFIG_SMP=y CONFIG_THUMB2_KERNEL=y -CONFIG_ARCH_FORCE_MAX_ORDER=12 +CONFIG_ARCH_FORCE_MAX_ORDER=11 CONFIG_VFP=y CONFIG_NEON=y CONFIG_MODULES=y diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1023e896d46b..cb5c6aa3254e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1476,22 +1476,22 @@ config XEN # include/linux/mmzone.h requires the following to be true: # -# MAX_ORDER - 1 + PAGE_SHIFT <= SECTION_SIZE_BITS +# MAX_ORDER + PAGE_SHIFT <= SECTION_SIZE_BITS # -# so the maximum value of MAX_ORDER is SECTION_SIZE_BITS + 1 - PAGE_SHIFT: +# so the maximum value of MAX_ORDER is SECTION_SIZE_BITS - PAGE_SHIFT: # # | SECTION_SIZE_BITS | PAGE_SHIFT | max MAX_ORDER | default MAX_ORDER | # ----+-------------------+--------------+-----------------+--------------------+ -# 4K | 27 | 12 | 16 | 11 | -# 16K | 27 | 14 | 14 | 12 | -# 64K | 29 | 16 | 14 | 14 | +# 4K | 27 | 12 | 15 | 10 | +# 16K | 27 | 14 | 13 | 11 | +# 64K | 29 | 16 | 13 | 13 | config ARCH_FORCE_MAX_ORDER int "Maximum zone order" if ARM64_4K_PAGES || ARM64_16K_PAGES - default "14" if ARM64_64K_PAGES - range 12 14 if ARM64_16K_PAGES - default "12" if ARM64_16K_PAGES - range 11 16 if ARM64_4K_PAGES - default "11" + default "13" if ARM64_64K_PAGES + range 11 13 if ARM64_16K_PAGES + default "11" if ARM64_16K_PAGES + range 10 15 if ARM64_4K_PAGES + default "10" help The kernel memory allocator divides physically contiguous memory blocks into "zones", where each zone is a power of two number of @@ -1500,14 +1500,11 @@ config ARCH_FORCE_MAX_ORDER blocks of physically contiguous memory, then you may need to increase this value. - This config option is actually maximum order plus one. For example, - a value of 11 means that the largest free memory block is 2^10 pages. - We make sure that we can allocate up to a HugePage size for each configuration. Hence we have : - MAX_ORDER = (PMD_SHIFT - PAGE_SHIFT) + 1 => PAGE_SHIFT - 2 + MAX_ORDER = PMD_SHIFT - PAGE_SHIFT => PAGE_SHIFT - 3 - However for 4K, we choose a higher default value, 11 as opposed to 10, giving us + However for 4K, we choose a higher default value, 10 as opposed to 9, giving us 4M allocations matching the default size used by generic code. config UNMAP_KERNEL_AT_EL0 diff --git a/arch/arm64/include/asm/sparsemem.h b/arch/arm64/include/asm/sparsemem.h index 4b73463423c3..5f5437621029 100644 --- a/arch/arm64/include/asm/sparsemem.h +++ b/arch/arm64/include/asm/sparsemem.h @@ -10,7 +10,7 @@ /* * Section size must be at least 512MB for 64K base * page size config. Otherwise it will be less than - * (MAX_ORDER - 1) and the build process will fail. + * MAX_ORDER and the build process will fail. */ #ifdef CONFIG_ARM64_64K_PAGES #define SECTION_SIZE_BITS 29 diff --git a/arch/arm64/kvm/hyp/include/nvhe/gfp.h b/arch/arm64/kvm/hyp/include/nvhe/gfp.h index 0a048dc06a7d..fe5472a184a3 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/gfp.h +++ b/arch/arm64/kvm/hyp/include/nvhe/gfp.h @@ -16,7 +16,7 @@ struct hyp_pool { * API at EL2. */ hyp_spinlock_t lock; - struct list_head free_area[MAX_ORDER]; + struct list_head free_area[MAX_ORDER + 1]; phys_addr_t range_start; phys_addr_t range_end; unsigned short max_order; diff --git a/arch/arm64/kvm/hyp/nvhe/page_alloc.c b/arch/arm64/kvm/hyp/nvhe/page_alloc.c index 803ba3222e75..b1e392186a0f 100644 --- a/arch/arm64/kvm/hyp/nvhe/page_alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/page_alloc.c @@ -110,7 +110,7 @@ static void __hyp_attach_page(struct hyp_pool *pool, * after coalescing, so make sure to mark it HYP_NO_ORDER proactively. */ p->order = HYP_NO_ORDER; - for (; (order + 1) < pool->max_order; order++) { + for (; (order + 1) <= pool->max_order; order++) { buddy = __find_buddy_avail(pool, p, order); if (!buddy) break; @@ -203,9 +203,9 @@ void *hyp_alloc_pages(struct hyp_pool *pool, unsigned short order) hyp_spin_lock(&pool->lock); /* Look for a high-enough-order page */ - while (i < pool->max_order && list_empty(&pool->free_area[i])) + while (i <= pool->max_order && list_empty(&pool->free_area[i])) i++; - if (i >= pool->max_order) { + if (i > pool->max_order) { hyp_spin_unlock(&pool->lock); return NULL; } @@ -228,8 +228,8 @@ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, int i; hyp_spin_lock_init(&pool->lock); - pool->max_order = min(MAX_ORDER, get_order((nr_pages + 1) << PAGE_SHIFT)); - for (i = 0; i < pool->max_order; i++) + pool->max_order = min(MAX_ORDER, get_order(nr_pages << PAGE_SHIFT)); + for (i = 0; i <= pool->max_order; i++) INIT_LIST_HEAD(&pool->free_area[i]); pool->range_start = phys; pool->range_end = phys + (nr_pages << PAGE_SHIFT); diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index dba02da6fa34..c694fac43bed 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -334,7 +334,7 @@ config HIGHMEM config ARCH_FORCE_MAX_ORDER int "Maximum zone order" - default "11" + default "10" config DRAM_BASE hex "DRAM start addr (the same with memory-section in dts)" diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index d7e4a24e8644..0d2f41fa56ee 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -202,10 +202,10 @@ config IA64_CYCLONE If you're unsure, answer N. config ARCH_FORCE_MAX_ORDER - int "MAX_ORDER (11 - 17)" if !HUGETLB_PAGE - range 11 17 if !HUGETLB_PAGE - default "17" if HUGETLB_PAGE - default "11" + int "MAX_ORDER (10 - 16)" if !HUGETLB_PAGE + range 10 16 if !HUGETLB_PAGE + default "16" if HUGETLB_PAGE + default "10" config SMP bool "Symmetric multi-processing support" diff --git a/arch/ia64/include/asm/sparsemem.h b/arch/ia64/include/asm/sparsemem.h index 84e8ce387b69..a58f8b466d96 100644 --- a/arch/ia64/include/asm/sparsemem.h +++ b/arch/ia64/include/asm/sparsemem.h @@ -12,9 +12,9 @@ #define SECTION_SIZE_BITS (30) #define MAX_PHYSMEM_BITS (50) #ifdef CONFIG_ARCH_FORCE_MAX_ORDER -#if ((CONFIG_ARCH_FORCE_MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS) +#if (CONFIG_ARCH_FORCE_MAX_ORDER + PAGE_SHIFT > SECTION_SIZE_BITS) #undef SECTION_SIZE_BITS -#define SECTION_SIZE_BITS (CONFIG_ARCH_FORCE_MAX_ORDER - 1 + PAGE_SHIFT) +#define SECTION_SIZE_BITS (CONFIG_ARCH_FORCE_MAX_ORDER + PAGE_SHIFT) #endif #endif diff --git a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c index 380d2f3966c9..e8dd4323fb86 100644 --- a/arch/ia64/mm/hugetlbpage.c +++ b/arch/ia64/mm/hugetlbpage.c @@ -170,7 +170,7 @@ static int __init hugetlb_setup_sz(char *str) size = memparse(str, &str); if (*str || !is_power_of_2(size) || !(tr_pages & size) || size <= PAGE_SIZE || - size >= (1UL << PAGE_SHIFT << MAX_ORDER)) { + size > (1UL << PAGE_SHIFT << MAX_ORDER)) { printk(KERN_WARNING "Invalid huge page size specified\n"); return 1; } diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 7fd51257e0ed..272a3a12c98d 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -420,12 +420,12 @@ config NODES_SHIFT config ARCH_FORCE_MAX_ORDER int "Maximum zone order" - range 14 64 if PAGE_SIZE_64KB - default "14" if PAGE_SIZE_64KB - range 12 64 if PAGE_SIZE_16KB - default "12" if PAGE_SIZE_16KB - range 11 64 - default "11" + range 13 63 if PAGE_SIZE_64KB + default "13" if PAGE_SIZE_64KB + range 11 63 if PAGE_SIZE_16KB + default "11" if PAGE_SIZE_16KB + range 10 63 + default "10" help The kernel memory allocator divides physically contiguous memory blocks into "zones", where each zone is a power of two number of @@ -434,9 +434,6 @@ config ARCH_FORCE_MAX_ORDER blocks of physically contiguous memory, then you may need to increase this value. - This config option is actually maximum order plus one. For example, - a value of 11 means that the largest free memory block is 2^10 pages. - The page size is not necessarily 4KB. Keep this in mind when choosing a value for this option. diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu index 9380f6e3bb66..c9df6572133f 100644 --- a/arch/m68k/Kconfig.cpu +++ b/arch/m68k/Kconfig.cpu @@ -400,7 +400,7 @@ config SINGLE_MEMORY_CHUNK config ARCH_FORCE_MAX_ORDER int "Maximum zone order" if ADVANCED depends on !SINGLE_MEMORY_CHUNK - default "11" + default "10" help The kernel memory allocator divides physically contiguous memory blocks into "zones", where each zone is a power of two number of @@ -413,9 +413,6 @@ config ARCH_FORCE_MAX_ORDER value also defines the minimal size of the hole that allows freeing unused memory map. - This config option is actually maximum order plus one. For example, - a value of 11 means that the largest free memory block is 2^10 pages. - config 060_WRITETHROUGH bool "Use write-through caching for 68060 supervisor accesses" depends on ADVANCED && M68060 diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index e2f3ca73f40d..3e8b765b8c7b 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -2137,14 +2137,14 @@ endchoice config ARCH_FORCE_MAX_ORDER int "Maximum zone order" - range 14 64 if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_64KB - default "14" if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_64KB - range 13 64 if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_32KB - default "13" if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_32KB - range 12 64 if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_16KB - default "12" if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_16KB - range 0 64 - default "11" + range 13 63 if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_64KB + default "13" if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_64KB + range 12 63 if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_32KB + default "12" if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_32KB + range 11 63 if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_16KB + default "11" if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_16KB + range 0 63 + default "10" help The kernel memory allocator divides physically contiguous memory blocks into "zones", where each zone is a power of two number of @@ -2153,9 +2153,6 @@ config ARCH_FORCE_MAX_ORDER blocks of physically contiguous memory, then you may need to increase this value. - This config option is actually maximum order plus one. For example, - a value of 11 means that the largest free memory block is 2^10 pages. - The page size is not necessarily 4KB. Keep this in mind when choosing a value for this option. diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig index a582f72104f3..89708b95978c 100644 --- a/arch/nios2/Kconfig +++ b/arch/nios2/Kconfig @@ -46,8 +46,8 @@ source "kernel/Kconfig.hz" config ARCH_FORCE_MAX_ORDER int "Maximum zone order" - range 9 20 - default "11" + range 8 19 + default "10" help The kernel memory allocator divides physically contiguous memory blocks into "zones", where each zone is a power of two number of @@ -56,9 +56,6 @@ config ARCH_FORCE_MAX_ORDER blocks of physically contiguous memory, then you may need to increase this value. - This config option is actually maximum order plus one. For example, - a value of 11 means that the largest free memory block is 2^10 pages. - endmenu source "arch/nios2/platform/Kconfig.platform" diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 49c6d36b2b3e..24d56536b269 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -897,18 +897,18 @@ config DATA_SHIFT config ARCH_FORCE_MAX_ORDER int "Maximum zone order" - range 8 9 if PPC64 && PPC_64K_PAGES - default "9" if PPC64 && PPC_64K_PAGES - range 13 13 if PPC64 && !PPC_64K_PAGES - default "13" if PPC64 && !PPC_64K_PAGES - range 9 64 if PPC32 && PPC_16K_PAGES - default "9" if PPC32 && PPC_16K_PAGES - range 7 64 if PPC32 && PPC_64K_PAGES - default "7" if PPC32 && PPC_64K_PAGES - range 5 64 if PPC32 && PPC_256K_PAGES - default "5" if PPC32 && PPC_256K_PAGES - range 11 64 - default "11" + range 7 8 if PPC64 && PPC_64K_PAGES + default "8" if PPC64 && PPC_64K_PAGES + range 12 12 if PPC64 && !PPC_64K_PAGES + default "12" if PPC64 && !PPC_64K_PAGES + range 8 63 if PPC32 && PPC_16K_PAGES + default "8" if PPC32 && PPC_16K_PAGES + range 6 63 if PPC32 && PPC_64K_PAGES + default "6" if PPC32 && PPC_64K_PAGES + range 4 63 if PPC32 && PPC_256K_PAGES + default "4" if PPC32 && PPC_256K_PAGES + range 10 63 + default "10" help The kernel memory allocator divides physically contiguous memory blocks into "zones", where each zone is a power of two number of @@ -917,9 +917,6 @@ config ARCH_FORCE_MAX_ORDER blocks of physically contiguous memory, then you may need to increase this value. - This config option is actually maximum order plus one. For example, - a value of 11 means that the largest free memory block is 2^10 pages. - The page size is not necessarily 4KB. For example, on 64-bit systems, 64KB pages can be enabled via CONFIG_PPC_64K_PAGES. Keep this in mind when choosing a value for this option. diff --git a/arch/powerpc/configs/85xx/ge_imp3a_defconfig b/arch/powerpc/configs/85xx/ge_imp3a_defconfig index ea719898b581..6cb7e90d52c1 100644 --- a/arch/powerpc/configs/85xx/ge_imp3a_defconfig +++ b/arch/powerpc/configs/85xx/ge_imp3a_defconfig @@ -30,7 +30,7 @@ CONFIG_PREEMPT=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m CONFIG_MATH_EMULATION=y -CONFIG_ARCH_FORCE_MAX_ORDER=17 +CONFIG_ARCH_FORCE_MAX_ORDER=16 CONFIG_PCI=y CONFIG_PCIEPORTBUS=y CONFIG_PCI_MSI=y diff --git a/arch/powerpc/configs/fsl-emb-nonhw.config b/arch/powerpc/configs/fsl-emb-nonhw.config index ab8a8c4530d9..3009b0efaf34 100644 --- a/arch/powerpc/configs/fsl-emb-nonhw.config +++ b/arch/powerpc/configs/fsl-emb-nonhw.config @@ -41,7 +41,7 @@ CONFIG_FIXED_PHY=y CONFIG_FONT_8x16=y CONFIG_FONT_8x8=y CONFIG_FONTS=y -CONFIG_ARCH_FORCE_MAX_ORDER=13 +CONFIG_ARCH_FORCE_MAX_ORDER=12 CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_FRAME_WARN=1024 CONFIG_FTL=y diff --git a/arch/powerpc/mm/book3s64/iommu_api.c b/arch/powerpc/mm/book3s64/iommu_api.c index 7fcfba162e0d..81d7185e2ae8 100644 --- a/arch/powerpc/mm/book3s64/iommu_api.c +++ b/arch/powerpc/mm/book3s64/iommu_api.c @@ -97,7 +97,7 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua, } mmap_read_lock(mm); - chunk = (1UL << (PAGE_SHIFT + MAX_ORDER - 1)) / + chunk = (1UL << (PAGE_SHIFT + MAX_ORDER)) / sizeof(struct vm_area_struct *); chunk = min(chunk, entries); for (entry = 0; entry < entries; entry += chunk) { diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index f1ba8d1e8c1a..b900933507da 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -615,7 +615,7 @@ void __init gigantic_hugetlb_cma_reserve(void) order = mmu_psize_to_shift(MMU_PAGE_16G) - PAGE_SHIFT; if (order) { - VM_WARN_ON(order < MAX_ORDER); + VM_WARN_ON(order <= MAX_ORDER); hugetlb_cma_reserve(order); } } diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 4f6e20a35aa1..5a81f106068e 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -1740,7 +1740,7 @@ static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe) * DMA window can be larger than available memory, which will * cause errors later. */ - const u64 maxblock = 1UL << (PAGE_SHIFT + MAX_ORDER - 1); + const u64 maxblock = 1UL << (PAGE_SHIFT + MAX_ORDER); /* * We create the default window as big as we can. The constraint is diff --git a/arch/sh/configs/ecovec24_defconfig b/arch/sh/configs/ecovec24_defconfig index b52e14ccb450..4d655e8d4d74 100644 --- a/arch/sh/configs/ecovec24_defconfig +++ b/arch/sh/configs/ecovec24_defconfig @@ -8,7 +8,7 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH7724=y -CONFIG_ARCH_FORCE_MAX_ORDER=12 +CONFIG_ARCH_FORCE_MAX_ORDER=11 CONFIG_MEMORY_SIZE=0x10000000 CONFIG_FLATMEM_MANUAL=y CONFIG_SH_ECOVEC=y diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig index 411fdc0901f7..40271090bd7d 100644 --- a/arch/sh/mm/Kconfig +++ b/arch/sh/mm/Kconfig @@ -20,13 +20,13 @@ config PAGE_OFFSET config ARCH_FORCE_MAX_ORDER int "Maximum zone order" - range 9 64 if PAGE_SIZE_16KB - default "9" if PAGE_SIZE_16KB - range 7 64 if PAGE_SIZE_64KB - default "7" if PAGE_SIZE_64KB - range 11 64 - default "14" if !MMU - default "11" + range 8 63 if PAGE_SIZE_16KB + default "8" if PAGE_SIZE_16KB + range 6 63 if PAGE_SIZE_64KB + default "6" if PAGE_SIZE_64KB + range 10 63 + default "13" if !MMU + default "10" help The kernel memory allocator divides physically contiguous memory blocks into "zones", where each zone is a power of two number of @@ -35,9 +35,6 @@ config ARCH_FORCE_MAX_ORDER blocks of physically contiguous memory, then you may need to increase this value. - This config option is actually maximum order plus one. For example, - a value of 11 means that the largest free memory block is 2^10 pages. - The page size is not necessarily 4KB. Keep this in mind when choosing a value for this option. diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 84437a4c6545..e3242bf5a8df 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -271,7 +271,7 @@ config ARCH_SPARSEMEM_DEFAULT config ARCH_FORCE_MAX_ORDER int "Maximum zone order" - default "13" + default "12" help The kernel memory allocator divides physically contiguous memory blocks into "zones", where each zone is a power of two number of @@ -280,9 +280,6 @@ config ARCH_FORCE_MAX_ORDER blocks of physically contiguous memory, then you may need to increase this value. - This config option is actually maximum order plus one. For example, - a value of 13 means that the largest free memory block is 2^12 pages. - if SPARC64 || COMPILE_TEST source "kernel/power/Kconfig" endif diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c index 384480971805..7d91ca6aa675 100644 --- a/arch/sparc/kernel/pci_sun4v.c +++ b/arch/sparc/kernel/pci_sun4v.c @@ -193,7 +193,7 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size, size = IO_PAGE_ALIGN(size); order = get_order(size); - if (unlikely(order >= MAX_ORDER)) + if (unlikely(order > MAX_ORDER)) return NULL; npages = size >> IO_PAGE_SHIFT; diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index 5b4de4a89dec..08ffd17d5ec3 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c @@ -897,7 +897,7 @@ void __init cheetah_ecache_flush_init(void) /* Now allocate error trap reporting scoreboard. */ sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info)); - for (order = 0; order < MAX_ORDER; order++) { + for (order = 0; order <= MAX_ORDER; order++) { if ((PAGE_SIZE << order) >= sz) break; } diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c index dba8dffe2113..5e2931a18409 100644 --- a/arch/sparc/mm/tsb.c +++ b/arch/sparc/mm/tsb.c @@ -402,8 +402,8 @@ void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss) unsigned long new_rss_limit; gfp_t gfp_flags; - if (max_tsb_size > (PAGE_SIZE << (MAX_ORDER - 1))) - max_tsb_size = (PAGE_SIZE << (MAX_ORDER - 1)); + if (max_tsb_size > PAGE_SIZE << MAX_ORDER) + max_tsb_size = PAGE_SIZE << MAX_ORDER; new_cache_index = 0; for (new_size = 8192; new_size < max_tsb_size; new_size <<= 1UL) { diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c index 5e5a9c8e0e5d..8dcda617b8bf 100644 --- a/arch/um/kernel/um_arch.c +++ b/arch/um/kernel/um_arch.c @@ -368,10 +368,10 @@ int __init linux_main(int argc, char **argv) max_physmem = TASK_SIZE - uml_physmem - iomem_size - MIN_VMALLOC; /* - * Zones have to begin on a 1 << MAX_ORDER-1 page boundary, + * Zones have to begin on a 1 << MAX_ORDER page boundary, * so this makes sure that's true for highmem */ - max_physmem &= ~((1 << (PAGE_SHIFT + MAX_ORDER - 1)) - 1); + max_physmem &= ~((1 << (PAGE_SHIFT + MAX_ORDER)) - 1); if (physmem_size + iomem_size > max_physmem) { highmem = physmem_size + iomem_size - max_physmem; physmem_size -= highmem; diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index bcb0c5d2abc2..3eee334ba873 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -773,7 +773,7 @@ config HIGHMEM config ARCH_FORCE_MAX_ORDER int "Maximum zone order" - default "11" + default "10" help The kernel memory allocator divides physically contiguous memory blocks into "zones", where each zone is a power of two number of @@ -782,9 +782,6 @@ config ARCH_FORCE_MAX_ORDER blocks of physically contiguous memory, then you may need to increase this value. - This config option is actually maximum order plus one. For example, - a value of 11 means that the largest free memory block is 2^10 pages. - endmenu menu "Power management options" diff --git a/drivers/base/regmap/regmap-debugfs.c b/drivers/base/regmap/regmap-debugfs.c index 817eda2075aa..c491fabe3617 100644 --- a/drivers/base/regmap/regmap-debugfs.c +++ b/drivers/base/regmap/regmap-debugfs.c @@ -226,8 +226,8 @@ static ssize_t regmap_read_debugfs(struct regmap *map, unsigned int from, if (*ppos < 0 || !count) return -EINVAL; - if (count > (PAGE_SIZE << (MAX_ORDER - 1))) - count = PAGE_SIZE << (MAX_ORDER - 1); + if (count > (PAGE_SIZE << MAX_ORDER)) + count = PAGE_SIZE << MAX_ORDER; buf = kmalloc(count, GFP_KERNEL); if (!buf) @@ -373,8 +373,8 @@ static ssize_t regmap_reg_ranges_read_file(struct file *file, if (*ppos < 0 || !count) return -EINVAL; - if (count > (PAGE_SIZE << (MAX_ORDER - 1))) - count = PAGE_SIZE << (MAX_ORDER - 1); + if (count > (PAGE_SIZE << MAX_ORDER)) + count = PAGE_SIZE << MAX_ORDER; buf = kmalloc(count, GFP_KERNEL); if (!buf) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 90d2dfb6448e..cec2c20f5e59 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -3079,7 +3079,7 @@ static void raw_cmd_free(struct floppy_raw_cmd **ptr) } } -#define MAX_LEN (1UL << (MAX_ORDER - 1) << PAGE_SHIFT) +#define MAX_LEN (1UL << MAX_ORDER << PAGE_SHIFT) static int raw_cmd_copyin(int cmd, void __user *param, struct floppy_raw_cmd **rcmd) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index e2f25926eb51..bf095baca244 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -886,7 +886,7 @@ static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp) /* * The length of the ID shouldn't be assumed by software since * it may change in the future. The allocation size is limited - * to 1 << (PAGE_SHIFT + MAX_ORDER - 1) by the page allocator. + * to 1 << (PAGE_SHIFT + MAX_ORDER) by the page allocator. * If the allocation fails, simply return ENOMEM rather than * warning in the kernel log. */ diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 09586a837b1e..3df7a256e919 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -70,11 +70,11 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, HISI_ACC_SGL_ALIGN_SIZE); /* - * the pool may allocate a block of memory of size PAGE_SIZE * 2^(MAX_ORDER - 1), + * the pool may allocate a block of memory of size PAGE_SIZE * 2^MAX_ORDER, * block size may exceed 2^31 on ia64, so the max of block size is 2^31 */ - block_size = 1 << (PAGE_SHIFT + MAX_ORDER <= 32 ? - PAGE_SHIFT + MAX_ORDER - 1 : 31); + block_size = 1 << (PAGE_SHIFT + MAX_ORDER < 32 ? + PAGE_SHIFT + MAX_ORDER : 31); sgl_num_per_block = block_size / sgl_size; block_num = count / sgl_num_per_block; remain_sgl = count % sgl_num_per_block; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.c b/drivers/gpu/drm/i915/gem/i915_gem_internal.c index eae9e9f6d3bf..6bc26b4b06b8 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.c @@ -36,7 +36,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) struct sg_table *st; struct scatterlist *sg; unsigned int npages; /* restricted by sg_alloc_table */ - int max_order = MAX_ORDER - 1; + int max_order = MAX_ORDER; unsigned int max_segment; gfp_t gfp; diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c index defece0bcb81..99f39a5feca1 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c @@ -115,7 +115,7 @@ static int get_huge_pages(struct drm_i915_gem_object *obj) do { struct page *page; - GEM_BUG_ON(order >= MAX_ORDER); + GEM_BUG_ON(order > MAX_ORDER); page = alloc_pages(GFP | __GFP_ZERO, order); if (!page) goto err; diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index aa116a7bbae3..6c8585abe08d 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -65,11 +65,11 @@ module_param(page_pool_size, ulong, 0644); static atomic_long_t allocated_pages; -static struct ttm_pool_type global_write_combined[MAX_ORDER]; -static struct ttm_pool_type global_uncached[MAX_ORDER]; +static struct ttm_pool_type global_write_combined[MAX_ORDER + 1]; +static struct ttm_pool_type global_uncached[MAX_ORDER + 1]; -static struct ttm_pool_type global_dma32_write_combined[MAX_ORDER]; -static struct ttm_pool_type global_dma32_uncached[MAX_ORDER]; +static struct ttm_pool_type global_dma32_write_combined[MAX_ORDER + 1]; +static struct ttm_pool_type global_dma32_uncached[MAX_ORDER + 1]; static spinlock_t shrinker_lock; static struct list_head shrinker_list; @@ -405,7 +405,7 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, else gfp_flags |= GFP_HIGHUSER; - for (order = min_t(unsigned int, MAX_ORDER - 1, __fls(num_pages)); + for (order = min_t(unsigned int, MAX_ORDER, __fls(num_pages)); num_pages; order = min_t(unsigned int, order, __fls(num_pages))) { struct ttm_pool_type *pt; @@ -542,7 +542,7 @@ void ttm_pool_init(struct ttm_pool *pool, struct device *dev, if (use_dma_alloc) { for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) - for (j = 0; j < MAX_ORDER; ++j) + for (j = 0; j <= MAX_ORDER; ++j) ttm_pool_type_init(&pool->caching[i].orders[j], pool, i, j); } @@ -562,7 +562,7 @@ void ttm_pool_fini(struct ttm_pool *pool) if (pool->use_dma_alloc) { for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) - for (j = 0; j < MAX_ORDER; ++j) + for (j = 0; j <= MAX_ORDER; ++j) ttm_pool_type_fini(&pool->caching[i].orders[j]); } @@ -616,7 +616,7 @@ static void ttm_pool_debugfs_header(struct seq_file *m) unsigned int i; seq_puts(m, "\t "); - for (i = 0; i < MAX_ORDER; ++i) + for (i = 0; i <= MAX_ORDER; ++i) seq_printf(m, " ---%2u---", i); seq_puts(m, "\n"); } @@ -627,7 +627,7 @@ static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt, { unsigned int i; - for (i = 0; i < MAX_ORDER; ++i) + for (i = 0; i <= MAX_ORDER; ++i) seq_printf(m, " %8u", ttm_pool_type_count(&pt[i])); seq_puts(m, "\n"); } @@ -736,7 +736,7 @@ int ttm_pool_mgr_init(unsigned long num_pages) spin_lock_init(&shrinker_lock); INIT_LIST_HEAD(&shrinker_list); - for (i = 0; i < MAX_ORDER; ++i) { + for (i = 0; i <= MAX_ORDER; ++i) { ttm_pool_type_init(&global_write_combined[i], NULL, ttm_write_combined, i); ttm_pool_type_init(&global_uncached[i], NULL, ttm_uncached, i); @@ -769,7 +769,7 @@ void ttm_pool_mgr_fini(void) { unsigned int i; - for (i = 0; i < MAX_ORDER; ++i) { + for (i = 0; i <= MAX_ORDER; ++i) { ttm_pool_type_fini(&global_write_combined[i]); ttm_pool_type_fini(&global_uncached[i]); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 8d772ea8a583..b574c58a3487 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -182,7 +182,7 @@ #ifdef CONFIG_CMA_ALIGNMENT #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT) #else -#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1) +#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER) #endif /* diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index ac996fd6bd9c..7a9f0b0bddbd 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -736,7 +736,7 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, struct page **pages; unsigned int i = 0, nid = dev_to_node(dev); - order_mask &= GENMASK(MAX_ORDER - 1, 0); + order_mask &= GENMASK(MAX_ORDER, 0); if (!order_mask) return NULL; diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 586271b8aa39..85790b870877 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2440,8 +2440,8 @@ static bool its_parse_indirect_baser(struct its_node *its, * feature is not supported by hardware. */ new_order = max_t(u32, get_order(esz << ids), new_order); - if (new_order >= MAX_ORDER) { - new_order = MAX_ORDER - 1; + if (new_order > MAX_ORDER) { + new_order = MAX_ORDER; ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz); pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n", &its->phys_base, its_base_type_string[type], diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index cf077f9b30c3..733053c2eaa0 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -408,7 +408,7 @@ static void __cache_size_refresh(void) * If the allocation may fail we use __get_free_pages. Memory fragmentation * won't have a fatal effect here, but it just causes flushes of some other * buffers and more I/O will be performed. Don't use __get_free_pages if it - * always fails (i.e. order >= MAX_ORDER). + * always fails (i.e. order > MAX_ORDER). * * If the allocation shouldn't fail we use __vmalloc. This is only for the * initial reserve allocation, so there's no risk of wasting all vmalloc diff --git a/drivers/misc/genwqe/card_dev.c b/drivers/misc/genwqe/card_dev.c index d0e27438a73c..55fc5b80e649 100644 --- a/drivers/misc/genwqe/card_dev.c +++ b/drivers/misc/genwqe/card_dev.c @@ -443,7 +443,7 @@ static int genwqe_mmap(struct file *filp, struct vm_area_struct *vma) if (vsize == 0) return -EINVAL; - if (get_order(vsize) >= MAX_ORDER) + if (get_order(vsize) > MAX_ORDER) return -ENOMEM; dma_map = kzalloc(sizeof(struct dma_mapping), GFP_KERNEL); diff --git a/drivers/misc/genwqe/card_utils.c b/drivers/misc/genwqe/card_utils.c index ac29698d085a..1c798d6b2dfb 100644 --- a/drivers/misc/genwqe/card_utils.c +++ b/drivers/misc/genwqe/card_utils.c @@ -210,7 +210,7 @@ u32 genwqe_crc32(u8 *buff, size_t len, u32 init) void *__genwqe_alloc_consistent(struct genwqe_dev *cd, size_t size, dma_addr_t *dma_handle) { - if (get_order(size) >= MAX_ORDER) + if (get_order(size) > MAX_ORDER) return NULL; return dma_alloc_coherent(&cd->pci_dev->dev, size, dma_handle, @@ -308,7 +308,7 @@ int genwqe_alloc_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl, sgl->write = write; sgl->sgl_size = genwqe_sgl_size(sgl->nr_pages); - if (get_order(sgl->sgl_size) >= MAX_ORDER) { + if (get_order(sgl->sgl_size) > MAX_ORDER) { dev_err(&pci_dev->dev, "[%s] err: too much memory requested!\n", __func__); return ret; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 25be7f8ac7cd..3973ca6adf4c 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -1041,7 +1041,7 @@ static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) return; order = get_order(alloc_size); - if (order >= MAX_ORDER) { + if (order > MAX_ORDER) { if (net_ratelimit()) dev_warn(ring_to_dev(ring), "failed to allocate tx spare buffer, exceed to max order\n"); return; diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h index b35c9b6f913b..4e18b4cefa97 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.h +++ b/drivers/net/ethernet/ibm/ibmvnic.h @@ -75,7 +75,7 @@ * pool for the 4MB. Thus the 16 Rx and Tx queues require 32 * 5 = 160 * plus 16 for the TSO pools for a total of 176 LTB mappings per VNIC. */ -#define IBMVNIC_ONE_LTB_MAX ((u32)((1 << (MAX_ORDER - 1)) * PAGE_SIZE)) +#define IBMVNIC_ONE_LTB_MAX ((u32)((1 << MAX_ORDER) * PAGE_SIZE)) #define IBMVNIC_ONE_LTB_SIZE min((u32)(8 << 20), IBMVNIC_ONE_LTB_MAX) #define IBMVNIC_LTB_SET_SIZE (38 << 20) diff --git a/drivers/video/fbdev/hyperv_fb.c b/drivers/video/fbdev/hyperv_fb.c index ec3f6cf05f8c..34781dec3856 100644 --- a/drivers/video/fbdev/hyperv_fb.c +++ b/drivers/video/fbdev/hyperv_fb.c @@ -946,7 +946,7 @@ static phys_addr_t hvfb_get_phymem(struct hv_device *hdev, if (request_size == 0) return -1; - if (order < MAX_ORDER) { + if (order <= MAX_ORDER) { /* Call alloc_pages if the size is less than 2^MAX_ORDER */ page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); if (!page) @@ -977,7 +977,7 @@ static void hvfb_release_phymem(struct hv_device *hdev, { unsigned int order = get_order(size); - if (order < MAX_ORDER) + if (order <= MAX_ORDER) __free_pages(pfn_to_page(paddr >> PAGE_SHIFT), order); else dma_free_coherent(&hdev->device, diff --git a/drivers/video/fbdev/vermilion/vermilion.c b/drivers/video/fbdev/vermilion/vermilion.c index 0374ee6b6d03..32e74e02a02f 100644 --- a/drivers/video/fbdev/vermilion/vermilion.c +++ b/drivers/video/fbdev/vermilion/vermilion.c @@ -197,7 +197,7 @@ static int vmlfb_alloc_vram(struct vml_info *vinfo, va = &vinfo->vram[i]; order = 0; - while (requested > (PAGE_SIZE << order) && order < MAX_ORDER) + while (requested > (PAGE_SIZE << order) && order <= MAX_ORDER) order++; err = vmlfb_alloc_vram_area(va, order, 0); diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 3f78a3a1eb75..5b15936a5214 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -33,7 +33,7 @@ #define VIRTIO_BALLOON_FREE_PAGE_ALLOC_FLAG (__GFP_NORETRY | __GFP_NOWARN | \ __GFP_NOMEMALLOC) /* The order of free page blocks to report to host */ -#define VIRTIO_BALLOON_HINT_BLOCK_ORDER (MAX_ORDER - 1) +#define VIRTIO_BALLOON_HINT_BLOCK_ORDER MAX_ORDER /* The size of a free page block in bytes */ #define VIRTIO_BALLOON_HINT_BLOCK_BYTES \ (1 << (VIRTIO_BALLOON_HINT_BLOCK_ORDER + PAGE_SHIFT)) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 0c2892ec6817..835f6cc2fb66 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -1120,13 +1120,13 @@ static void virtio_mem_clear_fake_offline(unsigned long pfn, */ static void virtio_mem_fake_online(unsigned long pfn, unsigned long nr_pages) { - unsigned long order = MAX_ORDER - 1; + unsigned long order = MAX_ORDER; unsigned long i; /* * We might get called for ranges that don't cover properly aligned - * MAX_ORDER - 1 pages; however, we can only online properly aligned - * pages with an order of MAX_ORDER - 1 at maximum. + * MAX_ORDER pages; however, we can only online properly aligned + * pages with an order of MAX_ORDER at maximum. */ while (!IS_ALIGNED(pfn | nr_pages, 1 << order)) order--; @@ -1237,9 +1237,9 @@ static void virtio_mem_online_page(struct virtio_mem *vm, bool do_online; /* - * We can get called with any order up to MAX_ORDER - 1. If our - * subblock size is smaller than that and we have a mixture of plugged - * and unplugged subblocks within such a page, we have to process in + * We can get called with any order up to MAX_ORDER. If our subblock + * size is smaller than that and we have a mixture of plugged and + * unplugged subblocks within such a page, we have to process in * smaller granularity. In that case we'll adjust the order exactly once * within the loop. */ diff --git a/fs/ramfs/file-nommu.c b/fs/ramfs/file-nommu.c index 2f67516bb9bf..9fbb9b5256f7 100644 --- a/fs/ramfs/file-nommu.c +++ b/fs/ramfs/file-nommu.c @@ -70,7 +70,7 @@ int ramfs_nommu_expand_for_mapping(struct inode *inode, size_t newsize) /* make various checks */ order = get_order(newsize); - if (unlikely(order >= MAX_ORDER)) + if (unlikely(order > MAX_ORDER)) return -EFBIG; ret = inode_newsize_ok(inode, newsize); diff --git a/include/drm/ttm/ttm_pool.h b/include/drm/ttm/ttm_pool.h index ef09b23d29e3..8ce14f9d202a 100644 --- a/include/drm/ttm/ttm_pool.h +++ b/include/drm/ttm/ttm_pool.h @@ -72,7 +72,7 @@ struct ttm_pool { bool use_dma32; struct { - struct ttm_pool_type orders[MAX_ORDER]; + struct ttm_pool_type orders[MAX_ORDER + 1]; } caching[TTM_NUM_CACHING_TYPES]; }; diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 7c977d234aba..8fb7d91cd0b1 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -818,7 +818,7 @@ static inline unsigned huge_page_shift(struct hstate *h) static inline bool hstate_is_gigantic(struct hstate *h) { - return huge_page_order(h) >= MAX_ORDER; + return huge_page_order(h) > MAX_ORDER; } static inline unsigned int pages_per_huge_page(const struct hstate *h) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index bf8786d45b31..35b11cc210d9 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -26,11 +26,11 @@ /* Free memory management - zoned buddy allocator. */ #ifndef CONFIG_ARCH_FORCE_MAX_ORDER -#define MAX_ORDER 11 +#define MAX_ORDER 10 #else #define MAX_ORDER CONFIG_ARCH_FORCE_MAX_ORDER #endif -#define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1)) +#define MAX_ORDER_NR_PAGES (1 << MAX_ORDER) /* * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed @@ -93,7 +93,7 @@ static inline bool migratetype_is_mergeable(int mt) } #define for_each_migratetype_order(order, type) \ - for (order = 0; order < MAX_ORDER; order++) \ + for (order = 0; order <= MAX_ORDER; order++) \ for (type = 0; type < MIGRATE_TYPES; type++) extern int page_group_by_mobility_disabled; @@ -922,7 +922,7 @@ struct zone { CACHELINE_PADDING(_pad1_); /* free areas of different sizes */ - struct free_area free_area[MAX_ORDER]; + struct free_area free_area[MAX_ORDER + 1]; /* zone flags, see below */ unsigned long flags; @@ -1745,7 +1745,7 @@ static inline bool movable_only_nodes(nodemask_t *nodes) #define SECTION_BLOCKFLAGS_BITS \ ((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS) -#if (MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS +#if (MAX_ORDER + PAGE_SHIFT) > SECTION_SIZE_BITS #error Allocator MAX_ORDER exceeds SECTION_SIZE #endif diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h index 5f1ae07d724b..e83c4c095041 100644 --- a/include/linux/pageblock-flags.h +++ b/include/linux/pageblock-flags.h @@ -41,14 +41,14 @@ extern unsigned int pageblock_order; * Huge pages are a constant size, but don't exceed the maximum allocation * granularity. */ -#define pageblock_order min_t(unsigned int, HUGETLB_PAGE_ORDER, MAX_ORDER - 1) +#define pageblock_order min_t(unsigned int, HUGETLB_PAGE_ORDER, MAX_ORDER) #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ #else /* CONFIG_HUGETLB_PAGE */ /* If huge pages are not used, group by MAX_ORDER_NR_PAGES */ -#define pageblock_order (MAX_ORDER-1) +#define pageblock_order MAX_ORDER #endif /* CONFIG_HUGETLB_PAGE */ diff --git a/include/linux/slab.h b/include/linux/slab.h index 45af70315a94..aa4575ef2965 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -284,7 +284,7 @@ static inline unsigned int arch_slab_minalign(void) * (PAGE_SIZE*2). Larger requests are passed to the page allocator. */ #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) -#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT) #ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 5 #endif @@ -292,7 +292,7 @@ static inline unsigned int arch_slab_minalign(void) #ifdef CONFIG_SLUB #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) -#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT) #ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 3 #endif @@ -305,7 +305,7 @@ static inline unsigned int arch_slab_minalign(void) * be allocated from the same page. */ #define KMALLOC_SHIFT_HIGH PAGE_SHIFT -#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT) #ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 3 #endif diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 755f5f08ab38..90ce1dfd591c 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -474,7 +474,7 @@ static int __init crash_save_vmcoreinfo_init(void) VMCOREINFO_OFFSET(list_head, prev); VMCOREINFO_OFFSET(vmap_area, va_start); VMCOREINFO_OFFSET(vmap_area, list); - VMCOREINFO_LENGTH(zone.free_area, MAX_ORDER); + VMCOREINFO_LENGTH(zone.free_area, MAX_ORDER + 1); log_buf_vmcoreinfo_setup(); VMCOREINFO_LENGTH(free_area.free_list, MIGRATE_TYPES); VMCOREINFO_NUMBER(NR_FREE_PAGES); diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c index 4d40dcce7604..1acec2e22827 100644 --- a/kernel/dma/pool.c +++ b/kernel/dma/pool.c @@ -84,8 +84,8 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size, void *addr; int ret = -ENOMEM; - /* Cannot allocate larger than MAX_ORDER-1 */ - order = min(get_order(pool_size), MAX_ORDER-1); + /* Cannot allocate larger than MAX_ORDER */ + order = min(get_order(pool_size), MAX_ORDER); do { pool_size = 1 << (PAGE_SHIFT + order); @@ -190,7 +190,7 @@ static int __init dma_atomic_pool_init(void) /* * If coherent_pool was not used on the command line, default the pool - * sizes to 128KB per 1GB of memory, min 128KB, max MAX_ORDER-1. + * sizes to 128KB per 1GB of memory, min 128KB, max MAX_ORDER. */ if (!atomic_pool_size) { unsigned long pages = totalram_pages() / (SZ_1G / SZ_128K); diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index d6bbdb7830b2..a0433f37b024 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -609,8 +609,8 @@ static struct page *rb_alloc_aux_page(int node, int order) { struct page *page; - if (order >= MAX_ORDER) - order = MAX_ORDER - 1; + if (order > MAX_ORDER) + order = MAX_ORDER; do { page = alloc_pages_node(node, PERF_AUX_GFP, order); @@ -814,7 +814,7 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) size = sizeof(struct perf_buffer); size += nr_pages * sizeof(void *); - if (order_base_2(size) >= PAGE_SHIFT+MAX_ORDER) + if (order_base_2(size) > PAGE_SHIFT+MAX_ORDER) goto fail; node = (cpu == -1) ? cpu : cpu_to_node(cpu); diff --git a/mm/Kconfig b/mm/Kconfig index ca98b2072df5..969286ab14a1 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -346,9 +346,9 @@ config SHUFFLE_PAGE_ALLOCATOR the presence of a memory-side-cache. There are also incidental security benefits as it reduces the predictability of page allocations to compliment SLAB_FREELIST_RANDOM, but the - default granularity of shuffling on the "MAX_ORDER - 1" i.e, - 10th order of pages is selected based on cache utilization - benefits on x86. + default granularity of shuffling on the MAX_ORDER i.e, 10th + order of pages is selected based on cache utilization benefits + on x86. While the randomization improves cache utilization it may negatively impact workloads on platforms without a cache. For @@ -666,8 +666,8 @@ config HUGETLB_PAGE_SIZE_VARIABLE HUGETLB_PAGE_ORDER when there are multiple HugeTLB page sizes available on a platform. - Note that the pageblock_order cannot exceed MAX_ORDER - 1 and will be - clamped down to MAX_ORDER - 1. + Note that the pageblock_order cannot exceed MAX_ORDER and will be + clamped down to MAX_ORDER. config CONTIG_ALLOC def_bool (MEMORY_ISOLATION && COMPACTION) || CMA diff --git a/mm/compaction.c b/mm/compaction.c index 5a9501e0ae01..709136556b9e 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -583,7 +583,7 @@ static unsigned long isolate_freepages_block(struct compact_control *cc, if (PageCompound(page)) { const unsigned int order = compound_order(page); - if (likely(order < MAX_ORDER)) { + if (likely(order <= MAX_ORDER)) { blockpfn += (1UL << order) - 1; cursor += (1UL << order) - 1; } @@ -938,7 +938,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, * a valid page order. Consider only values in the * valid order range to prevent low_pfn overflow. */ - if (freepage_order > 0 && freepage_order < MAX_ORDER) + if (freepage_order > 0 && freepage_order <= MAX_ORDER) low_pfn += (1UL << freepage_order) - 1; continue; } @@ -954,7 +954,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, if (PageCompound(page) && !cc->alloc_contig) { const unsigned int order = compound_order(page); - if (likely(order < MAX_ORDER)) + if (likely(order <= MAX_ORDER)) low_pfn += (1UL << order) - 1; goto isolate_fail; } @@ -2124,7 +2124,7 @@ static enum compact_result __compact_finished(struct compact_control *cc) /* Direct compactor: Is a suitable page free? */ ret = COMPACT_NO_SUITABLE_PAGE; - for (order = cc->order; order < MAX_ORDER; order++) { + for (order = cc->order; order <= MAX_ORDER; order++) { struct free_area *area = &cc->zone->free_area[order]; bool can_steal; diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 4362021b1ce7..c54177aabebd 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -1086,7 +1086,7 @@ debug_vm_pgtable_alloc_huge_page(struct pgtable_debug_args *args, int order) struct page *page = NULL; #ifdef CONFIG_CONTIG_ALLOC - if (order >= MAX_ORDER) { + if (order > MAX_ORDER) { page = alloc_contig_pages((1 << order), GFP_KERNEL, first_online_node, NULL); if (page) { @@ -1096,7 +1096,7 @@ debug_vm_pgtable_alloc_huge_page(struct pgtable_debug_args *args, int order) } #endif - if (order < MAX_ORDER) + if (order <= MAX_ORDER) page = alloc_pages(GFP_KERNEL, order); return page; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 2d860e70fe88..1df386d13d24 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -467,7 +467,7 @@ static int __init hugepage_init(void) /* * hugepages can't be allocated by the buddy allocator */ - MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER); + MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_ORDER); /* * we use page->mapping and page->index in second tail page * as list_head: assuming THP order >= 2 diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 712e32b38295..9122e50ae02a 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2090,7 +2090,7 @@ pgoff_t hugetlb_basepage_index(struct page *page) pgoff_t index = page_index(page_head); unsigned long compound_idx; - if (compound_order(page_head) >= MAX_ORDER) + if (compound_order(page_head) > MAX_ORDER) compound_idx = page_to_pfn(page) - page_to_pfn(page_head); else compound_idx = page - page_head; @@ -4497,7 +4497,7 @@ static int __init default_hugepagesz_setup(char *s) * The number of default huge pages (for this size) could have been * specified as the first hugetlb parameter: hugepages=X. If so, * then default_hstate_max_huge_pages is set. If the default huge - * page size is gigantic (>= MAX_ORDER), then the pages must be + * page size is gigantic (> MAX_ORDER), then the pages must be * allocated here from bootmem allocator. */ if (default_hstate_max_huge_pages) { diff --git a/mm/kmsan/init.c b/mm/kmsan/init.c index 7fb794242fad..ffedf4dbc49d 100644 --- a/mm/kmsan/init.c +++ b/mm/kmsan/init.c @@ -96,7 +96,7 @@ void __init kmsan_init_shadow(void) struct metadata_page_pair { struct page *shadow, *origin; }; -static struct metadata_page_pair held_back[MAX_ORDER] __initdata; +static struct metadata_page_pair held_back[MAX_ORDER + 1] __initdata; /* * Eager metadata allocation. When the memblock allocator is freeing pages to @@ -211,8 +211,8 @@ static void kmsan_memblock_discard(void) * order=N-1, * - repeat. */ - collect.order = MAX_ORDER - 1; - for (int i = MAX_ORDER - 1; i >= 0; i--) { + collect.order = MAX_ORDER; + for (int i = MAX_ORDER; i >= 0; i--) { if (held_back[i].shadow) smallstack_push(&collect, held_back[i].shadow); if (held_back[i].origin) diff --git a/mm/memblock.c b/mm/memblock.c index 25fd0626a9e7..7911224b1ed3 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -2043,7 +2043,7 @@ static void __init __free_pages_memory(unsigned long start, unsigned long end) int order; while (start < end) { - order = min(MAX_ORDER - 1UL, __ffs(start)); + order = min_t(int, MAX_ORDER, __ffs(start)); while (start + (1UL << order) > end) order--; diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index db3b270254f1..c8f0a8c2d049 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -596,7 +596,7 @@ static void online_pages_range(unsigned long start_pfn, unsigned long nr_pages) unsigned long pfn; /* - * Online the pages in MAX_ORDER - 1 aligned chunks. The callback might + * Online the pages in MAX_ORDER aligned chunks. The callback might * decide to not expose all pages to the buddy (e.g., expose them * later). We account all pages as being online and belonging to this * zone ("present"). @@ -605,7 +605,7 @@ static void online_pages_range(unsigned long start_pfn, unsigned long nr_pages) * this and the first chunk to online will be pageblock_nr_pages. */ for (pfn = start_pfn; pfn < end_pfn;) { - int order = min(MAX_ORDER - 1UL, __ffs(pfn)); + int order = min_t(int, MAX_ORDER, __ffs(pfn)); (*online_page_callback)(pfn_to_page(pfn), order); pfn += (1UL << order); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 0936bde1d486..c3e49d028a7a 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1063,7 +1063,7 @@ buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, unsigned long higher_page_pfn; struct page *higher_page; - if (order >= MAX_ORDER - 2) + if (order >= MAX_ORDER - 1) return false; higher_page_pfn = buddy_pfn & pfn; @@ -1118,7 +1118,7 @@ static inline void __free_one_page(struct page *page, VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page); VM_BUG_ON_PAGE(bad_range(zone, page), page); - while (order < MAX_ORDER - 1) { + while (order < MAX_ORDER) { if (compaction_capture(capc, page, order, migratetype)) { __mod_zone_freepage_state(zone, -(1 << order), migratetype); @@ -2499,7 +2499,7 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, struct page *page; /* Find a page of the appropriate size in the preferred list */ - for (current_order = order; current_order < MAX_ORDER; ++current_order) { + for (current_order = order; current_order <= MAX_ORDER; ++current_order) { area = &(zone->free_area[current_order]); page = get_page_from_free_area(area, migratetype); if (!page) @@ -2871,7 +2871,7 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, continue; spin_lock_irqsave(&zone->lock, flags); - for (order = 0; order < MAX_ORDER; order++) { + for (order = 0; order <= MAX_ORDER; order++) { struct free_area *area = &(zone->free_area[order]); page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC); @@ -2955,7 +2955,7 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, * approximates finding the pageblock with the most free pages, which * would be too costly to do exactly. */ - for (current_order = MAX_ORDER - 1; current_order >= min_order; + for (current_order = MAX_ORDER; current_order >= min_order; --current_order) { area = &(zone->free_area[current_order]); fallback_mt = find_suitable_fallback(area, current_order, @@ -2981,7 +2981,7 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, return false; find_smallest: - for (current_order = order; current_order < MAX_ORDER; + for (current_order = order; current_order <= MAX_ORDER; current_order++) { area = &(zone->free_area[current_order]); fallback_mt = find_suitable_fallback(area, current_order, @@ -2994,7 +2994,7 @@ find_smallest: * This should not happen - we already found a suitable fallback * when looking for the largest page. */ - VM_BUG_ON(current_order == MAX_ORDER); + VM_BUG_ON(current_order > MAX_ORDER); do_steal: page = get_page_from_free_area(area, fallback_mt); @@ -3955,7 +3955,7 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, return true; /* For a high-order request, check at least one suitable page is free */ - for (o = order; o < MAX_ORDER; o++) { + for (o = order; o <= MAX_ORDER; o++) { struct free_area *area = &z->free_area[o]; int mt; @@ -5475,7 +5475,7 @@ struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid, * There are several places where we assume that the order value is sane * so bail out early if the request is out of bound. */ - if (WARN_ON_ONCE_GFP(order >= MAX_ORDER, gfp)) + if (WARN_ON_ONCE_GFP(order > MAX_ORDER, gfp)) return NULL; gfp &= gfp_allowed_mask; @@ -6205,8 +6205,8 @@ void __show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_i for_each_populated_zone(zone) { unsigned int order; - unsigned long nr[MAX_ORDER], flags, total = 0; - unsigned char types[MAX_ORDER]; + unsigned long nr[MAX_ORDER + 1], flags, total = 0; + unsigned char types[MAX_ORDER + 1]; if (zone_idx(zone) > max_zone_idx) continue; @@ -6216,7 +6216,7 @@ void __show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_i printk(KERN_CONT "%s: ", zone->name); spin_lock_irqsave(&zone->lock, flags); - for (order = 0; order < MAX_ORDER; order++) { + for (order = 0; order <= MAX_ORDER; order++) { struct free_area *area = &zone->free_area[order]; int type; @@ -6230,7 +6230,7 @@ void __show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_i } } spin_unlock_irqrestore(&zone->lock, flags); - for (order = 0; order < MAX_ORDER; order++) { + for (order = 0; order <= MAX_ORDER; order++) { printk(KERN_CONT "%lu*%lukB ", nr[order], K(1UL) << order); if (nr[order]) @@ -7581,7 +7581,7 @@ static inline void setup_usemap(struct zone *zone) {} /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ void __init set_pageblock_order(void) { - unsigned int order = MAX_ORDER - 1; + unsigned int order = MAX_ORDER; /* Check that pageblock_nr_pages has not already been setup */ if (pageblock_order) @@ -9076,7 +9076,7 @@ void *__init alloc_large_system_hash(const char *tablename, else table = memblock_alloc_raw(size, SMP_CACHE_BYTES); - } else if (get_order(size) >= MAX_ORDER || hashdist) { + } else if (get_order(size) > MAX_ORDER || hashdist) { table = vmalloc_huge(size, gfp_flags); virt = true; if (table) @@ -9290,7 +9290,7 @@ int alloc_contig_range(unsigned long start, unsigned long end, order = 0; outer_start = start; while (!PageBuddy(pfn_to_page(outer_start))) { - if (++order >= MAX_ORDER) { + if (++order > MAX_ORDER) { outer_start = start; break; } @@ -9540,7 +9540,7 @@ bool is_free_buddy_page(struct page *page) unsigned long pfn = page_to_pfn(page); unsigned int order; - for (order = 0; order < MAX_ORDER; order++) { + for (order = 0; order <= MAX_ORDER; order++) { struct page *page_head = page - (pfn & ((1 << order) - 1)); if (PageBuddy(page_head) && @@ -9548,7 +9548,7 @@ bool is_free_buddy_page(struct page *page) break; } - return order < MAX_ORDER; + return order <= MAX_ORDER; } EXPORT_SYMBOL(is_free_buddy_page); @@ -9599,7 +9599,7 @@ bool take_page_off_buddy(struct page *page) bool ret = false; spin_lock_irqsave(&zone->lock, flags); - for (order = 0; order < MAX_ORDER; order++) { + for (order = 0; order <= MAX_ORDER; order++) { struct page *page_head = page - (pfn & ((1 << order) - 1)); int page_order = buddy_order(page_head); diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 47fbc1696466..c6f3605e37ab 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -226,7 +226,7 @@ static void unset_migratetype_isolate(struct page *page, int migratetype) */ if (PageBuddy(page)) { order = buddy_order(page); - if (order >= pageblock_order && order < MAX_ORDER - 1) { + if (order >= pageblock_order && order < MAX_ORDER) { buddy = find_buddy_page_pfn(page, page_to_pfn(page), order, NULL); if (buddy && !is_migrate_isolate_page(buddy)) { @@ -290,11 +290,11 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages) * isolate_single_pageblock() * @migratetype: migrate type to set in error recovery. * - * Free and in-use pages can be as big as MAX_ORDER-1 and contain more than one + * Free and in-use pages can be as big as MAX_ORDER and contain more than one * pageblock. When not all pageblocks within a page are isolated at the same * time, free page accounting can go wrong. For example, in the case of - * MAX_ORDER-1 = pageblock_order + 1, a MAX_ORDER-1 page has two pagelbocks. - * [ MAX_ORDER-1 ] + * MAX_ORDER = pageblock_order + 1, a MAX_ORDER page has two pagelbocks. + * [ MAX_ORDER ] * [ pageblock0 | pageblock1 ] * When either pageblock is isolated, if it is a free page, the page is not * split into separate migratetype lists, which is supposed to; if it is an @@ -451,7 +451,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, * the free page to the right migratetype list. * * head_pfn is not used here as a hugetlb page order - * can be bigger than MAX_ORDER-1, but after it is + * can be bigger than MAX_ORDER, but after it is * freed, the free page order is not. Use pfn within * the range to find the head of the free page. */ @@ -459,7 +459,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, outer_pfn = pfn; while (!PageBuddy(pfn_to_page(outer_pfn))) { /* stop if we cannot find the free page */ - if (++order >= MAX_ORDER) + if (++order > MAX_ORDER) goto failed; outer_pfn &= ~0UL << order; } diff --git a/mm/page_owner.c b/mm/page_owner.c index 220cdeddc295..31169b3e7f06 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -315,7 +315,7 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, unsigned long freepage_order; freepage_order = buddy_order_unsafe(page); - if (freepage_order < MAX_ORDER) + if (freepage_order <= MAX_ORDER) pfn += (1UL << freepage_order) - 1; continue; } @@ -549,7 +549,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) if (PageBuddy(page)) { unsigned long freepage_order = buddy_order_unsafe(page); - if (freepage_order < MAX_ORDER) + if (freepage_order <= MAX_ORDER) pfn += (1UL << freepage_order) - 1; continue; } @@ -657,7 +657,7 @@ static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) if (PageBuddy(page)) { unsigned long order = buddy_order_unsafe(page); - if (order > 0 && order < MAX_ORDER) + if (order > 0 && order <= MAX_ORDER) pfn += (1UL << order) - 1; continue; } diff --git a/mm/page_reporting.c b/mm/page_reporting.c index 275b466de37b..b021f482a4cb 100644 --- a/mm/page_reporting.c +++ b/mm/page_reporting.c @@ -20,7 +20,7 @@ static int page_order_update_notify(const char *val, const struct kernel_param * * If param is set beyond this limit, order is set to default * pageblock_order value */ - return param_set_uint_minmax(val, kp, 0, MAX_ORDER-1); + return param_set_uint_minmax(val, kp, 0, MAX_ORDER); } static const struct kernel_param_ops page_reporting_param_ops = { @@ -276,7 +276,7 @@ page_reporting_process_zone(struct page_reporting_dev_info *prdev, return err; /* Process each free list starting from lowest order/mt */ - for (order = page_reporting_order; order < MAX_ORDER; order++) { + for (order = page_reporting_order; order <= MAX_ORDER; order++) { for (mt = 0; mt < MIGRATE_TYPES; mt++) { /* We do not pull pages from the isolate free list */ if (is_migrate_isolate(mt)) @@ -370,7 +370,7 @@ int page_reporting_register(struct page_reporting_dev_info *prdev) */ if (page_reporting_order == -1) { - if (prdev->order > 0 && prdev->order < MAX_ORDER) + if (prdev->order > 0 && prdev->order <= MAX_ORDER) page_reporting_order = prdev->order; else page_reporting_order = pageblock_order; diff --git a/mm/shuffle.h b/mm/shuffle.h index cec62984f7d3..a6bdf54f96f1 100644 --- a/mm/shuffle.h +++ b/mm/shuffle.h @@ -4,7 +4,7 @@ #define _MM_SHUFFLE_H #include -#define SHUFFLE_ORDER (MAX_ORDER-1) +#define SHUFFLE_ORDER MAX_ORDER #ifdef CONFIG_SHUFFLE_PAGE_ALLOCATOR DECLARE_STATIC_KEY_FALSE(page_alloc_shuffle_key); diff --git a/mm/slab.c b/mm/slab.c index edbe722fb906..6b7c172158e5 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -465,7 +465,7 @@ static int __init slab_max_order_setup(char *str) { get_option(&str, &slab_max_order); slab_max_order = slab_max_order < 0 ? 0 : - min(slab_max_order, MAX_ORDER - 1); + min(slab_max_order, MAX_ORDER); slab_max_order_set = true; return 1; diff --git a/mm/slub.c b/mm/slub.c index 32eb6b50fe18..f49d669ff604 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4171,8 +4171,8 @@ static inline int calculate_order(unsigned int size) /* * Doh this slab cannot be placed using slub_max_order. */ - order = calc_slab_order(size, 1, MAX_ORDER - 1, 1); - if (order < MAX_ORDER) + order = calc_slab_order(size, 1, MAX_ORDER, 1); + if (order <= MAX_ORDER) return order; return -ENOSYS; } @@ -4697,7 +4697,7 @@ __setup("slub_min_order=", setup_slub_min_order); static int __init setup_slub_max_order(char *str) { get_option(&str, (int *)&slub_max_order); - slub_max_order = min(slub_max_order, (unsigned int)MAX_ORDER - 1); + slub_max_order = min_t(unsigned int, slub_max_order, MAX_ORDER); return 1; } diff --git a/mm/vmscan.c b/mm/vmscan.c index 8faac4310cb5..98719e72b5e2 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -7002,7 +7002,7 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order, * scan_control uses s8 fields for order, priority, and reclaim_idx. * Confirm they are large enough for max values. */ - BUILD_BUG_ON(MAX_ORDER > S8_MAX); + BUILD_BUG_ON(MAX_ORDER >= S8_MAX); BUILD_BUG_ON(DEF_PRIORITY > S8_MAX); BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX); diff --git a/mm/vmstat.c b/mm/vmstat.c index 1ea6a5ce1c41..b7307627772d 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1055,7 +1055,7 @@ static void fill_contig_page_info(struct zone *zone, info->free_blocks_total = 0; info->free_blocks_suitable = 0; - for (order = 0; order < MAX_ORDER; order++) { + for (order = 0; order <= MAX_ORDER; order++) { unsigned long blocks; /* @@ -1088,7 +1088,7 @@ static int __fragmentation_index(unsigned int order, struct contig_page_info *in { unsigned long requested = 1UL << order; - if (WARN_ON_ONCE(order >= MAX_ORDER)) + if (WARN_ON_ONCE(order > MAX_ORDER)) return 0; if (!info->free_blocks_total) @@ -1462,7 +1462,7 @@ static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, int order; seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); - for (order = 0; order < MAX_ORDER; ++order) + for (order = 0; order <= MAX_ORDER; ++order) /* * Access to nr_free is lockless as nr_free is used only for * printing purposes. Use data_race to avoid KCSAN warning. @@ -1491,7 +1491,7 @@ static void pagetypeinfo_showfree_print(struct seq_file *m, pgdat->node_id, zone->name, migratetype_names[mtype]); - for (order = 0; order < MAX_ORDER; ++order) { + for (order = 0; order <= MAX_ORDER; ++order) { unsigned long freecount = 0; struct free_area *area; struct list_head *curr; @@ -1531,7 +1531,7 @@ static void pagetypeinfo_showfree(struct seq_file *m, void *arg) /* Print header */ seq_printf(m, "%-43s ", "Free pages count per migrate type at order"); - for (order = 0; order < MAX_ORDER; ++order) + for (order = 0; order <= MAX_ORDER; ++order) seq_printf(m, "%6d ", order); seq_putc(m, '\n'); @@ -2153,7 +2153,7 @@ static void unusable_show_print(struct seq_file *m, seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); - for (order = 0; order < MAX_ORDER; ++order) { + for (order = 0; order <= MAX_ORDER; ++order) { fill_contig_page_info(zone, order, &info); index = unusable_free_index(order, &info); seq_printf(m, "%d.%03d ", index / 1000, index % 1000); @@ -2205,7 +2205,7 @@ static void extfrag_show_print(struct seq_file *m, seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); - for (order = 0; order < MAX_ORDER; ++order) { + for (order = 0; order <= MAX_ORDER; ++order) { fill_contig_page_info(zone, order, &info); index = __fragmentation_index(order, &info); seq_printf(m, "%2d.%03d ", index / 1000, index % 1000); diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c index 854772dd52fd..9b66d6aeeb1a 100644 --- a/net/smc/smc_ib.c +++ b/net/smc/smc_ib.c @@ -843,7 +843,7 @@ long smc_ib_setup_per_ibdev(struct smc_ib_device *smcibdev) goto out; /* the calculated number of cq entries fits to mlx5 cq allocation */ cqe_size_order = cache_line_size() == 128 ? 7 : 6; - smc_order = MAX_ORDER - cqe_size_order - 1; + smc_order = MAX_ORDER - cqe_size_order; if (SMC_MAX_CQE + 2 > (0x00000001 << smc_order) * PAGE_SIZE) cqattr.cqe = (0x00000001 << smc_order) * PAGE_SIZE - 2; smcibdev->roce_cq_send = ib_create_cq(smcibdev->ibdev, diff --git a/security/integrity/ima/ima_crypto.c b/security/integrity/ima/ima_crypto.c index 64499056648a..51ad29940f05 100644 --- a/security/integrity/ima/ima_crypto.c +++ b/security/integrity/ima/ima_crypto.c @@ -38,7 +38,7 @@ static int param_set_bufsize(const char *val, const struct kernel_param *kp) size = memparse(val, NULL); order = get_order(size); - if (order >= MAX_ORDER) + if (order > MAX_ORDER) return -EINVAL; ima_maxorder = order; ima_bufsize = PAGE_SIZE << order; diff --git a/tools/testing/memblock/linux/mmzone.h b/tools/testing/memblock/linux/mmzone.h index e65f89b12f1c..134f8eab0768 100644 --- a/tools/testing/memblock/linux/mmzone.h +++ b/tools/testing/memblock/linux/mmzone.h @@ -17,10 +17,10 @@ enum zone_type { }; #define MAX_NR_ZONES __MAX_NR_ZONES -#define MAX_ORDER 11 -#define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1)) +#define MAX_ORDER 10 +#define MAX_ORDER_NR_PAGES (1 << MAX_ORDER) -#define pageblock_order (MAX_ORDER - 1) +#define pageblock_order MAX_ORDER #define pageblock_nr_pages BIT(pageblock_order) #define pageblock_align(pfn) ALIGN((pfn), pageblock_nr_pages) #define pageblock_start_pfn(pfn) ALIGN_DOWN((pfn), pageblock_nr_pages) -- cgit v1.2.3 From a734991ccaec1985fff42fb26bb6d789d35defb4 Mon Sep 17 00:00:00 2001 From: Axel Rasmussen Date: Tue, 14 Mar 2023 15:12:47 -0700 Subject: mm: userfaultfd: rename functions for clarity + consistency Patch series "mm: userfaultfd: refactor and add UFFDIO_CONTINUE_MODE_WP", v5. - Commits 1-3 refactor userfaultfd ioctl code without behavior changes, with the main goal of improving consistency and reducing the number of function args. - Commit 4 adds UFFDIO_CONTINUE_MODE_WP. This patch (of 4): The basic problem is, over time we've added new userfaultfd ioctls, and we've refactored the code so functions which used to handle only one case are now re-used to deal with several cases. While this happened, we didn't bother to rename the functions. Similarly, as we added new functions, we cargo-culted pieces of the now-inconsistent naming scheme, so those functions too ended up with names that don't make a lot of sense. A key point here is, "copy" in most userfaultfd code refers specifically to UFFDIO_COPY, where we allocate a new page and copy its contents from userspace. There are many functions with "copy" in the name that don't actually do this (at least in some cases). So, rename things into a consistent scheme. The high level idea is that the call stack for userfaultfd ioctls becomes: userfaultfd_ioctl -> userfaultfd_(particular ioctl) -> mfill_atomic_(particular kind of fill operation) -> mfill_atomic /* loops over pages in range */ -> mfill_atomic_pte /* deals with single pages */ -> mfill_atomic_pte_(particular kind of fill operation) -> mfill_atomic_install_pte There are of course some special cases (shmem, hugetlb), but this is the general structure which all function names now adhere to. Link: https://lkml.kernel.org/r/20230314221250.682452-1-axelrasmussen@google.com Link: https://lkml.kernel.org/r/20230314221250.682452-2-axelrasmussen@google.com Signed-off-by: Axel Rasmussen Acked-by: Peter Xu Acked-by: Mike Rapoport (IBM) Cc: Al Viro Cc: Hugh Dickins Cc: James Houghton Cc: Jan Kara Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Mike Kravetz Cc: Muchun Song Cc: Nadav Amit Cc: Shuah Khan Signed-off-by: Andrew Morton --- fs/userfaultfd.c | 18 ++++---- include/linux/hugetlb.h | 30 ++++++------- include/linux/userfaultfd_k.h | 18 ++++---- mm/hugetlb.c | 20 ++++----- mm/userfaultfd.c | 98 +++++++++++++++++++++---------------------- 5 files changed, 92 insertions(+), 92 deletions(-) (limited to 'include/linux') diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 881e9c82b9d1..4aedfd98e3f5 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1756,9 +1756,9 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP)) goto out; if (mmget_not_zero(ctx->mm)) { - ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src, - uffdio_copy.len, &ctx->mmap_changing, - uffdio_copy.mode); + ret = mfill_atomic_copy(ctx->mm, uffdio_copy.dst, uffdio_copy.src, + uffdio_copy.len, &ctx->mmap_changing, + uffdio_copy.mode); mmput(ctx->mm); } else { return -ESRCH; @@ -1808,9 +1808,9 @@ static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx, goto out; if (mmget_not_zero(ctx->mm)) { - ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start, - uffdio_zeropage.range.len, - &ctx->mmap_changing); + ret = mfill_atomic_zeropage(ctx->mm, uffdio_zeropage.range.start, + uffdio_zeropage.range.len, + &ctx->mmap_changing); mmput(ctx->mm); } else { return -ESRCH; @@ -1918,9 +1918,9 @@ static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) goto out; if (mmget_not_zero(ctx->mm)) { - ret = mcopy_continue(ctx->mm, uffdio_continue.range.start, - uffdio_continue.range.len, - &ctx->mmap_changing); + ret = mfill_atomic_continue(ctx->mm, uffdio_continue.range.start, + uffdio_continue.range.len, + &ctx->mmap_changing); mmput(ctx->mm); } else { return -ESRCH; diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 8fb7d91cd0b1..152434396c48 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -158,13 +158,13 @@ unsigned long hugetlb_total_pages(void); vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags); #ifdef CONFIG_USERFAULTFD -int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte, - struct vm_area_struct *dst_vma, - unsigned long dst_addr, - unsigned long src_addr, - enum mcopy_atomic_mode mode, - struct page **pagep, - bool wp_copy); +int hugetlb_mfill_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte, + struct vm_area_struct *dst_vma, + unsigned long dst_addr, + unsigned long src_addr, + enum mcopy_atomic_mode mode, + struct page **pagep, + bool wp_copy); #endif /* CONFIG_USERFAULTFD */ bool hugetlb_reserve_pages(struct inode *inode, long from, long to, struct vm_area_struct *vma, @@ -393,14 +393,14 @@ static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb, } #ifdef CONFIG_USERFAULTFD -static inline int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, - pte_t *dst_pte, - struct vm_area_struct *dst_vma, - unsigned long dst_addr, - unsigned long src_addr, - enum mcopy_atomic_mode mode, - struct page **pagep, - bool wp_copy) +static inline int hugetlb_mfill_atomic_pte(struct mm_struct *dst_mm, + pte_t *dst_pte, + struct vm_area_struct *dst_vma, + unsigned long dst_addr, + unsigned long src_addr, + enum mcopy_atomic_mode mode, + struct page **pagep, + bool wp_copy) { BUG(); return 0; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 0cf8880219da..ac178d810dc7 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -61,15 +61,15 @@ extern int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, unsigned long dst_addr, struct page *page, bool newly_allocated, bool wp_copy); -extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, - unsigned long src_start, unsigned long len, - atomic_t *mmap_changing, __u64 mode); -extern ssize_t mfill_zeropage(struct mm_struct *dst_mm, - unsigned long dst_start, - unsigned long len, - atomic_t *mmap_changing); -extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start, - unsigned long len, atomic_t *mmap_changing); +extern ssize_t mfill_atomic_copy(struct mm_struct *dst_mm, unsigned long dst_start, + unsigned long src_start, unsigned long len, + atomic_t *mmap_changing, __u64 mode); +extern ssize_t mfill_atomic_zeropage(struct mm_struct *dst_mm, + unsigned long dst_start, + unsigned long len, + atomic_t *mmap_changing); +extern ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long dst_start, + unsigned long len, atomic_t *mmap_changing); extern int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, unsigned long len, bool enable_wp, atomic_t *mmap_changing); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 9122e50ae02a..b1e474aa2fc5 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6156,17 +6156,17 @@ out_mutex: #ifdef CONFIG_USERFAULTFD /* - * Used by userfaultfd UFFDIO_COPY. Based on mcopy_atomic_pte with - * modifications for huge pages. + * Used by userfaultfd UFFDIO_* ioctls. Based on userfaultfd's mfill_atomic_pte + * with modifications for hugetlb pages. */ -int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, - pte_t *dst_pte, - struct vm_area_struct *dst_vma, - unsigned long dst_addr, - unsigned long src_addr, - enum mcopy_atomic_mode mode, - struct page **pagep, - bool wp_copy) +int hugetlb_mfill_atomic_pte(struct mm_struct *dst_mm, + pte_t *dst_pte, + struct vm_area_struct *dst_vma, + unsigned long dst_addr, + unsigned long src_addr, + enum mcopy_atomic_mode mode, + struct page **pagep, + bool wp_copy) { bool is_continue = (mode == MCOPY_ATOMIC_CONTINUE); struct hstate *h = hstate_vma(dst_vma); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 23cabd02ac52..874379ce271f 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -127,13 +127,13 @@ out_unlock: return ret; } -static int mcopy_atomic_pte(struct mm_struct *dst_mm, - pmd_t *dst_pmd, - struct vm_area_struct *dst_vma, - unsigned long dst_addr, - unsigned long src_addr, - struct page **pagep, - bool wp_copy) +static int mfill_atomic_pte_copy(struct mm_struct *dst_mm, + pmd_t *dst_pmd, + struct vm_area_struct *dst_vma, + unsigned long dst_addr, + unsigned long src_addr, + struct page **pagep, + bool wp_copy) { void *page_kaddr; int ret; @@ -204,10 +204,10 @@ out_release: goto out; } -static int mfill_zeropage_pte(struct mm_struct *dst_mm, - pmd_t *dst_pmd, - struct vm_area_struct *dst_vma, - unsigned long dst_addr) +static int mfill_atomic_pte_zeropage(struct mm_struct *dst_mm, + pmd_t *dst_pmd, + struct vm_area_struct *dst_vma, + unsigned long dst_addr) { pte_t _dst_pte, *dst_pte; spinlock_t *ptl; @@ -240,11 +240,11 @@ out_unlock: } /* Handles UFFDIO_CONTINUE for all shmem VMAs (shared or private). */ -static int mcontinue_atomic_pte(struct mm_struct *dst_mm, - pmd_t *dst_pmd, - struct vm_area_struct *dst_vma, - unsigned long dst_addr, - bool wp_copy) +static int mfill_atomic_pte_continue(struct mm_struct *dst_mm, + pmd_t *dst_pmd, + struct vm_area_struct *dst_vma, + unsigned long dst_addr, + bool wp_copy) { struct inode *inode = file_inode(dst_vma->vm_file); pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); @@ -307,10 +307,10 @@ static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address) #ifdef CONFIG_HUGETLB_PAGE /* - * __mcopy_atomic processing for HUGETLB vmas. Note that this routine is + * mfill_atomic processing for HUGETLB vmas. Note that this routine is * called with mmap_lock held, it will release mmap_lock before returning. */ -static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, +static __always_inline ssize_t mfill_atomic_hugetlb(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma, unsigned long dst_start, unsigned long src_start, @@ -411,7 +411,7 @@ retry: goto out_unlock; } - err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma, + err = hugetlb_mfill_atomic_pte(dst_mm, dst_pte, dst_vma, dst_addr, src_addr, mode, &page, wp_copy); @@ -463,7 +463,7 @@ out: } #else /* !CONFIG_HUGETLB_PAGE */ /* fail at build time if gcc attempts to use this */ -extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, +extern ssize_t mfill_atomic_hugetlb(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma, unsigned long dst_start, unsigned long src_start, @@ -484,8 +484,8 @@ static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, ssize_t err; if (mode == MCOPY_ATOMIC_CONTINUE) { - return mcontinue_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, - wp_copy); + return mfill_atomic_pte_continue(dst_mm, dst_pmd, dst_vma, + dst_addr, wp_copy); } /* @@ -500,11 +500,11 @@ static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, */ if (!(dst_vma->vm_flags & VM_SHARED)) { if (mode == MCOPY_ATOMIC_NORMAL) - err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma, - dst_addr, src_addr, page, - wp_copy); + err = mfill_atomic_pte_copy(dst_mm, dst_pmd, dst_vma, + dst_addr, src_addr, page, + wp_copy); else - err = mfill_zeropage_pte(dst_mm, dst_pmd, + err = mfill_atomic_pte_zeropage(dst_mm, dst_pmd, dst_vma, dst_addr); } else { err = shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, @@ -516,13 +516,13 @@ static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, return err; } -static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm, - unsigned long dst_start, - unsigned long src_start, - unsigned long len, - enum mcopy_atomic_mode mcopy_mode, - atomic_t *mmap_changing, - __u64 mode) +static __always_inline ssize_t mfill_atomic(struct mm_struct *dst_mm, + unsigned long dst_start, + unsigned long src_start, + unsigned long len, + enum mcopy_atomic_mode mcopy_mode, + atomic_t *mmap_changing, + __u64 mode) { struct vm_area_struct *dst_vma; ssize_t err; @@ -588,9 +588,9 @@ retry: * If this is a HUGETLB vma, pass off to appropriate routine */ if (is_vm_hugetlb_page(dst_vma)) - return __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start, - src_start, len, mcopy_mode, - wp_copy); + return mfill_atomic_hugetlb(dst_mm, dst_vma, dst_start, + src_start, len, mcopy_mode, + wp_copy); if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma)) goto out_unlock; @@ -688,26 +688,26 @@ out: return copied ? copied : err; } -ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start, - unsigned long src_start, unsigned long len, - atomic_t *mmap_changing, __u64 mode) +ssize_t mfill_atomic_copy(struct mm_struct *dst_mm, unsigned long dst_start, + unsigned long src_start, unsigned long len, + atomic_t *mmap_changing, __u64 mode) { - return __mcopy_atomic(dst_mm, dst_start, src_start, len, - MCOPY_ATOMIC_NORMAL, mmap_changing, mode); + return mfill_atomic(dst_mm, dst_start, src_start, len, + MCOPY_ATOMIC_NORMAL, mmap_changing, mode); } -ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start, - unsigned long len, atomic_t *mmap_changing) +ssize_t mfill_atomic_zeropage(struct mm_struct *dst_mm, unsigned long start, + unsigned long len, atomic_t *mmap_changing) { - return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_ZEROPAGE, - mmap_changing, 0); + return mfill_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_ZEROPAGE, + mmap_changing, 0); } -ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long start, - unsigned long len, atomic_t *mmap_changing) +ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long start, + unsigned long len, atomic_t *mmap_changing) { - return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_CONTINUE, - mmap_changing, 0); + return mfill_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_CONTINUE, + mmap_changing, 0); } long uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma, -- cgit v1.2.3 From 61c5004022f56c443b86800e8985d8803f3a22aa Mon Sep 17 00:00:00 2001 From: Axel Rasmussen Date: Tue, 14 Mar 2023 15:12:48 -0700 Subject: mm: userfaultfd: don't pass around both mm and vma Quite a few userfaultfd functions took both mm and vma pointers as arguments. Since the mm is trivially accessible via vma->vm_mm, there's no reason to pass both; it just needlessly extends the already long argument list. Get rid of the mm pointer, where possible, to shorten the argument list. Link: https://lkml.kernel.org/r/20230314221250.682452-3-axelrasmussen@google.com Signed-off-by: Axel Rasmussen Acked-by: Peter Xu Acked-by: Mike Rapoport (IBM) Cc: Al Viro Cc: Hugh Dickins Cc: James Houghton Cc: Jan Kara Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Mike Kravetz Cc: Muchun Song Cc: Nadav Amit Cc: Shuah Khan Signed-off-by: Andrew Morton --- fs/userfaultfd.c | 2 +- include/linux/hugetlb.h | 5 ++-- include/linux/shmem_fs.h | 4 +-- include/linux/userfaultfd_k.h | 4 +-- mm/hugetlb.c | 4 +-- mm/shmem.c | 7 +++-- mm/userfaultfd.c | 61 ++++++++++++++++++++----------------------- 7 files changed, 41 insertions(+), 46 deletions(-) (limited to 'include/linux') diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 4aedfd98e3f5..d8d432ca81e6 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1644,7 +1644,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, /* Reset ptes for the whole vma range if wr-protected */ if (userfaultfd_wp(vma)) - uffd_wp_range(mm, vma, start, vma_end - start, false); + uffd_wp_range(vma, start, vma_end - start, false); new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS; prev = vma_merge(&vmi, mm, prev, start, vma_end, new_flags, diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 152434396c48..3cb7cd853fa8 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -158,7 +158,7 @@ unsigned long hugetlb_total_pages(void); vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags); #ifdef CONFIG_USERFAULTFD -int hugetlb_mfill_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte, +int hugetlb_mfill_atomic_pte(pte_t *dst_pte, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, @@ -393,8 +393,7 @@ static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb, } #ifdef CONFIG_USERFAULTFD -static inline int hugetlb_mfill_atomic_pte(struct mm_struct *dst_mm, - pte_t *dst_pte, +static inline int hugetlb_mfill_atomic_pte(pte_t *dst_pte, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 50bf82b36995..922a2b45fe6f 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -152,14 +152,14 @@ extern void shmem_uncharge(struct inode *inode, long pages); #ifdef CONFIG_USERFAULTFD #ifdef CONFIG_SHMEM -extern int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, +extern int shmem_mfill_atomic_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, bool zeropage, bool wp_copy, struct page **pagep); #else /* !CONFIG_SHMEM */ -#define shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, \ +#define shmem_mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, \ src_addr, zeropage, wp_copy, pagep) ({ BUG(); 0; }) #endif /* CONFIG_SHMEM */ #endif /* CONFIG_USERFAULTFD */ diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index ac178d810dc7..9458cd94a508 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -56,7 +56,7 @@ enum mcopy_atomic_mode { MCOPY_ATOMIC_CONTINUE, }; -extern int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, +extern int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, bool newly_allocated, bool wp_copy); @@ -73,7 +73,7 @@ extern ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long dst extern int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, unsigned long len, bool enable_wp, atomic_t *mmap_changing); -extern long uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *vma, +extern long uffd_wp_range(struct vm_area_struct *vma, unsigned long start, unsigned long len, bool enable_wp); /* mm helpers */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index b1e474aa2fc5..6dc32cccbd9b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6159,8 +6159,7 @@ out_mutex: * Used by userfaultfd UFFDIO_* ioctls. Based on userfaultfd's mfill_atomic_pte * with modifications for hugetlb pages. */ -int hugetlb_mfill_atomic_pte(struct mm_struct *dst_mm, - pte_t *dst_pte, +int hugetlb_mfill_atomic_pte(pte_t *dst_pte, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, @@ -6168,6 +6167,7 @@ int hugetlb_mfill_atomic_pte(struct mm_struct *dst_mm, struct page **pagep, bool wp_copy) { + struct mm_struct *dst_mm = dst_vma->vm_mm; bool is_continue = (mode == MCOPY_ATOMIC_CONTINUE); struct hstate *h = hstate_vma(dst_vma); struct address_space *mapping = dst_vma->vm_file->f_mapping; diff --git a/mm/shmem.c b/mm/shmem.c index fa6e38f2f55f..9d13b9a64144 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2429,8 +2429,7 @@ static struct inode *shmem_get_inode(struct mnt_idmap *idmap, struct super_block } #ifdef CONFIG_USERFAULTFD -int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, - pmd_t *dst_pmd, +int shmem_mfill_atomic_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, @@ -2520,11 +2519,11 @@ int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, goto out_release; ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL, - gfp & GFP_RECLAIM_MASK, dst_mm); + gfp & GFP_RECLAIM_MASK, dst_vma->vm_mm); if (ret) goto out_release; - ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, + ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, &folio->page, true, wp_copy); if (ret) goto out_delete_from_cache; diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 874379ce271f..c3cc6cb04548 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -55,12 +55,13 @@ struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm, * This function handles both MCOPY_ATOMIC_NORMAL and _CONTINUE for both shmem * and anon, and for both shared and private VMAs. */ -int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, +int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, bool newly_allocated, bool wp_copy) { int ret; + struct mm_struct *dst_mm = dst_vma->vm_mm; pte_t _dst_pte, *dst_pte; bool writable = dst_vma->vm_flags & VM_WRITE; bool vm_shared = dst_vma->vm_flags & VM_SHARED; @@ -127,8 +128,7 @@ out_unlock: return ret; } -static int mfill_atomic_pte_copy(struct mm_struct *dst_mm, - pmd_t *dst_pmd, +static int mfill_atomic_pte_copy(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, @@ -190,10 +190,10 @@ static int mfill_atomic_pte_copy(struct mm_struct *dst_mm, __SetPageUptodate(page); ret = -ENOMEM; - if (mem_cgroup_charge(page_folio(page), dst_mm, GFP_KERNEL)) + if (mem_cgroup_charge(page_folio(page), dst_vma->vm_mm, GFP_KERNEL)) goto out_release; - ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, + ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, page, true, wp_copy); if (ret) goto out_release; @@ -204,8 +204,7 @@ out_release: goto out; } -static int mfill_atomic_pte_zeropage(struct mm_struct *dst_mm, - pmd_t *dst_pmd, +static int mfill_atomic_pte_zeropage(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr) { @@ -217,7 +216,7 @@ static int mfill_atomic_pte_zeropage(struct mm_struct *dst_mm, _dst_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr), dst_vma->vm_page_prot)); - dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); + dst_pte = pte_offset_map_lock(dst_vma->vm_mm, dst_pmd, dst_addr, &ptl); if (dst_vma->vm_file) { /* the shmem MAP_PRIVATE case requires checking the i_size */ inode = dst_vma->vm_file->f_inode; @@ -230,7 +229,7 @@ static int mfill_atomic_pte_zeropage(struct mm_struct *dst_mm, ret = -EEXIST; if (!pte_none(*dst_pte)) goto out_unlock; - set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); + set_pte_at(dst_vma->vm_mm, dst_addr, dst_pte, _dst_pte); /* No need to invalidate - it was non-present before */ update_mmu_cache(dst_vma, dst_addr, dst_pte); ret = 0; @@ -240,8 +239,7 @@ out_unlock: } /* Handles UFFDIO_CONTINUE for all shmem VMAs (shared or private). */ -static int mfill_atomic_pte_continue(struct mm_struct *dst_mm, - pmd_t *dst_pmd, +static int mfill_atomic_pte_continue(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, bool wp_copy) @@ -269,7 +267,7 @@ static int mfill_atomic_pte_continue(struct mm_struct *dst_mm, goto out_release; } - ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, + ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, page, false, wp_copy); if (ret) goto out_release; @@ -310,7 +308,7 @@ static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address) * mfill_atomic processing for HUGETLB vmas. Note that this routine is * called with mmap_lock held, it will release mmap_lock before returning. */ -static __always_inline ssize_t mfill_atomic_hugetlb(struct mm_struct *dst_mm, +static __always_inline ssize_t mfill_atomic_hugetlb( struct vm_area_struct *dst_vma, unsigned long dst_start, unsigned long src_start, @@ -318,6 +316,7 @@ static __always_inline ssize_t mfill_atomic_hugetlb(struct mm_struct *dst_mm, enum mcopy_atomic_mode mode, bool wp_copy) { + struct mm_struct *dst_mm = dst_vma->vm_mm; int vm_shared = dst_vma->vm_flags & VM_SHARED; ssize_t err; pte_t *dst_pte; @@ -411,7 +410,7 @@ retry: goto out_unlock; } - err = hugetlb_mfill_atomic_pte(dst_mm, dst_pte, dst_vma, + err = hugetlb_mfill_atomic_pte(dst_pte, dst_vma, dst_addr, src_addr, mode, &page, wp_copy); @@ -463,17 +462,15 @@ out: } #else /* !CONFIG_HUGETLB_PAGE */ /* fail at build time if gcc attempts to use this */ -extern ssize_t mfill_atomic_hugetlb(struct mm_struct *dst_mm, - struct vm_area_struct *dst_vma, - unsigned long dst_start, - unsigned long src_start, - unsigned long len, - enum mcopy_atomic_mode mode, - bool wp_copy); +extern ssize_t mfill_atomic_hugetlb(struct vm_area_struct *dst_vma, + unsigned long dst_start, + unsigned long src_start, + unsigned long len, + enum mcopy_atomic_mode mode, + bool wp_copy); #endif /* CONFIG_HUGETLB_PAGE */ -static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, - pmd_t *dst_pmd, +static __always_inline ssize_t mfill_atomic_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, @@ -484,7 +481,7 @@ static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, ssize_t err; if (mode == MCOPY_ATOMIC_CONTINUE) { - return mfill_atomic_pte_continue(dst_mm, dst_pmd, dst_vma, + return mfill_atomic_pte_continue(dst_pmd, dst_vma, dst_addr, wp_copy); } @@ -500,14 +497,14 @@ static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm, */ if (!(dst_vma->vm_flags & VM_SHARED)) { if (mode == MCOPY_ATOMIC_NORMAL) - err = mfill_atomic_pte_copy(dst_mm, dst_pmd, dst_vma, + err = mfill_atomic_pte_copy(dst_pmd, dst_vma, dst_addr, src_addr, page, wp_copy); else - err = mfill_atomic_pte_zeropage(dst_mm, dst_pmd, + err = mfill_atomic_pte_zeropage(dst_pmd, dst_vma, dst_addr); } else { - err = shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, + err = shmem_mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, src_addr, mode != MCOPY_ATOMIC_NORMAL, wp_copy, page); @@ -588,7 +585,7 @@ retry: * If this is a HUGETLB vma, pass off to appropriate routine */ if (is_vm_hugetlb_page(dst_vma)) - return mfill_atomic_hugetlb(dst_mm, dst_vma, dst_start, + return mfill_atomic_hugetlb(dst_vma, dst_start, src_start, len, mcopy_mode, wp_copy); @@ -641,7 +638,7 @@ retry: BUG_ON(pmd_none(*dst_pmd)); BUG_ON(pmd_trans_huge(*dst_pmd)); - err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, + err = mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, src_addr, &page, mcopy_mode, wp_copy); cond_resched(); @@ -710,7 +707,7 @@ ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long start, mmap_changing, 0); } -long uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma, +long uffd_wp_range(struct vm_area_struct *dst_vma, unsigned long start, unsigned long len, bool enable_wp) { unsigned int mm_cp_flags; @@ -732,7 +729,7 @@ long uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma, */ if (!enable_wp && vma_wants_manual_pte_write_upgrade(dst_vma)) mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE; - tlb_gather_mmu(&tlb, dst_mm); + tlb_gather_mmu(&tlb, dst_vma->vm_mm); ret = change_protection(&tlb, dst_vma, start, start + len, mm_cp_flags); tlb_finish_mmu(&tlb); @@ -788,7 +785,7 @@ int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, _start = max(dst_vma->vm_start, start); _end = min(dst_vma->vm_end, end); - err = uffd_wp_range(dst_mm, dst_vma, _start, _end - _start, enable_wp); + err = uffd_wp_range(dst_vma, _start, _end - _start, enable_wp); /* Return 0 on success, <0 on failures */ if (err < 0) -- cgit v1.2.3 From d9712937037e0ce887920f321429826e9dbfd960 Mon Sep 17 00:00:00 2001 From: Axel Rasmussen Date: Tue, 14 Mar 2023 15:12:49 -0700 Subject: mm: userfaultfd: combine 'mode' and 'wp_copy' arguments Many userfaultfd ioctl functions take both a 'mode' and a 'wp_copy' argument. In future commits we plan to plumb the flags through to more places, so we'd be proliferating the very long argument list even further. Let's take the time to simplify the argument list. Combine the two arguments into one - and generalize, so when we add more flags in the future, it doesn't imply more function arguments. Since the modes (copy, zeropage, continue) are mutually exclusive, store them as an integer value (0, 1, 2) in the low bits. Place combine-able flag bits in the high bits. This is quite similar to an earlier patch proposed by Nadav Amit ("userfaultfd: introduce uffd_flags" [1]). The main difference is that patch only handled flags, whereas this patch *also* combines the "mode" argument into the same type to shorten the argument list. [1]: https://lore.kernel.org/all/20220619233449.181323-2-namit@vmware.com/ Link: https://lkml.kernel.org/r/20230314221250.682452-4-axelrasmussen@google.com Signed-off-by: Axel Rasmussen Acked-by: James Houghton Acked-by: Peter Xu Acked-by: Mike Rapoport (IBM) Cc: Al Viro Cc: Hugh Dickins Cc: Jan Kara Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Mike Kravetz Cc: Muchun Song Cc: Shuah Khan Signed-off-by: Andrew Morton --- fs/userfaultfd.c | 5 ++- include/linux/hugetlb.h | 10 +++--- include/linux/shmem_fs.h | 5 +-- include/linux/userfaultfd_k.h | 46 +++++++++++++++++--------- mm/hugetlb.c | 12 +++---- mm/shmem.c | 7 ++-- mm/userfaultfd.c | 76 +++++++++++++++++++------------------------ 7 files changed, 84 insertions(+), 77 deletions(-) (limited to 'include/linux') diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index d8d432ca81e6..8971c3613cc6 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1729,6 +1729,7 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, struct uffdio_copy uffdio_copy; struct uffdio_copy __user *user_uffdio_copy; struct userfaultfd_wake_range range; + uffd_flags_t flags = 0; user_uffdio_copy = (struct uffdio_copy __user *) arg; @@ -1755,10 +1756,12 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, goto out; if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP)) goto out; + if (uffdio_copy.mode & UFFDIO_COPY_MODE_WP) + flags |= MFILL_ATOMIC_WP; if (mmget_not_zero(ctx->mm)) { ret = mfill_atomic_copy(ctx->mm, uffdio_copy.dst, uffdio_copy.src, uffdio_copy.len, &ctx->mmap_changing, - uffdio_copy.mode); + flags); mmput(ctx->mm); } else { return -ESRCH; diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 3cb7cd853fa8..2a758bcd6719 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -162,9 +162,8 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, - enum mcopy_atomic_mode mode, - struct page **pagep, - bool wp_copy); + uffd_flags_t flags, + struct page **pagep); #endif /* CONFIG_USERFAULTFD */ bool hugetlb_reserve_pages(struct inode *inode, long from, long to, struct vm_area_struct *vma, @@ -397,9 +396,8 @@ static inline int hugetlb_mfill_atomic_pte(pte_t *dst_pte, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, - enum mcopy_atomic_mode mode, - struct page **pagep, - bool wp_copy) + uffd_flags_t flags, + struct page **pagep) { BUG(); return 0; diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 922a2b45fe6f..3bb8d21edbb3 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -9,6 +9,7 @@ #include #include #include +#include /* inode in-kernel data */ @@ -156,11 +157,11 @@ extern int shmem_mfill_atomic_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, - bool zeropage, bool wp_copy, + uffd_flags_t flags, struct page **pagep); #else /* !CONFIG_SHMEM */ #define shmem_mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, \ - src_addr, zeropage, wp_copy, pagep) ({ BUG(); 0; }) + src_addr, flags, pagep) ({ BUG(); 0; }) #endif /* CONFIG_SHMEM */ #endif /* CONFIG_USERFAULTFD */ diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 9458cd94a508..4c477dece540 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -40,30 +40,44 @@ extern int sysctl_unprivileged_userfaultfd; extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); -/* - * The mode of operation for __mcopy_atomic and its helpers. - * - * This is almost an implementation detail (mcopy_atomic below doesn't take this - * as a parameter), but it's exposed here because memory-kind-specific - * implementations (e.g. hugetlbfs) need to know the mode of operation. - */ -enum mcopy_atomic_mode { - /* A normal copy_from_user into the destination range. */ - MCOPY_ATOMIC_NORMAL, - /* Don't copy; map the destination range to the zero page. */ - MCOPY_ATOMIC_ZEROPAGE, - /* Just install pte(s) with the existing page(s) in the page cache. */ - MCOPY_ATOMIC_CONTINUE, +/* A combined operation mode + behavior flags. */ +typedef unsigned int __bitwise uffd_flags_t; + +/* Mutually exclusive modes of operation. */ +enum mfill_atomic_mode { + MFILL_ATOMIC_COPY, + MFILL_ATOMIC_ZEROPAGE, + MFILL_ATOMIC_CONTINUE, + NR_MFILL_ATOMIC_MODES, }; +#define MFILL_ATOMIC_MODE_BITS (const_ilog2(NR_MFILL_ATOMIC_MODES - 1) + 1) +#define MFILL_ATOMIC_BIT(nr) BIT(MFILL_ATOMIC_MODE_BITS + (nr)) +#define MFILL_ATOMIC_FLAG(nr) ((__force uffd_flags_t) MFILL_ATOMIC_BIT(nr)) +#define MFILL_ATOMIC_MODE_MASK ((__force uffd_flags_t) (MFILL_ATOMIC_BIT(0) - 1)) + +static inline bool uffd_flags_mode_is(uffd_flags_t flags, enum mfill_atomic_mode expected) +{ + return (flags & MFILL_ATOMIC_MODE_MASK) == ((__force uffd_flags_t) expected); +} + +static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_atomic_mode mode) +{ + flags &= ~MFILL_ATOMIC_MODE_MASK; + return flags | ((__force uffd_flags_t) mode); +} + +/* Flags controlling behavior. These behavior changes are mode-independent. */ +#define MFILL_ATOMIC_WP MFILL_ATOMIC_FLAG(0) + extern int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, - bool newly_allocated, bool wp_copy); + bool newly_allocated, uffd_flags_t flags); extern ssize_t mfill_atomic_copy(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long src_start, unsigned long len, - atomic_t *mmap_changing, __u64 mode); + atomic_t *mmap_changing, uffd_flags_t flags); extern ssize_t mfill_atomic_zeropage(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long len, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 6dc32cccbd9b..8bfd07f4c143 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6163,12 +6163,12 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, - enum mcopy_atomic_mode mode, - struct page **pagep, - bool wp_copy) + uffd_flags_t flags, + struct page **pagep) { struct mm_struct *dst_mm = dst_vma->vm_mm; - bool is_continue = (mode == MCOPY_ATOMIC_CONTINUE); + bool is_continue = uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE); + bool wp_enabled = (flags & MFILL_ATOMIC_WP); struct hstate *h = hstate_vma(dst_vma); struct address_space *mapping = dst_vma->vm_file->f_mapping; pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr); @@ -6303,7 +6303,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, * For either: (1) CONTINUE on a non-shared VMA, or (2) UFFDIO_COPY * with wp flag set, don't set pte write bit. */ - if (wp_copy || (is_continue && !vm_shared)) + if (wp_enabled || (is_continue && !vm_shared)) writable = 0; else writable = dst_vma->vm_flags & VM_WRITE; @@ -6318,7 +6318,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, _dst_pte = huge_pte_mkdirty(_dst_pte); _dst_pte = pte_mkyoung(_dst_pte); - if (wp_copy) + if (wp_enabled) _dst_pte = huge_pte_mkuffd_wp(_dst_pte); set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); diff --git a/mm/shmem.c b/mm/shmem.c index 9d13b9a64144..b185c1db3009 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -76,7 +76,6 @@ static struct vfsmount *shm_mnt; #include #include #include -#include #include #include @@ -2433,7 +2432,7 @@ int shmem_mfill_atomic_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, - bool zeropage, bool wp_copy, + uffd_flags_t flags, struct page **pagep) { struct inode *inode = file_inode(dst_vma->vm_file); @@ -2465,7 +2464,7 @@ int shmem_mfill_atomic_pte(pmd_t *dst_pmd, if (!folio) goto out_unacct_blocks; - if (!zeropage) { /* COPY */ + if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) { page_kaddr = kmap_local_folio(folio, 0); /* * The read mmap_lock is held here. Despite the @@ -2524,7 +2523,7 @@ int shmem_mfill_atomic_pte(pmd_t *dst_pmd, goto out_release; ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, - &folio->page, true, wp_copy); + &folio->page, true, flags); if (ret) goto out_delete_from_cache; diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index c3cc6cb04548..a9b19b39413d 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -58,7 +58,7 @@ struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm, int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, - bool newly_allocated, bool wp_copy) + bool newly_allocated, uffd_flags_t flags) { int ret; struct mm_struct *dst_mm = dst_vma->vm_mm; @@ -77,7 +77,7 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd, writable = false; if (writable) _dst_pte = pte_mkwrite(_dst_pte); - if (wp_copy) + if (flags & MFILL_ATOMIC_WP) _dst_pte = pte_mkuffd_wp(_dst_pte); dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); @@ -132,8 +132,8 @@ static int mfill_atomic_pte_copy(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, - struct page **pagep, - bool wp_copy) + uffd_flags_t flags, + struct page **pagep) { void *page_kaddr; int ret; @@ -194,7 +194,7 @@ static int mfill_atomic_pte_copy(pmd_t *dst_pmd, goto out_release; ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, - page, true, wp_copy); + page, true, flags); if (ret) goto out_release; out: @@ -242,7 +242,7 @@ out_unlock: static int mfill_atomic_pte_continue(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, - bool wp_copy) + uffd_flags_t flags) { struct inode *inode = file_inode(dst_vma->vm_file); pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); @@ -268,7 +268,7 @@ static int mfill_atomic_pte_continue(pmd_t *dst_pmd, } ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, - page, false, wp_copy); + page, false, flags); if (ret) goto out_release; @@ -313,8 +313,7 @@ static __always_inline ssize_t mfill_atomic_hugetlb( unsigned long dst_start, unsigned long src_start, unsigned long len, - enum mcopy_atomic_mode mode, - bool wp_copy) + uffd_flags_t flags) { struct mm_struct *dst_mm = dst_vma->vm_mm; int vm_shared = dst_vma->vm_flags & VM_SHARED; @@ -334,7 +333,7 @@ static __always_inline ssize_t mfill_atomic_hugetlb( * by THP. Since we can not reliably insert a zero page, this * feature is not supported. */ - if (mode == MCOPY_ATOMIC_ZEROPAGE) { + if (uffd_flags_mode_is(flags, MFILL_ATOMIC_ZEROPAGE)) { mmap_read_unlock(dst_mm); return -EINVAL; } @@ -402,7 +401,7 @@ retry: goto out_unlock; } - if (mode != MCOPY_ATOMIC_CONTINUE && + if (!uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE) && !huge_pte_none_mostly(huge_ptep_get(dst_pte))) { err = -EEXIST; hugetlb_vma_unlock_read(dst_vma); @@ -410,9 +409,8 @@ retry: goto out_unlock; } - err = hugetlb_mfill_atomic_pte(dst_pte, dst_vma, - dst_addr, src_addr, mode, &page, - wp_copy); + err = hugetlb_mfill_atomic_pte(dst_pte, dst_vma, dst_addr, + src_addr, flags, &page); hugetlb_vma_unlock_read(dst_vma); mutex_unlock(&hugetlb_fault_mutex_table[hash]); @@ -466,23 +464,21 @@ extern ssize_t mfill_atomic_hugetlb(struct vm_area_struct *dst_vma, unsigned long dst_start, unsigned long src_start, unsigned long len, - enum mcopy_atomic_mode mode, - bool wp_copy); + uffd_flags_t flags); #endif /* CONFIG_HUGETLB_PAGE */ static __always_inline ssize_t mfill_atomic_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, - struct page **page, - enum mcopy_atomic_mode mode, - bool wp_copy) + uffd_flags_t flags, + struct page **pagep) { ssize_t err; - if (mode == MCOPY_ATOMIC_CONTINUE) { + if (uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE)) { return mfill_atomic_pte_continue(dst_pmd, dst_vma, - dst_addr, wp_copy); + dst_addr, flags); } /* @@ -496,18 +492,17 @@ static __always_inline ssize_t mfill_atomic_pte(pmd_t *dst_pmd, * and not in the radix tree. */ if (!(dst_vma->vm_flags & VM_SHARED)) { - if (mode == MCOPY_ATOMIC_NORMAL) + if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) err = mfill_atomic_pte_copy(dst_pmd, dst_vma, - dst_addr, src_addr, page, - wp_copy); + dst_addr, src_addr, + flags, pagep); else err = mfill_atomic_pte_zeropage(dst_pmd, dst_vma, dst_addr); } else { err = shmem_mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, src_addr, - mode != MCOPY_ATOMIC_NORMAL, - wp_copy, page); + flags, pagep); } return err; @@ -517,9 +512,8 @@ static __always_inline ssize_t mfill_atomic(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long src_start, unsigned long len, - enum mcopy_atomic_mode mcopy_mode, atomic_t *mmap_changing, - __u64 mode) + uffd_flags_t flags) { struct vm_area_struct *dst_vma; ssize_t err; @@ -527,7 +521,6 @@ static __always_inline ssize_t mfill_atomic(struct mm_struct *dst_mm, unsigned long src_addr, dst_addr; long copied; struct page *page; - bool wp_copy; /* * Sanitize the command parameters: @@ -577,8 +570,7 @@ retry: * validate 'mode' now that we know the dst_vma: don't allow * a wrprotect copy if the userfaultfd didn't register as WP. */ - wp_copy = mode & UFFDIO_COPY_MODE_WP; - if (wp_copy && !(dst_vma->vm_flags & VM_UFFD_WP)) + if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP)) goto out_unlock; /* @@ -586,12 +578,12 @@ retry: */ if (is_vm_hugetlb_page(dst_vma)) return mfill_atomic_hugetlb(dst_vma, dst_start, - src_start, len, mcopy_mode, - wp_copy); + src_start, len, flags); if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma)) goto out_unlock; - if (!vma_is_shmem(dst_vma) && mcopy_mode == MCOPY_ATOMIC_CONTINUE) + if (!vma_is_shmem(dst_vma) && + uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE)) goto out_unlock; /* @@ -639,7 +631,7 @@ retry: BUG_ON(pmd_trans_huge(*dst_pmd)); err = mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, - src_addr, &page, mcopy_mode, wp_copy); + src_addr, flags, &page); cond_resched(); if (unlikely(err == -ENOENT)) { @@ -687,24 +679,24 @@ out: ssize_t mfill_atomic_copy(struct mm_struct *dst_mm, unsigned long dst_start, unsigned long src_start, unsigned long len, - atomic_t *mmap_changing, __u64 mode) + atomic_t *mmap_changing, uffd_flags_t flags) { - return mfill_atomic(dst_mm, dst_start, src_start, len, - MCOPY_ATOMIC_NORMAL, mmap_changing, mode); + return mfill_atomic(dst_mm, dst_start, src_start, len, mmap_changing, + uffd_flags_set_mode(flags, MFILL_ATOMIC_COPY)); } ssize_t mfill_atomic_zeropage(struct mm_struct *dst_mm, unsigned long start, unsigned long len, atomic_t *mmap_changing) { - return mfill_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_ZEROPAGE, - mmap_changing, 0); + return mfill_atomic(dst_mm, start, 0, len, mmap_changing, + uffd_flags_set_mode(0, MFILL_ATOMIC_ZEROPAGE)); } ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long start, unsigned long len, atomic_t *mmap_changing) { - return mfill_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_CONTINUE, - mmap_changing, 0); + return mfill_atomic(dst_mm, start, 0, len, mmap_changing, + uffd_flags_set_mode(0, MFILL_ATOMIC_CONTINUE)); } long uffd_wp_range(struct vm_area_struct *dst_vma, -- cgit v1.2.3 From 0289184476c845968ad6ac9083c96cc0f75ca505 Mon Sep 17 00:00:00 2001 From: Axel Rasmussen Date: Tue, 14 Mar 2023 15:12:50 -0700 Subject: mm: userfaultfd: add UFFDIO_CONTINUE_MODE_WP to install WP PTEs UFFDIO_COPY already has UFFDIO_COPY_MODE_WP, so when installing a new PTE to resolve a missing fault, one can install a write-protected one. This is useful when using UFFDIO_REGISTER_MODE_{MISSING,WP} in combination. This was motivated by testing HugeTLB HGM [1], and in particular its interaction with userfaultfd features. Existing userfaultfd code supports using WP and MINOR modes together (i.e. you can register an area with both enabled), but without this CONTINUE flag the combination is in practice unusable. So, add an analogous UFFDIO_CONTINUE_MODE_WP, which does the same thing as UFFDIO_COPY_MODE_WP, but for *minor* faults. Update the selftest to do some very basic exercising of the new flag. Update Documentation/ to describe how these flags are used (neither the COPY nor the new CONTINUE versions of this mode flag were described there before). [1]: https://patchwork.kernel.org/project/linux-mm/cover/20230218002819.1486479-1-jthoughton@google.com/ Link: https://lkml.kernel.org/r/20230314221250.682452-5-axelrasmussen@google.com Signed-off-by: Axel Rasmussen Acked-by: Peter Xu Acked-by: Mike Rapoport (IBM) Cc: Al Viro Cc: Hugh Dickins Cc: Jan Kara Cc: Liam R. Howlett Cc: Matthew Wilcox (Oracle) Cc: Mike Kravetz Cc: Muchun Song Cc: Nadav Amit Cc: Shuah Khan Signed-off-by: Andrew Morton --- Documentation/admin-guide/mm/userfaultfd.rst | 8 ++++++++ fs/userfaultfd.c | 8 ++++++-- include/linux/userfaultfd_k.h | 3 ++- include/uapi/linux/userfaultfd.h | 7 +++++++ mm/userfaultfd.c | 5 +++-- tools/testing/selftests/mm/userfaultfd.c | 4 ++++ 6 files changed, 30 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst index bd2226299583..7c304e432205 100644 --- a/Documentation/admin-guide/mm/userfaultfd.rst +++ b/Documentation/admin-guide/mm/userfaultfd.rst @@ -236,6 +236,14 @@ newer kernels, one can also detect the feature UFFD_FEATURE_WP_UNPOPULATED and set the feature bit in advance to make sure none ptes will also be write protected even upon anonymous memory. +When using ``UFFDIO_REGISTER_MODE_WP`` in combination with either +``UFFDIO_REGISTER_MODE_MISSING`` or ``UFFDIO_REGISTER_MODE_MINOR``, when +resolving missing / minor faults with ``UFFDIO_COPY`` or ``UFFDIO_CONTINUE`` +respectively, it may be desirable for the new page / mapping to be +write-protected (so future writes will also result in a WP fault). These ioctls +support a mode flag (``UFFDIO_COPY_MODE_WP`` or ``UFFDIO_CONTINUE_MODE_WP`` +respectively) to configure the mapping this way. + QEMU/KVM ======== diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 8971c3613cc6..8395605790f6 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1893,6 +1893,7 @@ static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) struct uffdio_continue uffdio_continue; struct uffdio_continue __user *user_uffdio_continue; struct userfaultfd_wake_range range; + uffd_flags_t flags = 0; user_uffdio_continue = (struct uffdio_continue __user *)arg; @@ -1917,13 +1918,16 @@ static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) uffdio_continue.range.start) { goto out; } - if (uffdio_continue.mode & ~UFFDIO_CONTINUE_MODE_DONTWAKE) + if (uffdio_continue.mode & ~(UFFDIO_CONTINUE_MODE_DONTWAKE | + UFFDIO_CONTINUE_MODE_WP)) goto out; + if (uffdio_continue.mode & UFFDIO_CONTINUE_MODE_WP) + flags |= MFILL_ATOMIC_WP; if (mmget_not_zero(ctx->mm)) { ret = mfill_atomic_continue(ctx->mm, uffdio_continue.range.start, uffdio_continue.range.len, - &ctx->mmap_changing); + &ctx->mmap_changing, flags); mmput(ctx->mm); } else { return -ESRCH; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 4c477dece540..a2c53e98dfd6 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -83,7 +83,8 @@ extern ssize_t mfill_atomic_zeropage(struct mm_struct *dst_mm, unsigned long len, atomic_t *mmap_changing); extern ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long dst_start, - unsigned long len, atomic_t *mmap_changing); + unsigned long len, atomic_t *mmap_changing, + uffd_flags_t flags); extern int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, unsigned long len, bool enable_wp, atomic_t *mmap_changing); diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 90c958952bfc..66dd4cd277bd 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -305,6 +305,13 @@ struct uffdio_writeprotect { struct uffdio_continue { struct uffdio_range range; #define UFFDIO_CONTINUE_MODE_DONTWAKE ((__u64)1<<0) + /* + * UFFDIO_CONTINUE_MODE_WP will map the page write protected on + * the fly. UFFDIO_CONTINUE_MODE_WP is available only if the + * write protected ioctl is implemented for the range + * according to the uffdio_register.ioctls. + */ +#define UFFDIO_CONTINUE_MODE_WP ((__u64)1<<1) __u64 mode; /* diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index a9b19b39413d..7f1b5f8b712c 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -693,10 +693,11 @@ ssize_t mfill_atomic_zeropage(struct mm_struct *dst_mm, unsigned long start, } ssize_t mfill_atomic_continue(struct mm_struct *dst_mm, unsigned long start, - unsigned long len, atomic_t *mmap_changing) + unsigned long len, atomic_t *mmap_changing, + uffd_flags_t flags) { return mfill_atomic(dst_mm, start, 0, len, mmap_changing, - uffd_flags_set_mode(0, MFILL_ATOMIC_CONTINUE)); + uffd_flags_set_mode(flags, MFILL_ATOMIC_CONTINUE)); } long uffd_wp_range(struct vm_area_struct *dst_vma, diff --git a/tools/testing/selftests/mm/userfaultfd.c b/tools/testing/selftests/mm/userfaultfd.c index e030d63c031a..a96d126cb40e 100644 --- a/tools/testing/selftests/mm/userfaultfd.c +++ b/tools/testing/selftests/mm/userfaultfd.c @@ -585,6 +585,8 @@ static void continue_range(int ufd, __u64 start, __u64 len) req.range.start = start; req.range.len = len; req.mode = 0; + if (test_uffdio_wp) + req.mode |= UFFDIO_CONTINUE_MODE_WP; if (ioctl(ufd, UFFDIO_CONTINUE, &req)) err("UFFDIO_CONTINUE failed for address 0x%" PRIx64, @@ -1332,6 +1334,8 @@ static int userfaultfd_minor_test(void) uffdio_register.range.start = (unsigned long)area_dst_alias; uffdio_register.range.len = nr_pages * page_size; uffdio_register.mode = UFFDIO_REGISTER_MODE_MINOR; + if (test_uffdio_wp) + uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP; if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) err("register failure"); -- cgit v1.2.3 From 5d671eb4ef7e1423a78fc0c12dd419c1dc1f6967 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (IBM)" Date: Sun, 19 Mar 2023 13:42:14 +0200 Subject: mm: move get_page_from_free_area() to mm/page_alloc.c The get_page_from_free_area() helper is only used in mm/page_alloc.c so move it there to reduce noise in include/linux/mmzone.h Link: https://lkml.kernel.org/r/20230319114214.2133332-1-rppt@kernel.org Signed-off-by: Mike Rapoport (IBM) Reviewed-by: Lorenzo Stoakes Acked-by: Kirill A. Shutemov Reviewed-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 7 ------- mm/page_alloc.c | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 35b11cc210d9..2d3d78d01283 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -108,13 +108,6 @@ struct free_area { unsigned long nr_free; }; -static inline struct page *get_page_from_free_area(struct free_area *area, - int migratetype) -{ - return list_first_entry_or_null(&area->free_list[migratetype], - struct page, lru); -} - static inline bool free_area_empty(struct free_area *area, int migratetype) { return list_empty(&area->free_list[migratetype]); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c3e49d028a7a..4b09711b6f0f 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1048,6 +1048,13 @@ static inline void del_page_from_free_list(struct page *page, struct zone *zone, zone->free_area[order].nr_free--; } +static inline struct page *get_page_from_free_area(struct free_area *area, + int migratetype) +{ + return list_first_entry_or_null(&area->free_list[migratetype], + struct page, lru); +} + /* * If this is not the largest possible page, check if the buddy * of the next-highest order is free. If it is, it's possible -- cgit v1.2.3 From 9420f89db2dd611c5b436a13e13f74d65ecc3a6a Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (IBM)" Date: Tue, 21 Mar 2023 19:05:02 +0200 Subject: mm: move most of core MM initialization to mm/mm_init.c The bulk of memory management initialization code is spread all over mm/page_alloc.c and makes navigating through page allocator functionality difficult. Move most of the functions marked __init and __meminit to mm/mm_init.c to make it better localized and allow some more spare room before mm/page_alloc.c reaches 10k lines. No functional changes. Link: https://lkml.kernel.org/r/20230321170513.2401534-4-rppt@kernel.org Signed-off-by: Mike Rapoport (IBM) Acked-by: David Hildenbrand Acked-by: Vlastimil Babka Cc: Doug Berger Cc: Matthew Wilcox (Oracle) Cc: Mel Gorman Cc: Michal Hocko Cc: Thomas Bogendoerfer Signed-off-by: Andrew Morton --- include/linux/gfp.h | 5 - mm/cma.c | 1 + mm/internal.h | 38 +- mm/mm_init.c | 2304 +++++++++++++++++++++++++++++++++++++++++ mm/page_alloc.c | 2866 +++++---------------------------------------------- 5 files changed, 2614 insertions(+), 2600 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 65a78773dcca..7c554e4bd49f 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -361,9 +361,4 @@ extern struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, #endif void free_contig_range(unsigned long pfn, unsigned long nr_pages); -#ifdef CONFIG_CMA -/* CMA stuff */ -extern void init_cma_reserved_pageblock(struct page *page); -#endif - #endif /* __LINUX_GFP_H */ diff --git a/mm/cma.c b/mm/cma.c index a7263aa02c92..6268d6620254 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -33,6 +33,7 @@ #include #include +#include "internal.h" #include "cma.h" struct cma cma_areas[MAX_CMA_AREAS]; diff --git a/mm/internal.h b/mm/internal.h index 2a7ffd9962c4..22f1410a0ee3 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -202,6 +202,8 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); * in mm/page_alloc.c */ +extern char * const zone_names[MAX_NR_ZONES]; + /* * Structure for holding the mostly immutable allocation parameters passed * between functions involved in allocations, including the alloc_pages* @@ -366,7 +368,29 @@ extern void __putback_isolated_page(struct page *page, unsigned int order, extern void memblock_free_pages(struct page *page, unsigned long pfn, unsigned int order); extern void __free_pages_core(struct page *page, unsigned int order); + +static inline void prep_compound_head(struct page *page, unsigned int order) +{ + struct folio *folio = (struct folio *)page; + + set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); + set_compound_order(page, order); + atomic_set(&folio->_entire_mapcount, -1); + atomic_set(&folio->_nr_pages_mapped, 0); + atomic_set(&folio->_pincount, 0); +} + +static inline void prep_compound_tail(struct page *head, int tail_idx) +{ + struct page *p = head + tail_idx; + + p->mapping = TAIL_MAPPING; + set_compound_head(p, head); + set_page_private(p, 0); +} + extern void prep_compound_page(struct page *page, unsigned int order); + extern void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags); extern int user_min_free_kbytes; @@ -377,6 +401,7 @@ extern void free_unref_page_list(struct list_head *list); extern void zone_pcp_reset(struct zone *zone); extern void zone_pcp_disable(struct zone *zone); extern void zone_pcp_enable(struct zone *zone); +extern void zone_pcp_init(struct zone *zone); extern void *memmap_alloc(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, @@ -474,7 +499,12 @@ isolate_migratepages_range(struct compact_control *cc, int __alloc_contig_migrate_range(struct compact_control *cc, unsigned long start, unsigned long end); -#endif + +/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ +void init_cma_reserved_pageblock(struct page *page); + +#endif /* CONFIG_COMPACTION || CONFIG_CMA */ + int find_suitable_fallback(struct free_area *area, unsigned int order, int migratetype, bool only_stealable, bool *can_steal); @@ -658,6 +688,12 @@ static inline void vunmap_range_noflush(unsigned long start, unsigned long end) #endif /* !CONFIG_MMU */ /* Memory initialisation debug and verification */ +#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT +DECLARE_STATIC_KEY_TRUE(deferred_pages); + +bool __init deferred_grow_zone(struct zone *zone, unsigned int order); +#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ + enum mminit_level { MMINIT_WARNING, MMINIT_VERIFY, diff --git a/mm/mm_init.c b/mm/mm_init.c index c1883362e71d..68d0187c7886 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -14,7 +14,14 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include "internal.h" +#include "shuffle.h" #ifdef CONFIG_DEBUG_MEMORY_INIT int __meminitdata mminit_loglevel; @@ -198,3 +205,2300 @@ static int __init mm_sysfs_init(void) return 0; } postcore_initcall(mm_sysfs_init); + +static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; +static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; +static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; + +static unsigned long required_kernelcore __initdata; +static unsigned long required_kernelcore_percent __initdata; +static unsigned long required_movablecore __initdata; +static unsigned long required_movablecore_percent __initdata; + +static unsigned long nr_kernel_pages __initdata; +static unsigned long nr_all_pages __initdata; +static unsigned long dma_reserve __initdata; + +bool deferred_struct_pages __meminitdata; + +static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); + +static int __init cmdline_parse_core(char *p, unsigned long *core, + unsigned long *percent) +{ + unsigned long long coremem; + char *endptr; + + if (!p) + return -EINVAL; + + /* Value may be a percentage of total memory, otherwise bytes */ + coremem = simple_strtoull(p, &endptr, 0); + if (*endptr == '%') { + /* Paranoid check for percent values greater than 100 */ + WARN_ON(coremem > 100); + + *percent = coremem; + } else { + coremem = memparse(p, &p); + /* Paranoid check that UL is enough for the coremem value */ + WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); + + *core = coremem >> PAGE_SHIFT; + *percent = 0UL; + } + return 0; +} + +/* + * kernelcore=size sets the amount of memory for use for allocations that + * cannot be reclaimed or migrated. + */ +static int __init cmdline_parse_kernelcore(char *p) +{ + /* parse kernelcore=mirror */ + if (parse_option_str(p, "mirror")) { + mirrored_kernelcore = true; + return 0; + } + + return cmdline_parse_core(p, &required_kernelcore, + &required_kernelcore_percent); +} +early_param("kernelcore", cmdline_parse_kernelcore); + +/* + * movablecore=size sets the amount of memory for use for allocations that + * can be reclaimed or migrated. + */ +static int __init cmdline_parse_movablecore(char *p) +{ + return cmdline_parse_core(p, &required_movablecore, + &required_movablecore_percent); +} +early_param("movablecore", cmdline_parse_movablecore); + +/* + * early_calculate_totalpages() + * Sum pages in active regions for movable zone. + * Populate N_MEMORY for calculating usable_nodes. + */ +static unsigned long __init early_calculate_totalpages(void) +{ + unsigned long totalpages = 0; + unsigned long start_pfn, end_pfn; + int i, nid; + + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { + unsigned long pages = end_pfn - start_pfn; + + totalpages += pages; + if (pages) + node_set_state(nid, N_MEMORY); + } + return totalpages; +} + +/* + * This finds a zone that can be used for ZONE_MOVABLE pages. The + * assumption is made that zones within a node are ordered in monotonic + * increasing memory addresses so that the "highest" populated zone is used + */ +static void __init find_usable_zone_for_movable(void) +{ + int zone_index; + for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { + if (zone_index == ZONE_MOVABLE) + continue; + + if (arch_zone_highest_possible_pfn[zone_index] > + arch_zone_lowest_possible_pfn[zone_index]) + break; + } + + VM_BUG_ON(zone_index == -1); + movable_zone = zone_index; +} + +/* + * Find the PFN the Movable zone begins in each node. Kernel memory + * is spread evenly between nodes as long as the nodes have enough + * memory. When they don't, some nodes will have more kernelcore than + * others + */ +static void __init find_zone_movable_pfns_for_nodes(void) +{ + int i, nid; + unsigned long usable_startpfn; + unsigned long kernelcore_node, kernelcore_remaining; + /* save the state before borrow the nodemask */ + nodemask_t saved_node_state = node_states[N_MEMORY]; + unsigned long totalpages = early_calculate_totalpages(); + int usable_nodes = nodes_weight(node_states[N_MEMORY]); + struct memblock_region *r; + + /* Need to find movable_zone earlier when movable_node is specified. */ + find_usable_zone_for_movable(); + + /* + * If movable_node is specified, ignore kernelcore and movablecore + * options. + */ + if (movable_node_is_enabled()) { + for_each_mem_region(r) { + if (!memblock_is_hotpluggable(r)) + continue; + + nid = memblock_get_region_node(r); + + usable_startpfn = PFN_DOWN(r->base); + zone_movable_pfn[nid] = zone_movable_pfn[nid] ? + min(usable_startpfn, zone_movable_pfn[nid]) : + usable_startpfn; + } + + goto out2; + } + + /* + * If kernelcore=mirror is specified, ignore movablecore option + */ + if (mirrored_kernelcore) { + bool mem_below_4gb_not_mirrored = false; + + for_each_mem_region(r) { + if (memblock_is_mirror(r)) + continue; + + nid = memblock_get_region_node(r); + + usable_startpfn = memblock_region_memory_base_pfn(r); + + if (usable_startpfn < PHYS_PFN(SZ_4G)) { + mem_below_4gb_not_mirrored = true; + continue; + } + + zone_movable_pfn[nid] = zone_movable_pfn[nid] ? + min(usable_startpfn, zone_movable_pfn[nid]) : + usable_startpfn; + } + + if (mem_below_4gb_not_mirrored) + pr_warn("This configuration results in unmirrored kernel memory.\n"); + + goto out2; + } + + /* + * If kernelcore=nn% or movablecore=nn% was specified, calculate the + * amount of necessary memory. + */ + if (required_kernelcore_percent) + required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / + 10000UL; + if (required_movablecore_percent) + required_movablecore = (totalpages * 100 * required_movablecore_percent) / + 10000UL; + + /* + * If movablecore= was specified, calculate what size of + * kernelcore that corresponds so that memory usable for + * any allocation type is evenly spread. If both kernelcore + * and movablecore are specified, then the value of kernelcore + * will be used for required_kernelcore if it's greater than + * what movablecore would have allowed. + */ + if (required_movablecore) { + unsigned long corepages; + + /* + * Round-up so that ZONE_MOVABLE is at least as large as what + * was requested by the user + */ + required_movablecore = + roundup(required_movablecore, MAX_ORDER_NR_PAGES); + required_movablecore = min(totalpages, required_movablecore); + corepages = totalpages - required_movablecore; + + required_kernelcore = max(required_kernelcore, corepages); + } + + /* + * If kernelcore was not specified or kernelcore size is larger + * than totalpages, there is no ZONE_MOVABLE. + */ + if (!required_kernelcore || required_kernelcore >= totalpages) + goto out; + + /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ + usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; + +restart: + /* Spread kernelcore memory as evenly as possible throughout nodes */ + kernelcore_node = required_kernelcore / usable_nodes; + for_each_node_state(nid, N_MEMORY) { + unsigned long start_pfn, end_pfn; + + /* + * Recalculate kernelcore_node if the division per node + * now exceeds what is necessary to satisfy the requested + * amount of memory for the kernel + */ + if (required_kernelcore < kernelcore_node) + kernelcore_node = required_kernelcore / usable_nodes; + + /* + * As the map is walked, we track how much memory is usable + * by the kernel using kernelcore_remaining. When it is + * 0, the rest of the node is usable by ZONE_MOVABLE + */ + kernelcore_remaining = kernelcore_node; + + /* Go through each range of PFNs within this node */ + for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { + unsigned long size_pages; + + start_pfn = max(start_pfn, zone_movable_pfn[nid]); + if (start_pfn >= end_pfn) + continue; + + /* Account for what is only usable for kernelcore */ + if (start_pfn < usable_startpfn) { + unsigned long kernel_pages; + kernel_pages = min(end_pfn, usable_startpfn) + - start_pfn; + + kernelcore_remaining -= min(kernel_pages, + kernelcore_remaining); + required_kernelcore -= min(kernel_pages, + required_kernelcore); + + /* Continue if range is now fully accounted */ + if (end_pfn <= usable_startpfn) { + + /* + * Push zone_movable_pfn to the end so + * that if we have to rebalance + * kernelcore across nodes, we will + * not double account here + */ + zone_movable_pfn[nid] = end_pfn; + continue; + } + start_pfn = usable_startpfn; + } + + /* + * The usable PFN range for ZONE_MOVABLE is from + * start_pfn->end_pfn. Calculate size_pages as the + * number of pages used as kernelcore + */ + size_pages = end_pfn - start_pfn; + if (size_pages > kernelcore_remaining) + size_pages = kernelcore_remaining; + zone_movable_pfn[nid] = start_pfn + size_pages; + + /* + * Some kernelcore has been met, update counts and + * break if the kernelcore for this node has been + * satisfied + */ + required_kernelcore -= min(required_kernelcore, + size_pages); + kernelcore_remaining -= size_pages; + if (!kernelcore_remaining) + break; + } + } + + /* + * If there is still required_kernelcore, we do another pass with one + * less node in the count. This will push zone_movable_pfn[nid] further + * along on the nodes that still have memory until kernelcore is + * satisfied + */ + usable_nodes--; + if (usable_nodes && required_kernelcore > usable_nodes) + goto restart; + +out2: + /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ + for (nid = 0; nid < MAX_NUMNODES; nid++) { + unsigned long start_pfn, end_pfn; + + zone_movable_pfn[nid] = + roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); + + get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); + if (zone_movable_pfn[nid] >= end_pfn) + zone_movable_pfn[nid] = 0; + } + +out: + /* restore the node_state */ + node_states[N_MEMORY] = saved_node_state; +} + +static void __meminit __init_single_page(struct page *page, unsigned long pfn, + unsigned long zone, int nid) +{ + mm_zero_struct_page(page); + set_page_links(page, zone, nid, pfn); + init_page_count(page); + page_mapcount_reset(page); + page_cpupid_reset_last(page); + page_kasan_tag_reset(page); + + INIT_LIST_HEAD(&page->lru); +#ifdef WANT_PAGE_VIRTUAL + /* The shift won't overflow because ZONE_NORMAL is below 4G. */ + if (!is_highmem_idx(zone)) + set_page_address(page, __va(pfn << PAGE_SHIFT)); +#endif +} + +#ifdef CONFIG_NUMA +/* + * During memory init memblocks map pfns to nids. The search is expensive and + * this caches recent lookups. The implementation of __early_pfn_to_nid + * treats start/end as pfns. + */ +struct mminit_pfnnid_cache { + unsigned long last_start; + unsigned long last_end; + int last_nid; +}; + +static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; + +/* + * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. + */ +static int __meminit __early_pfn_to_nid(unsigned long pfn, + struct mminit_pfnnid_cache *state) +{ + unsigned long start_pfn, end_pfn; + int nid; + + if (state->last_start <= pfn && pfn < state->last_end) + return state->last_nid; + + nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); + if (nid != NUMA_NO_NODE) { + state->last_start = start_pfn; + state->last_end = end_pfn; + state->last_nid = nid; + } + + return nid; +} + +int __meminit early_pfn_to_nid(unsigned long pfn) +{ + static DEFINE_SPINLOCK(early_pfn_lock); + int nid; + + spin_lock(&early_pfn_lock); + nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); + if (nid < 0) + nid = first_online_node; + spin_unlock(&early_pfn_lock); + + return nid; +} +#endif /* CONFIG_NUMA */ + +#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT +static inline void pgdat_set_deferred_range(pg_data_t *pgdat) +{ + pgdat->first_deferred_pfn = ULONG_MAX; +} + +/* Returns true if the struct page for the pfn is initialised */ +static inline bool __meminit early_page_initialised(unsigned long pfn) +{ + int nid = early_pfn_to_nid(pfn); + + if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) + return false; + + return true; +} + +/* + * Returns true when the remaining initialisation should be deferred until + * later in the boot cycle when it can be parallelised. + */ +static bool __meminit +defer_init(int nid, unsigned long pfn, unsigned long end_pfn) +{ + static unsigned long prev_end_pfn, nr_initialised; + + if (early_page_ext_enabled()) + return false; + /* + * prev_end_pfn static that contains the end of previous zone + * No need to protect because called very early in boot before smp_init. + */ + if (prev_end_pfn != end_pfn) { + prev_end_pfn = end_pfn; + nr_initialised = 0; + } + + /* Always populate low zones for address-constrained allocations */ + if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) + return false; + + if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) + return true; + /* + * We start only with one section of pages, more pages are added as + * needed until the rest of deferred pages are initialized. + */ + nr_initialised++; + if ((nr_initialised > PAGES_PER_SECTION) && + (pfn & (PAGES_PER_SECTION - 1)) == 0) { + NODE_DATA(nid)->first_deferred_pfn = pfn; + return true; + } + return false; +} + +static void __meminit init_reserved_page(unsigned long pfn) +{ + pg_data_t *pgdat; + int nid, zid; + + if (early_page_initialised(pfn)) + return; + + nid = early_pfn_to_nid(pfn); + pgdat = NODE_DATA(nid); + + for (zid = 0; zid < MAX_NR_ZONES; zid++) { + struct zone *zone = &pgdat->node_zones[zid]; + + if (zone_spans_pfn(zone, pfn)) + break; + } + __init_single_page(pfn_to_page(pfn), pfn, zid, nid); +} +#else +static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} + +static inline bool early_page_initialised(unsigned long pfn) +{ + return true; +} + +static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) +{ + return false; +} + +static inline void init_reserved_page(unsigned long pfn) +{ +} +#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ + +/* + * Initialised pages do not have PageReserved set. This function is + * called for each range allocated by the bootmem allocator and + * marks the pages PageReserved. The remaining valid pages are later + * sent to the buddy page allocator. + */ +void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) +{ + unsigned long start_pfn = PFN_DOWN(start); + unsigned long end_pfn = PFN_UP(end); + + for (; start_pfn < end_pfn; start_pfn++) { + if (pfn_valid(start_pfn)) { + struct page *page = pfn_to_page(start_pfn); + + init_reserved_page(start_pfn); + + /* Avoid false-positive PageTail() */ + INIT_LIST_HEAD(&page->lru); + + /* + * no need for atomic set_bit because the struct + * page is not visible yet so nobody should + * access it yet. + */ + __SetPageReserved(page); + } + } +} + +/* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ +static bool __meminit +overlap_memmap_init(unsigned long zone, unsigned long *pfn) +{ + static struct memblock_region *r; + + if (mirrored_kernelcore && zone == ZONE_MOVABLE) { + if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { + for_each_mem_region(r) { + if (*pfn < memblock_region_memory_end_pfn(r)) + break; + } + } + if (*pfn >= memblock_region_memory_base_pfn(r) && + memblock_is_mirror(r)) { + *pfn = memblock_region_memory_end_pfn(r); + return true; + } + } + return false; +} + +/* + * Only struct pages that correspond to ranges defined by memblock.memory + * are zeroed and initialized by going through __init_single_page() during + * memmap_init_zone_range(). + * + * But, there could be struct pages that correspond to holes in + * memblock.memory. This can happen because of the following reasons: + * - physical memory bank size is not necessarily the exact multiple of the + * arbitrary section size + * - early reserved memory may not be listed in memblock.memory + * - memory layouts defined with memmap= kernel parameter may not align + * nicely with memmap sections + * + * Explicitly initialize those struct pages so that: + * - PG_Reserved is set + * - zone and node links point to zone and node that span the page if the + * hole is in the middle of a zone + * - zone and node links point to adjacent zone/node if the hole falls on + * the zone boundary; the pages in such holes will be prepended to the + * zone/node above the hole except for the trailing pages in the last + * section that will be appended to the zone/node below. + */ +static void __init init_unavailable_range(unsigned long spfn, + unsigned long epfn, + int zone, int node) +{ + unsigned long pfn; + u64 pgcnt = 0; + + for (pfn = spfn; pfn < epfn; pfn++) { + if (!pfn_valid(pageblock_start_pfn(pfn))) { + pfn = pageblock_end_pfn(pfn) - 1; + continue; + } + __init_single_page(pfn_to_page(pfn), pfn, zone, node); + __SetPageReserved(pfn_to_page(pfn)); + pgcnt++; + } + + if (pgcnt) + pr_info("On node %d, zone %s: %lld pages in unavailable ranges", + node, zone_names[zone], pgcnt); +} + +/* + * Initially all pages are reserved - free ones are freed + * up by memblock_free_all() once the early boot process is + * done. Non-atomic initialization, single-pass. + * + * All aligned pageblocks are initialized to the specified migratetype + * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related + * zone stats (e.g., nr_isolate_pageblock) are touched. + */ +void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, + unsigned long start_pfn, unsigned long zone_end_pfn, + enum meminit_context context, + struct vmem_altmap *altmap, int migratetype) +{ + unsigned long pfn, end_pfn = start_pfn + size; + struct page *page; + + if (highest_memmap_pfn < end_pfn - 1) + highest_memmap_pfn = end_pfn - 1; + +#ifdef CONFIG_ZONE_DEVICE + /* + * Honor reservation requested by the driver for this ZONE_DEVICE + * memory. We limit the total number of pages to initialize to just + * those that might contain the memory mapping. We will defer the + * ZONE_DEVICE page initialization until after we have released + * the hotplug lock. + */ + if (zone == ZONE_DEVICE) { + if (!altmap) + return; + + if (start_pfn == altmap->base_pfn) + start_pfn += altmap->reserve; + end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); + } +#endif + + for (pfn = start_pfn; pfn < end_pfn; ) { + /* + * There can be holes in boot-time mem_map[]s handed to this + * function. They do not exist on hotplugged memory. + */ + if (context == MEMINIT_EARLY) { + if (overlap_memmap_init(zone, &pfn)) + continue; + if (defer_init(nid, pfn, zone_end_pfn)) { + deferred_struct_pages = true; + break; + } + } + + page = pfn_to_page(pfn); + __init_single_page(page, pfn, zone, nid); + if (context == MEMINIT_HOTPLUG) + __SetPageReserved(page); + + /* + * Usually, we want to mark the pageblock MIGRATE_MOVABLE, + * such that unmovable allocations won't be scattered all + * over the place during system boot. + */ + if (pageblock_aligned(pfn)) { + set_pageblock_migratetype(page, migratetype); + cond_resched(); + } + pfn++; + } +} + +static void __init memmap_init_zone_range(struct zone *zone, + unsigned long start_pfn, + unsigned long end_pfn, + unsigned long *hole_pfn) +{ + unsigned long zone_start_pfn = zone->zone_start_pfn; + unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; + int nid = zone_to_nid(zone), zone_id = zone_idx(zone); + + start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); + end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); + + if (start_pfn >= end_pfn) + return; + + memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, + zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); + + if (*hole_pfn < start_pfn) + init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); + + *hole_pfn = end_pfn; +} + +static void __init memmap_init(void) +{ + unsigned long start_pfn, end_pfn; + unsigned long hole_pfn = 0; + int i, j, zone_id = 0, nid; + + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { + struct pglist_data *node = NODE_DATA(nid); + + for (j = 0; j < MAX_NR_ZONES; j++) { + struct zone *zone = node->node_zones + j; + + if (!populated_zone(zone)) + continue; + + memmap_init_zone_range(zone, start_pfn, end_pfn, + &hole_pfn); + zone_id = j; + } + } + +#ifdef CONFIG_SPARSEMEM + /* + * Initialize the memory map for hole in the range [memory_end, + * section_end]. + * Append the pages in this hole to the highest zone in the last + * node. + * The call to init_unavailable_range() is outside the ifdef to + * silence the compiler warining about zone_id set but not used; + * for FLATMEM it is a nop anyway + */ + end_pfn = round_up(end_pfn, PAGES_PER_SECTION); + if (hole_pfn < end_pfn) +#endif + init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); +} + +#ifdef CONFIG_ZONE_DEVICE +static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, + unsigned long zone_idx, int nid, + struct dev_pagemap *pgmap) +{ + + __init_single_page(page, pfn, zone_idx, nid); + + /* + * Mark page reserved as it will need to wait for onlining + * phase for it to be fully associated with a zone. + * + * We can use the non-atomic __set_bit operation for setting + * the flag as we are still initializing the pages. + */ + __SetPageReserved(page); + + /* + * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer + * and zone_device_data. It is a bug if a ZONE_DEVICE page is + * ever freed or placed on a driver-private list. + */ + page->pgmap = pgmap; + page->zone_device_data = NULL; + + /* + * Mark the block movable so that blocks are reserved for + * movable at startup. This will force kernel allocations + * to reserve their blocks rather than leaking throughout + * the address space during boot when many long-lived + * kernel allocations are made. + * + * Please note that MEMINIT_HOTPLUG path doesn't clear memmap + * because this is done early in section_activate() + */ + if (pageblock_aligned(pfn)) { + set_pageblock_migratetype(page, MIGRATE_MOVABLE); + cond_resched(); + } + + /* + * ZONE_DEVICE pages are released directly to the driver page allocator + * which will set the page count to 1 when allocating the page. + */ + if (pgmap->type == MEMORY_DEVICE_PRIVATE || + pgmap->type == MEMORY_DEVICE_COHERENT) + set_page_count(page, 0); +} + +/* + * With compound page geometry and when struct pages are stored in ram most + * tail pages are reused. Consequently, the amount of unique struct pages to + * initialize is a lot smaller that the total amount of struct pages being + * mapped. This is a paired / mild layering violation with explicit knowledge + * of how the sparse_vmemmap internals handle compound pages in the lack + * of an altmap. See vmemmap_populate_compound_pages(). + */ +static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, + unsigned long nr_pages) +{ + return is_power_of_2(sizeof(struct page)) && + !altmap ? 2 * (PAGE_SIZE / sizeof(struct page)) : nr_pages; +} + +static void __ref memmap_init_compound(struct page *head, + unsigned long head_pfn, + unsigned long zone_idx, int nid, + struct dev_pagemap *pgmap, + unsigned long nr_pages) +{ + unsigned long pfn, end_pfn = head_pfn + nr_pages; + unsigned int order = pgmap->vmemmap_shift; + + __SetPageHead(head); + for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { + struct page *page = pfn_to_page(pfn); + + __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + prep_compound_tail(head, pfn - head_pfn); + set_page_count(page, 0); + + /* + * The first tail page stores important compound page info. + * Call prep_compound_head() after the first tail page has + * been initialized, to not have the data overwritten. + */ + if (pfn == head_pfn + 1) + prep_compound_head(head, order); + } +} + +void __ref memmap_init_zone_device(struct zone *zone, + unsigned long start_pfn, + unsigned long nr_pages, + struct dev_pagemap *pgmap) +{ + unsigned long pfn, end_pfn = start_pfn + nr_pages; + struct pglist_data *pgdat = zone->zone_pgdat; + struct vmem_altmap *altmap = pgmap_altmap(pgmap); + unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); + unsigned long zone_idx = zone_idx(zone); + unsigned long start = jiffies; + int nid = pgdat->node_id; + + if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) + return; + + /* + * The call to memmap_init should have already taken care + * of the pages reserved for the memmap, so we can just jump to + * the end of that region and start processing the device pages. + */ + if (altmap) { + start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); + nr_pages = end_pfn - start_pfn; + } + + for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { + struct page *page = pfn_to_page(pfn); + + __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + + if (pfns_per_compound == 1) + continue; + + memmap_init_compound(page, pfn, zone_idx, nid, pgmap, + compound_nr_pages(altmap, pfns_per_compound)); + } + + pr_info("%s initialised %lu pages in %ums\n", __func__, + nr_pages, jiffies_to_msecs(jiffies - start)); +} +#endif + +/* + * The zone ranges provided by the architecture do not include ZONE_MOVABLE + * because it is sized independent of architecture. Unlike the other zones, + * the starting point for ZONE_MOVABLE is not fixed. It may be different + * in each node depending on the size of each node and how evenly kernelcore + * is distributed. This helper function adjusts the zone ranges + * provided by the architecture for a given node by using the end of the + * highest usable zone for ZONE_MOVABLE. This preserves the assumption that + * zones within a node are in order of monotonic increases memory addresses + */ +static void __init adjust_zone_range_for_zone_movable(int nid, + unsigned long zone_type, + unsigned long node_start_pfn, + unsigned long node_end_pfn, + unsigned long *zone_start_pfn, + unsigned long *zone_end_pfn) +{ + /* Only adjust if ZONE_MOVABLE is on this node */ + if (zone_movable_pfn[nid]) { + /* Size ZONE_MOVABLE */ + if (zone_type == ZONE_MOVABLE) { + *zone_start_pfn = zone_movable_pfn[nid]; + *zone_end_pfn = min(node_end_pfn, + arch_zone_highest_possible_pfn[movable_zone]); + + /* Adjust for ZONE_MOVABLE starting within this range */ + } else if (!mirrored_kernelcore && + *zone_start_pfn < zone_movable_pfn[nid] && + *zone_end_pfn > zone_movable_pfn[nid]) { + *zone_end_pfn = zone_movable_pfn[nid]; + + /* Check if this whole range is within ZONE_MOVABLE */ + } else if (*zone_start_pfn >= zone_movable_pfn[nid]) + *zone_start_pfn = *zone_end_pfn; + } +} + +/* + * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, + * then all holes in the requested range will be accounted for. + */ +unsigned long __init __absent_pages_in_range(int nid, + unsigned long range_start_pfn, + unsigned long range_end_pfn) +{ + unsigned long nr_absent = range_end_pfn - range_start_pfn; + unsigned long start_pfn, end_pfn; + int i; + + for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { + start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); + end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); + nr_absent -= end_pfn - start_pfn; + } + return nr_absent; +} + +/** + * absent_pages_in_range - Return number of page frames in holes within a range + * @start_pfn: The start PFN to start searching for holes + * @end_pfn: The end PFN to stop searching for holes + * + * Return: the number of pages frames in memory holes within a range. + */ +unsigned long __init absent_pages_in_range(unsigned long start_pfn, + unsigned long end_pfn) +{ + return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); +} + +/* Return the number of page frames in holes in a zone on a node */ +static unsigned long __init zone_absent_pages_in_node(int nid, + unsigned long zone_type, + unsigned long node_start_pfn, + unsigned long node_end_pfn) +{ + unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; + unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; + unsigned long zone_start_pfn, zone_end_pfn; + unsigned long nr_absent; + + /* When hotadd a new node from cpu_up(), the node should be empty */ + if (!node_start_pfn && !node_end_pfn) + return 0; + + zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); + zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); + + adjust_zone_range_for_zone_movable(nid, zone_type, + node_start_pfn, node_end_pfn, + &zone_start_pfn, &zone_end_pfn); + nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); + + /* + * ZONE_MOVABLE handling. + * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages + * and vice versa. + */ + if (mirrored_kernelcore && zone_movable_pfn[nid]) { + unsigned long start_pfn, end_pfn; + struct memblock_region *r; + + for_each_mem_region(r) { + start_pfn = clamp(memblock_region_memory_base_pfn(r), + zone_start_pfn, zone_end_pfn); + end_pfn = clamp(memblock_region_memory_end_pfn(r), + zone_start_pfn, zone_end_pfn); + + if (zone_type == ZONE_MOVABLE && + memblock_is_mirror(r)) + nr_absent += end_pfn - start_pfn; + + if (zone_type == ZONE_NORMAL && + !memblock_is_mirror(r)) + nr_absent += end_pfn - start_pfn; + } + } + + return nr_absent; +} + +/* + * Return the number of pages a zone spans in a node, including holes + * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() + */ +static unsigned long __init zone_spanned_pages_in_node(int nid, + unsigned long zone_type, + unsigned long node_start_pfn, + unsigned long node_end_pfn, + unsigned long *zone_start_pfn, + unsigned long *zone_end_pfn) +{ + unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; + unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; + /* When hotadd a new node from cpu_up(), the node should be empty */ + if (!node_start_pfn && !node_end_pfn) + return 0; + + /* Get the start and end of the zone */ + *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); + *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); + adjust_zone_range_for_zone_movable(nid, zone_type, + node_start_pfn, node_end_pfn, + zone_start_pfn, zone_end_pfn); + + /* Check that this node has pages within the zone's required range */ + if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) + return 0; + + /* Move the zone boundaries inside the node if necessary */ + *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); + *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); + + /* Return the spanned pages */ + return *zone_end_pfn - *zone_start_pfn; +} + +static void __init calculate_node_totalpages(struct pglist_data *pgdat, + unsigned long node_start_pfn, + unsigned long node_end_pfn) +{ + unsigned long realtotalpages = 0, totalpages = 0; + enum zone_type i; + + for (i = 0; i < MAX_NR_ZONES; i++) { + struct zone *zone = pgdat->node_zones + i; + unsigned long zone_start_pfn, zone_end_pfn; + unsigned long spanned, absent; + unsigned long size, real_size; + + spanned = zone_spanned_pages_in_node(pgdat->node_id, i, + node_start_pfn, + node_end_pfn, + &zone_start_pfn, + &zone_end_pfn); + absent = zone_absent_pages_in_node(pgdat->node_id, i, + node_start_pfn, + node_end_pfn); + + size = spanned; + real_size = size - absent; + + if (size) + zone->zone_start_pfn = zone_start_pfn; + else + zone->zone_start_pfn = 0; + zone->spanned_pages = size; + zone->present_pages = real_size; +#if defined(CONFIG_MEMORY_HOTPLUG) + zone->present_early_pages = real_size; +#endif + + totalpages += size; + realtotalpages += real_size; + } + + pgdat->node_spanned_pages = totalpages; + pgdat->node_present_pages = realtotalpages; + pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); +} + +static unsigned long __init calc_memmap_size(unsigned long spanned_pages, + unsigned long present_pages) +{ + unsigned long pages = spanned_pages; + + /* + * Provide a more accurate estimation if there are holes within + * the zone and SPARSEMEM is in use. If there are holes within the + * zone, each populated memory region may cost us one or two extra + * memmap pages due to alignment because memmap pages for each + * populated regions may not be naturally aligned on page boundary. + * So the (present_pages >> 4) heuristic is a tradeoff for that. + */ + if (spanned_pages > present_pages + (present_pages >> 4) && + IS_ENABLED(CONFIG_SPARSEMEM)) + pages = present_pages; + + return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static void pgdat_init_split_queue(struct pglist_data *pgdat) +{ + struct deferred_split *ds_queue = &pgdat->deferred_split_queue; + + spin_lock_init(&ds_queue->split_queue_lock); + INIT_LIST_HEAD(&ds_queue->split_queue); + ds_queue->split_queue_len = 0; +} +#else +static void pgdat_init_split_queue(struct pglist_data *pgdat) {} +#endif + +#ifdef CONFIG_COMPACTION +static void pgdat_init_kcompactd(struct pglist_data *pgdat) +{ + init_waitqueue_head(&pgdat->kcompactd_wait); +} +#else +static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} +#endif + +static void __meminit pgdat_init_internals(struct pglist_data *pgdat) +{ + int i; + + pgdat_resize_init(pgdat); + pgdat_kswapd_lock_init(pgdat); + + pgdat_init_split_queue(pgdat); + pgdat_init_kcompactd(pgdat); + + init_waitqueue_head(&pgdat->kswapd_wait); + init_waitqueue_head(&pgdat->pfmemalloc_wait); + + for (i = 0; i < NR_VMSCAN_THROTTLE; i++) + init_waitqueue_head(&pgdat->reclaim_wait[i]); + + pgdat_page_ext_init(pgdat); + lruvec_init(&pgdat->__lruvec); +} + +static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, + unsigned long remaining_pages) +{ + atomic_long_set(&zone->managed_pages, remaining_pages); + zone_set_nid(zone, nid); + zone->name = zone_names[idx]; + zone->zone_pgdat = NODE_DATA(nid); + spin_lock_init(&zone->lock); + zone_seqlock_init(zone); + zone_pcp_init(zone); +} + +static void __meminit zone_init_free_lists(struct zone *zone) +{ + unsigned int order, t; + for_each_migratetype_order(order, t) { + INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); + zone->free_area[order].nr_free = 0; + } +} + +void __meminit init_currently_empty_zone(struct zone *zone, + unsigned long zone_start_pfn, + unsigned long size) +{ + struct pglist_data *pgdat = zone->zone_pgdat; + int zone_idx = zone_idx(zone) + 1; + + if (zone_idx > pgdat->nr_zones) + pgdat->nr_zones = zone_idx; + + zone->zone_start_pfn = zone_start_pfn; + + mminit_dprintk(MMINIT_TRACE, "memmap_init", + "Initialising map node %d zone %lu pfns %lu -> %lu\n", + pgdat->node_id, + (unsigned long)zone_idx(zone), + zone_start_pfn, (zone_start_pfn + size)); + + zone_init_free_lists(zone); + zone->initialized = 1; +} + +#ifndef CONFIG_SPARSEMEM +/* + * Calculate the size of the zone->blockflags rounded to an unsigned long + * Start by making sure zonesize is a multiple of pageblock_order by rounding + * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally + * round what is now in bits to nearest long in bits, then return it in + * bytes. + */ +static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) +{ + unsigned long usemapsize; + + zonesize += zone_start_pfn & (pageblock_nr_pages-1); + usemapsize = roundup(zonesize, pageblock_nr_pages); + usemapsize = usemapsize >> pageblock_order; + usemapsize *= NR_PAGEBLOCK_BITS; + usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); + + return usemapsize / 8; +} + +static void __ref setup_usemap(struct zone *zone) +{ + unsigned long usemapsize = usemap_size(zone->zone_start_pfn, + zone->spanned_pages); + zone->pageblock_flags = NULL; + if (usemapsize) { + zone->pageblock_flags = + memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, + zone_to_nid(zone)); + if (!zone->pageblock_flags) + panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", + usemapsize, zone->name, zone_to_nid(zone)); + } +} +#else +static inline void setup_usemap(struct zone *zone) {} +#endif /* CONFIG_SPARSEMEM */ + +#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE + +/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ +void __init set_pageblock_order(void) +{ + unsigned int order = MAX_ORDER; + + /* Check that pageblock_nr_pages has not already been setup */ + if (pageblock_order) + return; + + /* Don't let pageblocks exceed the maximum allocation granularity. */ + if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order) + order = HUGETLB_PAGE_ORDER; + + /* + * Assume the largest contiguous order of interest is a huge page. + * This value may be variable depending on boot parameters on IA64 and + * powerpc. + */ + pageblock_order = order; +} +#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ + +/* + * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() + * is unused as pageblock_order is set at compile-time. See + * include/linux/pageblock-flags.h for the values of pageblock_order based on + * the kernel config + */ +void __init set_pageblock_order(void) +{ +} + +#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ + +/* + * Set up the zone data structures + * - init pgdat internals + * - init all zones belonging to this node + * + * NOTE: this function is only called during memory hotplug + */ +#ifdef CONFIG_MEMORY_HOTPLUG +void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) +{ + int nid = pgdat->node_id; + enum zone_type z; + int cpu; + + pgdat_init_internals(pgdat); + + if (pgdat->per_cpu_nodestats == &boot_nodestats) + pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); + + /* + * Reset the nr_zones, order and highest_zoneidx before reuse. + * Note that kswapd will init kswapd_highest_zoneidx properly + * when it starts in the near future. + */ + pgdat->nr_zones = 0; + pgdat->kswapd_order = 0; + pgdat->kswapd_highest_zoneidx = 0; + pgdat->node_start_pfn = 0; + for_each_online_cpu(cpu) { + struct per_cpu_nodestat *p; + + p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); + memset(p, 0, sizeof(*p)); + } + + for (z = 0; z < MAX_NR_ZONES; z++) + zone_init_internals(&pgdat->node_zones[z], z, nid, 0); +} +#endif + +/* + * Set up the zone data structures: + * - mark all pages reserved + * - mark all memory queues empty + * - clear the memory bitmaps + * + * NOTE: pgdat should get zeroed by caller. + * NOTE: this function is only called during early init. + */ +static void __init free_area_init_core(struct pglist_data *pgdat) +{ + enum zone_type j; + int nid = pgdat->node_id; + + pgdat_init_internals(pgdat); + pgdat->per_cpu_nodestats = &boot_nodestats; + + for (j = 0; j < MAX_NR_ZONES; j++) { + struct zone *zone = pgdat->node_zones + j; + unsigned long size, freesize, memmap_pages; + + size = zone->spanned_pages; + freesize = zone->present_pages; + + /* + * Adjust freesize so that it accounts for how much memory + * is used by this zone for memmap. This affects the watermark + * and per-cpu initialisations + */ + memmap_pages = calc_memmap_size(size, freesize); + if (!is_highmem_idx(j)) { + if (freesize >= memmap_pages) { + freesize -= memmap_pages; + if (memmap_pages) + pr_debug(" %s zone: %lu pages used for memmap\n", + zone_names[j], memmap_pages); + } else + pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n", + zone_names[j], memmap_pages, freesize); + } + + /* Account for reserved pages */ + if (j == 0 && freesize > dma_reserve) { + freesize -= dma_reserve; + pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve); + } + + if (!is_highmem_idx(j)) + nr_kernel_pages += freesize; + /* Charge for highmem memmap if there are enough kernel pages */ + else if (nr_kernel_pages > memmap_pages * 2) + nr_kernel_pages -= memmap_pages; + nr_all_pages += freesize; + + /* + * Set an approximate value for lowmem here, it will be adjusted + * when the bootmem allocator frees pages into the buddy system. + * And all highmem pages will be managed by the buddy system. + */ + zone_init_internals(zone, j, nid, freesize); + + if (!size) + continue; + + set_pageblock_order(); + setup_usemap(zone); + init_currently_empty_zone(zone, zone->zone_start_pfn, size); + } +} + +void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, + phys_addr_t min_addr, int nid, bool exact_nid) +{ + void *ptr; + + if (exact_nid) + ptr = memblock_alloc_exact_nid_raw(size, align, min_addr, + MEMBLOCK_ALLOC_ACCESSIBLE, + nid); + else + ptr = memblock_alloc_try_nid_raw(size, align, min_addr, + MEMBLOCK_ALLOC_ACCESSIBLE, + nid); + + if (ptr && size > 0) + page_init_poison(ptr, size); + + return ptr; +} + +#ifdef CONFIG_FLATMEM +static void __init alloc_node_mem_map(struct pglist_data *pgdat) +{ + unsigned long __maybe_unused start = 0; + unsigned long __maybe_unused offset = 0; + + /* Skip empty nodes */ + if (!pgdat->node_spanned_pages) + return; + + start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); + offset = pgdat->node_start_pfn - start; + /* ia64 gets its own node_mem_map, before this, without bootmem */ + if (!pgdat->node_mem_map) { + unsigned long size, end; + struct page *map; + + /* + * The zone's endpoints aren't required to be MAX_ORDER + * aligned but the node_mem_map endpoints must be in order + * for the buddy allocator to function correctly. + */ + end = pgdat_end_pfn(pgdat); + end = ALIGN(end, MAX_ORDER_NR_PAGES); + size = (end - start) * sizeof(struct page); + map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, + pgdat->node_id, false); + if (!map) + panic("Failed to allocate %ld bytes for node %d memory map\n", + size, pgdat->node_id); + pgdat->node_mem_map = map + offset; + } + pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", + __func__, pgdat->node_id, (unsigned long)pgdat, + (unsigned long)pgdat->node_mem_map); +#ifndef CONFIG_NUMA + /* + * With no DISCONTIG, the global mem_map is just set as node 0's + */ + if (pgdat == NODE_DATA(0)) { + mem_map = NODE_DATA(0)->node_mem_map; + if (page_to_pfn(mem_map) != pgdat->node_start_pfn) + mem_map -= offset; + } +#endif +} +#else +static inline void alloc_node_mem_map(struct pglist_data *pgdat) { } +#endif /* CONFIG_FLATMEM */ + +/** + * get_pfn_range_for_nid - Return the start and end page frames for a node + * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. + * @start_pfn: Passed by reference. On return, it will have the node start_pfn. + * @end_pfn: Passed by reference. On return, it will have the node end_pfn. + * + * It returns the start and end page frame of a node based on information + * provided by memblock_set_node(). If called for a node + * with no available memory, a warning is printed and the start and end + * PFNs will be 0. + */ +void __init get_pfn_range_for_nid(unsigned int nid, + unsigned long *start_pfn, unsigned long *end_pfn) +{ + unsigned long this_start_pfn, this_end_pfn; + int i; + + *start_pfn = -1UL; + *end_pfn = 0; + + for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { + *start_pfn = min(*start_pfn, this_start_pfn); + *end_pfn = max(*end_pfn, this_end_pfn); + } + + if (*start_pfn == -1UL) + *start_pfn = 0; +} + +static void __init free_area_init_node(int nid) +{ + pg_data_t *pgdat = NODE_DATA(nid); + unsigned long start_pfn = 0; + unsigned long end_pfn = 0; + + /* pg_data_t should be reset to zero when it's allocated */ + WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); + + get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); + + pgdat->node_id = nid; + pgdat->node_start_pfn = start_pfn; + pgdat->per_cpu_nodestats = NULL; + + if (start_pfn != end_pfn) { + pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, + (u64)start_pfn << PAGE_SHIFT, + end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); + } else { + pr_info("Initmem setup node %d as memoryless\n", nid); + } + + calculate_node_totalpages(pgdat, start_pfn, end_pfn); + + alloc_node_mem_map(pgdat); + pgdat_set_deferred_range(pgdat); + + free_area_init_core(pgdat); + lru_gen_init_pgdat(pgdat); +} + +/* Any regular or high memory on that node ? */ +static void check_for_memory(pg_data_t *pgdat, int nid) +{ + enum zone_type zone_type; + + for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { + struct zone *zone = &pgdat->node_zones[zone_type]; + if (populated_zone(zone)) { + if (IS_ENABLED(CONFIG_HIGHMEM)) + node_set_state(nid, N_HIGH_MEMORY); + if (zone_type <= ZONE_NORMAL) + node_set_state(nid, N_NORMAL_MEMORY); + break; + } + } +} + +#if MAX_NUMNODES > 1 +/* + * Figure out the number of possible node ids. + */ +void __init setup_nr_node_ids(void) +{ + unsigned int highest; + + highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); + nr_node_ids = highest + 1; +} +#endif + +static void __init free_area_init_memoryless_node(int nid) +{ + free_area_init_node(nid); +} + +/* + * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For + * such cases we allow max_zone_pfn sorted in the descending order + */ +bool __weak arch_has_descending_max_zone_pfns(void) +{ + return false; +} + +/** + * free_area_init - Initialise all pg_data_t and zone data + * @max_zone_pfn: an array of max PFNs for each zone + * + * This will call free_area_init_node() for each active node in the system. + * Using the page ranges provided by memblock_set_node(), the size of each + * zone in each node and their holes is calculated. If the maximum PFN + * between two adjacent zones match, it is assumed that the zone is empty. + * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed + * that arch_max_dma32_pfn has no pages. It is also assumed that a zone + * starts where the previous one ended. For example, ZONE_DMA32 starts + * at arch_max_dma_pfn. + */ +void __init free_area_init(unsigned long *max_zone_pfn) +{ + unsigned long start_pfn, end_pfn; + int i, nid, zone; + bool descending; + + /* Record where the zone boundaries are */ + memset(arch_zone_lowest_possible_pfn, 0, + sizeof(arch_zone_lowest_possible_pfn)); + memset(arch_zone_highest_possible_pfn, 0, + sizeof(arch_zone_highest_possible_pfn)); + + start_pfn = PHYS_PFN(memblock_start_of_DRAM()); + descending = arch_has_descending_max_zone_pfns(); + + for (i = 0; i < MAX_NR_ZONES; i++) { + if (descending) + zone = MAX_NR_ZONES - i - 1; + else + zone = i; + + if (zone == ZONE_MOVABLE) + continue; + + end_pfn = max(max_zone_pfn[zone], start_pfn); + arch_zone_lowest_possible_pfn[zone] = start_pfn; + arch_zone_highest_possible_pfn[zone] = end_pfn; + + start_pfn = end_pfn; + } + + /* Find the PFNs that ZONE_MOVABLE begins at in each node */ + memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); + find_zone_movable_pfns_for_nodes(); + + /* Print out the zone ranges */ + pr_info("Zone ranges:\n"); + for (i = 0; i < MAX_NR_ZONES; i++) { + if (i == ZONE_MOVABLE) + continue; + pr_info(" %-8s ", zone_names[i]); + if (arch_zone_lowest_possible_pfn[i] == + arch_zone_highest_possible_pfn[i]) + pr_cont("empty\n"); + else + pr_cont("[mem %#018Lx-%#018Lx]\n", + (u64)arch_zone_lowest_possible_pfn[i] + << PAGE_SHIFT, + ((u64)arch_zone_highest_possible_pfn[i] + << PAGE_SHIFT) - 1); + } + + /* Print out the PFNs ZONE_MOVABLE begins at in each node */ + pr_info("Movable zone start for each node\n"); + for (i = 0; i < MAX_NUMNODES; i++) { + if (zone_movable_pfn[i]) + pr_info(" Node %d: %#018Lx\n", i, + (u64)zone_movable_pfn[i] << PAGE_SHIFT); + } + + /* + * Print out the early node map, and initialize the + * subsection-map relative to active online memory ranges to + * enable future "sub-section" extensions of the memory map. + */ + pr_info("Early memory node ranges\n"); + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { + pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, + (u64)start_pfn << PAGE_SHIFT, + ((u64)end_pfn << PAGE_SHIFT) - 1); + subsection_map_init(start_pfn, end_pfn - start_pfn); + } + + /* Initialise every node */ + mminit_verify_pageflags_layout(); + setup_nr_node_ids(); + for_each_node(nid) { + pg_data_t *pgdat; + + if (!node_online(nid)) { + pr_info("Initializing node %d as memoryless\n", nid); + + /* Allocator not initialized yet */ + pgdat = arch_alloc_nodedata(nid); + if (!pgdat) + panic("Cannot allocate %zuB for node %d.\n", + sizeof(*pgdat), nid); + arch_refresh_nodedata(nid, pgdat); + free_area_init_memoryless_node(nid); + + /* + * We do not want to confuse userspace by sysfs + * files/directories for node without any memory + * attached to it, so this node is not marked as + * N_MEMORY and not marked online so that no sysfs + * hierarchy will be created via register_one_node for + * it. The pgdat will get fully initialized by + * hotadd_init_pgdat() when memory is hotplugged into + * this node. + */ + continue; + } + + pgdat = NODE_DATA(nid); + free_area_init_node(nid); + + /* Any memory on that node */ + if (pgdat->node_present_pages) + node_set_state(nid, N_MEMORY); + check_for_memory(pgdat, nid); + } + + memmap_init(); +} + +/** + * node_map_pfn_alignment - determine the maximum internode alignment + * + * This function should be called after node map is populated and sorted. + * It calculates the maximum power of two alignment which can distinguish + * all the nodes. + * + * For example, if all nodes are 1GiB and aligned to 1GiB, the return value + * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the + * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is + * shifted, 1GiB is enough and this function will indicate so. + * + * This is used to test whether pfn -> nid mapping of the chosen memory + * model has fine enough granularity to avoid incorrect mapping for the + * populated node map. + * + * Return: the determined alignment in pfn's. 0 if there is no alignment + * requirement (single node). + */ +unsigned long __init node_map_pfn_alignment(void) +{ + unsigned long accl_mask = 0, last_end = 0; + unsigned long start, end, mask; + int last_nid = NUMA_NO_NODE; + int i, nid; + + for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { + if (!start || last_nid < 0 || last_nid == nid) { + last_nid = nid; + last_end = end; + continue; + } + + /* + * Start with a mask granular enough to pin-point to the + * start pfn and tick off bits one-by-one until it becomes + * too coarse to separate the current node from the last. + */ + mask = ~((1 << __ffs(start)) - 1); + while (mask && last_end <= (start & (mask << 1))) + mask <<= 1; + + /* accumulate all internode masks */ + accl_mask |= mask; + } + + /* convert mask to number of pages */ + return ~accl_mask + 1; +} + +#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT +static void __init deferred_free_range(unsigned long pfn, + unsigned long nr_pages) +{ + struct page *page; + unsigned long i; + + if (!nr_pages) + return; + + page = pfn_to_page(pfn); + + /* Free a large naturally-aligned chunk if possible */ + if (nr_pages == pageblock_nr_pages && pageblock_aligned(pfn)) { + set_pageblock_migratetype(page, MIGRATE_MOVABLE); + __free_pages_core(page, pageblock_order); + return; + } + + for (i = 0; i < nr_pages; i++, page++, pfn++) { + if (pageblock_aligned(pfn)) + set_pageblock_migratetype(page, MIGRATE_MOVABLE); + __free_pages_core(page, 0); + } +} + +/* Completion tracking for deferred_init_memmap() threads */ +static atomic_t pgdat_init_n_undone __initdata; +static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); + +static inline void __init pgdat_init_report_one_done(void) +{ + if (atomic_dec_and_test(&pgdat_init_n_undone)) + complete(&pgdat_init_all_done_comp); +} + +/* + * Returns true if page needs to be initialized or freed to buddy allocator. + * + * We check if a current large page is valid by only checking the validity + * of the head pfn. + */ +static inline bool __init deferred_pfn_valid(unsigned long pfn) +{ + if (pageblock_aligned(pfn) && !pfn_valid(pfn)) + return false; + return true; +} + +/* + * Free pages to buddy allocator. Try to free aligned pages in + * pageblock_nr_pages sizes. + */ +static void __init deferred_free_pages(unsigned long pfn, + unsigned long end_pfn) +{ + unsigned long nr_free = 0; + + for (; pfn < end_pfn; pfn++) { + if (!deferred_pfn_valid(pfn)) { + deferred_free_range(pfn - nr_free, nr_free); + nr_free = 0; + } else if (pageblock_aligned(pfn)) { + deferred_free_range(pfn - nr_free, nr_free); + nr_free = 1; + } else { + nr_free++; + } + } + /* Free the last block of pages to allocator */ + deferred_free_range(pfn - nr_free, nr_free); +} + +/* + * Initialize struct pages. We minimize pfn page lookups and scheduler checks + * by performing it only once every pageblock_nr_pages. + * Return number of pages initialized. + */ +static unsigned long __init deferred_init_pages(struct zone *zone, + unsigned long pfn, + unsigned long end_pfn) +{ + int nid = zone_to_nid(zone); + unsigned long nr_pages = 0; + int zid = zone_idx(zone); + struct page *page = NULL; + + for (; pfn < end_pfn; pfn++) { + if (!deferred_pfn_valid(pfn)) { + page = NULL; + continue; + } else if (!page || pageblock_aligned(pfn)) { + page = pfn_to_page(pfn); + } else { + page++; + } + __init_single_page(page, pfn, zid, nid); + nr_pages++; + } + return (nr_pages); +} + +/* + * This function is meant to pre-load the iterator for the zone init. + * Specifically it walks through the ranges until we are caught up to the + * first_init_pfn value and exits there. If we never encounter the value we + * return false indicating there are no valid ranges left. + */ +static bool __init +deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, + unsigned long *spfn, unsigned long *epfn, + unsigned long first_init_pfn) +{ + u64 j; + + /* + * Start out by walking through the ranges in this zone that have + * already been initialized. We don't need to do anything with them + * so we just need to flush them out of the system. + */ + for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { + if (*epfn <= first_init_pfn) + continue; + if (*spfn < first_init_pfn) + *spfn = first_init_pfn; + *i = j; + return true; + } + + return false; +} + +/* + * Initialize and free pages. We do it in two loops: first we initialize + * struct page, then free to buddy allocator, because while we are + * freeing pages we can access pages that are ahead (computing buddy + * page in __free_one_page()). + * + * In order to try and keep some memory in the cache we have the loop + * broken along max page order boundaries. This way we will not cause + * any issues with the buddy page computation. + */ +static unsigned long __init +deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, + unsigned long *end_pfn) +{ + unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); + unsigned long spfn = *start_pfn, epfn = *end_pfn; + unsigned long nr_pages = 0; + u64 j = *i; + + /* First we loop through and initialize the page values */ + for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { + unsigned long t; + + if (mo_pfn <= *start_pfn) + break; + + t = min(mo_pfn, *end_pfn); + nr_pages += deferred_init_pages(zone, *start_pfn, t); + + if (mo_pfn < *end_pfn) { + *start_pfn = mo_pfn; + break; + } + } + + /* Reset values and now loop through freeing pages as needed */ + swap(j, *i); + + for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { + unsigned long t; + + if (mo_pfn <= spfn) + break; + + t = min(mo_pfn, epfn); + deferred_free_pages(spfn, t); + + if (mo_pfn <= epfn) + break; + } + + return nr_pages; +} + +static void __init +deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, + void *arg) +{ + unsigned long spfn, epfn; + struct zone *zone = arg; + u64 i; + + deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); + + /* + * Initialize and free pages in MAX_ORDER sized increments so that we + * can avoid introducing any issues with the buddy allocator. + */ + while (spfn < end_pfn) { + deferred_init_maxorder(&i, zone, &spfn, &epfn); + cond_resched(); + } +} + +/* An arch may override for more concurrency. */ +__weak int __init +deferred_page_init_max_threads(const struct cpumask *node_cpumask) +{ + return 1; +} + +/* Initialise remaining memory on a node */ +static int __init deferred_init_memmap(void *data) +{ + pg_data_t *pgdat = data; + const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); + unsigned long spfn = 0, epfn = 0; + unsigned long first_init_pfn, flags; + unsigned long start = jiffies; + struct zone *zone; + int zid, max_threads; + u64 i; + + /* Bind memory initialisation thread to a local node if possible */ + if (!cpumask_empty(cpumask)) + set_cpus_allowed_ptr(current, cpumask); + + pgdat_resize_lock(pgdat, &flags); + first_init_pfn = pgdat->first_deferred_pfn; + if (first_init_pfn == ULONG_MAX) { + pgdat_resize_unlock(pgdat, &flags); + pgdat_init_report_one_done(); + return 0; + } + + /* Sanity check boundaries */ + BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); + BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); + pgdat->first_deferred_pfn = ULONG_MAX; + + /* + * Once we unlock here, the zone cannot be grown anymore, thus if an + * interrupt thread must allocate this early in boot, zone must be + * pre-grown prior to start of deferred page initialization. + */ + pgdat_resize_unlock(pgdat, &flags); + + /* Only the highest zone is deferred so find it */ + for (zid = 0; zid < MAX_NR_ZONES; zid++) { + zone = pgdat->node_zones + zid; + if (first_init_pfn < zone_end_pfn(zone)) + break; + } + + /* If the zone is empty somebody else may have cleared out the zone */ + if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, + first_init_pfn)) + goto zone_empty; + + max_threads = deferred_page_init_max_threads(cpumask); + + while (spfn < epfn) { + unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); + struct padata_mt_job job = { + .thread_fn = deferred_init_memmap_chunk, + .fn_arg = zone, + .start = spfn, + .size = epfn_align - spfn, + .align = PAGES_PER_SECTION, + .min_chunk = PAGES_PER_SECTION, + .max_threads = max_threads, + }; + + padata_do_multithreaded(&job); + deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, + epfn_align); + } +zone_empty: + /* Sanity check that the next zone really is unpopulated */ + WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); + + pr_info("node %d deferred pages initialised in %ums\n", + pgdat->node_id, jiffies_to_msecs(jiffies - start)); + + pgdat_init_report_one_done(); + return 0; +} + +/* + * If this zone has deferred pages, try to grow it by initializing enough + * deferred pages to satisfy the allocation specified by order, rounded up to + * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments + * of SECTION_SIZE bytes by initializing struct pages in increments of + * PAGES_PER_SECTION * sizeof(struct page) bytes. + * + * Return true when zone was grown, otherwise return false. We return true even + * when we grow less than requested, to let the caller decide if there are + * enough pages to satisfy the allocation. + * + * Note: We use noinline because this function is needed only during boot, and + * it is called from a __ref function _deferred_grow_zone. This way we are + * making sure that it is not inlined into permanent text section. + */ +bool __init deferred_grow_zone(struct zone *zone, unsigned int order) +{ + unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); + pg_data_t *pgdat = zone->zone_pgdat; + unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; + unsigned long spfn, epfn, flags; + unsigned long nr_pages = 0; + u64 i; + + /* Only the last zone may have deferred pages */ + if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) + return false; + + pgdat_resize_lock(pgdat, &flags); + + /* + * If someone grew this zone while we were waiting for spinlock, return + * true, as there might be enough pages already. + */ + if (first_deferred_pfn != pgdat->first_deferred_pfn) { + pgdat_resize_unlock(pgdat, &flags); + return true; + } + + /* If the zone is empty somebody else may have cleared out the zone */ + if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, + first_deferred_pfn)) { + pgdat->first_deferred_pfn = ULONG_MAX; + pgdat_resize_unlock(pgdat, &flags); + /* Retry only once. */ + return first_deferred_pfn != ULONG_MAX; + } + + /* + * Initialize and free pages in MAX_ORDER sized increments so + * that we can avoid introducing any issues with the buddy + * allocator. + */ + while (spfn < epfn) { + /* update our first deferred PFN for this section */ + first_deferred_pfn = spfn; + + nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); + touch_nmi_watchdog(); + + /* We should only stop along section boundaries */ + if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) + continue; + + /* If our quota has been met we can stop here */ + if (nr_pages >= nr_pages_needed) + break; + } + + pgdat->first_deferred_pfn = spfn; + pgdat_resize_unlock(pgdat, &flags); + + return nr_pages > 0; +} + +#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ + +#ifdef CONFIG_CMA +void __init init_cma_reserved_pageblock(struct page *page) +{ + unsigned i = pageblock_nr_pages; + struct page *p = page; + + do { + __ClearPageReserved(p); + set_page_count(p, 0); + } while (++p, --i); + + set_pageblock_migratetype(page, MIGRATE_CMA); + set_page_refcounted(page); + __free_pages(page, pageblock_order); + + adjust_managed_page_count(page, pageblock_nr_pages); + page_zone(page)->cma_pages += pageblock_nr_pages; +} +#endif + +void __init page_alloc_init_late(void) +{ + struct zone *zone; + int nid; + +#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT + + /* There will be num_node_state(N_MEMORY) threads */ + atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); + for_each_node_state(nid, N_MEMORY) { + kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); + } + + /* Block until all are initialised */ + wait_for_completion(&pgdat_init_all_done_comp); + + /* + * We initialized the rest of the deferred pages. Permanently disable + * on-demand struct page initialization. + */ + static_branch_disable(&deferred_pages); + + /* Reinit limits that are based on free pages after the kernel is up */ + files_maxfiles_init(); +#endif + + buffer_init(); + + /* Discard memblock private memory */ + memblock_discard(); + + for_each_node_state(nid, N_MEMORY) + shuffle_free_memory(NODE_DATA(nid)); + + for_each_populated_zone(zone) + set_zone_contiguous(zone); +} + +#ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES +/* + * Returns the number of pages that arch has reserved but + * is not known to alloc_large_system_hash(). + */ +static unsigned long __init arch_reserved_kernel_pages(void) +{ + return 0; +} +#endif + +/* + * Adaptive scale is meant to reduce sizes of hash tables on large memory + * machines. As memory size is increased the scale is also increased but at + * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory + * quadruples the scale is increased by one, which means the size of hash table + * only doubles, instead of quadrupling as well. + * Because 32-bit systems cannot have large physical memory, where this scaling + * makes sense, it is disabled on such platforms. + */ +#if __BITS_PER_LONG > 32 +#define ADAPT_SCALE_BASE (64ul << 30) +#define ADAPT_SCALE_SHIFT 2 +#define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) +#endif + +/* + * allocate a large system hash table from bootmem + * - it is assumed that the hash table must contain an exact power-of-2 + * quantity of entries + * - limit is the number of hash buckets, not the total allocation size + */ +void *__init alloc_large_system_hash(const char *tablename, + unsigned long bucketsize, + unsigned long numentries, + int scale, + int flags, + unsigned int *_hash_shift, + unsigned int *_hash_mask, + unsigned long low_limit, + unsigned long high_limit) +{ + unsigned long long max = high_limit; + unsigned long log2qty, size; + void *table; + gfp_t gfp_flags; + bool virt; + bool huge; + + /* allow the kernel cmdline to have a say */ + if (!numentries) { + /* round applicable memory size up to nearest megabyte */ + numentries = nr_kernel_pages; + numentries -= arch_reserved_kernel_pages(); + + /* It isn't necessary when PAGE_SIZE >= 1MB */ + if (PAGE_SIZE < SZ_1M) + numentries = round_up(numentries, SZ_1M / PAGE_SIZE); + +#if __BITS_PER_LONG > 32 + if (!high_limit) { + unsigned long adapt; + + for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; + adapt <<= ADAPT_SCALE_SHIFT) + scale++; + } +#endif + + /* limit to 1 bucket per 2^scale bytes of low memory */ + if (scale > PAGE_SHIFT) + numentries >>= (scale - PAGE_SHIFT); + else + numentries <<= (PAGE_SHIFT - scale); + + /* Make sure we've got at least a 0-order allocation.. */ + if (unlikely(flags & HASH_SMALL)) { + /* Makes no sense without HASH_EARLY */ + WARN_ON(!(flags & HASH_EARLY)); + if (!(numentries >> *_hash_shift)) { + numentries = 1UL << *_hash_shift; + BUG_ON(!numentries); + } + } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) + numentries = PAGE_SIZE / bucketsize; + } + numentries = roundup_pow_of_two(numentries); + + /* limit allocation size to 1/16 total memory by default */ + if (max == 0) { + max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; + do_div(max, bucketsize); + } + max = min(max, 0x80000000ULL); + + if (numentries < low_limit) + numentries = low_limit; + if (numentries > max) + numentries = max; + + log2qty = ilog2(numentries); + + gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; + do { + virt = false; + size = bucketsize << log2qty; + if (flags & HASH_EARLY) { + if (flags & HASH_ZERO) + table = memblock_alloc(size, SMP_CACHE_BYTES); + else + table = memblock_alloc_raw(size, + SMP_CACHE_BYTES); + } else if (get_order(size) > MAX_ORDER || hashdist) { + table = vmalloc_huge(size, gfp_flags); + virt = true; + if (table) + huge = is_vm_area_hugepages(table); + } else { + /* + * If bucketsize is not a power-of-two, we may free + * some pages at the end of hash table which + * alloc_pages_exact() automatically does + */ + table = alloc_pages_exact(size, gfp_flags); + kmemleak_alloc(table, size, 1, gfp_flags); + } + } while (!table && size > PAGE_SIZE && --log2qty); + + if (!table) + panic("Failed to allocate %s hash table\n", tablename); + + pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", + tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, + virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); + + if (_hash_shift) + *_hash_shift = log2qty; + if (_hash_mask) + *_hash_mask = (1 << log2qty) - 1; + + return table; +} + +/** + * set_dma_reserve - set the specified number of pages reserved in the first zone + * @new_dma_reserve: The number of pages to mark reserved + * + * The per-cpu batchsize and zone watermarks are determined by managed_pages. + * In the DMA zone, a significant percentage may be consumed by kernel image + * and other unfreeable allocations which can skew the watermarks badly. This + * function may optionally be used to account for unfreeable pages in the + * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and + * smaller per-cpu batchsize. + */ +void __init set_dma_reserve(unsigned long new_dma_reserve) +{ + dma_reserve = new_dma_reserve; +} + +void __init memblock_free_pages(struct page *page, unsigned long pfn, + unsigned int order) +{ + if (!early_page_initialised(pfn)) + return; + if (!kmsan_memblock_free_pages(page, order)) { + /* KMSAN will take care of these pages. */ + return; + } + __free_pages_core(page, order); +} diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 33925488040f..8adc70b6034d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -72,9 +72,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -355,7 +353,7 @@ int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { [ZONE_MOVABLE] = 0, }; -static char * const zone_names[MAX_NR_ZONES] = { +char * const zone_names[MAX_NR_ZONES] = { #ifdef CONFIG_ZONE_DMA "DMA", #endif @@ -401,17 +399,6 @@ int user_min_free_kbytes = -1; int watermark_boost_factor __read_mostly = 15000; int watermark_scale_factor = 10; -static unsigned long nr_kernel_pages __initdata; -static unsigned long nr_all_pages __initdata; -static unsigned long dma_reserve __initdata; - -static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; -static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; -static unsigned long required_kernelcore __initdata; -static unsigned long required_kernelcore_percent __initdata; -static unsigned long required_movablecore __initdata; -static unsigned long required_movablecore_percent __initdata; -static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; bool mirrored_kernelcore __initdata_memblock; /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ @@ -427,86 +414,36 @@ EXPORT_SYMBOL(nr_online_nodes); int page_group_by_mobility_disabled __read_mostly; -bool deferred_struct_pages __meminitdata; - #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT /* * During boot we initialize deferred pages on-demand, as needed, but once * page_alloc_init_late() has finished, the deferred pages are all initialized, * and we can permanently disable that path. */ -static DEFINE_STATIC_KEY_TRUE(deferred_pages); +DEFINE_STATIC_KEY_TRUE(deferred_pages); static inline bool deferred_pages_enabled(void) { return static_branch_unlikely(&deferred_pages); } -/* Returns true if the struct page for the pfn is initialised */ -static inline bool __meminit early_page_initialised(unsigned long pfn) -{ - int nid = early_pfn_to_nid(pfn); - - if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) - return false; - - return true; -} - /* - * Returns true when the remaining initialisation should be deferred until - * later in the boot cycle when it can be parallelised. + * deferred_grow_zone() is __init, but it is called from + * get_page_from_freelist() during early boot until deferred_pages permanently + * disables this call. This is why we have refdata wrapper to avoid warning, + * and to ensure that the function body gets unloaded. */ -static bool __meminit -defer_init(int nid, unsigned long pfn, unsigned long end_pfn) +static bool __ref +_deferred_grow_zone(struct zone *zone, unsigned int order) { - static unsigned long prev_end_pfn, nr_initialised; - - if (early_page_ext_enabled()) - return false; - /* - * prev_end_pfn static that contains the end of previous zone - * No need to protect because called very early in boot before smp_init. - */ - if (prev_end_pfn != end_pfn) { - prev_end_pfn = end_pfn; - nr_initialised = 0; - } - - /* Always populate low zones for address-constrained allocations */ - if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) - return false; - - if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) - return true; - /* - * We start only with one section of pages, more pages are added as - * needed until the rest of deferred pages are initialized. - */ - nr_initialised++; - if ((nr_initialised > PAGES_PER_SECTION) && - (pfn & (PAGES_PER_SECTION - 1)) == 0) { - NODE_DATA(nid)->first_deferred_pfn = pfn; - return true; - } - return false; + return deferred_grow_zone(zone, order); } #else static inline bool deferred_pages_enabled(void) { return false; } - -static inline bool early_page_initialised(unsigned long pfn) -{ - return true; -} - -static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) -{ - return false; -} -#endif +#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ /* Return a pointer to the bitmap storing bits affecting a block of pages */ static inline unsigned long *get_pageblock_bitmap(const struct page *page, @@ -772,26 +709,6 @@ void free_compound_page(struct page *page) free_the_page(page, compound_order(page)); } -static void prep_compound_head(struct page *page, unsigned int order) -{ - struct folio *folio = (struct folio *)page; - - set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); - set_compound_order(page, order); - atomic_set(&folio->_entire_mapcount, -1); - atomic_set(&folio->_nr_pages_mapped, 0); - atomic_set(&folio->_pincount, 0); -} - -static void prep_compound_tail(struct page *head, int tail_idx) -{ - struct page *p = head + tail_idx; - - p->mapping = TAIL_MAPPING; - set_compound_head(p, head); - set_page_private(p, 0); -} - void prep_compound_page(struct page *page, unsigned int order) { int i; @@ -1608,80 +1525,6 @@ static void free_one_page(struct zone *zone, spin_unlock_irqrestore(&zone->lock, flags); } -static void __meminit __init_single_page(struct page *page, unsigned long pfn, - unsigned long zone, int nid) -{ - mm_zero_struct_page(page); - set_page_links(page, zone, nid, pfn); - init_page_count(page); - page_mapcount_reset(page); - page_cpupid_reset_last(page); - page_kasan_tag_reset(page); - - INIT_LIST_HEAD(&page->lru); -#ifdef WANT_PAGE_VIRTUAL - /* The shift won't overflow because ZONE_NORMAL is below 4G. */ - if (!is_highmem_idx(zone)) - set_page_address(page, __va(pfn << PAGE_SHIFT)); -#endif -} - -#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT -static void __meminit init_reserved_page(unsigned long pfn) -{ - pg_data_t *pgdat; - int nid, zid; - - if (early_page_initialised(pfn)) - return; - - nid = early_pfn_to_nid(pfn); - pgdat = NODE_DATA(nid); - - for (zid = 0; zid < MAX_NR_ZONES; zid++) { - struct zone *zone = &pgdat->node_zones[zid]; - - if (zone_spans_pfn(zone, pfn)) - break; - } - __init_single_page(pfn_to_page(pfn), pfn, zid, nid); -} -#else -static inline void init_reserved_page(unsigned long pfn) -{ -} -#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ - -/* - * Initialised pages do not have PageReserved set. This function is - * called for each range allocated by the bootmem allocator and - * marks the pages PageReserved. The remaining valid pages are later - * sent to the buddy page allocator. - */ -void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end) -{ - unsigned long start_pfn = PFN_DOWN(start); - unsigned long end_pfn = PFN_UP(end); - - for (; start_pfn < end_pfn; start_pfn++) { - if (pfn_valid(start_pfn)) { - struct page *page = pfn_to_page(start_pfn); - - init_reserved_page(start_pfn); - - /* Avoid false-positive PageTail() */ - INIT_LIST_HEAD(&page->lru); - - /* - * no need for atomic set_bit because the struct - * page is not visible yet so nobody should - * access it yet. - */ - __SetPageReserved(page); - } - } -} - static void __free_pages_ok(struct page *page, unsigned int order, fpi_t fpi_flags) { @@ -1740,70 +1583,6 @@ void __free_pages_core(struct page *page, unsigned int order) __free_pages_ok(page, order, FPI_TO_TAIL); } -#ifdef CONFIG_NUMA - -/* - * During memory init memblocks map pfns to nids. The search is expensive and - * this caches recent lookups. The implementation of __early_pfn_to_nid - * treats start/end as pfns. - */ -struct mminit_pfnnid_cache { - unsigned long last_start; - unsigned long last_end; - int last_nid; -}; - -static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; - -/* - * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. - */ -static int __meminit __early_pfn_to_nid(unsigned long pfn, - struct mminit_pfnnid_cache *state) -{ - unsigned long start_pfn, end_pfn; - int nid; - - if (state->last_start <= pfn && pfn < state->last_end) - return state->last_nid; - - nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); - if (nid != NUMA_NO_NODE) { - state->last_start = start_pfn; - state->last_end = end_pfn; - state->last_nid = nid; - } - - return nid; -} - -int __meminit early_pfn_to_nid(unsigned long pfn) -{ - static DEFINE_SPINLOCK(early_pfn_lock); - int nid; - - spin_lock(&early_pfn_lock); - nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); - if (nid < 0) - nid = first_online_node; - spin_unlock(&early_pfn_lock); - - return nid; -} -#endif /* CONFIG_NUMA */ - -void __init memblock_free_pages(struct page *page, unsigned long pfn, - unsigned int order) -{ - if (!early_page_initialised(pfn)) - return; - if (!kmsan_memblock_free_pages(page, order)) { - /* KMSAN will take care of these pages. */ - return; - } - __free_pages_core(page, order); -} - /* * Check that the whole (or subset of) a pageblock given by the interval of * [start_pfn, end_pfn) is valid and within the same zone, before scanning it @@ -1874,570 +1653,131 @@ void clear_zone_contiguous(struct zone *zone) zone->contiguous = false; } -#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT -static void __init deferred_free_range(unsigned long pfn, - unsigned long nr_pages) -{ - struct page *page; - unsigned long i; - - if (!nr_pages) - return; - - page = pfn_to_page(pfn); - - /* Free a large naturally-aligned chunk if possible */ - if (nr_pages == pageblock_nr_pages && pageblock_aligned(pfn)) { - set_pageblock_migratetype(page, MIGRATE_MOVABLE); - __free_pages_core(page, pageblock_order); - return; - } - - for (i = 0; i < nr_pages; i++, page++, pfn++) { - if (pageblock_aligned(pfn)) - set_pageblock_migratetype(page, MIGRATE_MOVABLE); - __free_pages_core(page, 0); - } -} - -/* Completion tracking for deferred_init_memmap() threads */ -static atomic_t pgdat_init_n_undone __initdata; -static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); - -static inline void __init pgdat_init_report_one_done(void) -{ - if (atomic_dec_and_test(&pgdat_init_n_undone)) - complete(&pgdat_init_all_done_comp); -} - /* - * Returns true if page needs to be initialized or freed to buddy allocator. + * The order of subdivision here is critical for the IO subsystem. + * Please do not alter this order without good reasons and regression + * testing. Specifically, as large blocks of memory are subdivided, + * the order in which smaller blocks are delivered depends on the order + * they're subdivided in this function. This is the primary factor + * influencing the order in which pages are delivered to the IO + * subsystem according to empirical testing, and this is also justified + * by considering the behavior of a buddy system containing a single + * large block of memory acted on by a series of small allocations. + * This behavior is a critical factor in sglist merging's success. * - * We check if a current large page is valid by only checking the validity - * of the head pfn. + * -- nyc */ -static inline bool __init deferred_pfn_valid(unsigned long pfn) +static inline void expand(struct zone *zone, struct page *page, + int low, int high, int migratetype) { - if (pageblock_aligned(pfn) && !pfn_valid(pfn)) - return false; - return true; -} - -/* - * Free pages to buddy allocator. Try to free aligned pages in - * pageblock_nr_pages sizes. - */ -static void __init deferred_free_pages(unsigned long pfn, - unsigned long end_pfn) -{ - unsigned long nr_free = 0; - - for (; pfn < end_pfn; pfn++) { - if (!deferred_pfn_valid(pfn)) { - deferred_free_range(pfn - nr_free, nr_free); - nr_free = 0; - } else if (pageblock_aligned(pfn)) { - deferred_free_range(pfn - nr_free, nr_free); - nr_free = 1; - } else { - nr_free++; - } - } - /* Free the last block of pages to allocator */ - deferred_free_range(pfn - nr_free, nr_free); -} + unsigned long size = 1 << high; -/* - * Initialize struct pages. We minimize pfn page lookups and scheduler checks - * by performing it only once every pageblock_nr_pages. - * Return number of pages initialized. - */ -static unsigned long __init deferred_init_pages(struct zone *zone, - unsigned long pfn, - unsigned long end_pfn) -{ - int nid = zone_to_nid(zone); - unsigned long nr_pages = 0; - int zid = zone_idx(zone); - struct page *page = NULL; + while (high > low) { + high--; + size >>= 1; + VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); - for (; pfn < end_pfn; pfn++) { - if (!deferred_pfn_valid(pfn)) { - page = NULL; + /* + * Mark as guard pages (or page), that will allow to + * merge back to allocator when buddy will be freed. + * Corresponding page table entries will not be touched, + * pages will stay not present in virtual address space + */ + if (set_page_guard(zone, &page[size], high, migratetype)) continue; - } else if (!page || pageblock_aligned(pfn)) { - page = pfn_to_page(pfn); - } else { - page++; - } - __init_single_page(page, pfn, zid, nid); - nr_pages++; + + add_to_free_list(&page[size], zone, high, migratetype); + set_buddy_order(&page[size], high); } - return (nr_pages); } -/* - * This function is meant to pre-load the iterator for the zone init. - * Specifically it walks through the ranges until we are caught up to the - * first_init_pfn value and exits there. If we never encounter the value we - * return false indicating there are no valid ranges left. - */ -static bool __init -deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, - unsigned long *spfn, unsigned long *epfn, - unsigned long first_init_pfn) +static void check_new_page_bad(struct page *page) { - u64 j; - - /* - * Start out by walking through the ranges in this zone that have - * already been initialized. We don't need to do anything with them - * so we just need to flush them out of the system. - */ - for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { - if (*epfn <= first_init_pfn) - continue; - if (*spfn < first_init_pfn) - *spfn = first_init_pfn; - *i = j; - return true; + if (unlikely(page->flags & __PG_HWPOISON)) { + /* Don't complain about hwpoisoned pages */ + page_mapcount_reset(page); /* remove PageBuddy */ + return; } - return false; + bad_page(page, + page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); } /* - * Initialize and free pages. We do it in two loops: first we initialize - * struct page, then free to buddy allocator, because while we are - * freeing pages we can access pages that are ahead (computing buddy - * page in __free_one_page()). - * - * In order to try and keep some memory in the cache we have the loop - * broken along max page order boundaries. This way we will not cause - * any issues with the buddy page computation. + * This page is about to be returned from the page allocator */ -static unsigned long __init -deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, - unsigned long *end_pfn) +static int check_new_page(struct page *page) { - unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); - unsigned long spfn = *start_pfn, epfn = *end_pfn; - unsigned long nr_pages = 0; - u64 j = *i; - - /* First we loop through and initialize the page values */ - for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { - unsigned long t; + if (likely(page_expected_state(page, + PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) + return 0; - if (mo_pfn <= *start_pfn) - break; + check_new_page_bad(page); + return 1; +} - t = min(mo_pfn, *end_pfn); - nr_pages += deferred_init_pages(zone, *start_pfn, t); +static inline bool check_new_pages(struct page *page, unsigned int order) +{ + if (is_check_pages_enabled()) { + for (int i = 0; i < (1 << order); i++) { + struct page *p = page + i; - if (mo_pfn < *end_pfn) { - *start_pfn = mo_pfn; - break; + if (unlikely(check_new_page(p))) + return true; } } - /* Reset values and now loop through freeing pages as needed */ - swap(j, *i); - - for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { - unsigned long t; - - if (mo_pfn <= spfn) - break; - - t = min(mo_pfn, epfn); - deferred_free_pages(spfn, t); - - if (mo_pfn <= epfn) - break; - } - - return nr_pages; + return false; } -static void __init -deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, - void *arg) +static inline bool should_skip_kasan_unpoison(gfp_t flags) { - unsigned long spfn, epfn; - struct zone *zone = arg; - u64 i; + /* Don't skip if a software KASAN mode is enabled. */ + if (IS_ENABLED(CONFIG_KASAN_GENERIC) || + IS_ENABLED(CONFIG_KASAN_SW_TAGS)) + return false; - deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); + /* Skip, if hardware tag-based KASAN is not enabled. */ + if (!kasan_hw_tags_enabled()) + return true; /* - * Initialize and free pages in MAX_ORDER sized increments so that we - * can avoid introducing any issues with the buddy allocator. + * With hardware tag-based KASAN enabled, skip if this has been + * requested via __GFP_SKIP_KASAN. */ - while (spfn < end_pfn) { - deferred_init_maxorder(&i, zone, &spfn, &epfn); - cond_resched(); - } + return flags & __GFP_SKIP_KASAN; } -/* An arch may override for more concurrency. */ -__weak int __init -deferred_page_init_max_threads(const struct cpumask *node_cpumask) +static inline bool should_skip_init(gfp_t flags) { - return 1; + /* Don't skip, if hardware tag-based KASAN is not enabled. */ + if (!kasan_hw_tags_enabled()) + return false; + + /* For hardware tag-based KASAN, skip if requested. */ + return (flags & __GFP_SKIP_ZERO); } -/* Initialise remaining memory on a node */ -static int __init deferred_init_memmap(void *data) +inline void post_alloc_hook(struct page *page, unsigned int order, + gfp_t gfp_flags) { - pg_data_t *pgdat = data; - const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); - unsigned long spfn = 0, epfn = 0; - unsigned long first_init_pfn, flags; - unsigned long start = jiffies; - struct zone *zone; - int zid, max_threads; - u64 i; - - /* Bind memory initialisation thread to a local node if possible */ - if (!cpumask_empty(cpumask)) - set_cpus_allowed_ptr(current, cpumask); - - pgdat_resize_lock(pgdat, &flags); - first_init_pfn = pgdat->first_deferred_pfn; - if (first_init_pfn == ULONG_MAX) { - pgdat_resize_unlock(pgdat, &flags); - pgdat_init_report_one_done(); - return 0; - } + bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) && + !should_skip_init(gfp_flags); + bool zero_tags = init && (gfp_flags & __GFP_ZEROTAGS); + int i; + + set_page_private(page, 0); + set_page_refcounted(page); - /* Sanity check boundaries */ - BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); - BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); - pgdat->first_deferred_pfn = ULONG_MAX; + arch_alloc_page(page, order); + debug_pagealloc_map_pages(page, 1 << order); /* - * Once we unlock here, the zone cannot be grown anymore, thus if an - * interrupt thread must allocate this early in boot, zone must be - * pre-grown prior to start of deferred page initialization. + * Page unpoisoning must happen before memory initialization. + * Otherwise, the poison pattern will be overwritten for __GFP_ZERO + * allocations and the page unpoisoning code will complain. */ - pgdat_resize_unlock(pgdat, &flags); - - /* Only the highest zone is deferred so find it */ - for (zid = 0; zid < MAX_NR_ZONES; zid++) { - zone = pgdat->node_zones + zid; - if (first_init_pfn < zone_end_pfn(zone)) - break; - } - - /* If the zone is empty somebody else may have cleared out the zone */ - if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, - first_init_pfn)) - goto zone_empty; - - max_threads = deferred_page_init_max_threads(cpumask); - - while (spfn < epfn) { - unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); - struct padata_mt_job job = { - .thread_fn = deferred_init_memmap_chunk, - .fn_arg = zone, - .start = spfn, - .size = epfn_align - spfn, - .align = PAGES_PER_SECTION, - .min_chunk = PAGES_PER_SECTION, - .max_threads = max_threads, - }; - - padata_do_multithreaded(&job); - deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, - epfn_align); - } -zone_empty: - /* Sanity check that the next zone really is unpopulated */ - WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); - - pr_info("node %d deferred pages initialised in %ums\n", - pgdat->node_id, jiffies_to_msecs(jiffies - start)); - - pgdat_init_report_one_done(); - return 0; -} - -/* - * If this zone has deferred pages, try to grow it by initializing enough - * deferred pages to satisfy the allocation specified by order, rounded up to - * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments - * of SECTION_SIZE bytes by initializing struct pages in increments of - * PAGES_PER_SECTION * sizeof(struct page) bytes. - * - * Return true when zone was grown, otherwise return false. We return true even - * when we grow less than requested, to let the caller decide if there are - * enough pages to satisfy the allocation. - * - * Note: We use noinline because this function is needed only during boot, and - * it is called from a __ref function _deferred_grow_zone. This way we are - * making sure that it is not inlined into permanent text section. - */ -static noinline bool __init -deferred_grow_zone(struct zone *zone, unsigned int order) -{ - unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); - pg_data_t *pgdat = zone->zone_pgdat; - unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; - unsigned long spfn, epfn, flags; - unsigned long nr_pages = 0; - u64 i; - - /* Only the last zone may have deferred pages */ - if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) - return false; - - pgdat_resize_lock(pgdat, &flags); - - /* - * If someone grew this zone while we were waiting for spinlock, return - * true, as there might be enough pages already. - */ - if (first_deferred_pfn != pgdat->first_deferred_pfn) { - pgdat_resize_unlock(pgdat, &flags); - return true; - } - - /* If the zone is empty somebody else may have cleared out the zone */ - if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, - first_deferred_pfn)) { - pgdat->first_deferred_pfn = ULONG_MAX; - pgdat_resize_unlock(pgdat, &flags); - /* Retry only once. */ - return first_deferred_pfn != ULONG_MAX; - } - - /* - * Initialize and free pages in MAX_ORDER sized increments so - * that we can avoid introducing any issues with the buddy - * allocator. - */ - while (spfn < epfn) { - /* update our first deferred PFN for this section */ - first_deferred_pfn = spfn; - - nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); - touch_nmi_watchdog(); - - /* We should only stop along section boundaries */ - if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) - continue; - - /* If our quota has been met we can stop here */ - if (nr_pages >= nr_pages_needed) - break; - } - - pgdat->first_deferred_pfn = spfn; - pgdat_resize_unlock(pgdat, &flags); - - return nr_pages > 0; -} - -/* - * deferred_grow_zone() is __init, but it is called from - * get_page_from_freelist() during early boot until deferred_pages permanently - * disables this call. This is why we have refdata wrapper to avoid warning, - * and to ensure that the function body gets unloaded. - */ -static bool __ref -_deferred_grow_zone(struct zone *zone, unsigned int order) -{ - return deferred_grow_zone(zone, order); -} - -#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ - -void __init page_alloc_init_late(void) -{ - struct zone *zone; - int nid; - -#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT - - /* There will be num_node_state(N_MEMORY) threads */ - atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); - for_each_node_state(nid, N_MEMORY) { - kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); - } - - /* Block until all are initialised */ - wait_for_completion(&pgdat_init_all_done_comp); - - /* - * We initialized the rest of the deferred pages. Permanently disable - * on-demand struct page initialization. - */ - static_branch_disable(&deferred_pages); - - /* Reinit limits that are based on free pages after the kernel is up */ - files_maxfiles_init(); -#endif - - buffer_init(); - - /* Discard memblock private memory */ - memblock_discard(); - - for_each_node_state(nid, N_MEMORY) - shuffle_free_memory(NODE_DATA(nid)); - - for_each_populated_zone(zone) - set_zone_contiguous(zone); -} - -#ifdef CONFIG_CMA -/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ -void __init init_cma_reserved_pageblock(struct page *page) -{ - unsigned i = pageblock_nr_pages; - struct page *p = page; - - do { - __ClearPageReserved(p); - set_page_count(p, 0); - } while (++p, --i); - - set_pageblock_migratetype(page, MIGRATE_CMA); - set_page_refcounted(page); - __free_pages(page, pageblock_order); - - adjust_managed_page_count(page, pageblock_nr_pages); - page_zone(page)->cma_pages += pageblock_nr_pages; -} -#endif - -/* - * The order of subdivision here is critical for the IO subsystem. - * Please do not alter this order without good reasons and regression - * testing. Specifically, as large blocks of memory are subdivided, - * the order in which smaller blocks are delivered depends on the order - * they're subdivided in this function. This is the primary factor - * influencing the order in which pages are delivered to the IO - * subsystem according to empirical testing, and this is also justified - * by considering the behavior of a buddy system containing a single - * large block of memory acted on by a series of small allocations. - * This behavior is a critical factor in sglist merging's success. - * - * -- nyc - */ -static inline void expand(struct zone *zone, struct page *page, - int low, int high, int migratetype) -{ - unsigned long size = 1 << high; - - while (high > low) { - high--; - size >>= 1; - VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); - - /* - * Mark as guard pages (or page), that will allow to - * merge back to allocator when buddy will be freed. - * Corresponding page table entries will not be touched, - * pages will stay not present in virtual address space - */ - if (set_page_guard(zone, &page[size], high, migratetype)) - continue; - - add_to_free_list(&page[size], zone, high, migratetype); - set_buddy_order(&page[size], high); - } -} - -static void check_new_page_bad(struct page *page) -{ - if (unlikely(page->flags & __PG_HWPOISON)) { - /* Don't complain about hwpoisoned pages */ - page_mapcount_reset(page); /* remove PageBuddy */ - return; - } - - bad_page(page, - page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP)); -} - -/* - * This page is about to be returned from the page allocator - */ -static int check_new_page(struct page *page) -{ - if (likely(page_expected_state(page, - PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON))) - return 0; - - check_new_page_bad(page); - return 1; -} - -static inline bool check_new_pages(struct page *page, unsigned int order) -{ - if (is_check_pages_enabled()) { - for (int i = 0; i < (1 << order); i++) { - struct page *p = page + i; - - if (unlikely(check_new_page(p))) - return true; - } - } - - return false; -} - -static inline bool should_skip_kasan_unpoison(gfp_t flags) -{ - /* Don't skip if a software KASAN mode is enabled. */ - if (IS_ENABLED(CONFIG_KASAN_GENERIC) || - IS_ENABLED(CONFIG_KASAN_SW_TAGS)) - return false; - - /* Skip, if hardware tag-based KASAN is not enabled. */ - if (!kasan_hw_tags_enabled()) - return true; - - /* - * With hardware tag-based KASAN enabled, skip if this has been - * requested via __GFP_SKIP_KASAN. - */ - return flags & __GFP_SKIP_KASAN; -} - -static inline bool should_skip_init(gfp_t flags) -{ - /* Don't skip, if hardware tag-based KASAN is not enabled. */ - if (!kasan_hw_tags_enabled()) - return false; - - /* For hardware tag-based KASAN, skip if requested. */ - return (flags & __GFP_SKIP_ZERO); -} - -inline void post_alloc_hook(struct page *page, unsigned int order, - gfp_t gfp_flags) -{ - bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) && - !should_skip_init(gfp_flags); - bool zero_tags = init && (gfp_flags & __GFP_ZEROTAGS); - int i; - - set_page_private(page, 0); - set_page_refcounted(page); - - arch_alloc_page(page, order); - debug_pagealloc_map_pages(page, 1 << order); - - /* - * Page unpoisoning must happen before memory initialization. - * Otherwise, the poison pattern will be overwritten for __GFP_ZERO - * allocations and the page unpoisoning code will complain. - */ - kernel_unpoison_pages(page, 1 << order); + kernel_unpoison_pages(page, 1 << order); /* * As memory initialization might be integrated into KASAN, @@ -6547,7 +5887,6 @@ static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonesta #define BOOT_PAGESET_BATCH 1 static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset); static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats); -static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); static void __build_all_zonelists(void *data) { @@ -6661,395 +6000,35 @@ void __ref build_all_zonelists(pg_data_t *pgdat) #endif } -/* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ -static bool __meminit -overlap_memmap_init(unsigned long zone, unsigned long *pfn) -{ - static struct memblock_region *r; - - if (mirrored_kernelcore && zone == ZONE_MOVABLE) { - if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { - for_each_mem_region(r) { - if (*pfn < memblock_region_memory_end_pfn(r)) - break; - } - } - if (*pfn >= memblock_region_memory_base_pfn(r) && - memblock_is_mirror(r)) { - *pfn = memblock_region_memory_end_pfn(r); - return true; - } - } - return false; -} - -/* - * Initially all pages are reserved - free ones are freed - * up by memblock_free_all() once the early boot process is - * done. Non-atomic initialization, single-pass. - * - * All aligned pageblocks are initialized to the specified migratetype - * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related - * zone stats (e.g., nr_isolate_pageblock) are touched. - */ -void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, - unsigned long start_pfn, unsigned long zone_end_pfn, - enum meminit_context context, - struct vmem_altmap *altmap, int migratetype) +static int zone_batchsize(struct zone *zone) { - unsigned long pfn, end_pfn = start_pfn + size; - struct page *page; +#ifdef CONFIG_MMU + int batch; - if (highest_memmap_pfn < end_pfn - 1) - highest_memmap_pfn = end_pfn - 1; + /* + * The number of pages to batch allocate is either ~0.1% + * of the zone or 1MB, whichever is smaller. The batch + * size is striking a balance between allocation latency + * and zone lock contention. + */ + batch = min(zone_managed_pages(zone) >> 10, SZ_1M / PAGE_SIZE); + batch /= 4; /* We effectively *= 4 below */ + if (batch < 1) + batch = 1; -#ifdef CONFIG_ZONE_DEVICE /* - * Honor reservation requested by the driver for this ZONE_DEVICE - * memory. We limit the total number of pages to initialize to just - * those that might contain the memory mapping. We will defer the - * ZONE_DEVICE page initialization until after we have released - * the hotplug lock. + * Clamp the batch to a 2^n - 1 value. Having a power + * of 2 value was found to be more likely to have + * suboptimal cache aliasing properties in some cases. + * + * For example if 2 tasks are alternately allocating + * batches of pages, one task can end up with a lot + * of pages of one half of the possible page colors + * and the other with pages of the other colors. */ - if (zone == ZONE_DEVICE) { - if (!altmap) - return; + batch = rounddown_pow_of_two(batch + batch/2) - 1; - if (start_pfn == altmap->base_pfn) - start_pfn += altmap->reserve; - end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); - } -#endif - - for (pfn = start_pfn; pfn < end_pfn; ) { - /* - * There can be holes in boot-time mem_map[]s handed to this - * function. They do not exist on hotplugged memory. - */ - if (context == MEMINIT_EARLY) { - if (overlap_memmap_init(zone, &pfn)) - continue; - if (defer_init(nid, pfn, zone_end_pfn)) { - deferred_struct_pages = true; - break; - } - } - - page = pfn_to_page(pfn); - __init_single_page(page, pfn, zone, nid); - if (context == MEMINIT_HOTPLUG) - __SetPageReserved(page); - - /* - * Usually, we want to mark the pageblock MIGRATE_MOVABLE, - * such that unmovable allocations won't be scattered all - * over the place during system boot. - */ - if (pageblock_aligned(pfn)) { - set_pageblock_migratetype(page, migratetype); - cond_resched(); - } - pfn++; - } -} - -#ifdef CONFIG_ZONE_DEVICE -static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, - unsigned long zone_idx, int nid, - struct dev_pagemap *pgmap) -{ - - __init_single_page(page, pfn, zone_idx, nid); - - /* - * Mark page reserved as it will need to wait for onlining - * phase for it to be fully associated with a zone. - * - * We can use the non-atomic __set_bit operation for setting - * the flag as we are still initializing the pages. - */ - __SetPageReserved(page); - - /* - * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer - * and zone_device_data. It is a bug if a ZONE_DEVICE page is - * ever freed or placed on a driver-private list. - */ - page->pgmap = pgmap; - page->zone_device_data = NULL; - - /* - * Mark the block movable so that blocks are reserved for - * movable at startup. This will force kernel allocations - * to reserve their blocks rather than leaking throughout - * the address space during boot when many long-lived - * kernel allocations are made. - * - * Please note that MEMINIT_HOTPLUG path doesn't clear memmap - * because this is done early in section_activate() - */ - if (pageblock_aligned(pfn)) { - set_pageblock_migratetype(page, MIGRATE_MOVABLE); - cond_resched(); - } - - /* - * ZONE_DEVICE pages are released directly to the driver page allocator - * which will set the page count to 1 when allocating the page. - */ - if (pgmap->type == MEMORY_DEVICE_PRIVATE || - pgmap->type == MEMORY_DEVICE_COHERENT) - set_page_count(page, 0); -} - -/* - * With compound page geometry and when struct pages are stored in ram most - * tail pages are reused. Consequently, the amount of unique struct pages to - * initialize is a lot smaller that the total amount of struct pages being - * mapped. This is a paired / mild layering violation with explicit knowledge - * of how the sparse_vmemmap internals handle compound pages in the lack - * of an altmap. See vmemmap_populate_compound_pages(). - */ -static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, - unsigned long nr_pages) -{ - return is_power_of_2(sizeof(struct page)) && - !altmap ? 2 * (PAGE_SIZE / sizeof(struct page)) : nr_pages; -} - -static void __ref memmap_init_compound(struct page *head, - unsigned long head_pfn, - unsigned long zone_idx, int nid, - struct dev_pagemap *pgmap, - unsigned long nr_pages) -{ - unsigned long pfn, end_pfn = head_pfn + nr_pages; - unsigned int order = pgmap->vmemmap_shift; - - __SetPageHead(head); - for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { - struct page *page = pfn_to_page(pfn); - - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); - prep_compound_tail(head, pfn - head_pfn); - set_page_count(page, 0); - - /* - * The first tail page stores important compound page info. - * Call prep_compound_head() after the first tail page has - * been initialized, to not have the data overwritten. - */ - if (pfn == head_pfn + 1) - prep_compound_head(head, order); - } -} - -void __ref memmap_init_zone_device(struct zone *zone, - unsigned long start_pfn, - unsigned long nr_pages, - struct dev_pagemap *pgmap) -{ - unsigned long pfn, end_pfn = start_pfn + nr_pages; - struct pglist_data *pgdat = zone->zone_pgdat; - struct vmem_altmap *altmap = pgmap_altmap(pgmap); - unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); - unsigned long zone_idx = zone_idx(zone); - unsigned long start = jiffies; - int nid = pgdat->node_id; - - if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) - return; - - /* - * The call to memmap_init should have already taken care - * of the pages reserved for the memmap, so we can just jump to - * the end of that region and start processing the device pages. - */ - if (altmap) { - start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); - nr_pages = end_pfn - start_pfn; - } - - for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { - struct page *page = pfn_to_page(pfn); - - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); - - if (pfns_per_compound == 1) - continue; - - memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(altmap, pfns_per_compound)); - } - - pr_info("%s initialised %lu pages in %ums\n", __func__, - nr_pages, jiffies_to_msecs(jiffies - start)); -} - -#endif -static void __meminit zone_init_free_lists(struct zone *zone) -{ - unsigned int order, t; - for_each_migratetype_order(order, t) { - INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); - zone->free_area[order].nr_free = 0; - } -} - -/* - * Only struct pages that correspond to ranges defined by memblock.memory - * are zeroed and initialized by going through __init_single_page() during - * memmap_init_zone_range(). - * - * But, there could be struct pages that correspond to holes in - * memblock.memory. This can happen because of the following reasons: - * - physical memory bank size is not necessarily the exact multiple of the - * arbitrary section size - * - early reserved memory may not be listed in memblock.memory - * - memory layouts defined with memmap= kernel parameter may not align - * nicely with memmap sections - * - * Explicitly initialize those struct pages so that: - * - PG_Reserved is set - * - zone and node links point to zone and node that span the page if the - * hole is in the middle of a zone - * - zone and node links point to adjacent zone/node if the hole falls on - * the zone boundary; the pages in such holes will be prepended to the - * zone/node above the hole except for the trailing pages in the last - * section that will be appended to the zone/node below. - */ -static void __init init_unavailable_range(unsigned long spfn, - unsigned long epfn, - int zone, int node) -{ - unsigned long pfn; - u64 pgcnt = 0; - - for (pfn = spfn; pfn < epfn; pfn++) { - if (!pfn_valid(pageblock_start_pfn(pfn))) { - pfn = pageblock_end_pfn(pfn) - 1; - continue; - } - __init_single_page(pfn_to_page(pfn), pfn, zone, node); - __SetPageReserved(pfn_to_page(pfn)); - pgcnt++; - } - - if (pgcnt) - pr_info("On node %d, zone %s: %lld pages in unavailable ranges", - node, zone_names[zone], pgcnt); -} - -static void __init memmap_init_zone_range(struct zone *zone, - unsigned long start_pfn, - unsigned long end_pfn, - unsigned long *hole_pfn) -{ - unsigned long zone_start_pfn = zone->zone_start_pfn; - unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; - int nid = zone_to_nid(zone), zone_id = zone_idx(zone); - - start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); - end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); - - if (start_pfn >= end_pfn) - return; - - memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, - zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); - - if (*hole_pfn < start_pfn) - init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); - - *hole_pfn = end_pfn; -} - -static void __init memmap_init(void) -{ - unsigned long start_pfn, end_pfn; - unsigned long hole_pfn = 0; - int i, j, zone_id = 0, nid; - - for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { - struct pglist_data *node = NODE_DATA(nid); - - for (j = 0; j < MAX_NR_ZONES; j++) { - struct zone *zone = node->node_zones + j; - - if (!populated_zone(zone)) - continue; - - memmap_init_zone_range(zone, start_pfn, end_pfn, - &hole_pfn); - zone_id = j; - } - } - -#ifdef CONFIG_SPARSEMEM - /* - * Initialize the memory map for hole in the range [memory_end, - * section_end]. - * Append the pages in this hole to the highest zone in the last - * node. - * The call to init_unavailable_range() is outside the ifdef to - * silence the compiler warining about zone_id set but not used; - * for FLATMEM it is a nop anyway - */ - end_pfn = round_up(end_pfn, PAGES_PER_SECTION); - if (hole_pfn < end_pfn) -#endif - init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); -} - -void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, - phys_addr_t min_addr, int nid, bool exact_nid) -{ - void *ptr; - - if (exact_nid) - ptr = memblock_alloc_exact_nid_raw(size, align, min_addr, - MEMBLOCK_ALLOC_ACCESSIBLE, - nid); - else - ptr = memblock_alloc_try_nid_raw(size, align, min_addr, - MEMBLOCK_ALLOC_ACCESSIBLE, - nid); - - if (ptr && size > 0) - page_init_poison(ptr, size); - - return ptr; -} - -static int zone_batchsize(struct zone *zone) -{ -#ifdef CONFIG_MMU - int batch; - - /* - * The number of pages to batch allocate is either ~0.1% - * of the zone or 1MB, whichever is smaller. The batch - * size is striking a balance between allocation latency - * and zone lock contention. - */ - batch = min(zone_managed_pages(zone) >> 10, SZ_1M / PAGE_SIZE); - batch /= 4; /* We effectively *= 4 below */ - if (batch < 1) - batch = 1; - - /* - * Clamp the batch to a 2^n - 1 value. Having a power - * of 2 value was found to be more likely to have - * suboptimal cache aliasing properties in some cases. - * - * For example if 2 tasks are alternately allocating - * batches of pages, one task can end up with a lot - * of pages of one half of the possible page colors - * and the other with pages of the other colors. - */ - batch = rounddown_pow_of_two(batch + batch/2) - 1; - - return batch; + return batch; #else /* The deferral and batching of frees should be suppressed under NOMMU @@ -7071,1352 +6050,210 @@ static int zone_batchsize(struct zone *zone) static int zone_highsize(struct zone *zone, int batch, int cpu_online) { -#ifdef CONFIG_MMU - int high; - int nr_split_cpus; - unsigned long total_pages; - - if (!percpu_pagelist_high_fraction) { - /* - * By default, the high value of the pcp is based on the zone - * low watermark so that if they are full then background - * reclaim will not be started prematurely. - */ - total_pages = low_wmark_pages(zone); - } else { - /* - * If percpu_pagelist_high_fraction is configured, the high - * value is based on a fraction of the managed pages in the - * zone. - */ - total_pages = zone_managed_pages(zone) / percpu_pagelist_high_fraction; - } - - /* - * Split the high value across all online CPUs local to the zone. Note - * that early in boot that CPUs may not be online yet and that during - * CPU hotplug that the cpumask is not yet updated when a CPU is being - * onlined. For memory nodes that have no CPUs, split pcp->high across - * all online CPUs to mitigate the risk that reclaim is triggered - * prematurely due to pages stored on pcp lists. - */ - nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online; - if (!nr_split_cpus) - nr_split_cpus = num_online_cpus(); - high = total_pages / nr_split_cpus; - - /* - * Ensure high is at least batch*4. The multiple is based on the - * historical relationship between high and batch. - */ - high = max(high, batch << 2); - - return high; -#else - return 0; -#endif -} - -/* - * pcp->high and pcp->batch values are related and generally batch is lower - * than high. They are also related to pcp->count such that count is lower - * than high, and as soon as it reaches high, the pcplist is flushed. - * - * However, guaranteeing these relations at all times would require e.g. write - * barriers here but also careful usage of read barriers at the read side, and - * thus be prone to error and bad for performance. Thus the update only prevents - * store tearing. Any new users of pcp->batch and pcp->high should ensure they - * can cope with those fields changing asynchronously, and fully trust only the - * pcp->count field on the local CPU with interrupts disabled. - * - * mutex_is_locked(&pcp_batch_high_lock) required when calling this function - * outside of boot time (or some other assurance that no concurrent updaters - * exist). - */ -static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, - unsigned long batch) -{ - WRITE_ONCE(pcp->batch, batch); - WRITE_ONCE(pcp->high, high); -} - -static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats) -{ - int pindex; - - memset(pcp, 0, sizeof(*pcp)); - memset(pzstats, 0, sizeof(*pzstats)); - - spin_lock_init(&pcp->lock); - for (pindex = 0; pindex < NR_PCP_LISTS; pindex++) - INIT_LIST_HEAD(&pcp->lists[pindex]); - - /* - * Set batch and high values safe for a boot pageset. A true percpu - * pageset's initialization will update them subsequently. Here we don't - * need to be as careful as pageset_update() as nobody can access the - * pageset yet. - */ - pcp->high = BOOT_PAGESET_HIGH; - pcp->batch = BOOT_PAGESET_BATCH; - pcp->free_factor = 0; -} - -static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high, - unsigned long batch) -{ - struct per_cpu_pages *pcp; - int cpu; - - for_each_possible_cpu(cpu) { - pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); - pageset_update(pcp, high, batch); - } -} - -/* - * Calculate and set new high and batch values for all per-cpu pagesets of a - * zone based on the zone's size. - */ -static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) -{ - int new_high, new_batch; - - new_batch = max(1, zone_batchsize(zone)); - new_high = zone_highsize(zone, new_batch, cpu_online); - - if (zone->pageset_high == new_high && - zone->pageset_batch == new_batch) - return; - - zone->pageset_high = new_high; - zone->pageset_batch = new_batch; - - __zone_set_pageset_high_and_batch(zone, new_high, new_batch); -} - -void __meminit setup_zone_pageset(struct zone *zone) -{ - int cpu; - - /* Size may be 0 on !SMP && !NUMA */ - if (sizeof(struct per_cpu_zonestat) > 0) - zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat); - - zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages); - for_each_possible_cpu(cpu) { - struct per_cpu_pages *pcp; - struct per_cpu_zonestat *pzstats; - - pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); - pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); - per_cpu_pages_init(pcp, pzstats); - } - - zone_set_pageset_high_and_batch(zone, 0); -} - -/* - * The zone indicated has a new number of managed_pages; batch sizes and percpu - * page high values need to be recalculated. - */ -static void zone_pcp_update(struct zone *zone, int cpu_online) -{ - mutex_lock(&pcp_batch_high_lock); - zone_set_pageset_high_and_batch(zone, cpu_online); - mutex_unlock(&pcp_batch_high_lock); -} - -/* - * Allocate per cpu pagesets and initialize them. - * Before this call only boot pagesets were available. - */ -void __init setup_per_cpu_pageset(void) -{ - struct pglist_data *pgdat; - struct zone *zone; - int __maybe_unused cpu; - - for_each_populated_zone(zone) - setup_zone_pageset(zone); - -#ifdef CONFIG_NUMA - /* - * Unpopulated zones continue using the boot pagesets. - * The numa stats for these pagesets need to be reset. - * Otherwise, they will end up skewing the stats of - * the nodes these zones are associated with. - */ - for_each_possible_cpu(cpu) { - struct per_cpu_zonestat *pzstats = &per_cpu(boot_zonestats, cpu); - memset(pzstats->vm_numa_event, 0, - sizeof(pzstats->vm_numa_event)); - } -#endif - - for_each_online_pgdat(pgdat) - pgdat->per_cpu_nodestats = - alloc_percpu(struct per_cpu_nodestat); -} - -static __meminit void zone_pcp_init(struct zone *zone) -{ - /* - * per cpu subsystem is not up at this point. The following code - * relies on the ability of the linker to provide the - * offset of a (static) per cpu variable into the per cpu area. - */ - zone->per_cpu_pageset = &boot_pageset; - zone->per_cpu_zonestats = &boot_zonestats; - zone->pageset_high = BOOT_PAGESET_HIGH; - zone->pageset_batch = BOOT_PAGESET_BATCH; - - if (populated_zone(zone)) - pr_debug(" %s zone: %lu pages, LIFO batch:%u\n", zone->name, - zone->present_pages, zone_batchsize(zone)); -} - -void __meminit init_currently_empty_zone(struct zone *zone, - unsigned long zone_start_pfn, - unsigned long size) -{ - struct pglist_data *pgdat = zone->zone_pgdat; - int zone_idx = zone_idx(zone) + 1; - - if (zone_idx > pgdat->nr_zones) - pgdat->nr_zones = zone_idx; - - zone->zone_start_pfn = zone_start_pfn; - - mminit_dprintk(MMINIT_TRACE, "memmap_init", - "Initialising map node %d zone %lu pfns %lu -> %lu\n", - pgdat->node_id, - (unsigned long)zone_idx(zone), - zone_start_pfn, (zone_start_pfn + size)); - - zone_init_free_lists(zone); - zone->initialized = 1; -} - -/** - * get_pfn_range_for_nid - Return the start and end page frames for a node - * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. - * @start_pfn: Passed by reference. On return, it will have the node start_pfn. - * @end_pfn: Passed by reference. On return, it will have the node end_pfn. - * - * It returns the start and end page frame of a node based on information - * provided by memblock_set_node(). If called for a node - * with no available memory, a warning is printed and the start and end - * PFNs will be 0. - */ -void __init get_pfn_range_for_nid(unsigned int nid, - unsigned long *start_pfn, unsigned long *end_pfn) -{ - unsigned long this_start_pfn, this_end_pfn; - int i; - - *start_pfn = -1UL; - *end_pfn = 0; - - for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { - *start_pfn = min(*start_pfn, this_start_pfn); - *end_pfn = max(*end_pfn, this_end_pfn); - } - - if (*start_pfn == -1UL) - *start_pfn = 0; -} - -/* - * This finds a zone that can be used for ZONE_MOVABLE pages. The - * assumption is made that zones within a node are ordered in monotonic - * increasing memory addresses so that the "highest" populated zone is used - */ -static void __init find_usable_zone_for_movable(void) -{ - int zone_index; - for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { - if (zone_index == ZONE_MOVABLE) - continue; - - if (arch_zone_highest_possible_pfn[zone_index] > - arch_zone_lowest_possible_pfn[zone_index]) - break; - } - - VM_BUG_ON(zone_index == -1); - movable_zone = zone_index; -} - -/* - * The zone ranges provided by the architecture do not include ZONE_MOVABLE - * because it is sized independent of architecture. Unlike the other zones, - * the starting point for ZONE_MOVABLE is not fixed. It may be different - * in each node depending on the size of each node and how evenly kernelcore - * is distributed. This helper function adjusts the zone ranges - * provided by the architecture for a given node by using the end of the - * highest usable zone for ZONE_MOVABLE. This preserves the assumption that - * zones within a node are in order of monotonic increases memory addresses - */ -static void __init adjust_zone_range_for_zone_movable(int nid, - unsigned long zone_type, - unsigned long node_start_pfn, - unsigned long node_end_pfn, - unsigned long *zone_start_pfn, - unsigned long *zone_end_pfn) -{ - /* Only adjust if ZONE_MOVABLE is on this node */ - if (zone_movable_pfn[nid]) { - /* Size ZONE_MOVABLE */ - if (zone_type == ZONE_MOVABLE) { - *zone_start_pfn = zone_movable_pfn[nid]; - *zone_end_pfn = min(node_end_pfn, - arch_zone_highest_possible_pfn[movable_zone]); - - /* Adjust for ZONE_MOVABLE starting within this range */ - } else if (!mirrored_kernelcore && - *zone_start_pfn < zone_movable_pfn[nid] && - *zone_end_pfn > zone_movable_pfn[nid]) { - *zone_end_pfn = zone_movable_pfn[nid]; - - /* Check if this whole range is within ZONE_MOVABLE */ - } else if (*zone_start_pfn >= zone_movable_pfn[nid]) - *zone_start_pfn = *zone_end_pfn; - } -} - -/* - * Return the number of pages a zone spans in a node, including holes - * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() - */ -static unsigned long __init zone_spanned_pages_in_node(int nid, - unsigned long zone_type, - unsigned long node_start_pfn, - unsigned long node_end_pfn, - unsigned long *zone_start_pfn, - unsigned long *zone_end_pfn) -{ - unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; - unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; - /* When hotadd a new node from cpu_up(), the node should be empty */ - if (!node_start_pfn && !node_end_pfn) - return 0; - - /* Get the start and end of the zone */ - *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); - *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); - adjust_zone_range_for_zone_movable(nid, zone_type, - node_start_pfn, node_end_pfn, - zone_start_pfn, zone_end_pfn); - - /* Check that this node has pages within the zone's required range */ - if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) - return 0; - - /* Move the zone boundaries inside the node if necessary */ - *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); - *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); - - /* Return the spanned pages */ - return *zone_end_pfn - *zone_start_pfn; -} - -/* - * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, - * then all holes in the requested range will be accounted for. - */ -unsigned long __init __absent_pages_in_range(int nid, - unsigned long range_start_pfn, - unsigned long range_end_pfn) -{ - unsigned long nr_absent = range_end_pfn - range_start_pfn; - unsigned long start_pfn, end_pfn; - int i; - - for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { - start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); - end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); - nr_absent -= end_pfn - start_pfn; - } - return nr_absent; -} - -/** - * absent_pages_in_range - Return number of page frames in holes within a range - * @start_pfn: The start PFN to start searching for holes - * @end_pfn: The end PFN to stop searching for holes - * - * Return: the number of pages frames in memory holes within a range. - */ -unsigned long __init absent_pages_in_range(unsigned long start_pfn, - unsigned long end_pfn) -{ - return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); -} - -/* Return the number of page frames in holes in a zone on a node */ -static unsigned long __init zone_absent_pages_in_node(int nid, - unsigned long zone_type, - unsigned long node_start_pfn, - unsigned long node_end_pfn) -{ - unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; - unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; - unsigned long zone_start_pfn, zone_end_pfn; - unsigned long nr_absent; - - /* When hotadd a new node from cpu_up(), the node should be empty */ - if (!node_start_pfn && !node_end_pfn) - return 0; - - zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); - zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); - - adjust_zone_range_for_zone_movable(nid, zone_type, - node_start_pfn, node_end_pfn, - &zone_start_pfn, &zone_end_pfn); - nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); - - /* - * ZONE_MOVABLE handling. - * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages - * and vice versa. - */ - if (mirrored_kernelcore && zone_movable_pfn[nid]) { - unsigned long start_pfn, end_pfn; - struct memblock_region *r; - - for_each_mem_region(r) { - start_pfn = clamp(memblock_region_memory_base_pfn(r), - zone_start_pfn, zone_end_pfn); - end_pfn = clamp(memblock_region_memory_end_pfn(r), - zone_start_pfn, zone_end_pfn); - - if (zone_type == ZONE_MOVABLE && - memblock_is_mirror(r)) - nr_absent += end_pfn - start_pfn; - - if (zone_type == ZONE_NORMAL && - !memblock_is_mirror(r)) - nr_absent += end_pfn - start_pfn; - } - } - - return nr_absent; -} - -static void __init calculate_node_totalpages(struct pglist_data *pgdat, - unsigned long node_start_pfn, - unsigned long node_end_pfn) -{ - unsigned long realtotalpages = 0, totalpages = 0; - enum zone_type i; - - for (i = 0; i < MAX_NR_ZONES; i++) { - struct zone *zone = pgdat->node_zones + i; - unsigned long zone_start_pfn, zone_end_pfn; - unsigned long spanned, absent; - unsigned long size, real_size; - - spanned = zone_spanned_pages_in_node(pgdat->node_id, i, - node_start_pfn, - node_end_pfn, - &zone_start_pfn, - &zone_end_pfn); - absent = zone_absent_pages_in_node(pgdat->node_id, i, - node_start_pfn, - node_end_pfn); - - size = spanned; - real_size = size - absent; - - if (size) - zone->zone_start_pfn = zone_start_pfn; - else - zone->zone_start_pfn = 0; - zone->spanned_pages = size; - zone->present_pages = real_size; -#if defined(CONFIG_MEMORY_HOTPLUG) - zone->present_early_pages = real_size; -#endif - - totalpages += size; - realtotalpages += real_size; - } - - pgdat->node_spanned_pages = totalpages; - pgdat->node_present_pages = realtotalpages; - pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); -} - -#ifndef CONFIG_SPARSEMEM -/* - * Calculate the size of the zone->blockflags rounded to an unsigned long - * Start by making sure zonesize is a multiple of pageblock_order by rounding - * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally - * round what is now in bits to nearest long in bits, then return it in - * bytes. - */ -static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) -{ - unsigned long usemapsize; - - zonesize += zone_start_pfn & (pageblock_nr_pages-1); - usemapsize = roundup(zonesize, pageblock_nr_pages); - usemapsize = usemapsize >> pageblock_order; - usemapsize *= NR_PAGEBLOCK_BITS; - usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); - - return usemapsize / 8; -} - -static void __ref setup_usemap(struct zone *zone) -{ - unsigned long usemapsize = usemap_size(zone->zone_start_pfn, - zone->spanned_pages); - zone->pageblock_flags = NULL; - if (usemapsize) { - zone->pageblock_flags = - memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, - zone_to_nid(zone)); - if (!zone->pageblock_flags) - panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", - usemapsize, zone->name, zone_to_nid(zone)); - } -} -#else -static inline void setup_usemap(struct zone *zone) {} -#endif /* CONFIG_SPARSEMEM */ - -#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE - -/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ -void __init set_pageblock_order(void) -{ - unsigned int order = MAX_ORDER; - - /* Check that pageblock_nr_pages has not already been setup */ - if (pageblock_order) - return; - - /* Don't let pageblocks exceed the maximum allocation granularity. */ - if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order) - order = HUGETLB_PAGE_ORDER; - - /* - * Assume the largest contiguous order of interest is a huge page. - * This value may be variable depending on boot parameters on IA64 and - * powerpc. - */ - pageblock_order = order; -} -#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ - -/* - * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() - * is unused as pageblock_order is set at compile-time. See - * include/linux/pageblock-flags.h for the values of pageblock_order based on - * the kernel config - */ -void __init set_pageblock_order(void) -{ -} - -#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ - -static unsigned long __init calc_memmap_size(unsigned long spanned_pages, - unsigned long present_pages) -{ - unsigned long pages = spanned_pages; - - /* - * Provide a more accurate estimation if there are holes within - * the zone and SPARSEMEM is in use. If there are holes within the - * zone, each populated memory region may cost us one or two extra - * memmap pages due to alignment because memmap pages for each - * populated regions may not be naturally aligned on page boundary. - * So the (present_pages >> 4) heuristic is a tradeoff for that. - */ - if (spanned_pages > present_pages + (present_pages >> 4) && - IS_ENABLED(CONFIG_SPARSEMEM)) - pages = present_pages; - - return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; -} - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -static void pgdat_init_split_queue(struct pglist_data *pgdat) -{ - struct deferred_split *ds_queue = &pgdat->deferred_split_queue; - - spin_lock_init(&ds_queue->split_queue_lock); - INIT_LIST_HEAD(&ds_queue->split_queue); - ds_queue->split_queue_len = 0; -} -#else -static void pgdat_init_split_queue(struct pglist_data *pgdat) {} -#endif - -#ifdef CONFIG_COMPACTION -static void pgdat_init_kcompactd(struct pglist_data *pgdat) -{ - init_waitqueue_head(&pgdat->kcompactd_wait); -} -#else -static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} -#endif - -static void __meminit pgdat_init_internals(struct pglist_data *pgdat) -{ - int i; - - pgdat_resize_init(pgdat); - pgdat_kswapd_lock_init(pgdat); - - pgdat_init_split_queue(pgdat); - pgdat_init_kcompactd(pgdat); - - init_waitqueue_head(&pgdat->kswapd_wait); - init_waitqueue_head(&pgdat->pfmemalloc_wait); - - for (i = 0; i < NR_VMSCAN_THROTTLE; i++) - init_waitqueue_head(&pgdat->reclaim_wait[i]); - - pgdat_page_ext_init(pgdat); - lruvec_init(&pgdat->__lruvec); -} - -static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, - unsigned long remaining_pages) -{ - atomic_long_set(&zone->managed_pages, remaining_pages); - zone_set_nid(zone, nid); - zone->name = zone_names[idx]; - zone->zone_pgdat = NODE_DATA(nid); - spin_lock_init(&zone->lock); - zone_seqlock_init(zone); - zone_pcp_init(zone); -} - -/* - * Set up the zone data structures - * - init pgdat internals - * - init all zones belonging to this node - * - * NOTE: this function is only called during memory hotplug - */ -#ifdef CONFIG_MEMORY_HOTPLUG -void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) -{ - int nid = pgdat->node_id; - enum zone_type z; - int cpu; - - pgdat_init_internals(pgdat); - - if (pgdat->per_cpu_nodestats == &boot_nodestats) - pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); - - /* - * Reset the nr_zones, order and highest_zoneidx before reuse. - * Note that kswapd will init kswapd_highest_zoneidx properly - * when it starts in the near future. - */ - pgdat->nr_zones = 0; - pgdat->kswapd_order = 0; - pgdat->kswapd_highest_zoneidx = 0; - pgdat->node_start_pfn = 0; - for_each_online_cpu(cpu) { - struct per_cpu_nodestat *p; - - p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); - memset(p, 0, sizeof(*p)); - } - - for (z = 0; z < MAX_NR_ZONES; z++) - zone_init_internals(&pgdat->node_zones[z], z, nid, 0); -} -#endif - -/* - * Set up the zone data structures: - * - mark all pages reserved - * - mark all memory queues empty - * - clear the memory bitmaps - * - * NOTE: pgdat should get zeroed by caller. - * NOTE: this function is only called during early init. - */ -static void __init free_area_init_core(struct pglist_data *pgdat) -{ - enum zone_type j; - int nid = pgdat->node_id; - - pgdat_init_internals(pgdat); - pgdat->per_cpu_nodestats = &boot_nodestats; - - for (j = 0; j < MAX_NR_ZONES; j++) { - struct zone *zone = pgdat->node_zones + j; - unsigned long size, freesize, memmap_pages; - - size = zone->spanned_pages; - freesize = zone->present_pages; - - /* - * Adjust freesize so that it accounts for how much memory - * is used by this zone for memmap. This affects the watermark - * and per-cpu initialisations - */ - memmap_pages = calc_memmap_size(size, freesize); - if (!is_highmem_idx(j)) { - if (freesize >= memmap_pages) { - freesize -= memmap_pages; - if (memmap_pages) - pr_debug(" %s zone: %lu pages used for memmap\n", - zone_names[j], memmap_pages); - } else - pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n", - zone_names[j], memmap_pages, freesize); - } - - /* Account for reserved pages */ - if (j == 0 && freesize > dma_reserve) { - freesize -= dma_reserve; - pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve); - } - - if (!is_highmem_idx(j)) - nr_kernel_pages += freesize; - /* Charge for highmem memmap if there are enough kernel pages */ - else if (nr_kernel_pages > memmap_pages * 2) - nr_kernel_pages -= memmap_pages; - nr_all_pages += freesize; - - /* - * Set an approximate value for lowmem here, it will be adjusted - * when the bootmem allocator frees pages into the buddy system. - * And all highmem pages will be managed by the buddy system. - */ - zone_init_internals(zone, j, nid, freesize); - - if (!size) - continue; - - set_pageblock_order(); - setup_usemap(zone); - init_currently_empty_zone(zone, zone->zone_start_pfn, size); - } -} - -#ifdef CONFIG_FLATMEM -static void __init alloc_node_mem_map(struct pglist_data *pgdat) -{ - unsigned long __maybe_unused start = 0; - unsigned long __maybe_unused offset = 0; - - /* Skip empty nodes */ - if (!pgdat->node_spanned_pages) - return; - - start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); - offset = pgdat->node_start_pfn - start; - /* ia64 gets its own node_mem_map, before this, without bootmem */ - if (!pgdat->node_mem_map) { - unsigned long size, end; - struct page *map; - - /* - * The zone's endpoints aren't required to be MAX_ORDER - * aligned but the node_mem_map endpoints must be in order - * for the buddy allocator to function correctly. - */ - end = pgdat_end_pfn(pgdat); - end = ALIGN(end, MAX_ORDER_NR_PAGES); - size = (end - start) * sizeof(struct page); - map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, - pgdat->node_id, false); - if (!map) - panic("Failed to allocate %ld bytes for node %d memory map\n", - size, pgdat->node_id); - pgdat->node_mem_map = map + offset; - } - pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", - __func__, pgdat->node_id, (unsigned long)pgdat, - (unsigned long)pgdat->node_mem_map); -#ifndef CONFIG_NUMA - /* - * With no DISCONTIG, the global mem_map is just set as node 0's - */ - if (pgdat == NODE_DATA(0)) { - mem_map = NODE_DATA(0)->node_mem_map; - if (page_to_pfn(mem_map) != pgdat->node_start_pfn) - mem_map -= offset; - } -#endif -} -#else -static inline void alloc_node_mem_map(struct pglist_data *pgdat) { } -#endif /* CONFIG_FLATMEM */ - -#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT -static inline void pgdat_set_deferred_range(pg_data_t *pgdat) -{ - pgdat->first_deferred_pfn = ULONG_MAX; -} -#else -static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} -#endif - -static void __init free_area_init_node(int nid) -{ - pg_data_t *pgdat = NODE_DATA(nid); - unsigned long start_pfn = 0; - unsigned long end_pfn = 0; - - /* pg_data_t should be reset to zero when it's allocated */ - WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); - - get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); - - pgdat->node_id = nid; - pgdat->node_start_pfn = start_pfn; - pgdat->per_cpu_nodestats = NULL; - - if (start_pfn != end_pfn) { - pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, - (u64)start_pfn << PAGE_SHIFT, - end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); - } else { - pr_info("Initmem setup node %d as memoryless\n", nid); - } - - calculate_node_totalpages(pgdat, start_pfn, end_pfn); - - alloc_node_mem_map(pgdat); - pgdat_set_deferred_range(pgdat); - - free_area_init_core(pgdat); - lru_gen_init_pgdat(pgdat); -} - -static void __init free_area_init_memoryless_node(int nid) -{ - free_area_init_node(nid); -} - -#if MAX_NUMNODES > 1 -/* - * Figure out the number of possible node ids. - */ -void __init setup_nr_node_ids(void) -{ - unsigned int highest; - - highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); - nr_node_ids = highest + 1; -} -#endif - -/** - * node_map_pfn_alignment - determine the maximum internode alignment - * - * This function should be called after node map is populated and sorted. - * It calculates the maximum power of two alignment which can distinguish - * all the nodes. - * - * For example, if all nodes are 1GiB and aligned to 1GiB, the return value - * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the - * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is - * shifted, 1GiB is enough and this function will indicate so. - * - * This is used to test whether pfn -> nid mapping of the chosen memory - * model has fine enough granularity to avoid incorrect mapping for the - * populated node map. - * - * Return: the determined alignment in pfn's. 0 if there is no alignment - * requirement (single node). - */ -unsigned long __init node_map_pfn_alignment(void) -{ - unsigned long accl_mask = 0, last_end = 0; - unsigned long start, end, mask; - int last_nid = NUMA_NO_NODE; - int i, nid; - - for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { - if (!start || last_nid < 0 || last_nid == nid) { - last_nid = nid; - last_end = end; - continue; - } - - /* - * Start with a mask granular enough to pin-point to the - * start pfn and tick off bits one-by-one until it becomes - * too coarse to separate the current node from the last. - */ - mask = ~((1 << __ffs(start)) - 1); - while (mask && last_end <= (start & (mask << 1))) - mask <<= 1; - - /* accumulate all internode masks */ - accl_mask |= mask; - } - - /* convert mask to number of pages */ - return ~accl_mask + 1; -} - -/* - * early_calculate_totalpages() - * Sum pages in active regions for movable zone. - * Populate N_MEMORY for calculating usable_nodes. - */ -static unsigned long __init early_calculate_totalpages(void) -{ - unsigned long totalpages = 0; - unsigned long start_pfn, end_pfn; - int i, nid; - - for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { - unsigned long pages = end_pfn - start_pfn; - - totalpages += pages; - if (pages) - node_set_state(nid, N_MEMORY); - } - return totalpages; -} - -/* - * Find the PFN the Movable zone begins in each node. Kernel memory - * is spread evenly between nodes as long as the nodes have enough - * memory. When they don't, some nodes will have more kernelcore than - * others - */ -static void __init find_zone_movable_pfns_for_nodes(void) -{ - int i, nid; - unsigned long usable_startpfn; - unsigned long kernelcore_node, kernelcore_remaining; - /* save the state before borrow the nodemask */ - nodemask_t saved_node_state = node_states[N_MEMORY]; - unsigned long totalpages = early_calculate_totalpages(); - int usable_nodes = nodes_weight(node_states[N_MEMORY]); - struct memblock_region *r; - - /* Need to find movable_zone earlier when movable_node is specified. */ - find_usable_zone_for_movable(); - - /* - * If movable_node is specified, ignore kernelcore and movablecore - * options. - */ - if (movable_node_is_enabled()) { - for_each_mem_region(r) { - if (!memblock_is_hotpluggable(r)) - continue; - - nid = memblock_get_region_node(r); - - usable_startpfn = PFN_DOWN(r->base); - zone_movable_pfn[nid] = zone_movable_pfn[nid] ? - min(usable_startpfn, zone_movable_pfn[nid]) : - usable_startpfn; - } - - goto out2; - } - - /* - * If kernelcore=mirror is specified, ignore movablecore option - */ - if (mirrored_kernelcore) { - bool mem_below_4gb_not_mirrored = false; - - for_each_mem_region(r) { - if (memblock_is_mirror(r)) - continue; - - nid = memblock_get_region_node(r); - - usable_startpfn = memblock_region_memory_base_pfn(r); - - if (usable_startpfn < PHYS_PFN(SZ_4G)) { - mem_below_4gb_not_mirrored = true; - continue; - } - - zone_movable_pfn[nid] = zone_movable_pfn[nid] ? - min(usable_startpfn, zone_movable_pfn[nid]) : - usable_startpfn; - } - - if (mem_below_4gb_not_mirrored) - pr_warn("This configuration results in unmirrored kernel memory.\n"); - - goto out2; - } - - /* - * If kernelcore=nn% or movablecore=nn% was specified, calculate the - * amount of necessary memory. - */ - if (required_kernelcore_percent) - required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / - 10000UL; - if (required_movablecore_percent) - required_movablecore = (totalpages * 100 * required_movablecore_percent) / - 10000UL; - - /* - * If movablecore= was specified, calculate what size of - * kernelcore that corresponds so that memory usable for - * any allocation type is evenly spread. If both kernelcore - * and movablecore are specified, then the value of kernelcore - * will be used for required_kernelcore if it's greater than - * what movablecore would have allowed. - */ - if (required_movablecore) { - unsigned long corepages; - - /* - * Round-up so that ZONE_MOVABLE is at least as large as what - * was requested by the user - */ - required_movablecore = - roundup(required_movablecore, MAX_ORDER_NR_PAGES); - required_movablecore = min(totalpages, required_movablecore); - corepages = totalpages - required_movablecore; - - required_kernelcore = max(required_kernelcore, corepages); - } - - /* - * If kernelcore was not specified or kernelcore size is larger - * than totalpages, there is no ZONE_MOVABLE. - */ - if (!required_kernelcore || required_kernelcore >= totalpages) - goto out; - - /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ - usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; - -restart: - /* Spread kernelcore memory as evenly as possible throughout nodes */ - kernelcore_node = required_kernelcore / usable_nodes; - for_each_node_state(nid, N_MEMORY) { - unsigned long start_pfn, end_pfn; +#ifdef CONFIG_MMU + int high; + int nr_split_cpus; + unsigned long total_pages; + if (!percpu_pagelist_high_fraction) { /* - * Recalculate kernelcore_node if the division per node - * now exceeds what is necessary to satisfy the requested - * amount of memory for the kernel + * By default, the high value of the pcp is based on the zone + * low watermark so that if they are full then background + * reclaim will not be started prematurely. */ - if (required_kernelcore < kernelcore_node) - kernelcore_node = required_kernelcore / usable_nodes; - + total_pages = low_wmark_pages(zone); + } else { /* - * As the map is walked, we track how much memory is usable - * by the kernel using kernelcore_remaining. When it is - * 0, the rest of the node is usable by ZONE_MOVABLE + * If percpu_pagelist_high_fraction is configured, the high + * value is based on a fraction of the managed pages in the + * zone. */ - kernelcore_remaining = kernelcore_node; - - /* Go through each range of PFNs within this node */ - for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { - unsigned long size_pages; - - start_pfn = max(start_pfn, zone_movable_pfn[nid]); - if (start_pfn >= end_pfn) - continue; - - /* Account for what is only usable for kernelcore */ - if (start_pfn < usable_startpfn) { - unsigned long kernel_pages; - kernel_pages = min(end_pfn, usable_startpfn) - - start_pfn; - - kernelcore_remaining -= min(kernel_pages, - kernelcore_remaining); - required_kernelcore -= min(kernel_pages, - required_kernelcore); - - /* Continue if range is now fully accounted */ - if (end_pfn <= usable_startpfn) { - - /* - * Push zone_movable_pfn to the end so - * that if we have to rebalance - * kernelcore across nodes, we will - * not double account here - */ - zone_movable_pfn[nid] = end_pfn; - continue; - } - start_pfn = usable_startpfn; - } - - /* - * The usable PFN range for ZONE_MOVABLE is from - * start_pfn->end_pfn. Calculate size_pages as the - * number of pages used as kernelcore - */ - size_pages = end_pfn - start_pfn; - if (size_pages > kernelcore_remaining) - size_pages = kernelcore_remaining; - zone_movable_pfn[nid] = start_pfn + size_pages; - - /* - * Some kernelcore has been met, update counts and - * break if the kernelcore for this node has been - * satisfied - */ - required_kernelcore -= min(required_kernelcore, - size_pages); - kernelcore_remaining -= size_pages; - if (!kernelcore_remaining) - break; - } + total_pages = zone_managed_pages(zone) / percpu_pagelist_high_fraction; } /* - * If there is still required_kernelcore, we do another pass with one - * less node in the count. This will push zone_movable_pfn[nid] further - * along on the nodes that still have memory until kernelcore is - * satisfied + * Split the high value across all online CPUs local to the zone. Note + * that early in boot that CPUs may not be online yet and that during + * CPU hotplug that the cpumask is not yet updated when a CPU is being + * onlined. For memory nodes that have no CPUs, split pcp->high across + * all online CPUs to mitigate the risk that reclaim is triggered + * prematurely due to pages stored on pcp lists. */ - usable_nodes--; - if (usable_nodes && required_kernelcore > usable_nodes) - goto restart; - -out2: - /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ - for (nid = 0; nid < MAX_NUMNODES; nid++) { - unsigned long start_pfn, end_pfn; - - zone_movable_pfn[nid] = - roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); - - get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); - if (zone_movable_pfn[nid] >= end_pfn) - zone_movable_pfn[nid] = 0; - } - -out: - /* restore the node_state */ - node_states[N_MEMORY] = saved_node_state; -} + nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online; + if (!nr_split_cpus) + nr_split_cpus = num_online_cpus(); + high = total_pages / nr_split_cpus; -/* Any regular or high memory on that node ? */ -static void check_for_memory(pg_data_t *pgdat, int nid) -{ - enum zone_type zone_type; + /* + * Ensure high is at least batch*4. The multiple is based on the + * historical relationship between high and batch. + */ + high = max(high, batch << 2); - for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { - struct zone *zone = &pgdat->node_zones[zone_type]; - if (populated_zone(zone)) { - if (IS_ENABLED(CONFIG_HIGHMEM)) - node_set_state(nid, N_HIGH_MEMORY); - if (zone_type <= ZONE_NORMAL) - node_set_state(nid, N_NORMAL_MEMORY); - break; - } - } + return high; +#else + return 0; +#endif } /* - * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For - * such cases we allow max_zone_pfn sorted in the descending order + * pcp->high and pcp->batch values are related and generally batch is lower + * than high. They are also related to pcp->count such that count is lower + * than high, and as soon as it reaches high, the pcplist is flushed. + * + * However, guaranteeing these relations at all times would require e.g. write + * barriers here but also careful usage of read barriers at the read side, and + * thus be prone to error and bad for performance. Thus the update only prevents + * store tearing. Any new users of pcp->batch and pcp->high should ensure they + * can cope with those fields changing asynchronously, and fully trust only the + * pcp->count field on the local CPU with interrupts disabled. + * + * mutex_is_locked(&pcp_batch_high_lock) required when calling this function + * outside of boot time (or some other assurance that no concurrent updaters + * exist). */ -bool __weak arch_has_descending_max_zone_pfns(void) +static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, + unsigned long batch) { - return false; + WRITE_ONCE(pcp->batch, batch); + WRITE_ONCE(pcp->high, high); } -/** - * free_area_init - Initialise all pg_data_t and zone data - * @max_zone_pfn: an array of max PFNs for each zone - * - * This will call free_area_init_node() for each active node in the system. - * Using the page ranges provided by memblock_set_node(), the size of each - * zone in each node and their holes is calculated. If the maximum PFN - * between two adjacent zones match, it is assumed that the zone is empty. - * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed - * that arch_max_dma32_pfn has no pages. It is also assumed that a zone - * starts where the previous one ended. For example, ZONE_DMA32 starts - * at arch_max_dma_pfn. - */ -void __init free_area_init(unsigned long *max_zone_pfn) +static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats) { - unsigned long start_pfn, end_pfn; - int i, nid, zone; - bool descending; - - /* Record where the zone boundaries are */ - memset(arch_zone_lowest_possible_pfn, 0, - sizeof(arch_zone_lowest_possible_pfn)); - memset(arch_zone_highest_possible_pfn, 0, - sizeof(arch_zone_highest_possible_pfn)); - - start_pfn = PHYS_PFN(memblock_start_of_DRAM()); - descending = arch_has_descending_max_zone_pfns(); - - for (i = 0; i < MAX_NR_ZONES; i++) { - if (descending) - zone = MAX_NR_ZONES - i - 1; - else - zone = i; - - if (zone == ZONE_MOVABLE) - continue; + int pindex; - end_pfn = max(max_zone_pfn[zone], start_pfn); - arch_zone_lowest_possible_pfn[zone] = start_pfn; - arch_zone_highest_possible_pfn[zone] = end_pfn; + memset(pcp, 0, sizeof(*pcp)); + memset(pzstats, 0, sizeof(*pzstats)); - start_pfn = end_pfn; - } + spin_lock_init(&pcp->lock); + for (pindex = 0; pindex < NR_PCP_LISTS; pindex++) + INIT_LIST_HEAD(&pcp->lists[pindex]); - /* Find the PFNs that ZONE_MOVABLE begins at in each node */ - memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); - find_zone_movable_pfns_for_nodes(); + /* + * Set batch and high values safe for a boot pageset. A true percpu + * pageset's initialization will update them subsequently. Here we don't + * need to be as careful as pageset_update() as nobody can access the + * pageset yet. + */ + pcp->high = BOOT_PAGESET_HIGH; + pcp->batch = BOOT_PAGESET_BATCH; + pcp->free_factor = 0; +} - /* Print out the zone ranges */ - pr_info("Zone ranges:\n"); - for (i = 0; i < MAX_NR_ZONES; i++) { - if (i == ZONE_MOVABLE) - continue; - pr_info(" %-8s ", zone_names[i]); - if (arch_zone_lowest_possible_pfn[i] == - arch_zone_highest_possible_pfn[i]) - pr_cont("empty\n"); - else - pr_cont("[mem %#018Lx-%#018Lx]\n", - (u64)arch_zone_lowest_possible_pfn[i] - << PAGE_SHIFT, - ((u64)arch_zone_highest_possible_pfn[i] - << PAGE_SHIFT) - 1); - } +static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high, + unsigned long batch) +{ + struct per_cpu_pages *pcp; + int cpu; - /* Print out the PFNs ZONE_MOVABLE begins at in each node */ - pr_info("Movable zone start for each node\n"); - for (i = 0; i < MAX_NUMNODES; i++) { - if (zone_movable_pfn[i]) - pr_info(" Node %d: %#018Lx\n", i, - (u64)zone_movable_pfn[i] << PAGE_SHIFT); + for_each_possible_cpu(cpu) { + pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); + pageset_update(pcp, high, batch); } +} - /* - * Print out the early node map, and initialize the - * subsection-map relative to active online memory ranges to - * enable future "sub-section" extensions of the memory map. - */ - pr_info("Early memory node ranges\n"); - for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { - pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, - (u64)start_pfn << PAGE_SHIFT, - ((u64)end_pfn << PAGE_SHIFT) - 1); - subsection_map_init(start_pfn, end_pfn - start_pfn); - } - - /* Initialise every node */ - mminit_verify_pageflags_layout(); - setup_nr_node_ids(); - for_each_node(nid) { - pg_data_t *pgdat; - - if (!node_online(nid)) { - pr_info("Initializing node %d as memoryless\n", nid); - - /* Allocator not initialized yet */ - pgdat = arch_alloc_nodedata(nid); - if (!pgdat) - panic("Cannot allocate %zuB for node %d.\n", - sizeof(*pgdat), nid); - arch_refresh_nodedata(nid, pgdat); - free_area_init_memoryless_node(nid); +/* + * Calculate and set new high and batch values for all per-cpu pagesets of a + * zone based on the zone's size. + */ +static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) +{ + int new_high, new_batch; - /* - * We do not want to confuse userspace by sysfs - * files/directories for node without any memory - * attached to it, so this node is not marked as - * N_MEMORY and not marked online so that no sysfs - * hierarchy will be created via register_one_node for - * it. The pgdat will get fully initialized by - * hotadd_init_pgdat() when memory is hotplugged into - * this node. - */ - continue; - } + new_batch = max(1, zone_batchsize(zone)); + new_high = zone_highsize(zone, new_batch, cpu_online); - pgdat = NODE_DATA(nid); - free_area_init_node(nid); + if (zone->pageset_high == new_high && + zone->pageset_batch == new_batch) + return; - /* Any memory on that node */ - if (pgdat->node_present_pages) - node_set_state(nid, N_MEMORY); - check_for_memory(pgdat, nid); - } + zone->pageset_high = new_high; + zone->pageset_batch = new_batch; - memmap_init(); + __zone_set_pageset_high_and_batch(zone, new_high, new_batch); } -static int __init cmdline_parse_core(char *p, unsigned long *core, - unsigned long *percent) +void __meminit setup_zone_pageset(struct zone *zone) { - unsigned long long coremem; - char *endptr; - - if (!p) - return -EINVAL; + int cpu; - /* Value may be a percentage of total memory, otherwise bytes */ - coremem = simple_strtoull(p, &endptr, 0); - if (*endptr == '%') { - /* Paranoid check for percent values greater than 100 */ - WARN_ON(coremem > 100); + /* Size may be 0 on !SMP && !NUMA */ + if (sizeof(struct per_cpu_zonestat) > 0) + zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat); - *percent = coremem; - } else { - coremem = memparse(p, &p); - /* Paranoid check that UL is enough for the coremem value */ - WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); + zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages); + for_each_possible_cpu(cpu) { + struct per_cpu_pages *pcp; + struct per_cpu_zonestat *pzstats; - *core = coremem >> PAGE_SHIFT; - *percent = 0UL; + pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); + pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); + per_cpu_pages_init(pcp, pzstats); } - return 0; + + zone_set_pageset_high_and_batch(zone, 0); } /* - * kernelcore=size sets the amount of memory for use for allocations that - * cannot be reclaimed or migrated. + * The zone indicated has a new number of managed_pages; batch sizes and percpu + * page high values need to be recalculated. */ -static int __init cmdline_parse_kernelcore(char *p) +static void zone_pcp_update(struct zone *zone, int cpu_online) { - /* parse kernelcore=mirror */ - if (parse_option_str(p, "mirror")) { - mirrored_kernelcore = true; - return 0; - } - - return cmdline_parse_core(p, &required_kernelcore, - &required_kernelcore_percent); + mutex_lock(&pcp_batch_high_lock); + zone_set_pageset_high_and_batch(zone, cpu_online); + mutex_unlock(&pcp_batch_high_lock); } /* - * movablecore=size sets the amount of memory for use for allocations that - * can be reclaimed or migrated. + * Allocate per cpu pagesets and initialize them. + * Before this call only boot pagesets were available. */ -static int __init cmdline_parse_movablecore(char *p) +void __init setup_per_cpu_pageset(void) { - return cmdline_parse_core(p, &required_movablecore, - &required_movablecore_percent); + struct pglist_data *pgdat; + struct zone *zone; + int __maybe_unused cpu; + + for_each_populated_zone(zone) + setup_zone_pageset(zone); + +#ifdef CONFIG_NUMA + /* + * Unpopulated zones continue using the boot pagesets. + * The numa stats for these pagesets need to be reset. + * Otherwise, they will end up skewing the stats of + * the nodes these zones are associated with. + */ + for_each_possible_cpu(cpu) { + struct per_cpu_zonestat *pzstats = &per_cpu(boot_zonestats, cpu); + memset(pzstats->vm_numa_event, 0, + sizeof(pzstats->vm_numa_event)); + } +#endif + + for_each_online_pgdat(pgdat) + pgdat->per_cpu_nodestats = + alloc_percpu(struct per_cpu_nodestat); } -early_param("kernelcore", cmdline_parse_kernelcore); -early_param("movablecore", cmdline_parse_movablecore); +__meminit void zone_pcp_init(struct zone *zone) +{ + /* + * per cpu subsystem is not up at this point. The following code + * relies on the ability of the linker to provide the + * offset of a (static) per cpu variable into the per cpu area. + */ + zone->per_cpu_pageset = &boot_pageset; + zone->per_cpu_zonestats = &boot_zonestats; + zone->pageset_high = BOOT_PAGESET_HIGH; + zone->pageset_batch = BOOT_PAGESET_BATCH; + + if (populated_zone(zone)) + pr_debug(" %s zone: %lu pages, LIFO batch:%u\n", zone->name, + zone->present_pages, zone_batchsize(zone)); +} void adjust_managed_page_count(struct page *page, long count) { @@ -8516,22 +6353,6 @@ void __init mem_init_print_info(void) ); } -/** - * set_dma_reserve - set the specified number of pages reserved in the first zone - * @new_dma_reserve: The number of pages to mark reserved - * - * The per-cpu batchsize and zone watermarks are determined by managed_pages. - * In the DMA zone, a significant percentage may be consumed by kernel image - * and other unfreeable allocations which can skew the watermarks badly. This - * function may optionally be used to account for unfreeable pages in the - * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and - * smaller per-cpu batchsize. - */ -void __init set_dma_reserve(unsigned long new_dma_reserve) -{ - dma_reserve = new_dma_reserve; -} - static int page_alloc_cpu_dead(unsigned int cpu) { struct zone *zone; @@ -8976,149 +6797,6 @@ out: return ret; } -#ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES -/* - * Returns the number of pages that arch has reserved but - * is not known to alloc_large_system_hash(). - */ -static unsigned long __init arch_reserved_kernel_pages(void) -{ - return 0; -} -#endif - -/* - * Adaptive scale is meant to reduce sizes of hash tables on large memory - * machines. As memory size is increased the scale is also increased but at - * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory - * quadruples the scale is increased by one, which means the size of hash table - * only doubles, instead of quadrupling as well. - * Because 32-bit systems cannot have large physical memory, where this scaling - * makes sense, it is disabled on such platforms. - */ -#if __BITS_PER_LONG > 32 -#define ADAPT_SCALE_BASE (64ul << 30) -#define ADAPT_SCALE_SHIFT 2 -#define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) -#endif - -/* - * allocate a large system hash table from bootmem - * - it is assumed that the hash table must contain an exact power-of-2 - * quantity of entries - * - limit is the number of hash buckets, not the total allocation size - */ -void *__init alloc_large_system_hash(const char *tablename, - unsigned long bucketsize, - unsigned long numentries, - int scale, - int flags, - unsigned int *_hash_shift, - unsigned int *_hash_mask, - unsigned long low_limit, - unsigned long high_limit) -{ - unsigned long long max = high_limit; - unsigned long log2qty, size; - void *table; - gfp_t gfp_flags; - bool virt; - bool huge; - - /* allow the kernel cmdline to have a say */ - if (!numentries) { - /* round applicable memory size up to nearest megabyte */ - numentries = nr_kernel_pages; - numentries -= arch_reserved_kernel_pages(); - - /* It isn't necessary when PAGE_SIZE >= 1MB */ - if (PAGE_SIZE < SZ_1M) - numentries = round_up(numentries, SZ_1M / PAGE_SIZE); - -#if __BITS_PER_LONG > 32 - if (!high_limit) { - unsigned long adapt; - - for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; - adapt <<= ADAPT_SCALE_SHIFT) - scale++; - } -#endif - - /* limit to 1 bucket per 2^scale bytes of low memory */ - if (scale > PAGE_SHIFT) - numentries >>= (scale - PAGE_SHIFT); - else - numentries <<= (PAGE_SHIFT - scale); - - /* Make sure we've got at least a 0-order allocation.. */ - if (unlikely(flags & HASH_SMALL)) { - /* Makes no sense without HASH_EARLY */ - WARN_ON(!(flags & HASH_EARLY)); - if (!(numentries >> *_hash_shift)) { - numentries = 1UL << *_hash_shift; - BUG_ON(!numentries); - } - } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) - numentries = PAGE_SIZE / bucketsize; - } - numentries = roundup_pow_of_two(numentries); - - /* limit allocation size to 1/16 total memory by default */ - if (max == 0) { - max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; - do_div(max, bucketsize); - } - max = min(max, 0x80000000ULL); - - if (numentries < low_limit) - numentries = low_limit; - if (numentries > max) - numentries = max; - - log2qty = ilog2(numentries); - - gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; - do { - virt = false; - size = bucketsize << log2qty; - if (flags & HASH_EARLY) { - if (flags & HASH_ZERO) - table = memblock_alloc(size, SMP_CACHE_BYTES); - else - table = memblock_alloc_raw(size, - SMP_CACHE_BYTES); - } else if (get_order(size) > MAX_ORDER || hashdist) { - table = vmalloc_huge(size, gfp_flags); - virt = true; - if (table) - huge = is_vm_area_hugepages(table); - } else { - /* - * If bucketsize is not a power-of-two, we may free - * some pages at the end of hash table which - * alloc_pages_exact() automatically does - */ - table = alloc_pages_exact(size, gfp_flags); - kmemleak_alloc(table, size, 1, gfp_flags); - } - } while (!table && size > PAGE_SIZE && --log2qty); - - if (!table) - panic("Failed to allocate %s hash table\n", tablename); - - pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", - tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, - virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); - - if (_hash_shift) - *_hash_shift = log2qty; - if (_hash_mask) - *_hash_mask = (1 << log2qty) - 1; - - return table; -} - #ifdef CONFIG_CONTIG_ALLOC #if defined(CONFIG_DYNAMIC_DEBUG) || \ (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) -- cgit v1.2.3 From c4fbed4b0284d91797ee3cf60983b94c84c396b6 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (IBM)" Date: Tue, 21 Mar 2023 19:05:04 +0200 Subject: mm/page_alloc: rename page_alloc_init() to page_alloc_init_cpuhp() The page_alloc_init() name is really misleading because all this function does is sets up CPU hotplug callbacks for the page allocator. Rename it to page_alloc_init_cpuhp() so that name will reflect what the function does. Link: https://lkml.kernel.org/r/20230321170513.2401534-6-rppt@kernel.org Signed-off-by: Mike Rapoport (IBM) Reviewed-by: David Hildenbrand Reviewed-by: Vlastimil Babka Cc: Doug Berger Cc: Matthew Wilcox (Oracle) Cc: Mel Gorman Cc: Michal Hocko Cc: Thomas Bogendoerfer Signed-off-by: Andrew Morton --- include/linux/gfp.h | 2 +- init/main.c | 2 +- mm/page_alloc.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 7c554e4bd49f..ed8cb537c6a7 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -319,7 +319,7 @@ extern void page_frag_free(void *addr); #define __free_page(page) __free_pages((page), 0) #define free_page(addr) free_pages((addr), 0) -void page_alloc_init(void); +void page_alloc_init_cpuhp(void); void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); void drain_all_pages(struct zone *zone); void drain_local_pages(struct zone *zone); diff --git a/init/main.c b/init/main.c index bb87b789c543..f285bc53c345 100644 --- a/init/main.c +++ b/init/main.c @@ -973,7 +973,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void) boot_cpu_hotplug_init(); build_all_zonelists(NULL); - page_alloc_init(); + page_alloc_init_cpuhp(); pr_notice("Kernel command line: %s\n", saved_command_line); /* parameters may set static keys */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3abc994bcfa1..1131b87586d5 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6393,7 +6393,7 @@ static int page_alloc_cpu_online(unsigned int cpu) return 0; } -void __init page_alloc_init(void) +void __init page_alloc_init_cpuhp(void) { int ret; -- cgit v1.2.3 From b7ec1bf3e7b9dd9d3335c937f5d834680d74addf Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (IBM)" Date: Tue, 21 Mar 2023 19:05:06 +0200 Subject: init,mm: move mm_init() to mm/mm_init.c and rename it to mm_core_init() Make mm_init() a part of mm/ codebase. mm_core_init() better describes what the function does and does not clash with mm_init() in kernel/fork.c Link: https://lkml.kernel.org/r/20230321170513.2401534-8-rppt@kernel.org Signed-off-by: Mike Rapoport (IBM) Acked-by: David Hildenbrand Reviewed-by: Vlastimil Babka Cc: Doug Berger Cc: Matthew Wilcox (Oracle) Cc: Mel Gorman Cc: Michal Hocko Cc: Thomas Bogendoerfer Signed-off-by: Andrew Morton --- include/linux/mm.h | 1 + init/main.c | 71 ++-------------------------------------------------- mm/mm_init.c | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 69 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 1f79667824eb..6be179cec081 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -38,6 +38,7 @@ struct pt_regs; extern int sysctl_page_lock_unfairness; +void mm_core_init(void); void init_mm_internals(void); #ifndef CONFIG_NUMA /* Don't use mapnrs, do it properly */ diff --git a/init/main.c b/init/main.c index c240cce5bc86..f1b2d8d80987 100644 --- a/init/main.c +++ b/init/main.c @@ -807,73 +807,6 @@ static inline void initcall_debug_enable(void) } #endif -/* Report memory auto-initialization states for this boot. */ -static void __init report_meminit(void) -{ - const char *stack; - - if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN)) - stack = "all(pattern)"; - else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO)) - stack = "all(zero)"; - else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL)) - stack = "byref_all(zero)"; - else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF)) - stack = "byref(zero)"; - else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER)) - stack = "__user(zero)"; - else - stack = "off"; - - pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n", - stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off", - want_init_on_free() ? "on" : "off"); - if (want_init_on_free()) - pr_info("mem auto-init: clearing system memory may take some time...\n"); -} - -/* - * Set up kernel memory allocators - */ -static void __init mm_init(void) -{ - /* Initializations relying on SMP setup */ - build_all_zonelists(NULL); - page_alloc_init_cpuhp(); - - /* - * page_ext requires contiguous pages, - * bigger than MAX_ORDER unless SPARSEMEM. - */ - page_ext_init_flatmem(); - init_mem_debugging_and_hardening(); - kfence_alloc_pool(); - report_meminit(); - kmsan_init_shadow(); - stack_depot_early_init(); - mem_init(); - mem_init_print_info(); - kmem_cache_init(); - /* - * page_owner must be initialized after buddy is ready, and also after - * slab is ready so that stack_depot_init() works properly - */ - page_ext_init_flatmem_late(); - kmemleak_init(); - pgtable_init(); - debug_objects_mem_init(); - vmalloc_init(); - /* If no deferred init page_ext now, as vmap is fully initialized */ - if (!deferred_struct_pages) - page_ext_init(); - /* Should be run before the first non-init thread is created */ - init_espfix_bsp(); - /* Should be run after espfix64 is set up. */ - pti_init(); - kmsan_init_runtime(); - mm_cache_init(); -} - #ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT, randomize_kstack_offset); @@ -997,13 +930,13 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void) /* * These use large bootmem allocations and must precede - * kmem_cache_init() + * initalization of page allocator */ setup_log_buf(0); vfs_caches_init_early(); sort_main_extable(); trap_init(); - mm_init(); + mm_core_init(); poking_init(); ftrace_init(); diff --git a/mm/mm_init.c b/mm/mm_init.c index 2e60c7186132..bba73f1fb277 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -20,9 +20,15 @@ #include #include #include +#include +#include +#include +#include #include "internal.h" #include "shuffle.h" +#include + #ifdef CONFIG_DEBUG_MEMORY_INIT int __meminitdata mminit_loglevel; @@ -2524,3 +2530,70 @@ void __init memblock_free_pages(struct page *page, unsigned long pfn, } __free_pages_core(page, order); } + +/* Report memory auto-initialization states for this boot. */ +static void __init report_meminit(void) +{ + const char *stack; + + if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN)) + stack = "all(pattern)"; + else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO)) + stack = "all(zero)"; + else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL)) + stack = "byref_all(zero)"; + else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF)) + stack = "byref(zero)"; + else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER)) + stack = "__user(zero)"; + else + stack = "off"; + + pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n", + stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off", + want_init_on_free() ? "on" : "off"); + if (want_init_on_free()) + pr_info("mem auto-init: clearing system memory may take some time...\n"); +} + +/* + * Set up kernel memory allocators + */ +void __init mm_core_init(void) +{ + /* Initializations relying on SMP setup */ + build_all_zonelists(NULL); + page_alloc_init_cpuhp(); + + /* + * page_ext requires contiguous pages, + * bigger than MAX_ORDER unless SPARSEMEM. + */ + page_ext_init_flatmem(); + init_mem_debugging_and_hardening(); + kfence_alloc_pool(); + report_meminit(); + kmsan_init_shadow(); + stack_depot_early_init(); + mem_init(); + mem_init_print_info(); + kmem_cache_init(); + /* + * page_owner must be initialized after buddy is ready, and also after + * slab is ready so that stack_depot_init() works properly + */ + page_ext_init_flatmem_late(); + kmemleak_init(); + pgtable_init(); + debug_objects_mem_init(); + vmalloc_init(); + /* If no deferred init page_ext now, as vmap is fully initialized */ + if (!deferred_struct_pages) + page_ext_init(); + /* Should be run before the first non-init thread is created */ + init_espfix_bsp(); + /* Should be run after espfix64 is set up. */ + pti_init(); + kmsan_init_runtime(); + mm_cache_init(); +} -- cgit v1.2.3 From 4cd1e9edf60efb20fad35cf5e9ade7ad75b34cd1 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (IBM)" Date: Tue, 21 Mar 2023 19:05:07 +0200 Subject: mm: call {ptlock,pgtable}_cache_init() directly from mm_core_init() and drop pgtable_init() as it has no real value and its name is misleading. Link: https://lkml.kernel.org/r/20230321170513.2401534-9-rppt@kernel.org Signed-off-by: Mike Rapoport (IBM) Reviewed-by: David Hildenbrand Reviewed-by: Vlastimil Babka Cc: Doug Berger Cc: Matthew Wilcox (Oracle) Cc: Mel Gorman Cc: Michal Hocko Cc: Thomas Bogendoerfer Cc: Sergei Shtylyov Signed-off-by: Andrew Morton --- include/linux/mm.h | 6 ------ mm/mm_init.c | 3 ++- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 6be179cec081..dfff8bcaa766 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2637,12 +2637,6 @@ static inline bool ptlock_init(struct page *page) { return true; } static inline void ptlock_free(struct page *page) {} #endif /* USE_SPLIT_PTE_PTLOCKS */ -static inline void pgtable_init(void) -{ - ptlock_cache_init(); - pgtable_cache_init(); -} - static inline bool pgtable_pte_page_ctor(struct page *page) { if (!ptlock_init(page)) diff --git a/mm/mm_init.c b/mm/mm_init.c index bba73f1fb277..f1475413394d 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2584,7 +2584,8 @@ void __init mm_core_init(void) */ page_ext_init_flatmem_late(); kmemleak_init(); - pgtable_init(); + ptlock_cache_init(); + pgtable_cache_init(); debug_objects_mem_init(); vmalloc_init(); /* If no deferred init page_ext now, as vmap is fully initialized */ -- cgit v1.2.3 From f2fc4b44ec2bb94c51c7ae1af9b1177d72705992 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (IBM)" Date: Tue, 21 Mar 2023 19:05:08 +0200 Subject: mm: move init_mem_debugging_and_hardening() to mm/mm_init.c init_mem_debugging_and_hardening() is only called from mm_core_init(). Move it close to the caller, make it static and rename it to mem_debugging_and_hardening_init() for consistency with surrounding convention. Link: https://lkml.kernel.org/r/20230321170513.2401534-10-rppt@kernel.org Signed-off-by: Mike Rapoport (IBM) Acked-by: David Hildenbrand Reviewed-by: Vlastimil Babka Cc: Doug Berger Cc: Matthew Wilcox (Oracle) Cc: Mel Gorman Cc: Michal Hocko Cc: Thomas Bogendoerfer Signed-off-by: Andrew Morton --- include/linux/mm.h | 1 - mm/internal.h | 8 +++++ mm/mm_init.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++- mm/page_alloc.c | 95 ------------------------------------------------------ 4 files changed, 98 insertions(+), 97 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index dfff8bcaa766..0fa6a6f94eda 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3251,7 +3251,6 @@ extern int apply_to_existing_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, pte_fn_t fn, void *data); -extern void __init init_mem_debugging_and_hardening(void); #ifdef CONFIG_PAGE_POISONING extern void __kernel_poison_pages(struct page *page, int numpages); extern void __kernel_unpoison_pages(struct page *page, int numpages); diff --git a/mm/internal.h b/mm/internal.h index 22f1410a0ee3..a2934c610182 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -204,6 +204,14 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); extern char * const zone_names[MAX_NR_ZONES]; +/* perform sanity checks on struct pages being allocated or freed */ +DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); + +static inline bool is_check_pages_enabled(void) +{ + return static_branch_unlikely(&check_pages_enabled); +} + /* * Structure for holding the mostly immutable allocation parameters passed * between functions involved in allocations, including the alloc_pages* diff --git a/mm/mm_init.c b/mm/mm_init.c index f1475413394d..43f6d3ed24ef 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2531,6 +2531,95 @@ void __init memblock_free_pages(struct page *page, unsigned long pfn, __free_pages_core(page, order); } +static bool _init_on_alloc_enabled_early __read_mostly + = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); +static int __init early_init_on_alloc(char *buf) +{ + + return kstrtobool(buf, &_init_on_alloc_enabled_early); +} +early_param("init_on_alloc", early_init_on_alloc); + +static bool _init_on_free_enabled_early __read_mostly + = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); +static int __init early_init_on_free(char *buf) +{ + return kstrtobool(buf, &_init_on_free_enabled_early); +} +early_param("init_on_free", early_init_on_free); + +DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); + +/* + * Enable static keys related to various memory debugging and hardening options. + * Some override others, and depend on early params that are evaluated in the + * order of appearance. So we need to first gather the full picture of what was + * enabled, and then make decisions. + */ +static void __init mem_debugging_and_hardening_init(void) +{ + bool page_poisoning_requested = false; + bool want_check_pages = false; + +#ifdef CONFIG_PAGE_POISONING + /* + * Page poisoning is debug page alloc for some arches. If + * either of those options are enabled, enable poisoning. + */ + if (page_poisoning_enabled() || + (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && + debug_pagealloc_enabled())) { + static_branch_enable(&_page_poisoning_enabled); + page_poisoning_requested = true; + want_check_pages = true; + } +#endif + + if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && + page_poisoning_requested) { + pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " + "will take precedence over init_on_alloc and init_on_free\n"); + _init_on_alloc_enabled_early = false; + _init_on_free_enabled_early = false; + } + + if (_init_on_alloc_enabled_early) { + want_check_pages = true; + static_branch_enable(&init_on_alloc); + } else { + static_branch_disable(&init_on_alloc); + } + + if (_init_on_free_enabled_early) { + want_check_pages = true; + static_branch_enable(&init_on_free); + } else { + static_branch_disable(&init_on_free); + } + + if (IS_ENABLED(CONFIG_KMSAN) && + (_init_on_alloc_enabled_early || _init_on_free_enabled_early)) + pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n"); + +#ifdef CONFIG_DEBUG_PAGEALLOC + if (debug_pagealloc_enabled()) { + want_check_pages = true; + static_branch_enable(&_debug_pagealloc_enabled); + + if (debug_guardpage_minorder()) + static_branch_enable(&_debug_guardpage_enabled); + } +#endif + + /* + * Any page debugging or hardening option also enables sanity checking + * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's + * enabled already. + */ + if (!IS_ENABLED(CONFIG_DEBUG_VM) && want_check_pages) + static_branch_enable(&check_pages_enabled); +} + /* Report memory auto-initialization states for this boot. */ static void __init report_meminit(void) { @@ -2570,7 +2659,7 @@ void __init mm_core_init(void) * bigger than MAX_ORDER unless SPARSEMEM. */ page_ext_init_flatmem(); - init_mem_debugging_and_hardening(); + mem_debugging_and_hardening_init(); kfence_alloc_pool(); report_meminit(); kmsan_init_shadow(); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 1131b87586d5..94bf3b7b5c49 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -240,31 +240,6 @@ EXPORT_SYMBOL(init_on_alloc); DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); EXPORT_SYMBOL(init_on_free); -/* perform sanity checks on struct pages being allocated or freed */ -static DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); - -static inline bool is_check_pages_enabled(void) -{ - return static_branch_unlikely(&check_pages_enabled); -} - -static bool _init_on_alloc_enabled_early __read_mostly - = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); -static int __init early_init_on_alloc(char *buf) -{ - - return kstrtobool(buf, &_init_on_alloc_enabled_early); -} -early_param("init_on_alloc", early_init_on_alloc); - -static bool _init_on_free_enabled_early __read_mostly - = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); -static int __init early_init_on_free(char *buf) -{ - return kstrtobool(buf, &_init_on_free_enabled_early); -} -early_param("init_on_free", early_init_on_free); - /* * A cached value of the page's pageblock's migratetype, used when the page is * put on a pcplist. Used to avoid the pageblock migratetype lookup when @@ -798,76 +773,6 @@ static inline void clear_page_guard(struct zone *zone, struct page *page, unsigned int order, int migratetype) {} #endif -/* - * Enable static keys related to various memory debugging and hardening options. - * Some override others, and depend on early params that are evaluated in the - * order of appearance. So we need to first gather the full picture of what was - * enabled, and then make decisions. - */ -void __init init_mem_debugging_and_hardening(void) -{ - bool page_poisoning_requested = false; - bool want_check_pages = false; - -#ifdef CONFIG_PAGE_POISONING - /* - * Page poisoning is debug page alloc for some arches. If - * either of those options are enabled, enable poisoning. - */ - if (page_poisoning_enabled() || - (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && - debug_pagealloc_enabled())) { - static_branch_enable(&_page_poisoning_enabled); - page_poisoning_requested = true; - want_check_pages = true; - } -#endif - - if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && - page_poisoning_requested) { - pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " - "will take precedence over init_on_alloc and init_on_free\n"); - _init_on_alloc_enabled_early = false; - _init_on_free_enabled_early = false; - } - - if (_init_on_alloc_enabled_early) { - want_check_pages = true; - static_branch_enable(&init_on_alloc); - } else { - static_branch_disable(&init_on_alloc); - } - - if (_init_on_free_enabled_early) { - want_check_pages = true; - static_branch_enable(&init_on_free); - } else { - static_branch_disable(&init_on_free); - } - - if (IS_ENABLED(CONFIG_KMSAN) && - (_init_on_alloc_enabled_early || _init_on_free_enabled_early)) - pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n"); - -#ifdef CONFIG_DEBUG_PAGEALLOC - if (debug_pagealloc_enabled()) { - want_check_pages = true; - static_branch_enable(&_debug_pagealloc_enabled); - - if (debug_guardpage_minorder()) - static_branch_enable(&_debug_guardpage_enabled); - } -#endif - - /* - * Any page debugging or hardening option also enables sanity checking - * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's - * enabled already. - */ - if (!IS_ENABLED(CONFIG_DEBUG_VM) && want_check_pages) - static_branch_enable(&check_pages_enabled); -} - static inline void set_buddy_order(struct page *page, unsigned int order) { set_page_private(page, order); -- cgit v1.2.3 From de57807e6f267a658a046dbca44dc40fe806d60f Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (IBM)" Date: Tue, 21 Mar 2023 19:05:09 +0200 Subject: init,mm: fold late call to page_ext_init() to page_alloc_init_late() When deferred initialization of struct pages is enabled, page_ext_init() must be called after all the deferred initialization is done, but there is no point to keep it a separate call from kernel_init_freeable() right after page_alloc_init_late(). Fold the call to page_ext_init() into page_alloc_init_late() and localize deferred_struct_pages variable. Link: https://lkml.kernel.org/r/20230321170513.2401534-11-rppt@kernel.org Signed-off-by: Mike Rapoport (IBM) Reviewed-by: David Hildenbrand Reviewed-by: Vlastimil Babka Cc: Doug Berger Cc: Matthew Wilcox (Oracle) Cc: Mel Gorman Cc: Michal Hocko Cc: Thomas Bogendoerfer Signed-off-by: Andrew Morton --- include/linux/page_ext.h | 2 -- init/main.c | 4 ---- mm/mm_init.c | 6 +++++- 3 files changed, 5 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index bc2e39090a1f..67314f648aeb 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -29,8 +29,6 @@ struct page_ext_operations { bool need_shared_flags; }; -extern bool deferred_struct_pages; - #ifdef CONFIG_PAGE_EXTENSION /* diff --git a/init/main.c b/init/main.c index f1b2d8d80987..43bc4c82dc58 100644 --- a/init/main.c +++ b/init/main.c @@ -62,7 +62,6 @@ #include #include #include -#include #include #include #include @@ -1565,9 +1564,6 @@ static noinline void __init kernel_init_freeable(void) padata_init(); page_alloc_init_late(); - /* Initialize page ext after all struct pages are initialized. */ - if (deferred_struct_pages) - page_ext_init(); do_basic_setup(); diff --git a/mm/mm_init.c b/mm/mm_init.c index 43f6d3ed24ef..ff70da11e797 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -225,7 +225,7 @@ static unsigned long nr_kernel_pages __initdata; static unsigned long nr_all_pages __initdata; static unsigned long dma_reserve __initdata; -bool deferred_struct_pages __meminitdata; +static bool deferred_struct_pages __meminitdata; static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); @@ -2358,6 +2358,10 @@ void __init page_alloc_init_late(void) for_each_populated_zone(zone) set_zone_contiguous(zone); + + /* Initialize page ext after all struct pages are initialized. */ + if (deferred_struct_pages) + page_ext_init(); } #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES -- cgit v1.2.3 From eb8589b4f8c107c346421881963c0ee0b8367c2c Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (IBM)" Date: Tue, 21 Mar 2023 19:05:10 +0200 Subject: mm: move mem_init_print_info() to mm_init.c mem_init_print_info() is only called from mm_core_init(). Move it close to the caller and make it static. Link: https://lkml.kernel.org/r/20230321170513.2401534-12-rppt@kernel.org Signed-off-by: Mike Rapoport (IBM) Acked-by: David Hildenbrand Reviewed-by: Vlastimil Babka Cc: Doug Berger Cc: Matthew Wilcox (Oracle) Cc: Mel Gorman Cc: Michal Hocko Cc: Thomas Bogendoerfer Signed-off-by: Andrew Morton --- include/linux/mm.h | 1 - mm/internal.h | 1 + mm/mm_init.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ mm/page_alloc.c | 53 ----------------------------------------------------- 4 files changed, 54 insertions(+), 54 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 0fa6a6f94eda..c3abad7e6ccb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2780,7 +2780,6 @@ extern unsigned long free_reserved_area(void *start, void *end, int poison, const char *s); extern void adjust_managed_page_count(struct page *page, long count); -extern void mem_init_print_info(void); extern void reserve_bootmem_region(phys_addr_t start, phys_addr_t end); diff --git a/mm/internal.h b/mm/internal.h index a2934c610182..7cea7d833c50 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -201,6 +201,7 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); /* * in mm/page_alloc.c */ +#define K(x) ((x) << (PAGE_SHIFT-10)) extern char * const zone_names[MAX_NR_ZONES]; diff --git a/mm/mm_init.c b/mm/mm_init.c index ff70da11e797..8adadf51bbd2 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include "internal.h" #include "shuffle.h" @@ -2649,6 +2651,57 @@ static void __init report_meminit(void) pr_info("mem auto-init: clearing system memory may take some time...\n"); } +static void __init mem_init_print_info(void) +{ + unsigned long physpages, codesize, datasize, rosize, bss_size; + unsigned long init_code_size, init_data_size; + + physpages = get_num_physpages(); + codesize = _etext - _stext; + datasize = _edata - _sdata; + rosize = __end_rodata - __start_rodata; + bss_size = __bss_stop - __bss_start; + init_data_size = __init_end - __init_begin; + init_code_size = _einittext - _sinittext; + + /* + * Detect special cases and adjust section sizes accordingly: + * 1) .init.* may be embedded into .data sections + * 2) .init.text.* may be out of [__init_begin, __init_end], + * please refer to arch/tile/kernel/vmlinux.lds.S. + * 3) .rodata.* may be embedded into .text or .data sections. + */ +#define adj_init_size(start, end, size, pos, adj) \ + do { \ + if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \ + size -= adj; \ + } while (0) + + adj_init_size(__init_begin, __init_end, init_data_size, + _sinittext, init_code_size); + adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); + adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); + adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); + adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); + +#undef adj_init_size + + pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" +#ifdef CONFIG_HIGHMEM + ", %luK highmem" +#endif + ")\n", + K(nr_free_pages()), K(physpages), + codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K, + (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K, + K(physpages - totalram_pages() - totalcma_pages), + K(totalcma_pages) +#ifdef CONFIG_HIGHMEM + , K(totalhigh_pages()) +#endif + ); +} + /* * Set up kernel memory allocators */ diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 94bf3b7b5c49..d0eb280ec7e4 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5246,8 +5246,6 @@ static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask return !node_isset(nid, *nodemask); } -#define K(x) ((x) << (PAGE_SHIFT-10)) - static void show_migration_types(unsigned char type) { static const char types[MIGRATE_TYPES] = { @@ -6207,57 +6205,6 @@ unsigned long free_reserved_area(void *start, void *end, int poison, const char return pages; } -void __init mem_init_print_info(void) -{ - unsigned long physpages, codesize, datasize, rosize, bss_size; - unsigned long init_code_size, init_data_size; - - physpages = get_num_physpages(); - codesize = _etext - _stext; - datasize = _edata - _sdata; - rosize = __end_rodata - __start_rodata; - bss_size = __bss_stop - __bss_start; - init_data_size = __init_end - __init_begin; - init_code_size = _einittext - _sinittext; - - /* - * Detect special cases and adjust section sizes accordingly: - * 1) .init.* may be embedded into .data sections - * 2) .init.text.* may be out of [__init_begin, __init_end], - * please refer to arch/tile/kernel/vmlinux.lds.S. - * 3) .rodata.* may be embedded into .text or .data sections. - */ -#define adj_init_size(start, end, size, pos, adj) \ - do { \ - if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \ - size -= adj; \ - } while (0) - - adj_init_size(__init_begin, __init_end, init_data_size, - _sinittext, init_code_size); - adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); - adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); - adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); - adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); - -#undef adj_init_size - - pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" -#ifdef CONFIG_HIGHMEM - ", %luK highmem" -#endif - ")\n", - K(nr_free_pages()), K(physpages), - codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K, - (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K, - K(physpages - totalram_pages() - totalcma_pages), - K(totalcma_pages) -#ifdef CONFIG_HIGHMEM - , K(totalhigh_pages()) -#endif - ); -} - static int page_alloc_cpu_dead(unsigned int cpu) { struct zone *zone; -- cgit v1.2.3 From d5d2c02a4980c2e22037679457bf2d921b86a503 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (IBM)" Date: Tue, 21 Mar 2023 19:05:11 +0200 Subject: mm: move kmem_cache_init() declaration to mm/slab.h kmem_cache_init() is called only from mm_core_init(), there is no need to declare it in include/linux/slab.h Move kmem_cache_init() declaration to mm/slab.h Link: https://lkml.kernel.org/r/20230321170513.2401534-13-rppt@kernel.org Signed-off-by: Mike Rapoport (IBM) Reviewed-by: David Hildenbrand Reviewed-by: Vlastimil Babka Cc: Doug Berger Cc: Matthew Wilcox (Oracle) Cc: Mel Gorman Cc: Michal Hocko Cc: Thomas Bogendoerfer Signed-off-by: Andrew Morton --- include/linux/slab.h | 1 - mm/mm_init.c | 1 + mm/slab.h | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/slab.h b/include/linux/slab.h index aa4575ef2965..f8b1d63c63a3 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -167,7 +167,6 @@ struct mem_cgroup; /* * struct kmem_cache related prototypes */ -void __init kmem_cache_init(void); bool slab_is_available(void); struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, diff --git a/mm/mm_init.c b/mm/mm_init.c index 8adadf51bbd2..53fb8e9d1e3b 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -27,6 +27,7 @@ #include #include #include "internal.h" +#include "slab.h" #include "shuffle.h" #include diff --git a/mm/slab.h b/mm/slab.h index 43966aa5fadf..3f8df2244f5a 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -4,6 +4,7 @@ /* * Internal slab definitions */ +void __init kmem_cache_init(void); /* Reuses the bits in struct page */ struct slab { -- cgit v1.2.3 From b671491199acd3609f87754d268f2f5cb10de18d Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (IBM)" Date: Tue, 21 Mar 2023 19:05:12 +0200 Subject: mm: move vmalloc_init() declaration to mm/internal.h vmalloc_init() is called only from mm_core_init(), there is no need to declare it in include/linux/vmalloc.h Move vmalloc_init() declaration to mm/internal.h Link: https://lkml.kernel.org/r/20230321170513.2401534-14-rppt@kernel.org Signed-off-by: Mike Rapoport (IBM) Reviewed-by: David Hildenbrand Reviewed-by: Vlastimil Babka Cc: Doug Berger Cc: Matthew Wilcox (Oracle) Cc: Mel Gorman Cc: Michal Hocko Cc: Thomas Bogendoerfer Signed-off-by: Andrew Morton --- include/linux/vmalloc.h | 4 ---- mm/internal.h | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 69250efa03d1..351fc7697214 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -131,12 +131,8 @@ extern void *vm_map_ram(struct page **pages, unsigned int count, int node); extern void vm_unmap_aliases(void); #ifdef CONFIG_MMU -extern void __init vmalloc_init(void); extern unsigned long vmalloc_nr_pages(void); #else -static inline void vmalloc_init(void) -{ -} static inline unsigned long vmalloc_nr_pages(void) { return 0; } #endif diff --git a/mm/internal.h b/mm/internal.h index 7cea7d833c50..82ba61d0ed6a 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -879,9 +879,14 @@ size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, * mm/vmalloc.c */ #ifdef CONFIG_MMU +void __init vmalloc_init(void); int vmap_pages_range_noflush(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, unsigned int page_shift); #else +static inline void vmalloc_init(void) +{ +} + static inline int vmap_pages_range_noflush(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, unsigned int page_shift) -- cgit v1.2.3 From bd23024b9774e681cbe6cc3afcb24244dfcb2390 Mon Sep 17 00:00:00 2001 From: Tomas Mudrunka Date: Tue, 21 Mar 2023 11:34:30 +0100 Subject: mm/memtest: add results of early memtest to /proc/meminfo Currently the memtest results were only presented in dmesg. When running a large fleet of devices without ECC RAM it's currently not easy to do bulk monitoring for memory corruption. You have to parse dmesg, but that's a ring buffer so the error might disappear after some time. In general I do not consider dmesg to be a great API to query RAM status. In several companies I've seen such errors remain undetected and cause issues for way too long. So I think it makes sense to provide a monitoring API, so that we can safely detect and act upon them. This adds /proc/meminfo entry which can be easily used by scripts. Link: https://lkml.kernel.org/r/20230321103430.7130-1-tomas.mudrunka@gmail.com Signed-off-by: Tomas Mudrunka Cc: Jonathan Corbet Cc: Mike Rapoport (IBM) Signed-off-by: Andrew Morton --- Documentation/filesystems/proc.rst | 8 ++++++++ fs/proc/meminfo.c | 13 +++++++++++++ include/linux/memblock.h | 2 ++ mm/memtest.c | 6 ++++++ 4 files changed, 29 insertions(+) (limited to 'include/linux') diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index 9d5fd9424e8b..8740362f31c6 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -996,6 +996,7 @@ Example output. You may not have all of these fields. VmallocUsed: 40444 kB VmallocChunk: 0 kB Percpu: 29312 kB + EarlyMemtestBad: 0 kB HardwareCorrupted: 0 kB AnonHugePages: 4149248 kB ShmemHugePages: 0 kB @@ -1146,6 +1147,13 @@ VmallocChunk Percpu Memory allocated to the percpu allocator used to back percpu allocations. This stat excludes the cost of metadata. +EarlyMemtestBad + The amount of RAM/memory in kB, that was identified as corrupted + by early memtest. If memtest was not run, this field will not + be displayed at all. Size is never rounded down to 0 kB. + That means if 0 kB is reported, you can safely assume + there was at least one pass of memtest and none of the passes + found a single faulty byte of RAM. HardwareCorrupted The amount of RAM/memory in KB, the kernel identifies as corrupted. diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index 440960110a42..b43d0bd42762 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -131,6 +132,18 @@ static int meminfo_proc_show(struct seq_file *m, void *v) show_val_kb(m, "VmallocChunk: ", 0ul); show_val_kb(m, "Percpu: ", pcpu_nr_pages()); +#ifdef CONFIG_MEMTEST + if (early_memtest_done) { + unsigned long early_memtest_bad_size_kb; + + early_memtest_bad_size_kb = early_memtest_bad_size>>10; + if (early_memtest_bad_size && !early_memtest_bad_size_kb) + early_memtest_bad_size_kb = 1; + /* When 0 is reported, it means there actually was a successful test */ + seq_printf(m, "EarlyMemtestBad: %5lu kB\n", early_memtest_bad_size_kb); + } +#endif + #ifdef CONFIG_MEMORY_FAILURE seq_printf(m, "HardwareCorrupted: %5lu kB\n", atomic_long_read(&num_poisoned_pages) << (PAGE_SHIFT - 10)); diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 50ad19662a32..f82ee3fac1cd 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -597,6 +597,8 @@ extern int hashdist; /* Distribute hashes across NUMA nodes? */ #endif #ifdef CONFIG_MEMTEST +extern phys_addr_t early_memtest_bad_size; /* Size of faulty ram found by memtest */ +extern bool early_memtest_done; /* Was early memtest done? */ extern void early_memtest(phys_addr_t start, phys_addr_t end); #else static inline void early_memtest(phys_addr_t start, phys_addr_t end) diff --git a/mm/memtest.c b/mm/memtest.c index f53ace709ccd..57149dfee438 100644 --- a/mm/memtest.c +++ b/mm/memtest.c @@ -4,6 +4,9 @@ #include #include +bool early_memtest_done; +phys_addr_t early_memtest_bad_size; + static u64 patterns[] __initdata = { /* The first entry has to be 0 to leave memtest with zeroed memory */ 0, @@ -30,6 +33,7 @@ static void __init reserve_bad_mem(u64 pattern, phys_addr_t start_bad, phys_addr pr_info(" %016llx bad mem addr %pa - %pa reserved\n", cpu_to_be64(pattern), &start_bad, &end_bad); memblock_reserve(start_bad, end_bad - start_bad); + early_memtest_bad_size += (end_bad - start_bad); } static void __init memtest(u64 pattern, phys_addr_t start_phys, phys_addr_t size) @@ -61,6 +65,8 @@ static void __init memtest(u64 pattern, phys_addr_t start_phys, phys_addr_t size } if (start_bad) reserve_bad_mem(pattern, start_bad, last_bad + incr); + + early_memtest_done = true; } static void __init do_one_pass(u64 pattern, phys_addr_t start, phys_addr_t end) -- cgit v1.2.3 From 28d8b812e97b31231e95864f36a6b32f4b307daa Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sun, 12 Mar 2023 23:40:13 +0000 Subject: mm: remove unused vmf_insert_mixed_prot() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "Remove drm/ttm-specific mm changes". Functionality was added specifically for the DRM TTM driver to support mapping memory for VM_MIXEDMAP VMAs with customised protection flags, however this has now been rolled back as issues were found with this approach. This series removes the mm changes too, retaining some of the useful comments. This patch (of 3): The sole user of vmf_insert_mixed_prot(), the drm ttm module, stopped using this in commit f91142c62161 ("drm/ttm: nuke VM_MIXEDMAP on BO mappings v3") citing use of VM_MIXEDMAP in this case being terribly broken. Remove this now-dead code and references to it, but retain the useful description of the prot != vma->vm_page_prot case, moving it to vmf_insert_pfn_prot() instead. Link: https://lkml.kernel.org/r/cover.1678661628.git.lstoakes@gmail.com Link: https://lkml.kernel.org/r/a069644388e6f1593a7020d15840e6fc9f39bcaf.1678661628.git.lstoakes@gmail.com Signed-off-by: Lorenzo Stoakes Cc: Christian König Cc: Dan Williams Cc: Jason Gunthorpe Cc: Kirill A. Shutemov Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Thomas Hellström Cc: Aaron Tomlin Cc: Christoph Lameter Cc: Frederic Weisbecker Cc: Heiko Carstens Cc: Huacai Chen Cc: Marcelo Tosatti Cc: Peter Xu Cc: "Russell King (Oracle)" Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 -- include/linux/mm_types.h | 7 +----- mm/memory.c | 57 +++++++++++++++--------------------------------- 3 files changed, 19 insertions(+), 47 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index c3abad7e6ccb..55d02356146c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3179,8 +3179,6 @@ vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot); vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, pfn_t pfn); -vm_fault_t vmf_insert_mixed_prot(struct vm_area_struct *vma, unsigned long addr, - pfn_t pfn, pgprot_t pgprot); vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, unsigned long addr, pfn_t pfn); int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len); diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 0722859c3647..651a5c256732 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -485,12 +485,7 @@ struct vm_area_struct { within vm_mm. */ struct mm_struct *vm_mm; /* The address space we belong to. */ - - /* - * Access permissions of this VMA. - * See vmf_insert_mixed_prot() for discussion. - */ - pgprot_t vm_page_prot; + pgprot_t vm_page_prot; /* Access permissions of this VMA. */ /* * Flags, see mm.h. diff --git a/mm/memory.c b/mm/memory.c index 0ee13d3de88c..7716f2cc4b68 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2165,8 +2165,20 @@ out_unlock: * vmf_insert_pfn_prot should only be used if using multiple VMAs is * impractical. * - * See vmf_insert_mixed_prot() for a discussion of the implication of using - * a value of @pgprot different from that of @vma->vm_page_prot. + * pgprot typically only differs from @vma->vm_page_prot when drivers set + * caching- and encryption bits different than those of @vma->vm_page_prot, + * because the caching- or encryption mode may not be known at mmap() time. + * + * This is ok as long as @vma->vm_page_prot is not used by the core vm + * to set caching and encryption bits for those vmas (except for COW pages). + * This is ensured by core vm only modifying these page table entries using + * functions that don't touch caching- or encryption bits, using pte_modify() + * if needed. (See for example mprotect()). + * + * Also when new page-table entries are created, this is only done using the + * fault() callback, and never using the value of vma->vm_page_prot, + * except for page-table entries that point to anonymous pages as the result + * of COW. * * Context: Process context. May allocate using %GFP_KERNEL. * Return: vm_fault_t value. @@ -2241,9 +2253,9 @@ static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn) } static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma, - unsigned long addr, pfn_t pfn, pgprot_t pgprot, - bool mkwrite) + unsigned long addr, pfn_t pfn, bool mkwrite) { + pgprot_t pgprot = vma->vm_page_prot; int err; BUG_ON(!vm_mixed_ok(vma, pfn)); @@ -2286,43 +2298,10 @@ static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma, return VM_FAULT_NOPAGE; } -/** - * vmf_insert_mixed_prot - insert single pfn into user vma with specified pgprot - * @vma: user vma to map to - * @addr: target user address of this page - * @pfn: source kernel pfn - * @pgprot: pgprot flags for the inserted page - * - * This is exactly like vmf_insert_mixed(), except that it allows drivers - * to override pgprot on a per-page basis. - * - * Typically this function should be used by drivers to set caching- and - * encryption bits different than those of @vma->vm_page_prot, because - * the caching- or encryption mode may not be known at mmap() time. - * This is ok as long as @vma->vm_page_prot is not used by the core vm - * to set caching and encryption bits for those vmas (except for COW pages). - * This is ensured by core vm only modifying these page table entries using - * functions that don't touch caching- or encryption bits, using pte_modify() - * if needed. (See for example mprotect()). - * Also when new page-table entries are created, this is only done using the - * fault() callback, and never using the value of vma->vm_page_prot, - * except for page-table entries that point to anonymous pages as the result - * of COW. - * - * Context: Process context. May allocate using %GFP_KERNEL. - * Return: vm_fault_t value. - */ -vm_fault_t vmf_insert_mixed_prot(struct vm_area_struct *vma, unsigned long addr, - pfn_t pfn, pgprot_t pgprot) -{ - return __vm_insert_mixed(vma, addr, pfn, pgprot, false); -} -EXPORT_SYMBOL(vmf_insert_mixed_prot); - vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, pfn_t pfn) { - return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, false); + return __vm_insert_mixed(vma, addr, pfn, false); } EXPORT_SYMBOL(vmf_insert_mixed); @@ -2334,7 +2313,7 @@ EXPORT_SYMBOL(vmf_insert_mixed); vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, unsigned long addr, pfn_t pfn) { - return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, true); + return __vm_insert_mixed(vma, addr, pfn, true); } EXPORT_SYMBOL(vmf_insert_mixed_mkwrite); -- cgit v1.2.3 From 7b806d229ef151c2997c041b791dfa89593e0e1b Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Sun, 12 Mar 2023 23:40:14 +0000 Subject: mm: remove vmf_insert_pfn_xxx_prot() for huge page-table entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This functionality's sole user, the drm ttm module, removed support for it in commit 0d979509539e ("drm/ttm: remove ttm_bo_vm_insert_huge()") as the whole approach is currently unworkable without a PMD/PUD special bit and updates to GUP. Link: https://lkml.kernel.org/r/604c2ad79659d4b8a6e3e1611c6219d5d3233988.1678661628.git.lstoakes@gmail.com Signed-off-by: Lorenzo Stoakes Cc: Christian König Cc: Dan Williams Cc: Jason Gunthorpe Cc: Kirill A. Shutemov Cc: Matthew Wilcox (Oracle) Cc: Michal Hocko Cc: Thomas Hellström Cc: Aaron Tomlin Cc: Christoph Lameter Cc: Frederic Weisbecker Cc: Heiko Carstens Cc: Huacai Chen Cc: Marcelo Tosatti Cc: Peter Xu Cc: "Russell King (Oracle)" Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/huge_mm.h | 39 ++------------------------------------- mm/huge_memory.c | 31 +++++++++++++------------------ 2 files changed, 15 insertions(+), 55 deletions(-) (limited to 'include/linux') diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 9a3a3af2dd80..20284387b841 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -39,44 +39,9 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, pgprot_t newprot, unsigned long cp_flags); -vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn, - pgprot_t pgprot, bool write); -/** - * vmf_insert_pfn_pmd - insert a pmd size pfn - * @vmf: Structure describing the fault - * @pfn: pfn to insert - * @pgprot: page protection to use - * @write: whether it's a write fault - * - * Insert a pmd size pfn. See vmf_insert_pfn() for additional info. - * - * Return: vm_fault_t value. - */ -static inline vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, - bool write) -{ - return vmf_insert_pfn_pmd_prot(vmf, pfn, vmf->vma->vm_page_prot, write); -} -vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn, - pgprot_t pgprot, bool write); - -/** - * vmf_insert_pfn_pud - insert a pud size pfn - * @vmf: Structure describing the fault - * @pfn: pfn to insert - * @pgprot: page protection to use - * @write: whether it's a write fault - * - * Insert a pud size pfn. See vmf_insert_pfn() for additional info. - * - * Return: vm_fault_t value. - */ -static inline vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, - bool write) -{ - return vmf_insert_pfn_pud_prot(vmf, pfn, vmf->vma->vm_page_prot, write); -} +vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write); +vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write); enum transparent_hugepage_flag { TRANSPARENT_HUGEPAGE_UNSUPPORTED, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1df386d13d24..81a5689806af 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -885,23 +885,20 @@ out_unlock: } /** - * vmf_insert_pfn_pmd_prot - insert a pmd size pfn + * vmf_insert_pfn_pmd - insert a pmd size pfn * @vmf: Structure describing the fault * @pfn: pfn to insert - * @pgprot: page protection to use * @write: whether it's a write fault * - * Insert a pmd size pfn. See vmf_insert_pfn() for additional info and - * also consult the vmf_insert_mixed_prot() documentation when - * @pgprot != @vmf->vma->vm_page_prot. + * Insert a pmd size pfn. See vmf_insert_pfn() for additional info. * * Return: vm_fault_t value. */ -vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn, - pgprot_t pgprot, bool write) +vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write) { unsigned long addr = vmf->address & PMD_MASK; struct vm_area_struct *vma = vmf->vma; + pgprot_t pgprot = vma->vm_page_prot; pgtable_t pgtable = NULL; /* @@ -929,7 +926,7 @@ vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn, insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); return VM_FAULT_NOPAGE; } -EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd_prot); +EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd); #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) @@ -940,9 +937,10 @@ static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) } static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, - pud_t *pud, pfn_t pfn, pgprot_t prot, bool write) + pud_t *pud, pfn_t pfn, bool write) { struct mm_struct *mm = vma->vm_mm; + pgprot_t prot = vma->vm_page_prot; pud_t entry; spinlock_t *ptl; @@ -976,23 +974,20 @@ out_unlock: } /** - * vmf_insert_pfn_pud_prot - insert a pud size pfn + * vmf_insert_pfn_pud - insert a pud size pfn * @vmf: Structure describing the fault * @pfn: pfn to insert - * @pgprot: page protection to use * @write: whether it's a write fault * - * Insert a pud size pfn. See vmf_insert_pfn() for additional info and - * also consult the vmf_insert_mixed_prot() documentation when - * @pgprot != @vmf->vma->vm_page_prot. + * Insert a pud size pfn. See vmf_insert_pfn() for additional info. * * Return: vm_fault_t value. */ -vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn, - pgprot_t pgprot, bool write) +vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write) { unsigned long addr = vmf->address & PUD_MASK; struct vm_area_struct *vma = vmf->vma; + pgprot_t pgprot = vma->vm_page_prot; /* * If we had pud_special, we could avoid all these restrictions, @@ -1010,10 +1005,10 @@ vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn, track_pfn_insert(vma, &pgprot, pfn); - insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write); + insert_pfn_pud(vma, addr, vmf->pud, pfn, write); return VM_FAULT_NOPAGE; } -EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud_prot); +EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud); #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ static void touch_pmd(struct vm_area_struct *vma, unsigned long addr, -- cgit v1.2.3 From 3f6dac0fd1b83178137e7b4e722d8f29612cbec1 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Tue, 21 Mar 2023 03:24:15 +0300 Subject: mm/page_alloc: make deferred page init free pages in MAX_ORDER blocks Normal page init path frees pages during the boot in MAX_ORDER chunks, but deferred page init path does it in pageblock blocks. Change deferred page init path to work in MAX_ORDER blocks. For cases when MAX_ORDER is larger than pageblock, set migrate type to MIGRATE_MOVABLE for all pageblocks covered by the page. Link: https://lkml.kernel.org/r/20230321002415.20843-1-kirill.shutemov@linux.intel.com Signed-off-by: Kirill A. Shutemov Reviewed-by: Vlastimil Babka Acked-by: David Hildenbrand Acked-by: Mel Gorman Acked-by: Mike Rapoport (IBM) Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 2 ++ mm/mm_init.c | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 2d3d78d01283..2d22e47dc1eb 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -32,6 +32,8 @@ #endif #define MAX_ORDER_NR_PAGES (1 << MAX_ORDER) +#define IS_MAX_ORDER_ALIGNED(pfn) IS_ALIGNED(pfn, MAX_ORDER_NR_PAGES) + /* * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed * costly to service. That is between allocation orders which should diff --git a/mm/mm_init.c b/mm/mm_init.c index 53fb8e9d1e3b..c88fbe74137c 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1951,9 +1951,10 @@ static void __init deferred_free_range(unsigned long pfn, page = pfn_to_page(pfn); /* Free a large naturally-aligned chunk if possible */ - if (nr_pages == pageblock_nr_pages && pageblock_aligned(pfn)) { - set_pageblock_migratetype(page, MIGRATE_MOVABLE); - __free_pages_core(page, pageblock_order); + if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) { + for (i = 0; i < nr_pages; i += pageblock_nr_pages) + set_pageblock_migratetype(page + i, MIGRATE_MOVABLE); + __free_pages_core(page, MAX_ORDER); return; } @@ -1977,19 +1978,19 @@ static inline void __init pgdat_init_report_one_done(void) /* * Returns true if page needs to be initialized or freed to buddy allocator. * - * We check if a current large page is valid by only checking the validity + * We check if a current MAX_ORDER block is valid by only checking the validity * of the head pfn. */ static inline bool __init deferred_pfn_valid(unsigned long pfn) { - if (pageblock_aligned(pfn) && !pfn_valid(pfn)) + if (IS_MAX_ORDER_ALIGNED(pfn) && !pfn_valid(pfn)) return false; return true; } /* * Free pages to buddy allocator. Try to free aligned pages in - * pageblock_nr_pages sizes. + * MAX_ORDER_NR_PAGES sizes. */ static void __init deferred_free_pages(unsigned long pfn, unsigned long end_pfn) @@ -2000,7 +2001,7 @@ static void __init deferred_free_pages(unsigned long pfn, if (!deferred_pfn_valid(pfn)) { deferred_free_range(pfn - nr_free, nr_free); nr_free = 0; - } else if (pageblock_aligned(pfn)) { + } else if (IS_MAX_ORDER_ALIGNED(pfn)) { deferred_free_range(pfn - nr_free, nr_free); nr_free = 1; } else { @@ -2013,7 +2014,7 @@ static void __init deferred_free_pages(unsigned long pfn, /* * Initialize struct pages. We minimize pfn page lookups and scheduler checks - * by performing it only once every pageblock_nr_pages. + * by performing it only once every MAX_ORDER_NR_PAGES. * Return number of pages initialized. */ static unsigned long __init deferred_init_pages(struct zone *zone, @@ -2029,7 +2030,7 @@ static unsigned long __init deferred_init_pages(struct zone *zone, if (!deferred_pfn_valid(pfn)) { page = NULL; continue; - } else if (!page || pageblock_aligned(pfn)) { + } else if (!page || IS_MAX_ORDER_ALIGNED(pfn)) { page = pfn_to_page(pfn); } else { page++; -- cgit v1.2.3 From 4f80818b4a58c9458dce0df7cce9abe107da445e Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Wed, 22 Mar 2023 18:57:03 +0000 Subject: iov_iter: add copy_page_to_iter_nofault() Provide a means to copy a page to user space from an iterator, aborting if a page fault would occur. This supports compound pages, but may be passed a tail page with an offset extending further into the compound page, so we cannot pass a folio. This allows for this function to be called from atomic context and _try_ to user pages if they are faulted in, aborting if not. The function does not use _copy_to_iter() in order to not specify might_fault(), this is similar to copy_page_from_iter_atomic(). This is being added in order that an iteratable form of vread() can be implemented while holding spinlocks. Link: https://lkml.kernel.org/r/19734729defb0f498a76bdec1bef3ac48a3af3e8.1679511146.git.lstoakes@gmail.com Signed-off-by: Lorenzo Stoakes Reviewed-by: Baoquan He Cc: Alexander Viro Cc: David Hildenbrand Cc: Jens Axboe Cc: Jiri Olsa Cc: Liu Shixin Cc: Matthew Wilcox (Oracle) Cc: Uladzislau Rezki (Sony) Signed-off-by: Andrew Morton --- include/linux/uio.h | 2 ++ lib/iov_iter.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) (limited to 'include/linux') diff --git a/include/linux/uio.h b/include/linux/uio.h index 27e3fd942960..29eb18bb6feb 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -173,6 +173,8 @@ static inline size_t copy_folio_to_iter(struct folio *folio, size_t offset, { return copy_page_to_iter(&folio->page, offset, bytes, i); } +size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, + size_t bytes, struct iov_iter *i); static __always_inline __must_check size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 274014e4eafe..34dd6bdf2fba 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -172,6 +172,18 @@ static int copyout(void __user *to, const void *from, size_t n) return n; } +static int copyout_nofault(void __user *to, const void *from, size_t n) +{ + long res; + + if (should_fail_usercopy()) + return n; + + res = copy_to_user_nofault(to, from, n); + + return res < 0 ? n : res; +} + static int copyin(void *to, const void __user *from, size_t n) { size_t res = n; @@ -734,6 +746,42 @@ size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, } EXPORT_SYMBOL(copy_page_to_iter); +size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, size_t bytes, + struct iov_iter *i) +{ + size_t res = 0; + + if (!page_copy_sane(page, offset, bytes)) + return 0; + if (WARN_ON_ONCE(i->data_source)) + return 0; + if (unlikely(iov_iter_is_pipe(i))) + return copy_page_to_iter_pipe(page, offset, bytes, i); + page += offset / PAGE_SIZE; // first subpage + offset %= PAGE_SIZE; + while (1) { + void *kaddr = kmap_local_page(page); + size_t n = min(bytes, (size_t)PAGE_SIZE - offset); + + iterate_and_advance(i, n, base, len, off, + copyout_nofault(base, kaddr + offset + off, len), + memcpy(base, kaddr + offset + off, len) + ) + kunmap_local(kaddr); + res += n; + bytes -= n; + if (!bytes || !n) + break; + offset += n; + if (offset == PAGE_SIZE) { + page++; + offset = 0; + } + } + return res; +} +EXPORT_SYMBOL(copy_page_to_iter_nofault); + size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, struct iov_iter *i) { -- cgit v1.2.3 From 4c91c07c93bbbdd7f2d9de2beb7ee5c2a48ad8e7 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Wed, 22 Mar 2023 18:57:04 +0000 Subject: mm: vmalloc: convert vread() to vread_iter() Having previously laid the foundation for converting vread() to an iterator function, pull the trigger and do so. This patch attempts to provide minimal refactoring and to reflect the existing logic as best we can, for example we continue to zero portions of memory not read, as before. Overall, there should be no functional difference other than a performance improvement in /proc/kcore access to vmalloc regions. Now we have eliminated the need for a bounce buffer in read_kcore_iter(), we dispense with it, and try to write to user memory optimistically but with faults disabled via copy_page_to_iter_nofault(). We already have preemption disabled by holding a spin lock. We continue faulting in until the operation is complete. Additionally, we must account for the fact that at any point a copy may fail (most likely due to a fault not being able to occur), we exit indicating fewer bytes retrieved than expected. [sfr@canb.auug.org.au: fix sparc64 warning] Link: https://lkml.kernel.org/r/20230320144721.663280c3@canb.auug.org.au [lstoakes@gmail.com: redo Stephen's sparc build fix] Link: https://lkml.kernel.org/r/8506cbc667c39205e65a323f750ff9c11a463798.1679566220.git.lstoakes@gmail.com [akpm@linux-foundation.org: unbreak uio.h includes] Link: https://lkml.kernel.org/r/941f88bc5ab928e6656e1e2593b91bf0f8c81e1b.1679511146.git.lstoakes@gmail.com Signed-off-by: Lorenzo Stoakes Signed-off-by: Stephen Rothwell Reviewed-by: Baoquan He Cc: Alexander Viro Cc: David Hildenbrand Cc: Jens Axboe Cc: Jiri Olsa Cc: Liu Shixin Cc: Matthew Wilcox (Oracle) Cc: Uladzislau Rezki (Sony) Signed-off-by: Andrew Morton --- fs/proc/kcore.c | 44 ++++----- include/linux/vmalloc.h | 3 +- mm/nommu.c | 10 +-- mm/vmalloc.c | 235 ++++++++++++++++++++++++++++++------------------ 4 files changed, 177 insertions(+), 115 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/kcore.c b/fs/proc/kcore.c index 08b795fd80b4..25b44b303b35 100644 --- a/fs/proc/kcore.c +++ b/fs/proc/kcore.c @@ -307,13 +307,9 @@ static void append_kcore_note(char *notes, size_t *i, const char *name, *i = ALIGN(*i + descsz, 4); } -static ssize_t -read_kcore_iter(struct kiocb *iocb, struct iov_iter *iter) +static ssize_t read_kcore_iter(struct kiocb *iocb, struct iov_iter *iter) { - struct file *file = iocb->ki_filp; - char *buf = file->private_data; loff_t *fpos = &iocb->ki_pos; - size_t phdrs_offset, notes_offset, data_offset; size_t page_offline_frozen = 1; size_t phdrs_len, notes_len; @@ -507,13 +503,30 @@ read_kcore_iter(struct kiocb *iocb, struct iov_iter *iter) switch (m->type) { case KCORE_VMALLOC: - vread(buf, (char *)start, tsz); - /* we have to zero-fill user buffer even if no read */ - if (copy_to_iter(buf, tsz, iter) != tsz) { - ret = -EFAULT; - goto out; + { + const char *src = (char *)start; + size_t read = 0, left = tsz; + + /* + * vmalloc uses spinlocks, so we optimistically try to + * read memory. If this fails, fault pages in and try + * again until we are done. + */ + while (true) { + read += vread_iter(iter, src, left); + if (read == tsz) + break; + + src += read; + left -= read; + + if (fault_in_iov_iter_writeable(iter, left)) { + ret = -EFAULT; + goto out; + } } break; + } case KCORE_USER: /* User page is handled prior to normal kernel page: */ if (copy_to_iter((char *)start, tsz, iter) != tsz) { @@ -582,10 +595,6 @@ static int open_kcore(struct inode *inode, struct file *filp) if (ret) return ret; - filp->private_data = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!filp->private_data) - return -ENOMEM; - if (kcore_need_update) kcore_update_ram(); if (i_size_read(inode) != proc_root_kcore->size) { @@ -596,16 +605,9 @@ static int open_kcore(struct inode *inode, struct file *filp) return 0; } -static int release_kcore(struct inode *inode, struct file *file) -{ - kfree(file->private_data); - return 0; -} - static const struct proc_ops kcore_proc_ops = { .proc_read_iter = read_kcore_iter, .proc_open = open_kcore, - .proc_release = release_kcore, .proc_lseek = default_llseek, }; diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 351fc7697214..c720be70c8dd 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -14,6 +14,7 @@ struct vm_area_struct; /* vma defining user mapping in mm_types.h */ struct notifier_block; /* in notifier.h */ +struct iov_iter; /* in uio.h */ /* bits in flags of vmalloc's vm_struct below */ #define VM_IOREMAP 0x00000001 /* ioremap() and friends */ @@ -247,7 +248,7 @@ static inline void set_vm_flush_reset_perms(void *addr) #endif /* for /proc/kcore */ -extern long vread(char *buf, char *addr, unsigned long count); +extern long vread_iter(struct iov_iter *iter, const char *addr, size_t count); /* * Internals. Don't use.. diff --git a/mm/nommu.c b/mm/nommu.c index 57ba243c6a37..f670d9979a26 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -36,6 +36,7 @@ #include #include +#include #include #include #include @@ -198,14 +199,13 @@ unsigned long vmalloc_to_pfn(const void *addr) } EXPORT_SYMBOL(vmalloc_to_pfn); -long vread(char *buf, char *addr, unsigned long count) +long vread_iter(struct iov_iter *iter, const char *addr, size_t count) { /* Don't allow overflow */ - if ((unsigned long) buf + count < count) - count = -(unsigned long) buf; + if ((unsigned long) addr + count < count) + count = -(unsigned long) addr; - memcpy(buf, addr, count); - return count; + return copy_to_iter(addr, count, iter); } /* diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 978194dc2bb8..5291c6f02cf7 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -33,11 +33,11 @@ #include #include #include +#include #include #include #include #include -#include #include #include #include @@ -3442,62 +3442,96 @@ void *vmalloc_32_user(unsigned long size) EXPORT_SYMBOL(vmalloc_32_user); /* - * small helper routine , copy contents to buf from addr. - * If the page is not present, fill zero. + * Atomically zero bytes in the iterator. + * + * Returns the number of zeroed bytes. */ +static size_t zero_iter(struct iov_iter *iter, size_t count) +{ + size_t remains = count; + + while (remains > 0) { + size_t num, copied; + + num = remains < PAGE_SIZE ? remains : PAGE_SIZE; + copied = copy_page_to_iter_nofault(ZERO_PAGE(0), 0, num, iter); + remains -= copied; + + if (copied < num) + break; + } -static int aligned_vread(char *buf, char *addr, unsigned long count) + return count - remains; +} + +/* + * small helper routine, copy contents to iter from addr. + * If the page is not present, fill zero. + * + * Returns the number of copied bytes. + */ +static size_t aligned_vread_iter(struct iov_iter *iter, + const char *addr, size_t count) { - struct page *p; - int copied = 0; + size_t remains = count; + struct page *page; - while (count) { + while (remains > 0) { unsigned long offset, length; + size_t copied = 0; offset = offset_in_page(addr); length = PAGE_SIZE - offset; - if (length > count) - length = count; - p = vmalloc_to_page(addr); + if (length > remains) + length = remains; + page = vmalloc_to_page(addr); /* - * To do safe access to this _mapped_ area, we need - * lock. But adding lock here means that we need to add - * overhead of vmalloc()/vfree() calls for this _debug_ - * interface, rarely used. Instead of that, we'll use - * kmap() and get small overhead in this access function. + * To do safe access to this _mapped_ area, we need lock. But + * adding lock here means that we need to add overhead of + * vmalloc()/vfree() calls for this _debug_ interface, rarely + * used. Instead of that, we'll use an local mapping via + * copy_page_to_iter_nofault() and accept a small overhead in + * this access function. */ - if (p) { - /* We can expect USER0 is not used -- see vread() */ - void *map = kmap_atomic(p); - memcpy(buf, map + offset, length); - kunmap_atomic(map); - } else - memset(buf, 0, length); + if (page) + copied = copy_page_to_iter_nofault(page, offset, + length, iter); + else + copied = zero_iter(iter, length); - addr += length; - buf += length; - copied += length; - count -= length; + addr += copied; + remains -= copied; + + if (copied != length) + break; } - return copied; + + return count - remains; } -static void vmap_ram_vread(char *buf, char *addr, int count, unsigned long flags) +/* + * Read from a vm_map_ram region of memory. + * + * Returns the number of copied bytes. + */ +static size_t vmap_ram_vread_iter(struct iov_iter *iter, const char *addr, + size_t count, unsigned long flags) { char *start; struct vmap_block *vb; unsigned long offset; - unsigned int rs, re, n; + unsigned int rs, re; + size_t remains, n; /* * If it's area created by vm_map_ram() interface directly, but * not further subdividing and delegating management to vmap_block, * handle it here. */ - if (!(flags & VMAP_BLOCK)) { - aligned_vread(buf, addr, count); - return; - } + if (!(flags & VMAP_BLOCK)) + return aligned_vread_iter(iter, addr, count); + + remains = count; /* * Area is split into regions and tracked with vmap_block, read out @@ -3505,50 +3539,64 @@ static void vmap_ram_vread(char *buf, char *addr, int count, unsigned long flags */ vb = xa_load(&vmap_blocks, addr_to_vb_idx((unsigned long)addr)); if (!vb) - goto finished; + goto finished_zero; spin_lock(&vb->lock); if (bitmap_empty(vb->used_map, VMAP_BBMAP_BITS)) { spin_unlock(&vb->lock); - goto finished; + goto finished_zero; } + for_each_set_bitrange(rs, re, vb->used_map, VMAP_BBMAP_BITS) { - if (!count) - break; + size_t copied; + + if (remains == 0) + goto finished; + start = vmap_block_vaddr(vb->va->va_start, rs); - while (addr < start) { - if (count == 0) - goto unlock; - *buf = '\0'; - buf++; - addr++; - count--; + + if (addr < start) { + size_t to_zero = min_t(size_t, start - addr, remains); + size_t zeroed = zero_iter(iter, to_zero); + + addr += zeroed; + remains -= zeroed; + + if (remains == 0 || zeroed != to_zero) + goto finished; } + /*it could start reading from the middle of used region*/ offset = offset_in_page(addr); n = ((re - rs + 1) << PAGE_SHIFT) - offset; - if (n > count) - n = count; - aligned_vread(buf, start+offset, n); + if (n > remains) + n = remains; + + copied = aligned_vread_iter(iter, start + offset, n); - buf += n; - addr += n; - count -= n; + addr += copied; + remains -= copied; + + if (copied != n) + goto finished; } -unlock: + spin_unlock(&vb->lock); -finished: +finished_zero: /* zero-fill the left dirty or free regions */ - if (count) - memset(buf, 0, count); + return count - remains + zero_iter(iter, remains); +finished: + /* We couldn't copy/zero everything */ + spin_unlock(&vb->lock); + return count - remains; } /** - * vread() - read vmalloc area in a safe way. - * @buf: buffer for reading data - * @addr: vm address. - * @count: number of bytes to be read. + * vread_iter() - read vmalloc area in a safe way to an iterator. + * @iter: the iterator to which data should be written. + * @addr: vm address. + * @count: number of bytes to be read. * * This function checks that addr is a valid vmalloc'ed area, and * copy data from that area to a given buffer. If the given memory range @@ -3568,13 +3616,12 @@ finished: * (same number as @count) or %0 if [addr...addr+count) doesn't * include any intersection with valid vmalloc area */ -long vread(char *buf, char *addr, unsigned long count) +long vread_iter(struct iov_iter *iter, const char *addr, size_t count) { struct vmap_area *va; struct vm_struct *vm; - char *vaddr, *buf_start = buf; - unsigned long buflen = count; - unsigned long n, size, flags; + char *vaddr; + size_t n, size, flags, remains; addr = kasan_reset_tag(addr); @@ -3582,18 +3629,22 @@ long vread(char *buf, char *addr, unsigned long count) if ((unsigned long) addr + count < count) count = -(unsigned long) addr; + remains = count; + spin_lock(&vmap_area_lock); va = find_vmap_area_exceed_addr((unsigned long)addr); if (!va) - goto finished; + goto finished_zero; /* no intersects with alive vmap_area */ - if ((unsigned long)addr + count <= va->va_start) - goto finished; + if ((unsigned long)addr + remains <= va->va_start) + goto finished_zero; list_for_each_entry_from(va, &vmap_area_list, list) { - if (!count) - break; + size_t copied; + + if (remains == 0) + goto finished; vm = va->vm; flags = va->flags & VMAP_FLAGS_MASK; @@ -3608,6 +3659,7 @@ long vread(char *buf, char *addr, unsigned long count) if (vm && (vm->flags & VM_UNINITIALIZED)) continue; + /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ smp_rmb(); @@ -3616,38 +3668,45 @@ long vread(char *buf, char *addr, unsigned long count) if (addr >= vaddr + size) continue; - while (addr < vaddr) { - if (count == 0) + + if (addr < vaddr) { + size_t to_zero = min_t(size_t, vaddr - addr, remains); + size_t zeroed = zero_iter(iter, to_zero); + + addr += zeroed; + remains -= zeroed; + + if (remains == 0 || zeroed != to_zero) goto finished; - *buf = '\0'; - buf++; - addr++; - count--; } + n = vaddr + size - addr; - if (n > count) - n = count; + if (n > remains) + n = remains; if (flags & VMAP_RAM) - vmap_ram_vread(buf, addr, n, flags); + copied = vmap_ram_vread_iter(iter, addr, n, flags); else if (!(vm->flags & VM_IOREMAP)) - aligned_vread(buf, addr, n); + copied = aligned_vread_iter(iter, addr, n); else /* IOREMAP area is treated as memory hole */ - memset(buf, 0, n); - buf += n; - addr += n; - count -= n; + copied = zero_iter(iter, n); + + addr += copied; + remains -= copied; + + if (copied != n) + goto finished; } -finished: - spin_unlock(&vmap_area_lock); - if (buf == buf_start) - return 0; +finished_zero: + spin_unlock(&vmap_area_lock); /* zero-fill memory holes */ - if (buf != buf_start + buflen) - memset(buf, 0, buflen - (buf - buf_start)); + return count - remains + zero_iter(iter, remains); +finished: + /* Nothing remains, or We couldn't copy/zero everything. */ + spin_unlock(&vmap_area_lock); - return buflen; + return count - remains; } /** -- cgit v1.2.3 From 20cce633f4254cc0df39665449726e3172518f6c Mon Sep 17 00:00:00 2001 From: Michel Lespinasse Date: Mon, 27 Feb 2023 09:36:09 -0800 Subject: mm: rcu safe VMA freeing This prepares for page faults handling under VMA lock, looking up VMAs under protection of an rcu read lock, instead of the usual mmap read lock. Link: https://lkml.kernel.org/r/20230227173632.3292573-11-surenb@google.com Signed-off-by: Michel Lespinasse Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/mm_types.h | 13 ++++++++++--- kernel/fork.c | 20 +++++++++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 651a5c256732..f203e2562e00 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -480,9 +480,16 @@ struct anon_vma_name { struct vm_area_struct { /* The first cache line has the info for VMA tree walking. */ - unsigned long vm_start; /* Our start address within vm_mm. */ - unsigned long vm_end; /* The first byte after our end address - within vm_mm. */ + union { + struct { + /* VMA covers [vm_start; vm_end) addresses within mm */ + unsigned long vm_start; + unsigned long vm_end; + }; +#ifdef CONFIG_PER_VMA_LOCK + struct rcu_head vm_rcu; /* Used for deferred freeing. */ +#endif + }; struct mm_struct *vm_mm; /* The address space we belong to. */ pgprot_t vm_page_prot; /* Access permissions of this VMA. */ diff --git a/kernel/fork.c b/kernel/fork.c index cea99f003f24..93ec6e14bb65 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -479,12 +479,30 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) return new; } -void vm_area_free(struct vm_area_struct *vma) +static void __vm_area_free(struct vm_area_struct *vma) { free_anon_vma_name(vma); kmem_cache_free(vm_area_cachep, vma); } +#ifdef CONFIG_PER_VMA_LOCK +static void vm_area_free_rcu_cb(struct rcu_head *head) +{ + struct vm_area_struct *vma = container_of(head, struct vm_area_struct, + vm_rcu); + __vm_area_free(vma); +} +#endif + +void vm_area_free(struct vm_area_struct *vma) +{ +#ifdef CONFIG_PER_VMA_LOCK + call_rcu(&vma->vm_rcu, vm_area_free_rcu_cb); +#else + __vm_area_free(vma); +#endif +} + static void account_kernel_stack(struct task_struct *tsk, int account) { if (IS_ENABLED(CONFIG_VMAP_STACK)) { -- cgit v1.2.3 From 438b6e12cd603a9416ca7c5462ab75b2a8c4d5b9 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Mon, 27 Feb 2023 09:36:10 -0800 Subject: mm: move mmap_lock assert function definitions Move mmap_lock assert function definitions up so that they can be used by other mmap_lock routines. Link: https://lkml.kernel.org/r/20230227173632.3292573-12-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/mmap_lock.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 96e113e23d04..e49ba91bb1f0 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -60,6 +60,18 @@ static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write) #endif /* CONFIG_TRACING */ +static inline void mmap_assert_locked(struct mm_struct *mm) +{ + lockdep_assert_held(&mm->mmap_lock); + VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); +} + +static inline void mmap_assert_write_locked(struct mm_struct *mm) +{ + lockdep_assert_held_write(&mm->mmap_lock); + VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); +} + static inline void mmap_init_lock(struct mm_struct *mm) { init_rwsem(&mm->mmap_lock); @@ -150,18 +162,6 @@ static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) up_read_non_owner(&mm->mmap_lock); } -static inline void mmap_assert_locked(struct mm_struct *mm) -{ - lockdep_assert_held(&mm->mmap_lock); - VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); -} - -static inline void mmap_assert_write_locked(struct mm_struct *mm) -{ - lockdep_assert_held_write(&mm->mmap_lock); - VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); -} - static inline int mmap_lock_is_contended(struct mm_struct *mm) { return rwsem_is_contended(&mm->mmap_lock); -- cgit v1.2.3 From 5e31275cc997f8ec5d9e8d65fe9840ebed89db19 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Mon, 27 Feb 2023 09:36:11 -0800 Subject: mm: add per-VMA lock and helper functions to control it Introduce per-VMA locking. The lock implementation relies on a per-vma and per-mm sequence counters to note exclusive locking: - read lock - (implemented by vma_start_read) requires the vma (vm_lock_seq) and mm (mm_lock_seq) sequence counters to differ. If they match then there must be a vma exclusive lock held somewhere. - read unlock - (implemented by vma_end_read) is a trivial vma->lock unlock. - write lock - (vma_start_write) requires the mmap_lock to be held exclusively and the current mm counter is assigned to the vma counter. This will allow multiple vmas to be locked under a single mmap_lock write lock (e.g. during vma merging). The vma counter is modified under exclusive vma lock. - write unlock - (vma_end_write_all) is a batch release of all vma locks held. It doesn't pair with a specific vma_start_write! It is done before exclusive mmap_lock is released by incrementing mm sequence counter (mm_lock_seq). - write downgrade - if the mmap_lock is downgraded to the read lock, all vma write locks are released as well (effectivelly same as write unlock). Link: https://lkml.kernel.org/r/20230227173632.3292573-13-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/mm.h | 82 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/mm_types.h | 8 +++++ include/linux/mmap_lock.h | 13 ++++++++ kernel/fork.c | 4 +++ mm/init-mm.c | 3 ++ 5 files changed, 110 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 55d02356146c..53bfb42663ea 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -624,6 +624,87 @@ struct vm_operations_struct { unsigned long addr); }; +#ifdef CONFIG_PER_VMA_LOCK +static inline void vma_init_lock(struct vm_area_struct *vma) +{ + init_rwsem(&vma->lock); + vma->vm_lock_seq = -1; +} + +/* + * Try to read-lock a vma. The function is allowed to occasionally yield false + * locked result to avoid performance overhead, in which case we fall back to + * using mmap_lock. The function should never yield false unlocked result. + */ +static inline bool vma_start_read(struct vm_area_struct *vma) +{ + /* Check before locking. A race might cause false locked result. */ + if (vma->vm_lock_seq == READ_ONCE(vma->vm_mm->mm_lock_seq)) + return false; + + if (unlikely(down_read_trylock(&vma->lock) == 0)) + return false; + + /* + * Overflow might produce false locked result. + * False unlocked result is impossible because we modify and check + * vma->vm_lock_seq under vma->lock protection and mm->mm_lock_seq + * modification invalidates all existing locks. + */ + if (unlikely(vma->vm_lock_seq == READ_ONCE(vma->vm_mm->mm_lock_seq))) { + up_read(&vma->lock); + return false; + } + return true; +} + +static inline void vma_end_read(struct vm_area_struct *vma) +{ + rcu_read_lock(); /* keeps vma alive till the end of up_read */ + up_read(&vma->lock); + rcu_read_unlock(); +} + +static inline void vma_start_write(struct vm_area_struct *vma) +{ + int mm_lock_seq; + + mmap_assert_write_locked(vma->vm_mm); + + /* + * current task is holding mmap_write_lock, both vma->vm_lock_seq and + * mm->mm_lock_seq can't be concurrently modified. + */ + mm_lock_seq = READ_ONCE(vma->vm_mm->mm_lock_seq); + if (vma->vm_lock_seq == mm_lock_seq) + return; + + down_write(&vma->lock); + vma->vm_lock_seq = mm_lock_seq; + up_write(&vma->lock); +} + +static inline void vma_assert_write_locked(struct vm_area_struct *vma) +{ + mmap_assert_write_locked(vma->vm_mm); + /* + * current task is holding mmap_write_lock, both vma->vm_lock_seq and + * mm->mm_lock_seq can't be concurrently modified. + */ + VM_BUG_ON_VMA(vma->vm_lock_seq != READ_ONCE(vma->vm_mm->mm_lock_seq), vma); +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline void vma_init_lock(struct vm_area_struct *vma) {} +static inline bool vma_start_read(struct vm_area_struct *vma) + { return false; } +static inline void vma_end_read(struct vm_area_struct *vma) {} +static inline void vma_start_write(struct vm_area_struct *vma) {} +static inline void vma_assert_write_locked(struct vm_area_struct *vma) {} + +#endif /* CONFIG_PER_VMA_LOCK */ + static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) { static const struct vm_operations_struct dummy_vm_ops = {}; @@ -632,6 +713,7 @@ static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) vma->vm_mm = mm; vma->vm_ops = &dummy_vm_ops; INIT_LIST_HEAD(&vma->anon_vma_chain); + vma_init_lock(vma); } /* Use when VMA is not part of the VMA tree and needs no locking */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index f203e2562e00..843a45893991 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -503,6 +503,11 @@ struct vm_area_struct { vm_flags_t __private __vm_flags; }; +#ifdef CONFIG_PER_VMA_LOCK + int vm_lock_seq; + struct rw_semaphore lock; +#endif + /* * For areas with an address space and backing store, * linkage into the address_space->i_mmap interval tree. @@ -639,6 +644,9 @@ struct mm_struct { * init_mm.mmlist, and are protected * by mmlist_lock */ +#ifdef CONFIG_PER_VMA_LOCK + int mm_lock_seq; +#endif unsigned long hiwater_rss; /* High-watermark of RSS usage */ diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index e49ba91bb1f0..aab8f1b28d26 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -72,6 +72,17 @@ static inline void mmap_assert_write_locked(struct mm_struct *mm) VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm); } +#ifdef CONFIG_PER_VMA_LOCK +static inline void vma_end_write_all(struct mm_struct *mm) +{ + mmap_assert_write_locked(mm); + /* No races during update due to exclusive mmap_lock being held */ + WRITE_ONCE(mm->mm_lock_seq, mm->mm_lock_seq + 1); +} +#else +static inline void vma_end_write_all(struct mm_struct *mm) {} +#endif + static inline void mmap_init_lock(struct mm_struct *mm) { init_rwsem(&mm->mmap_lock); @@ -114,12 +125,14 @@ static inline bool mmap_write_trylock(struct mm_struct *mm) static inline void mmap_write_unlock(struct mm_struct *mm) { __mmap_lock_trace_released(mm, true); + vma_end_write_all(mm); up_write(&mm->mmap_lock); } static inline void mmap_write_downgrade(struct mm_struct *mm) { __mmap_lock_trace_acquire_returned(mm, false, true); + vma_end_write_all(mm); downgrade_write(&mm->mmap_lock); } diff --git a/kernel/fork.c b/kernel/fork.c index 93ec6e14bb65..0907776d8ed4 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -474,6 +474,7 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) */ data_race(memcpy(new, orig, sizeof(*new))); INIT_LIST_HEAD(&new->anon_vma_chain); + vma_init_lock(new); dup_anon_vma_name(orig, new); } return new; @@ -1208,6 +1209,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, seqcount_init(&mm->write_protect_seq); mmap_init_lock(mm); INIT_LIST_HEAD(&mm->mmlist); +#ifdef CONFIG_PER_VMA_LOCK + mm->mm_lock_seq = 0; +#endif mm_pgtables_bytes_init(mm); mm->map_count = 0; mm->locked_vm = 0; diff --git a/mm/init-mm.c b/mm/init-mm.c index c9327abb771c..33269314e060 100644 --- a/mm/init-mm.c +++ b/mm/init-mm.c @@ -37,6 +37,9 @@ struct mm_struct init_mm = { .page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock), .arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock), .mmlist = LIST_HEAD_INIT(init_mm.mmlist), +#ifdef CONFIG_PER_VMA_LOCK + .mm_lock_seq = 0, +#endif .user_ns = &init_user_ns, .cpu_bitmap = CPU_BITS_NONE, #ifdef CONFIG_IOMMU_SVA -- cgit v1.2.3 From c732293331a22187d59cd485879276a1da398387 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Mon, 27 Feb 2023 09:36:12 -0800 Subject: mm: mark VMA as being written when changing vm_flags Updates to vm_flags have to be done with VMA marked as being written for preventing concurrent page faults or other modifications. Link: https://lkml.kernel.org/r/20230227173632.3292573-14-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/mm.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 53bfb42663ea..d0a6c99aba09 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -727,28 +727,28 @@ static inline void vm_flags_init(struct vm_area_struct *vma, static inline void vm_flags_reset(struct vm_area_struct *vma, vm_flags_t flags) { - mmap_assert_write_locked(vma->vm_mm); + vma_start_write(vma); vm_flags_init(vma, flags); } static inline void vm_flags_reset_once(struct vm_area_struct *vma, vm_flags_t flags) { - mmap_assert_write_locked(vma->vm_mm); + vma_start_write(vma); WRITE_ONCE(ACCESS_PRIVATE(vma, __vm_flags), flags); } static inline void vm_flags_set(struct vm_area_struct *vma, vm_flags_t flags) { - mmap_assert_write_locked(vma->vm_mm); + vma_start_write(vma); ACCESS_PRIVATE(vma, __vm_flags) |= flags; } static inline void vm_flags_clear(struct vm_area_struct *vma, vm_flags_t flags) { - mmap_assert_write_locked(vma->vm_mm); + vma_start_write(vma); ACCESS_PRIVATE(vma, __vm_flags) &= ~flags; } @@ -769,7 +769,7 @@ static inline void __vm_flags_mod(struct vm_area_struct *vma, static inline void vm_flags_mod(struct vm_area_struct *vma, vm_flags_t set, vm_flags_t clear) { - mmap_assert_write_locked(vma->vm_mm); + vma_start_write(vma); __vm_flags_mod(vma, set, clear); } -- cgit v1.2.3 From 55fd6fccad3172c0feaaa817f0a1283629ff183e Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Mon, 27 Feb 2023 09:36:14 -0800 Subject: mm/khugepaged: write-lock VMA while collapsing a huge page Protect VMA from concurrent page fault handler while collapsing a huge page. Page fault handler needs a stable PMD to use PTL and relies on per-VMA lock to prevent concurrent PMD changes. pmdp_collapse_flush(), set_huge_pmd() and collapse_and_free_pmd() can modify a PMD, which will not be detected by a page fault handler without proper locking. Before this patch, page tables can be walked under any one of the mmap_lock, the mapping lock, and the anon_vma lock; so when khugepaged unlinks and frees page tables, it must ensure that all of those either are locked or don't exist. This patch adds a fourth lock under which page tables can be traversed, and so khugepaged must also lock out that one. [surenb@google.com: vm_lock/i_mmap_rwsem inversion in retract_page_tables] Link: https://lkml.kernel.org/r/20230303213250.3555716-1-surenb@google.com [surenb@google.com: build fix] Link: https://lkml.kernel.org/r/CAJuCfpFjWhtzRE1X=J+_JjgJzNKhq-=JT8yTBSTHthwp0pqWZw@mail.gmail.com Link: https://lkml.kernel.org/r/20230227173632.3292573-16-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/mm.h | 41 ++++++++++++++++++++++++++++++----------- mm/khugepaged.c | 8 ++++++++ mm/rmap.c | 31 ++++++++++++++++--------------- 3 files changed, 54 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index d0a6c99aba09..d6a2abc51e3d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -665,18 +665,23 @@ static inline void vma_end_read(struct vm_area_struct *vma) rcu_read_unlock(); } -static inline void vma_start_write(struct vm_area_struct *vma) +static bool __is_vma_write_locked(struct vm_area_struct *vma, int *mm_lock_seq) { - int mm_lock_seq; - mmap_assert_write_locked(vma->vm_mm); /* * current task is holding mmap_write_lock, both vma->vm_lock_seq and * mm->mm_lock_seq can't be concurrently modified. */ - mm_lock_seq = READ_ONCE(vma->vm_mm->mm_lock_seq); - if (vma->vm_lock_seq == mm_lock_seq) + *mm_lock_seq = READ_ONCE(vma->vm_mm->mm_lock_seq); + return (vma->vm_lock_seq == *mm_lock_seq); +} + +static inline void vma_start_write(struct vm_area_struct *vma) +{ + int mm_lock_seq; + + if (__is_vma_write_locked(vma, &mm_lock_seq)) return; down_write(&vma->lock); @@ -684,14 +689,26 @@ static inline void vma_start_write(struct vm_area_struct *vma) up_write(&vma->lock); } +static inline bool vma_try_start_write(struct vm_area_struct *vma) +{ + int mm_lock_seq; + + if (__is_vma_write_locked(vma, &mm_lock_seq)) + return true; + + if (!down_write_trylock(&vma->vm_lock->lock)) + return false; + + vma->vm_lock_seq = mm_lock_seq; + up_write(&vma->vm_lock->lock); + return true; +} + static inline void vma_assert_write_locked(struct vm_area_struct *vma) { - mmap_assert_write_locked(vma->vm_mm); - /* - * current task is holding mmap_write_lock, both vma->vm_lock_seq and - * mm->mm_lock_seq can't be concurrently modified. - */ - VM_BUG_ON_VMA(vma->vm_lock_seq != READ_ONCE(vma->vm_mm->mm_lock_seq), vma); + int mm_lock_seq; + + VM_BUG_ON_VMA(!__is_vma_write_locked(vma, &mm_lock_seq), vma); } #else /* CONFIG_PER_VMA_LOCK */ @@ -701,6 +718,8 @@ static inline bool vma_start_read(struct vm_area_struct *vma) { return false; } static inline void vma_end_read(struct vm_area_struct *vma) {} static inline void vma_start_write(struct vm_area_struct *vma) {} +static inline bool vma_try_start_write(struct vm_area_struct *vma) + { return true; } static inline void vma_assert_write_locked(struct vm_area_struct *vma) {} #endif /* CONFIG_PER_VMA_LOCK */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index c7317678cb10..bee7fd7db380 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1056,6 +1056,7 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address, if (result != SCAN_SUCCEED) goto out_up_write; + vma_start_write(vma); anon_vma_lock_write(vma->anon_vma); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, address, @@ -1517,6 +1518,9 @@ int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, goto drop_hpage; } + /* Lock the vma before taking i_mmap and page table locks */ + vma_start_write(vma); + /* * We need to lock the mapping so that from here on, only GUP-fast and * hardware page walks can access the parts of the page tables that @@ -1694,6 +1698,10 @@ static int retract_page_tables(struct address_space *mapping, pgoff_t pgoff, result = SCAN_PTE_MAPPED_HUGEPAGE; if ((cc->is_khugepaged || is_target) && mmap_write_trylock(mm)) { + /* trylock for the same lock inversion as above */ + if (!vma_try_start_write(vma)) + goto unlock_next; + /* * Re-check whether we have an ->anon_vma, because * collapse_and_free_pmd() requires that either no diff --git a/mm/rmap.c b/mm/rmap.c index 1ea2756b2797..ba901c416785 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -25,21 +25,22 @@ * mapping->invalidate_lock (in filemap_fault) * page->flags PG_locked (lock_page) * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below) - * mapping->i_mmap_rwsem - * anon_vma->rwsem - * mm->page_table_lock or pte_lock - * swap_lock (in swap_duplicate, swap_info_get) - * mmlist_lock (in mmput, drain_mmlist and others) - * mapping->private_lock (in block_dirty_folio) - * folio_lock_memcg move_lock (in block_dirty_folio) - * i_pages lock (widely used) - * lruvec->lru_lock (in folio_lruvec_lock_irq) - * inode->i_lock (in set_page_dirty's __mark_inode_dirty) - * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) - * sb_lock (within inode_lock in fs/fs-writeback.c) - * i_pages lock (widely used, in set_page_dirty, - * in arch-dependent flush_dcache_mmap_lock, - * within bdi.wb->list_lock in __sync_single_inode) + * vma_start_write + * mapping->i_mmap_rwsem + * anon_vma->rwsem + * mm->page_table_lock or pte_lock + * swap_lock (in swap_duplicate, swap_info_get) + * mmlist_lock (in mmput, drain_mmlist and others) + * mapping->private_lock (in block_dirty_folio) + * folio_lock_memcg move_lock (in block_dirty_folio) + * i_pages lock (widely used) + * lruvec->lru_lock (in folio_lruvec_lock_irq) + * inode->i_lock (in set_page_dirty's __mark_inode_dirty) + * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) + * sb_lock (within inode_lock in fs/fs-writeback.c) + * i_pages lock (widely used, in set_page_dirty, + * in arch-dependent flush_dcache_mmap_lock, + * within bdi.wb->list_lock in __sync_single_inode) * * anon_vma->rwsem,mapping->i_mmap_rwsem (memory_failure, collect_procs_anon) * ->tasklist_lock -- cgit v1.2.3 From 457f67be5910a2b5f1fda8af06bfe4d3492a0a4f Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Mon, 27 Feb 2023 09:36:21 -0800 Subject: mm: introduce vma detached flag Per-vma locking mechanism will search for VMA under RCU protection and then after locking it, has to ensure it was not removed from the VMA tree after we found it. To make this check efficient, introduce a vma->detached flag to mark VMAs which were removed from the VMA tree. Link: https://lkml.kernel.org/r/20230227173632.3292573-23-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/mm.h | 11 +++++++++++ include/linux/mm_types.h | 3 +++ mm/mmap.c | 2 ++ 3 files changed, 16 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index d6a2abc51e3d..d0f289bfef01 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -711,6 +711,14 @@ static inline void vma_assert_write_locked(struct vm_area_struct *vma) VM_BUG_ON_VMA(!__is_vma_write_locked(vma, &mm_lock_seq), vma); } +static inline void vma_mark_detached(struct vm_area_struct *vma, bool detached) +{ + /* When detaching vma should be write-locked */ + if (detached) + vma_assert_write_locked(vma); + vma->detached = detached; +} + #else /* CONFIG_PER_VMA_LOCK */ static inline void vma_init_lock(struct vm_area_struct *vma) {} @@ -721,6 +729,8 @@ static inline void vma_start_write(struct vm_area_struct *vma) {} static inline bool vma_try_start_write(struct vm_area_struct *vma) { return true; } static inline void vma_assert_write_locked(struct vm_area_struct *vma) {} +static inline void vma_mark_detached(struct vm_area_struct *vma, + bool detached) {} #endif /* CONFIG_PER_VMA_LOCK */ @@ -732,6 +742,7 @@ static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) vma->vm_mm = mm; vma->vm_ops = &dummy_vm_ops; INIT_LIST_HEAD(&vma->anon_vma_chain); + vma_mark_detached(vma, false); vma_init_lock(vma); } diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 843a45893991..3248ae45cb2e 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -506,6 +506,9 @@ struct vm_area_struct { #ifdef CONFIG_PER_VMA_LOCK int vm_lock_seq; struct rw_semaphore lock; + + /* Flag to indicate areas detached from the mm->mm_mt tree */ + bool detached; #endif /* diff --git a/mm/mmap.c b/mm/mmap.c index 18aed0ea6bd3..b42f58591b9a 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -599,6 +599,7 @@ static inline void vma_complete(struct vma_prepare *vp, if (vp->remove) { again: + vma_mark_detached(vp->remove, true); if (vp->file) { uprobe_munmap(vp->remove, vp->remove->vm_start, vp->remove->vm_end); @@ -2276,6 +2277,7 @@ static inline int munmap_sidetree(struct vm_area_struct *vma, if (mas_store_gfp(mas_detach, vma, GFP_KERNEL)) return -ENOMEM; + vma_mark_detached(vma, true); if (vma->vm_flags & VM_LOCKED) vma->vm_mm->locked_vm -= vma_pages(vma); -- cgit v1.2.3 From 50ee32537206140e4cf6e47024be29a84d458d49 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Mon, 27 Feb 2023 09:36:22 -0800 Subject: mm: introduce lock_vma_under_rcu to be used from arch-specific code Introduce lock_vma_under_rcu function to lookup and lock a VMA during page fault handling. When VMA is not found, can't be locked or changes after being locked, the function returns NULL. The lookup is performed under RCU protection to prevent the found VMA from being destroyed before the VMA lock is acquired. VMA lock statistics are updated according to the results. For now only anonymous VMAs can be searched this way. In other cases the function returns NULL. Link: https://lkml.kernel.org/r/20230227173632.3292573-24-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/mm.h | 3 +++ mm/memory.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index d0f289bfef01..0619f9fa7581 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -719,6 +719,9 @@ static inline void vma_mark_detached(struct vm_area_struct *vma, bool detached) vma->detached = detached; } +struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, + unsigned long address); + #else /* CONFIG_PER_VMA_LOCK */ static inline void vma_init_lock(struct vm_area_struct *vma) {} diff --git a/mm/memory.c b/mm/memory.c index 8ca78ae8bba7..1235de20af73 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5232,6 +5232,52 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, } EXPORT_SYMBOL_GPL(handle_mm_fault); +#ifdef CONFIG_PER_VMA_LOCK +/* + * Lookup and lock a VMA under RCU protection. Returned VMA is guaranteed to be + * stable and not isolated. If the VMA is not found or is being modified the + * function returns NULL. + */ +struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, + unsigned long address) +{ + MA_STATE(mas, &mm->mm_mt, address, address); + struct vm_area_struct *vma; + + rcu_read_lock(); +retry: + vma = mas_walk(&mas); + if (!vma) + goto inval; + + /* Only anonymous vmas are supported for now */ + if (!vma_is_anonymous(vma)) + goto inval; + + if (!vma_start_read(vma)) + goto inval; + + /* Check since vm_start/vm_end might change before we lock the VMA */ + if (unlikely(address < vma->vm_start || address >= vma->vm_end)) { + vma_end_read(vma); + goto inval; + } + + /* Check if the VMA got isolated after we found it */ + if (vma->detached) { + vma_end_read(vma); + /* The area was replaced with another one */ + goto retry; + } + + rcu_read_unlock(); + return vma; +inval: + rcu_read_unlock(); + return NULL; +} +#endif /* CONFIG_PER_VMA_LOCK */ + #ifndef __PAGETABLE_P4D_FOLDED /* * Allocate p4d page table. -- cgit v1.2.3 From 55324e46eb8b886ecd08c9a089d3a694c705b3b0 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Mon, 27 Feb 2023 09:36:24 -0800 Subject: mm: add FAULT_FLAG_VMA_LOCK flag Add a new flag to distinguish page faults handled under protection of per-vma lock. [surenb@google.com: document FAULT_FLAG_VMA_LOCK flag] Link: https://lkml.kernel.org/r/20230301022720.1380780-2-surenb@google.com Link: https://lore.kernel.org/all/20230301113648.7c279865@canb.auug.org.au/ Link: https://lkml.kernel.org/r/20230227173632.3292573-26-surenb@google.com Signed-off-by: Suren Baghdasaryan Reviewed-by: Laurent Dufour Cc: Dan Carpenter Cc: David Hildenbrand Cc: David Howells Cc: Hugh Dickins Cc: Jason Gunthorpe Cc: Liam R. Howlett Cc: Mathieu Desnoyers Cc: Matthew Wilcox Cc: Pavel Tatashin Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/mm.h | 3 ++- include/linux/mm_types.h | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 0619f9fa7581..1876825d6700 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -479,7 +479,8 @@ static inline bool fault_flag_allow_retry_first(enum fault_flag flags) { FAULT_FLAG_USER, "USER" }, \ { FAULT_FLAG_REMOTE, "REMOTE" }, \ { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ - { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" } + { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ + { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" } /* * vm_fault is filled by the pagefault handler and passed to the vma's diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 3248ae45cb2e..3a6cf4c2edd0 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1049,6 +1049,7 @@ typedef struct { * mapped after the fault. * @FAULT_FLAG_ORIG_PTE_VALID: whether the fault has vmf->orig_pte cached. * We should only access orig_pte if this flag set. + * @FAULT_FLAG_VMA_LOCK: The fault is handled under VMA lock. * * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify * whether we would allow page faults to retry by specifying these two @@ -1086,6 +1087,7 @@ enum fault_flag { FAULT_FLAG_INTERRUPTIBLE = 1 << 9, FAULT_FLAG_UNSHARE = 1 << 10, FAULT_FLAG_ORIG_PTE_VALID = 1 << 11, + FAULT_FLAG_VMA_LOCK = 1 << 12, }; typedef unsigned int __bitwise zap_flags_t; -- cgit v1.2.3 From 52f238653e452e0fda61e880f263a173d219acd1 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Mon, 27 Feb 2023 09:36:27 -0800 Subject: mm: introduce per-VMA lock statistics Add a new CONFIG_PER_VMA_LOCK_STATS config option to dump extra statistics about handling page fault under VMA lock. Link: https://lkml.kernel.org/r/20230227173632.3292573-29-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/vm_event_item.h | 6 ++++++ include/linux/vmstat.h | 6 ++++++ mm/Kconfig.debug | 6 ++++++ mm/memory.c | 2 ++ mm/vmstat.c | 6 ++++++ 5 files changed, 26 insertions(+) (limited to 'include/linux') diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 7f5d1caf5890..8abfa1240040 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -149,6 +149,12 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, #ifdef CONFIG_X86 DIRECT_MAP_LEVEL2_SPLIT, DIRECT_MAP_LEVEL3_SPLIT, +#endif +#ifdef CONFIG_PER_VMA_LOCK_STATS + VMA_LOCK_SUCCESS, + VMA_LOCK_ABORT, + VMA_LOCK_RETRY, + VMA_LOCK_MISS, #endif NR_VM_EVENT_ITEMS }; diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index 19cf5b6892ce..fed855bae6d8 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -125,6 +125,12 @@ static inline void vm_events_fold_cpu(int cpu) #define count_vm_tlb_events(x, y) do { (void)(y); } while (0) #endif +#ifdef CONFIG_PER_VMA_LOCK_STATS +#define count_vm_vma_lock_event(x) count_vm_event(x) +#else +#define count_vm_vma_lock_event(x) do {} while (0) +#endif + #define __count_zid_vm_events(item, zid, delta) \ __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta) diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index c3547a373c9c..4965a7333a3f 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -279,3 +279,9 @@ config DEBUG_KMEMLEAK_AUTO_SCAN If unsure, say Y. +config PER_VMA_LOCK_STATS + bool "Statistics for per-vma locks" + depends on PER_VMA_LOCK + default y + help + Statistics for per-vma locks. diff --git a/mm/memory.c b/mm/memory.c index 55ac9cdfd398..9999574a9636 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5284,6 +5284,7 @@ retry: /* Check if the VMA got isolated after we found it */ if (vma->detached) { vma_end_read(vma); + count_vm_vma_lock_event(VMA_LOCK_MISS); /* The area was replaced with another one */ goto retry; } @@ -5292,6 +5293,7 @@ retry: return vma; inval: rcu_read_unlock(); + count_vm_vma_lock_event(VMA_LOCK_ABORT); return NULL; } #endif /* CONFIG_PER_VMA_LOCK */ diff --git a/mm/vmstat.c b/mm/vmstat.c index b7307627772d..c28046371b45 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1399,6 +1399,12 @@ const char * const vmstat_text[] = { "direct_map_level2_splits", "direct_map_level3_splits", #endif +#ifdef CONFIG_PER_VMA_LOCK_STATS + "vma_lock_success", + "vma_lock_abort", + "vma_lock_retry", + "vma_lock_miss", +#endif #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */ }; #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */ -- cgit v1.2.3 From 0d2ebf9c3f7822e7ba3e4792ea3b6b19aa2da34a Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Mon, 27 Feb 2023 09:36:31 -0800 Subject: mm/mmap: free vm_area_struct without call_rcu in exit_mmap call_rcu() can take a long time when callback offloading is enabled. Its use in the vm_area_free can cause regressions in the exit path when multiple VMAs are being freed. Because exit_mmap() is called only after the last mm user drops its refcount, the page fault handlers can't be racing with it. Any other possible user like oom-reaper or process_mrelease are already synchronized using mmap_lock. Therefore exit_mmap() can free VMAs directly, without the use of call_rcu(). Expose __vm_area_free() and use it from exit_mmap() to avoid possible call_rcu() floods and performance regressions caused by it. Link: https://lkml.kernel.org/r/20230227173632.3292573-33-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 ++ kernel/fork.c | 2 +- mm/mmap.c | 11 +++++++---- 3 files changed, 10 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 1876825d6700..f31c75dc190e 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -257,6 +257,8 @@ void setup_initial_init_mm(void *start_code, void *end_code, struct vm_area_struct *vm_area_alloc(struct mm_struct *); struct vm_area_struct *vm_area_dup(struct vm_area_struct *); void vm_area_free(struct vm_area_struct *); +/* Use only if VMA has no other users */ +void __vm_area_free(struct vm_area_struct *vma); #ifndef CONFIG_MMU extern struct rb_root nommu_region_tree; diff --git a/kernel/fork.c b/kernel/fork.c index 76fcc08984d4..f86f7e917954 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -480,7 +480,7 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) return new; } -static void __vm_area_free(struct vm_area_struct *vma) +void __vm_area_free(struct vm_area_struct *vma) { free_anon_vma_name(vma); kmem_cache_free(vm_area_cachep, vma); diff --git a/mm/mmap.c b/mm/mmap.c index b42f58591b9a..511f656eb423 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -133,7 +133,7 @@ void unlink_file_vma(struct vm_area_struct *vma) /* * Close a vm structure and free it. */ -static void remove_vma(struct vm_area_struct *vma) +static void remove_vma(struct vm_area_struct *vma, bool unreachable) { might_sleep(); if (vma->vm_ops && vma->vm_ops->close) @@ -141,7 +141,10 @@ static void remove_vma(struct vm_area_struct *vma) if (vma->vm_file) fput(vma->vm_file); mpol_put(vma_policy(vma)); - vm_area_free(vma); + if (unreachable) + __vm_area_free(vma); + else + vm_area_free(vma); } static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi, @@ -2145,7 +2148,7 @@ static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas) if (vma->vm_flags & VM_ACCOUNT) nr_accounted += nrpages; vm_stat_account(mm, vma->vm_flags, -nrpages); - remove_vma(vma); + remove_vma(vma, false); } vm_unacct_memory(nr_accounted); validate_mm(mm); @@ -3078,7 +3081,7 @@ void exit_mmap(struct mm_struct *mm) do { if (vma->vm_flags & VM_ACCOUNT) nr_accounted += vma_pages(vma); - remove_vma(vma); + remove_vma(vma, true); count++; cond_resched(); } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL); -- cgit v1.2.3 From c7f8f31c00d187a2c71a241c7f2bd6aa102a4e6f Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Mon, 27 Feb 2023 09:36:32 -0800 Subject: mm: separate vma->lock from vm_area_struct vma->lock being part of the vm_area_struct causes performance regression during page faults because during contention its count and owner fields are constantly updated and having other parts of vm_area_struct used during page fault handling next to them causes constant cache line bouncing. Fix that by moving the lock outside of the vm_area_struct. All attempts to keep vma->lock inside vm_area_struct in a separate cache line still produce performance regression especially on NUMA machines. Smallest regression was achieved when lock is placed in the fourth cache line but that bloats vm_area_struct to 256 bytes. Considering performance and memory impact, separate lock looks like the best option. It increases memory footprint of each VMA but that can be optimized later if the new size causes issues. Note that after this change vma_init() does not allocate or initialize vma->lock anymore. A number of drivers allocate a pseudo VMA on the stack but they never use the VMA's lock, therefore it does not need to be allocated. The future drivers which might need the VMA lock should use vm_area_alloc()/vm_area_free() to allocate the VMA. Link: https://lkml.kernel.org/r/20230227173632.3292573-34-surenb@google.com Signed-off-by: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/mm.h | 23 +++++++-------- include/linux/mm_types.h | 6 +++- kernel/fork.c | 73 ++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 74 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index f31c75dc190e..f5487a86b154 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -628,12 +628,6 @@ struct vm_operations_struct { }; #ifdef CONFIG_PER_VMA_LOCK -static inline void vma_init_lock(struct vm_area_struct *vma) -{ - init_rwsem(&vma->lock); - vma->vm_lock_seq = -1; -} - /* * Try to read-lock a vma. The function is allowed to occasionally yield false * locked result to avoid performance overhead, in which case we fall back to @@ -645,17 +639,17 @@ static inline bool vma_start_read(struct vm_area_struct *vma) if (vma->vm_lock_seq == READ_ONCE(vma->vm_mm->mm_lock_seq)) return false; - if (unlikely(down_read_trylock(&vma->lock) == 0)) + if (unlikely(down_read_trylock(&vma->vm_lock->lock) == 0)) return false; /* * Overflow might produce false locked result. * False unlocked result is impossible because we modify and check - * vma->vm_lock_seq under vma->lock protection and mm->mm_lock_seq + * vma->vm_lock_seq under vma->vm_lock protection and mm->mm_lock_seq * modification invalidates all existing locks. */ if (unlikely(vma->vm_lock_seq == READ_ONCE(vma->vm_mm->mm_lock_seq))) { - up_read(&vma->lock); + up_read(&vma->vm_lock->lock); return false; } return true; @@ -664,7 +658,7 @@ static inline bool vma_start_read(struct vm_area_struct *vma) static inline void vma_end_read(struct vm_area_struct *vma) { rcu_read_lock(); /* keeps vma alive till the end of up_read */ - up_read(&vma->lock); + up_read(&vma->vm_lock->lock); rcu_read_unlock(); } @@ -687,9 +681,9 @@ static inline void vma_start_write(struct vm_area_struct *vma) if (__is_vma_write_locked(vma, &mm_lock_seq)) return; - down_write(&vma->lock); + down_write(&vma->vm_lock->lock); vma->vm_lock_seq = mm_lock_seq; - up_write(&vma->lock); + up_write(&vma->vm_lock->lock); } static inline bool vma_try_start_write(struct vm_area_struct *vma) @@ -740,6 +734,10 @@ static inline void vma_mark_detached(struct vm_area_struct *vma, #endif /* CONFIG_PER_VMA_LOCK */ +/* + * WARNING: vma_init does not initialize vma->vm_lock. + * Use vm_area_alloc()/vm_area_free() if vma needs locking. + */ static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) { static const struct vm_operations_struct dummy_vm_ops = {}; @@ -749,7 +747,6 @@ static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) vma->vm_ops = &dummy_vm_ops; INIT_LIST_HEAD(&vma->anon_vma_chain); vma_mark_detached(vma, false); - vma_init_lock(vma); } /* Use when VMA is not part of the VMA tree and needs no locking */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 3a6cf4c2edd0..1b3031e95215 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -471,6 +471,10 @@ struct anon_vma_name { char name[]; }; +struct vma_lock { + struct rw_semaphore lock; +}; + /* * This struct describes a virtual memory area. There is one of these * per VM-area/task. A VM area is any part of the process virtual memory @@ -505,7 +509,7 @@ struct vm_area_struct { #ifdef CONFIG_PER_VMA_LOCK int vm_lock_seq; - struct rw_semaphore lock; + struct vma_lock *vm_lock; /* Flag to indicate areas detached from the mm->mm_mt tree */ bool detached; diff --git a/kernel/fork.c b/kernel/fork.c index f86f7e917954..c75088367bb4 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -451,13 +451,49 @@ static struct kmem_cache *vm_area_cachep; /* SLAB cache for mm_struct structures (tsk->mm) */ static struct kmem_cache *mm_cachep; +#ifdef CONFIG_PER_VMA_LOCK + +/* SLAB cache for vm_area_struct.lock */ +static struct kmem_cache *vma_lock_cachep; + +static bool vma_lock_alloc(struct vm_area_struct *vma) +{ + vma->vm_lock = kmem_cache_alloc(vma_lock_cachep, GFP_KERNEL); + if (!vma->vm_lock) + return false; + + init_rwsem(&vma->vm_lock->lock); + vma->vm_lock_seq = -1; + + return true; +} + +static inline void vma_lock_free(struct vm_area_struct *vma) +{ + kmem_cache_free(vma_lock_cachep, vma->vm_lock); +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline bool vma_lock_alloc(struct vm_area_struct *vma) { return true; } +static inline void vma_lock_free(struct vm_area_struct *vma) {} + +#endif /* CONFIG_PER_VMA_LOCK */ + struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) { struct vm_area_struct *vma; vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); - if (vma) - vma_init(vma, mm); + if (!vma) + return NULL; + + vma_init(vma, mm); + if (!vma_lock_alloc(vma)) { + kmem_cache_free(vm_area_cachep, vma); + return NULL; + } + return vma; } @@ -465,24 +501,30 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) { struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); - if (new) { - ASSERT_EXCLUSIVE_WRITER(orig->vm_flags); - ASSERT_EXCLUSIVE_WRITER(orig->vm_file); - /* - * orig->shared.rb may be modified concurrently, but the clone - * will be reinitialized. - */ - data_race(memcpy(new, orig, sizeof(*new))); - INIT_LIST_HEAD(&new->anon_vma_chain); - vma_init_lock(new); - dup_anon_vma_name(orig, new); + if (!new) + return NULL; + + ASSERT_EXCLUSIVE_WRITER(orig->vm_flags); + ASSERT_EXCLUSIVE_WRITER(orig->vm_file); + /* + * orig->shared.rb may be modified concurrently, but the clone + * will be reinitialized. + */ + data_race(memcpy(new, orig, sizeof(*new))); + if (!vma_lock_alloc(new)) { + kmem_cache_free(vm_area_cachep, new); + return NULL; } + INIT_LIST_HEAD(&new->anon_vma_chain); + dup_anon_vma_name(orig, new); + return new; } void __vm_area_free(struct vm_area_struct *vma) { free_anon_vma_name(vma); + vma_lock_free(vma); kmem_cache_free(vm_area_cachep, vma); } @@ -493,7 +535,7 @@ static void vm_area_free_rcu_cb(struct rcu_head *head) vm_rcu); /* The vma should not be locked while being destroyed. */ - VM_BUG_ON_VMA(rwsem_is_locked(&vma->lock), vma); + VM_BUG_ON_VMA(rwsem_is_locked(&vma->vm_lock->lock), vma); __vm_area_free(vma); } #endif @@ -3152,6 +3194,9 @@ void __init proc_caches_init(void) NULL); vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC|SLAB_ACCOUNT); +#ifdef CONFIG_PER_VMA_LOCK + vma_lock_cachep = KMEM_CACHE(vma_lock, SLAB_PANIC|SLAB_ACCOUNT); +#endif mmap_init(); nsproxy_cache_init(); } -- cgit v1.2.3 From ef6a22b70f6d90449a5c797b8968a682824e2011 Mon Sep 17 00:00:00 2001 From: Mel Gorman Date: Wed, 1 Mar 2023 17:49:00 +0530 Subject: sched/numa: apply the scan delay to every new vma Pach series "sched/numa: Enhance vma scanning", v3. The patchset proposes one of the enhancements to numa vma scanning suggested by Mel. This is continuation of [3]. Reposting the rebased patchset to akpm mm-unstable tree (March 1) Existing mechanism of scan period involves, scan period derived from per-thread stats. Process Adaptive autoNUMA [1] proposed to gather NUMA fault stats at per-process level to capture aplication behaviour better. During that course of discussion, Mel proposed several ideas to enhance current numa balancing. One of the suggestion was below Track what threads access a VMA. The suggestion was to use an unsigned long pid_mask and use the lower bits to tag approximately what threads access a VMA. Skip VMAs that did not trap a fault. This would be approximate because of PID collisions but would reduce scanning of areas the thread is not interested in. The above suggestion intends not to penalize threads that has no interest in the vma, thus reduce scanning overhead. V3 changes are mostly based on PeterZ comments (details below in changes) Summary of patchset: Current patchset implements: 1. Delay the vma scanning logic for newly created VMA's so that additional overhead of scanning is not incurred for short lived tasks (implementation by Mel) 2. Store the information of tasks accessing VMA in 2 windows. It is regularly cleared in (4*sysctl_numa_balancing_scan_delay) interval. The above time is derived from experimenting (Suggested by PeterZ) to balance between frequent clearing vs obsolete access data 3. hash_32 used to encode task index accessing VMA information 4. VMA's acess information is used to skip scanning for the tasks which had not accessed VMA Changes since V2: patch1: - Renaming of structure, macro to function, - Add explanation to heuristics - Adding more details from result (PeterZ) Patch2: - Usage of test and set bit (PeterZ) - Move storing access PID info to numa_migrate_prep() - Add a note on fainess among tasks allowed to scan (PeterZ) Patch3: - Maintain two windows of access PID information (PeterZ supported implementation and Gave idea to extend to N if needed) Patch4: - Apply hash_32 function to track VMA accessing PIDs (PeterZ) Changes since RFC V1: - Include Mel's vma scan delay patch - Change the accessing pid store logic (Thanks Mel) - Fencing structure / code to NUMA_BALANCING (David, Mel) - Adding clearing access PID logic (Mel) - Descriptive change log ( Mike Rapoport) Things to ponder over: ========================================== - Improvement to clearing accessing PIDs logic (discussed in-detail in patch3 itself (Done in this patchset by implementing 2 window history) - Current scan period is not changed in the patchset, so we do see frequent tries to scan. Relaxing scan period dynamically could improve results further. [1] sched/numa: Process Adaptive autoNUMA Link: https://lore.kernel.org/lkml/20220128052851.17162-1-bharata@amd.com/T/ [2] RFC V1 Link: https://lore.kernel.org/all/cover.1673610485.git.raghavendra.kt@amd.com/ [3] V2 Link: https://lore.kernel.org/lkml/cover.1675159422.git.raghavendra.kt@amd.com/ Results: Summary: Huge autonuma cost reduction seen in mmtest. Kernbench improvement is more than 5% and huge system time (80%+) improvement from mmtest autonuma. (dbench had huge std deviation to post) kernbench =========== 6.2.0-mmunstable-base 6.2.0-mmunstable-patched Amean user-256 22002.51 ( 0.00%) 22649.95 * -2.94%* Amean syst-256 10162.78 ( 0.00%) 8214.13 * 19.17%* Amean elsp-256 160.74 ( 0.00%) 156.92 * 2.38%* Duration User 66017.43 67959.84 Duration System 30503.15 24657.03 Duration Elapsed 504.61 493.12 6.2.0-mmunstable-base 6.2.0-mmunstable-patched Ops NUMA alloc hit 1738835089.00 1738780310.00 Ops NUMA alloc local 1738834448.00 1738779711.00 Ops NUMA base-page range updates 477310.00 392566.00 Ops NUMA PTE updates 477310.00 392566.00 Ops NUMA hint faults 96817.00 87555.00 Ops NUMA hint local faults % 10150.00 2192.00 Ops NUMA hint local percent 10.48 2.50 Ops NUMA pages migrated 86660.00 85363.00 Ops AutoNUMA cost 489.07 442.14 autonumabench =============== 6.2.0-mmunstable-base 6.2.0-mmunstable-patched Amean syst-NUMA01 399.50 ( 0.00%) 52.05 * 86.97%* Amean syst-NUMA01_THREADLOCAL 0.21 ( 0.00%) 0.22 * -5.41%* Amean syst-NUMA02 0.80 ( 0.00%) 0.78 * 2.68%* Amean syst-NUMA02_SMT 0.65 ( 0.00%) 0.68 * -3.95%* Amean elsp-NUMA01 313.26 ( 0.00%) 313.11 * 0.05%* Amean elsp-NUMA01_THREADLOCAL 1.06 ( 0.00%) 1.08 * -1.76%* Amean elsp-NUMA02 3.19 ( 0.00%) 3.24 * -1.52%* Amean elsp-NUMA02_SMT 3.72 ( 0.00%) 3.61 * 2.92%* Duration User 396433.47 324835.96 Duration System 2808.70 376.66 Duration Elapsed 2258.61 2258.12 6.2.0-mmunstable-base 6.2.0-mmunstable-patched Ops NUMA alloc hit 59921806.00 49623489.00 Ops NUMA alloc miss 0.00 0.00 Ops NUMA interleave hit 0.00 0.00 Ops NUMA alloc local 59920880.00 49622594.00 Ops NUMA base-page range updates 152259275.00 50075.00 Ops NUMA PTE updates 152259275.00 50075.00 Ops NUMA PMD updates 0.00 0.00 Ops NUMA hint faults 154660352.00 39014.00 Ops NUMA hint local faults % 138550501.00 23139.00 Ops NUMA hint local percent 89.58 59.31 Ops NUMA pages migrated 8179067.00 14147.00 Ops AutoNUMA cost 774522.98 195.69 This patch (of 4): Currently whenever a new task is created we wait for sysctl_numa_balancing_scan_delay to avoid unnessary scanning overhead. Extend the same logic to new or very short-lived VMAs. [raghavendra.kt@amd.com: add initialization in vm_area_dup())] Link: https://lkml.kernel.org/r/cover.1677672277.git.raghavendra.kt@amd.com Link: https://lkml.kernel.org/r/7a6fbba87c8b51e67efd3e74285bb4cb311a16ca.1677672277.git.raghavendra.kt@amd.com Signed-off-by: Mel Gorman Signed-off-by: Raghavendra K T Cc: Bharata B Rao Cc: David Hildenbrand Cc: Ingo Molnar Cc: Mike Rapoport Cc: Peter Zijlstra Cc: Disha Talreja Signed-off-by: Andrew Morton --- include/linux/mm.h | 16 ++++++++++++++++ include/linux/mm_types.h | 7 +++++++ kernel/fork.c | 2 ++ kernel/sched/fair.c | 19 +++++++++++++++++++ 4 files changed, 44 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index f5487a86b154..9f08a11b355c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -29,6 +29,7 @@ #include #include #include +#include struct mempolicy; struct anon_vma; @@ -627,6 +628,20 @@ struct vm_operations_struct { unsigned long addr); }; +#ifdef CONFIG_NUMA_BALANCING +static inline void vma_numab_state_init(struct vm_area_struct *vma) +{ + vma->numab_state = NULL; +} +static inline void vma_numab_state_free(struct vm_area_struct *vma) +{ + kfree(vma->numab_state); +} +#else +static inline void vma_numab_state_init(struct vm_area_struct *vma) {} +static inline void vma_numab_state_free(struct vm_area_struct *vma) {} +#endif /* CONFIG_NUMA_BALANCING */ + #ifdef CONFIG_PER_VMA_LOCK /* * Try to read-lock a vma. The function is allowed to occasionally yield false @@ -747,6 +762,7 @@ static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm) vma->vm_ops = &dummy_vm_ops; INIT_LIST_HEAD(&vma->anon_vma_chain); vma_mark_detached(vma, false); + vma_numab_state_init(vma); } /* Use when VMA is not part of the VMA tree and needs no locking */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 1b3031e95215..3e1a42673769 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -475,6 +475,10 @@ struct vma_lock { struct rw_semaphore lock; }; +struct vma_numab_state { + unsigned long next_scan; +}; + /* * This struct describes a virtual memory area. There is one of these * per VM-area/task. A VM area is any part of the process virtual memory @@ -560,6 +564,9 @@ struct vm_area_struct { #endif #ifdef CONFIG_NUMA struct mempolicy *vm_policy; /* NUMA policy for the VMA */ +#endif +#ifdef CONFIG_NUMA_BALANCING + struct vma_numab_state *numab_state; /* NUMA Balancing state */ #endif struct vm_userfaultfd_ctx vm_userfaultfd_ctx; } __randomize_layout; diff --git a/kernel/fork.c b/kernel/fork.c index c75088367bb4..2bde99037652 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -516,6 +516,7 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) return NULL; } INIT_LIST_HEAD(&new->anon_vma_chain); + vma_numab_state_init(new); dup_anon_vma_name(orig, new); return new; @@ -523,6 +524,7 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) void __vm_area_free(struct vm_area_struct *vma) { + vma_numab_state_free(vma); free_anon_vma_name(vma); vma_lock_free(vma); kmem_cache_free(vm_area_cachep, vma); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 6986ea31c984..7072de1686d5 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3027,6 +3027,25 @@ static void task_numa_work(struct callback_head *work) if (!vma_is_accessible(vma)) continue; + /* Initialise new per-VMA NUMAB state. */ + if (!vma->numab_state) { + vma->numab_state = kzalloc(sizeof(struct vma_numab_state), + GFP_KERNEL); + if (!vma->numab_state) + continue; + + vma->numab_state->next_scan = now + + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); + } + + /* + * Scanning the VMA's of short lived tasks add more overhead. So + * delay the scan for new VMAs. + */ + if (mm->numa_scan_seq && time_before(jiffies, + vma->numab_state->next_scan)) + continue; + do { start = max(start, vma->vm_start); end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); -- cgit v1.2.3 From fc137c0ddab29b591db6a091dc6d7ce20ccb73f2 Mon Sep 17 00:00:00 2001 From: Raghavendra K T Date: Wed, 1 Mar 2023 17:49:01 +0530 Subject: sched/numa: enhance vma scanning logic During Numa scanning make sure only relevant vmas of the tasks are scanned. Before: All the tasks of a process participate in scanning the vma even if they do not access vma in it's lifespan. Now: Except cases of first few unconditional scans, if a process do not touch vma (exluding false positive cases of PID collisions) tasks no longer scan all vma Logic used: 1) 6 bits of PID used to mark active bit in vma numab status during fault to remember PIDs accessing vma. (Thanks Mel) 2) Subsequently in scan path, vma scanning is skipped if current PID had not accessed vma. 3) First two times we do allow unconditional scan to preserve earlier behaviour of scanning. Acknowledgement to Bharata B Rao for initial patch to store pid information and Peter Zijlstra (Usage of test and set bit) Link: https://lkml.kernel.org/r/092f03105c7c1d3450f4636b1ea350407f07640e.1677672277.git.raghavendra.kt@amd.com Signed-off-by: Raghavendra K T Suggested-by: Mel Gorman Cc: David Hildenbrand Cc: Disha Talreja Cc: Ingo Molnar Cc: Mike Rapoport Signed-off-by: Andrew Morton --- include/linux/mm.h | 14 ++++++++++++++ include/linux/mm_types.h | 1 + kernel/sched/fair.c | 19 +++++++++++++++++++ mm/memory.c | 3 +++ 4 files changed, 37 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 9f08a11b355c..215327daffae 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1686,6 +1686,16 @@ static inline int xchg_page_access_time(struct page *page, int time) last_time = page_cpupid_xchg_last(page, time >> PAGE_ACCESS_TIME_BUCKETS); return last_time << PAGE_ACCESS_TIME_BUCKETS; } + +static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) +{ + unsigned int pid_bit; + + pid_bit = current->pid % BITS_PER_LONG; + if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->access_pids)) { + __set_bit(pid_bit, &vma->numab_state->access_pids); + } +} #else /* !CONFIG_NUMA_BALANCING */ static inline int page_cpupid_xchg_last(struct page *page, int cpupid) { @@ -1735,6 +1745,10 @@ static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) { return false; } + +static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) +{ +} #endif /* CONFIG_NUMA_BALANCING */ #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 3e1a42673769..f8cbd8efc7cb 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -477,6 +477,7 @@ struct vma_lock { struct vma_numab_state { unsigned long next_scan; + unsigned long access_pids; }; /* diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 7072de1686d5..ef27b5931480 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2928,6 +2928,21 @@ static void reset_ptenuma_scan(struct task_struct *p) p->mm->numa_scan_offset = 0; } +static bool vma_is_accessed(struct vm_area_struct *vma) +{ + /* + * Allow unconditional access first two times, so that all the (pages) + * of VMAs get prot_none fault introduced irrespective of accesses. + * This is also done to avoid any side effect of task scanning + * amplifying the unfairness of disjoint set of VMAs' access. + */ + if (READ_ONCE(current->mm->numa_scan_seq) < 2) + return true; + + return test_bit(current->pid % BITS_PER_LONG, + &vma->numab_state->access_pids); +} + /* * The expensive part of numa migration is done from task_work context. * Triggered from task_tick_numa(). @@ -3046,6 +3061,10 @@ static void task_numa_work(struct callback_head *work) vma->numab_state->next_scan)) continue; + /* Do not scan the VMA if task has not accessed */ + if (!vma_is_accessed(vma)) + continue; + do { start = max(start, vma->vm_start); end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); diff --git a/mm/memory.c b/mm/memory.c index 9999574a9636..f77fccb5310c 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4661,6 +4661,9 @@ int numa_migrate_prep(struct page *page, struct vm_area_struct *vma, { get_page(page); + /* Record the current PID acceesing VMA */ + vma_set_access_pid_bit(vma); + count_vm_numa_event(NUMA_HINT_FAULTS); if (page_nid == numa_node_id()) { count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); -- cgit v1.2.3 From 20f586486b87dcfe10b8c79398e24e720885588a Mon Sep 17 00:00:00 2001 From: Raghavendra K T Date: Wed, 1 Mar 2023 17:49:02 +0530 Subject: sched/numa: implement access PID reset logic This helps to ensure that only recently accessed PIDs scan the VMAs. Current implementation: (idea supported by PeterZ) 1. Accessing PID information is maintained in two windows. access_pids[1] being newest. 2. Reset old access PID info i.e. access_pid[0] every (4 * sysctl_numa_balancing_scan_delay) interval after initial scan delay period expires. The above interval seemed to be experimentally optimum since it avoids frequent reset of access info as well as helps clearing the old access info regularly. The reset logic is implemented in scan path. Link: https://lkml.kernel.org/r/f7a675f66d1442d048b4216b2baf94515012c405.1677672277.git.raghavendra.kt@amd.com Signed-off-by: Raghavendra K T Suggested-by: Mel Gorman Cc: Bharata B Rao Cc: David Hildenbrand Cc: Disha Talreja Cc: Ingo Molnar Cc: Mike Rapoport Cc: Peter Zijlstra Signed-off-by: Andrew Morton --- include/linux/mm.h | 4 ++-- include/linux/mm_types.h | 3 ++- kernel/sched/fair.c | 23 +++++++++++++++++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 215327daffae..e05a878e186e 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1692,8 +1692,8 @@ static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) unsigned int pid_bit; pid_bit = current->pid % BITS_PER_LONG; - if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->access_pids)) { - __set_bit(pid_bit, &vma->numab_state->access_pids); + if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->access_pids[1])) { + __set_bit(pid_bit, &vma->numab_state->access_pids[1]); } } #else /* !CONFIG_NUMA_BALANCING */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index f8cbd8efc7cb..092f842a854f 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -477,7 +477,8 @@ struct vma_lock { struct vma_numab_state { unsigned long next_scan; - unsigned long access_pids; + unsigned long next_pid_reset; + unsigned long access_pids[2]; }; /* diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ef27b5931480..a962d4b60cd7 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2930,6 +2930,7 @@ static void reset_ptenuma_scan(struct task_struct *p) static bool vma_is_accessed(struct vm_area_struct *vma) { + unsigned long pids; /* * Allow unconditional access first two times, so that all the (pages) * of VMAs get prot_none fault introduced irrespective of accesses. @@ -2939,10 +2940,12 @@ static bool vma_is_accessed(struct vm_area_struct *vma) if (READ_ONCE(current->mm->numa_scan_seq) < 2) return true; - return test_bit(current->pid % BITS_PER_LONG, - &vma->numab_state->access_pids); + pids = vma->numab_state->access_pids[0] | vma->numab_state->access_pids[1]; + return test_bit(current->pid % BITS_PER_LONG, &pids); } +#define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay) + /* * The expensive part of numa migration is done from task_work context. * Triggered from task_tick_numa(). @@ -3051,6 +3054,10 @@ static void task_numa_work(struct callback_head *work) vma->numab_state->next_scan = now + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); + + /* Reset happens after 4 times scan delay of scan start */ + vma->numab_state->next_pid_reset = vma->numab_state->next_scan + + msecs_to_jiffies(VMA_PID_RESET_PERIOD); } /* @@ -3065,6 +3072,18 @@ static void task_numa_work(struct callback_head *work) if (!vma_is_accessed(vma)) continue; + /* + * RESET access PIDs regularly for old VMAs. Resetting after checking + * vma for recent access to avoid clearing PID info before access.. + */ + if (mm->numa_scan_seq && + time_after(jiffies, vma->numab_state->next_pid_reset)) { + vma->numab_state->next_pid_reset = vma->numab_state->next_pid_reset + + msecs_to_jiffies(VMA_PID_RESET_PERIOD); + vma->numab_state->access_pids[0] = READ_ONCE(vma->numab_state->access_pids[1]); + vma->numab_state->access_pids[1] = 0; + } + do { start = max(start, vma->vm_start); end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); -- cgit v1.2.3 From d46031f40e0f7f7bf63914bb3f2e404ad3886ecd Mon Sep 17 00:00:00 2001 From: Raghavendra K T Date: Wed, 1 Mar 2023 17:49:03 +0530 Subject: sched/numa: use hash_32 to mix up PIDs accessing VMA before: last 6 bits of PID is used as index to store information about tasks accessing VMA's. after: hash_32 is used to take of cases where tasks are created over a period of time, and thus improve collision probability. Result: The patch series overall improves autonuma cost. Kernbench around more than 5% improvement and system time in mmtest autonuma showed more than 80% improvement Link: https://lkml.kernel.org/r/d5a9f75513300caed74e5c8570bba9317b963c2b.1677672277.git.raghavendra.kt@amd.com Signed-off-by: Raghavendra K T Suggested-by: Peter Zijlstra Cc: Bharata B Rao Cc: David Hildenbrand Cc: Disha Talreja Cc: Ingo Molnar Cc: Mel Gorman Cc: Mike Rapoport Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 +- kernel/sched/fair.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index e05a878e186e..e249208f8fbe 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1691,7 +1691,7 @@ static inline void vma_set_access_pid_bit(struct vm_area_struct *vma) { unsigned int pid_bit; - pid_bit = current->pid % BITS_PER_LONG; + pid_bit = hash_32(current->pid, ilog2(BITS_PER_LONG)); if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->access_pids[1])) { __set_bit(pid_bit, &vma->numab_state->access_pids[1]); } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a962d4b60cd7..db6fc9d978ae 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2941,7 +2941,7 @@ static bool vma_is_accessed(struct vm_area_struct *vma) return true; pids = vma->numab_state->access_pids[0] | vma->numab_state->access_pids[1]; - return test_bit(current->pid % BITS_PER_LONG, &pids); + return test_bit(hash_32(current->pid, ilog2(BITS_PER_LONG)), &pids); } #define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay) -- cgit v1.2.3 From d0a4564bd024eaa81cab8c7255e7c44230bdd8a2 Mon Sep 17 00:00:00 2001 From: Olivier Moysan Date: Tue, 13 Dec 2022 11:27:07 +0100 Subject: pwm: stm32: Enforce settings for PWM capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PWM capture assumes that the input selector is set to default input and that the slave mode is disabled. Force reset state for TISEL and SMCR registers to match this requirement. Note that slave mode disabling is not a pre-requisite by itself for capture mode, as hardware supports it for PWM capture. However, the current implementation of the driver does not allow slave mode for PWM capture. Setting slave mode for PWM capture results in wrong capture values. Signed-off-by: Olivier Moysan Acked-by: Lee Jones Acked-by: Uwe Kleine-König Signed-off-by: Thierry Reding --- drivers/pwm/pwm-stm32.c | 4 ++++ include/linux/mfd/stm32-timers.h | 1 + 2 files changed, 5 insertions(+) (limited to 'include/linux') diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c index a482f7e0e4ab..62e397aeb9aa 100644 --- a/drivers/pwm/pwm-stm32.c +++ b/drivers/pwm/pwm-stm32.c @@ -207,6 +207,10 @@ static int stm32_pwm_capture(struct pwm_chip *chip, struct pwm_device *pwm, regmap_write(priv->regmap, TIM_ARR, priv->max_arr); regmap_write(priv->regmap, TIM_PSC, psc); + /* Reset input selector to its default input and disable slave mode */ + regmap_write(priv->regmap, TIM_TISEL, 0x0); + regmap_write(priv->regmap, TIM_SMCR, 0x0); + /* Map TI1 or TI2 PWM input to IC1 & IC2 (or TI3/4 to IC3 & IC4) */ regmap_update_bits(priv->regmap, pwm->hwpwm < 2 ? TIM_CCMR1 : TIM_CCMR2, diff --git a/include/linux/mfd/stm32-timers.h b/include/linux/mfd/stm32-timers.h index 5f5c43fd69dd..1b94325febb3 100644 --- a/include/linux/mfd/stm32-timers.h +++ b/include/linux/mfd/stm32-timers.h @@ -31,6 +31,7 @@ #define TIM_BDTR 0x44 /* Break and Dead-Time Reg */ #define TIM_DCR 0x48 /* DMA control register */ #define TIM_DMAR 0x4C /* DMA register for transfer */ +#define TIM_TISEL 0x68 /* Input Selection */ #define TIM_CR1_CEN BIT(0) /* Counter Enable */ #define TIM_CR1_DIR BIT(4) /* Counter Direction */ -- cgit v1.2.3 From fe4e5efa401fe15eab9259e358730145449e83a2 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Sat, 1 Apr 2023 10:15:29 +0100 Subject: dma-mapping: provide a fallback dma_default_coherent dma_default_coherent was decleared unconditionally at kernel/dma/mapping.c but only decleared when any of non-coherent options is enabled in dma-map-ops.h. Guard the declaration in mapping.c with non-coherent options and provide a fallback definition. Signed-off-by: Jiaxun Yang Signed-off-by: Christoph Hellwig --- include/linux/dma-map-ops.h | 2 ++ kernel/dma/mapping.c | 4 ++++ 2 files changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 41bf4bdb117a..31f114f486c4 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -269,6 +269,8 @@ static inline bool dev_is_dma_coherent(struct device *dev) return dev->dma_coherent; } #else +#define dma_default_coherent true + static inline bool dev_is_dma_coherent(struct device *dev) { return true; diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 68106e3791f6..80f9663ffe26 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -17,7 +17,11 @@ #include "debug.h" #include "direct.h" +#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) bool dma_default_coherent; +#endif /* * Managed DMA API -- cgit v1.2.3 From 890a3ee3ce416e2f1aed41718a8c1d42c82cf1b2 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 23 Mar 2023 17:50:29 +0200 Subject: kernel.h: split the hexadecimal related helpers to hex.h For the sake of cleaning up the kernel.h split the hexadecimal related helpers to own header called 'hex.h'. Link: https://lkml.kernel.org/r/20230323155029.40000-1-andriy.shevchenko@linux.intel.com Signed-off-by: Andy Shevchenko Cc: Rasmus Villemoes Signed-off-by: Andrew Morton --- include/linux/hex.h | 35 +++++++++++++++++++++++++++++++++++ include/linux/kernel.h | 29 +---------------------------- 2 files changed, 36 insertions(+), 28 deletions(-) create mode 100644 include/linux/hex.h (limited to 'include/linux') diff --git a/include/linux/hex.h b/include/linux/hex.h new file mode 100644 index 000000000000..2618382e5b0c --- /dev/null +++ b/include/linux/hex.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_HEX_H +#define _LINUX_HEX_H + +#include + +extern const char hex_asc[]; +#define hex_asc_lo(x) hex_asc[((x) & 0x0f)] +#define hex_asc_hi(x) hex_asc[((x) & 0xf0) >> 4] + +static inline char *hex_byte_pack(char *buf, u8 byte) +{ + *buf++ = hex_asc_hi(byte); + *buf++ = hex_asc_lo(byte); + return buf; +} + +extern const char hex_asc_upper[]; +#define hex_asc_upper_lo(x) hex_asc_upper[((x) & 0x0f)] +#define hex_asc_upper_hi(x) hex_asc_upper[((x) & 0xf0) >> 4] + +static inline char *hex_byte_pack_upper(char *buf, u8 byte) +{ + *buf++ = hex_asc_upper_hi(byte); + *buf++ = hex_asc_upper_lo(byte); + return buf; +} + +extern int hex_to_bin(unsigned char ch); +extern int __must_check hex2bin(u8 *dst, const char *src, size_t count); +extern char *bin2hex(char *dst, const void *src, size_t count); + +bool mac_pton(const char *s, u8 *mac); + +#endif diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 40bce7495af8..0d91e0af0125 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -263,34 +264,6 @@ extern enum system_states { SYSTEM_SUSPEND, } system_state; -extern const char hex_asc[]; -#define hex_asc_lo(x) hex_asc[((x) & 0x0f)] -#define hex_asc_hi(x) hex_asc[((x) & 0xf0) >> 4] - -static inline char *hex_byte_pack(char *buf, u8 byte) -{ - *buf++ = hex_asc_hi(byte); - *buf++ = hex_asc_lo(byte); - return buf; -} - -extern const char hex_asc_upper[]; -#define hex_asc_upper_lo(x) hex_asc_upper[((x) & 0x0f)] -#define hex_asc_upper_hi(x) hex_asc_upper[((x) & 0xf0) >> 4] - -static inline char *hex_byte_pack_upper(char *buf, u8 byte) -{ - *buf++ = hex_asc_upper_hi(byte); - *buf++ = hex_asc_upper_lo(byte); - return buf; -} - -extern int hex_to_bin(unsigned char ch); -extern int __must_check hex2bin(u8 *dst, const char *src, size_t count); -extern char *bin2hex(char *dst, const void *src, size_t count); - -bool mac_pton(const char *s, u8 *mac); - /* * General tracing related utility functions - trace_printk(), * tracing_on/tracing_off and tracing_start()/tracing_stop -- cgit v1.2.3 From 7982722ff7286596a1e2f343ec4219e309ddc82a Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 7 Mar 2023 16:44:15 -0600 Subject: x86/kexec: remove unnecessary arch_kexec_kernel_image_load() Patch series "kexec: Remove unnecessary arch hook", v2. There are no arch-specific things in arch_kexec_kernel_image_load(), so remove it and just use the generic version. This patch (of 2): The x86 implementation of arch_kexec_kernel_image_load() is functionally identical to the generic arch_kexec_kernel_image_load(): arch_kexec_kernel_image_load # x86 if (!image->fops || !image->fops->load) return ERR_PTR(-ENOEXEC); return image->fops->load(image, image->kernel_buf, ...) arch_kexec_kernel_image_load # generic kexec_image_load_default if (!image->fops || !image->fops->load) return ERR_PTR(-ENOEXEC); return image->fops->load(image, image->kernel_buf, ...) Remove the x86-specific version and use the generic arch_kexec_kernel_image_load(). No functional change intended. Link: https://lkml.kernel.org/r/20230307224416.907040-1-helgaas@kernel.org Link: https://lkml.kernel.org/r/20230307224416.907040-2-helgaas@kernel.org Signed-off-by: Bjorn Helgaas Reviewed-by: Simon Horman Acked-by: Baoquan He Cc: Borislav Petkov (AMD) Cc: Dave Hansen Cc: Eric Biederman Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/x86/include/asm/kexec.h | 3 --- arch/x86/kernel/machine_kexec_64.c | 11 ----------- include/linux/kexec.h | 2 -- 3 files changed, 16 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h index a3760ca796aa..5b77bbc28f96 100644 --- a/arch/x86/include/asm/kexec.h +++ b/arch/x86/include/asm/kexec.h @@ -200,9 +200,6 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, const Elf_Shdr *symtab); #define arch_kexec_apply_relocations_add arch_kexec_apply_relocations_add -void *arch_kexec_kernel_image_load(struct kimage *image); -#define arch_kexec_kernel_image_load arch_kexec_kernel_image_load - int arch_kimage_file_post_load_cleanup(struct kimage *image); #define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup #endif diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c index 0611fd83858e..1a3e2c05a8a5 100644 --- a/arch/x86/kernel/machine_kexec_64.c +++ b/arch/x86/kernel/machine_kexec_64.c @@ -374,17 +374,6 @@ void machine_kexec(struct kimage *image) /* arch-dependent functionality related to kexec file-based syscall */ #ifdef CONFIG_KEXEC_FILE -void *arch_kexec_kernel_image_load(struct kimage *image) -{ - if (!image->fops || !image->fops->load) - return ERR_PTR(-ENOEXEC); - - return image->fops->load(image, image->kernel_buf, - image->kernel_buf_len, image->initrd_buf, - image->initrd_buf_len, image->cmdline_buf, - image->cmdline_buf_len); -} - /* * Apply purgatory relocations. * diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 6883c5922701..4746bc9d39c9 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -207,12 +207,10 @@ static inline int arch_kimage_file_post_load_cleanup(struct kimage *image) } #endif -#ifndef arch_kexec_kernel_image_load static inline void *arch_kexec_kernel_image_load(struct kimage *image) { return kexec_image_load_default(image); } -#endif #ifdef CONFIG_KEXEC_SIG #ifdef CONFIG_SIGNED_PE_FILE_VERIFICATION -- cgit v1.2.3 From fb15abdca64503511bb32cb6ff70da306f24fa06 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 7 Mar 2023 16:44:16 -0600 Subject: kexec: remove unnecessary arch_kexec_kernel_image_load() arch_kexec_kernel_image_load() only calls kexec_image_load_default(), and there are no arch-specific implementations. Remove the unnecessary arch_kexec_kernel_image_load() and make kexec_image_load_default() static. No functional change intended. Link: https://lkml.kernel.org/r/20230307224416.907040-3-helgaas@kernel.org Signed-off-by: Bjorn Helgaas Reviewed-by: Simon Horman Acked-by: Baoquan He Cc: Borislav Petkov (AMD) Cc: Dave Hansen Cc: Eric Biederman Cc: "H. Peter Anvin" Cc: Ingo Molnar Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- include/linux/kexec.h | 6 ------ kernel/kexec_file.c | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 4746bc9d39c9..22b5cd24f581 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -190,7 +190,6 @@ int kexec_purgatory_get_set_symbol(struct kimage *image, const char *name, void *buf, unsigned int size, bool get_value); void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name); -void *kexec_image_load_default(struct kimage *image); #ifndef arch_kexec_kernel_image_probe static inline int @@ -207,11 +206,6 @@ static inline int arch_kimage_file_post_load_cleanup(struct kimage *image) } #endif -static inline void *arch_kexec_kernel_image_load(struct kimage *image) -{ - return kexec_image_load_default(image); -} - #ifdef CONFIG_KEXEC_SIG #ifdef CONFIG_SIGNED_PE_FILE_VERIFICATION int kexec_kernel_verify_pe_sig(const char *kernel, unsigned long kernel_len); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index f1a0e4e3fb5c..f989f5f1933b 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -65,7 +65,7 @@ int kexec_image_probe_default(struct kimage *image, void *buf, return ret; } -void *kexec_image_load_default(struct kimage *image) +static void *kexec_image_load_default(struct kimage *image) { if (!image->fops || !image->fops->load) return ERR_PTR(-ENOEXEC); @@ -249,8 +249,8 @@ kimage_file_prepare_segments(struct kimage *image, int kernel_fd, int initrd_fd, /* IMA needs to pass the measurement list to the next kernel. */ ima_add_kexec_buffer(image); - /* Call arch image load handlers */ - ldata = arch_kexec_kernel_image_load(image); + /* Call image load handler */ + ldata = kexec_image_load_default(image); if (IS_ERR(ldata)) { ret = PTR_ERR(ldata); -- cgit v1.2.3 From 38416c28e16890b52fdd5eb73479299ec3f062f3 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Fri, 31 Mar 2023 15:40:28 +0300 Subject: iio: light: Add gain-time-scale helpers Some light sensors can adjust both the HW-gain and integration time. There are cases where adjusting the integration time has similar impact to the scale of the reported values as gain setting has. IIO users do typically expect to handle scale by a single writable 'scale' entry. Driver should then adjust the gain/time accordingly. It however is difficult for a driver to know whether it should change gain or integration time to meet the requested scale. Usually it is preferred to have longer integration time which usually improves accuracy, but there may be use-cases where long measurement times can be an issue. Thus it can be preferable to allow also changing the integration time - but mitigate the scale impact by also changing the gain underneath. Eg, if integration time change doubles the measured values, the driver can reduce the HW-gain to half. The theory of the computations of gain-time-scale is simple. However, some people (undersigned) got that implemented wrong for more than once. Add some gain-time-scale helpers in order to not dublicate errors in all drivers needing these computations. Signed-off-by: Matti Vaittinen Link: https://lore.kernel.org/r/268d418e7cffcdaa2ece6738478bbc57692c213e.1680263956.git.mazziesaccount@gmail.com Signed-off-by: Jonathan Cameron --- drivers/iio/Kconfig | 3 + drivers/iio/Makefile | 1 + drivers/iio/industrialio-gts-helper.c | 1077 +++++++++++++++++++++++++++++++++ include/linux/iio/iio-gts-helper.h | 206 +++++++ 4 files changed, 1287 insertions(+) create mode 100644 drivers/iio/industrialio-gts-helper.c create mode 100644 include/linux/iio/iio-gts-helper.h (limited to 'include/linux') diff --git a/drivers/iio/Kconfig b/drivers/iio/Kconfig index b190846c3dc2..52eb46ef84c1 100644 --- a/drivers/iio/Kconfig +++ b/drivers/iio/Kconfig @@ -30,6 +30,9 @@ config IIO_CONFIGFS (e.g. software triggers). For more info see Documentation/iio/iio_configfs.rst. +config IIO_GTS_HELPER + tristate + config IIO_TRIGGER bool "Enable triggered sampling support" help diff --git a/drivers/iio/Makefile b/drivers/iio/Makefile index 3be08cdadd7e..9622347a1c1b 100644 --- a/drivers/iio/Makefile +++ b/drivers/iio/Makefile @@ -9,6 +9,7 @@ industrialio-$(CONFIG_IIO_BUFFER) += industrialio-buffer.o industrialio-$(CONFIG_IIO_TRIGGER) += industrialio-trigger.o obj-$(CONFIG_IIO_CONFIGFS) += industrialio-configfs.o +obj-$(CONFIG_IIO_GTS_HELPER) += industrialio-gts-helper.o obj-$(CONFIG_IIO_SW_DEVICE) += industrialio-sw-device.o obj-$(CONFIG_IIO_SW_TRIGGER) += industrialio-sw-trigger.o obj-$(CONFIG_IIO_TRIGGERED_EVENT) += industrialio-triggered-event.o diff --git a/drivers/iio/industrialio-gts-helper.c b/drivers/iio/industrialio-gts-helper.c new file mode 100644 index 000000000000..8bb68975b259 --- /dev/null +++ b/drivers/iio/industrialio-gts-helper.c @@ -0,0 +1,1077 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* gain-time-scale conversion helpers for IIO light sensors + * + * Copyright (c) 2023 Matti Vaittinen + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/** + * iio_gts_get_gain - Convert scale to total gain + * + * Internal helper for converting scale to total gain. + * + * @max: Maximum linearized scale. As an example, when scale is created + * in magnitude of NANOs and max scale is 64.1 - The linearized + * scale is 64 100 000 000. + * @scale: Linearized scale to compute the gain for. + * + * Return: (floored) gain corresponding to the scale. -EINVAL if scale + * is invalid. + */ +static int iio_gts_get_gain(const u64 max, const u64 scale) +{ + u64 full = max; + int tmp = 1; + + if (scale > full || !scale) + return -EINVAL; + + if (U64_MAX - full < scale) { + /* Risk of overflow */ + if (full - scale < scale) + return 1; + + full -= scale; + tmp++; + } + + while (full > scale * (u64)tmp) + tmp++; + + return tmp; +} + +/** + * gain_get_scale_fraction - get the gain or time based on scale and known one + * + * @max: Maximum linearized scale. As an example, when scale is created + * in magnitude of NANOs and max scale is 64.1 - The linearized + * scale is 64 100 000 000. + * @scale: Linearized scale to compute the gain/time for. + * @known: Either integration time or gain depending on which one is known + * @unknown: Pointer to variable where the computed gain/time is stored + * + * Internal helper for computing unknown fraction of total gain. + * Compute either gain or time based on scale and either the gain or time + * depending on which one is known. + * + * Return: 0 on success. + */ +static int gain_get_scale_fraction(const u64 max, u64 scale, int known, + int *unknown) +{ + int tot_gain; + + tot_gain = iio_gts_get_gain(max, scale); + if (tot_gain < 0) + return tot_gain; + + *unknown = tot_gain / known; + + /* We require total gain to be exact multiple of known * unknown */ + if (!*unknown || *unknown * known != tot_gain) + return -EINVAL; + + return 0; +} + +static int iio_gts_delinearize(u64 lin_scale, unsigned long scaler, + int *scale_whole, int *scale_nano) +{ + int frac; + + if (scaler > NANO) + return -EOVERFLOW; + + if (!scaler) + return -EINVAL; + + frac = do_div(lin_scale, scaler); + + *scale_whole = lin_scale; + *scale_nano = frac * (NANO / scaler); + + return 0; +} + +static int iio_gts_linearize(int scale_whole, int scale_nano, + unsigned long scaler, u64 *lin_scale) +{ + /* + * Expect scale to be (mostly) NANO or MICRO. Divide divider instead of + * multiplication followed by division to avoid overflow. + */ + if (scaler > NANO || !scaler) + return -EINVAL; + + *lin_scale = (u64)scale_whole * (u64)scaler + + (u64)(scale_nano / (NANO / scaler)); + + return 0; +} + +/** + * iio_gts_total_gain_to_scale - convert gain to scale + * @gts: Gain time scale descriptor + * @total_gain: the gain to be converted + * @scale_int: Pointer to integral part of the scale (typically val1) + * @scale_nano: Pointer to fractional part of the scale (nano or ppb) + * + * Convert the total gain value to scale. NOTE: This does not separate gain + * generated by HW-gain or integration time. It is up to caller to decide what + * part of the total gain is due to integration time and what due to HW-gain. + * + * Return: 0 on success. Negative errno on failure. + */ +int iio_gts_total_gain_to_scale(struct iio_gts *gts, int total_gain, + int *scale_int, int *scale_nano) +{ + u64 tmp; + + tmp = gts->max_scale; + + do_div(tmp, total_gain); + + return iio_gts_delinearize(tmp, NANO, scale_int, scale_nano); +} +EXPORT_SYMBOL_NS_GPL(iio_gts_total_gain_to_scale, IIO_GTS_HELPER); + +/** + * iio_gts_purge_avail_scale_table - free-up the available scale tables + * @gts: Gain time scale descriptor + * + * Free the space reserved by iio_gts_build_avail_scale_table(). + */ +static void iio_gts_purge_avail_scale_table(struct iio_gts *gts) +{ + int i; + + if (gts->per_time_avail_scale_tables) { + for (i = 0; i < gts->num_itime; i++) + kfree(gts->per_time_avail_scale_tables[i]); + + kfree(gts->per_time_avail_scale_tables); + gts->per_time_avail_scale_tables = NULL; + } + + kfree(gts->avail_all_scales_table); + gts->avail_all_scales_table = NULL; + + gts->num_avail_all_scales = 0; +} + +static int iio_gts_gain_cmp(const void *a, const void *b) +{ + return *(int *)a - *(int *)b; +} + +static int gain_to_scaletables(struct iio_gts *gts, int **gains, int **scales) +{ + int ret, i, j, new_idx, time_idx; + int *all_gains; + size_t gain_bytes; + + for (i = 0; i < gts->num_itime; i++) { + /* + * Sort the tables for nice output and for easier finding of + * unique values. + */ + sort(gains[i], gts->num_hwgain, sizeof(int), iio_gts_gain_cmp, + NULL); + + /* Convert gains to scales */ + for (j = 0; j < gts->num_hwgain; j++) { + ret = iio_gts_total_gain_to_scale(gts, gains[i][j], + &scales[i][2 * j], + &scales[i][2 * j + 1]); + if (ret) + return ret; + } + } + + gain_bytes = array_size(gts->num_hwgain, sizeof(int)); + all_gains = kcalloc(gts->num_itime, gain_bytes, GFP_KERNEL); + if (!all_gains) + return -ENOMEM; + + /* + * We assume all the gains for same integration time were unique. + * It is likely the first time table had greatest time multiplier as + * the times are in the order of preference and greater times are + * usually preferred. Hence we start from the last table which is likely + * to have the smallest total gains. + */ + time_idx = gts->num_itime - 1; + memcpy(all_gains, gains[time_idx], gain_bytes); + new_idx = gts->num_hwgain; + + while (time_idx--) { + for (j = 0; j < gts->num_hwgain; j++) { + int candidate = gains[time_idx][j]; + int chk; + + if (candidate > all_gains[new_idx - 1]) { + all_gains[new_idx] = candidate; + new_idx++; + + continue; + } + for (chk = 0; chk < new_idx; chk++) + if (candidate <= all_gains[chk]) + break; + + if (candidate == all_gains[chk]) + continue; + + memmove(&all_gains[chk + 1], &all_gains[chk], + (new_idx - chk) * sizeof(int)); + all_gains[chk] = candidate; + new_idx++; + } + } + + gts->avail_all_scales_table = kcalloc(new_idx, 2 * sizeof(int), + GFP_KERNEL); + if (!gts->avail_all_scales_table) { + ret = -ENOMEM; + goto free_out; + } + gts->num_avail_all_scales = new_idx; + + for (i = 0; i < gts->num_avail_all_scales; i++) { + ret = iio_gts_total_gain_to_scale(gts, all_gains[i], + >s->avail_all_scales_table[i * 2], + >s->avail_all_scales_table[i * 2 + 1]); + + if (ret) { + kfree(gts->avail_all_scales_table); + gts->num_avail_all_scales = 0; + goto free_out; + } + } + +free_out: + kfree(all_gains); + + return ret; +} + +/** + * iio_gts_build_avail_scale_table - create tables of available scales + * @gts: Gain time scale descriptor + * + * Build the tables which can represent the available scales based on the + * originally given gain and time tables. When both time and gain tables are + * given this results: + * 1. A set of tables representing available scales for each supported + * integration time. + * 2. A single table listing all the unique scales that any combination of + * supported gains and times can provide. + * + * NOTE: Space allocated for the tables must be freed using + * iio_gts_purge_avail_scale_table() when the tables are no longer needed. + * + * Return: 0 on success. + */ +static int iio_gts_build_avail_scale_table(struct iio_gts *gts) +{ + int **per_time_gains, **per_time_scales, i, j, ret = -ENOMEM; + + per_time_gains = kcalloc(gts->num_itime, sizeof(*per_time_gains), GFP_KERNEL); + if (!per_time_gains) + return ret; + + per_time_scales = kcalloc(gts->num_itime, sizeof(*per_time_scales), GFP_KERNEL); + if (!per_time_scales) + goto free_gains; + + for (i = 0; i < gts->num_itime; i++) { + per_time_scales[i] = kcalloc(gts->num_hwgain, 2 * sizeof(int), + GFP_KERNEL); + if (!per_time_scales[i]) + goto err_free_out; + + per_time_gains[i] = kcalloc(gts->num_hwgain, sizeof(int), + GFP_KERNEL); + if (!per_time_gains[i]) { + kfree(per_time_scales[i]); + goto err_free_out; + } + + for (j = 0; j < gts->num_hwgain; j++) + per_time_gains[i][j] = gts->hwgain_table[j].gain * + gts->itime_table[i].mul; + } + + ret = gain_to_scaletables(gts, per_time_gains, per_time_scales); + if (ret) + goto err_free_out; + + kfree(per_time_gains); + gts->per_time_avail_scale_tables = per_time_scales; + + return 0; + +err_free_out: + for (i--; i; i--) { + kfree(per_time_scales[i]); + kfree(per_time_gains[i]); + } + kfree(per_time_scales); +free_gains: + kfree(per_time_gains); + + return ret; +} + +/** + * iio_gts_build_avail_time_table - build table of available integration times + * @gts: Gain time scale descriptor + * + * Build the table which can represent the available times to be returned + * to users using the read_avail-callback. + * + * NOTE: Space allocated for the tables must be freed using + * iio_gts_purge_avail_time_table() when the tables are no longer needed. + * + * Return: 0 on success. + */ +static int iio_gts_build_avail_time_table(struct iio_gts *gts) +{ + int *times, i, j, idx = 0; + + if (!gts->num_itime) + return 0; + + times = kcalloc(gts->num_itime, sizeof(int), GFP_KERNEL); + if (!times) + return -ENOMEM; + + /* Sort times from all tables to one and remove duplicates */ + for (i = gts->num_itime - 1; i >= 0; i--) { + int new = gts->itime_table[i].time_us; + + if (times[idx] < new) { + times[idx++] = new; + continue; + } + + for (j = 0; j <= idx; j++) { + if (times[j] > new) { + memmove(×[j + 1], ×[j], + (idx - j) * sizeof(int)); + times[j] = new; + idx++; + } + } + } + gts->avail_time_tables = times; + /* + * This is just to survive a unlikely corner-case where times in the + * given time table were not unique. Else we could just trust the + * gts->num_itime. + */ + gts->num_avail_time_tables = idx; + + return 0; +} + +/** + * iio_gts_purge_avail_time_table - free-up the available integration time table + * @gts: Gain time scale descriptor + * + * Free the space reserved by iio_gts_build_avail_time_table(). + */ +static void iio_gts_purge_avail_time_table(struct iio_gts *gts) +{ + if (gts->num_avail_time_tables) { + kfree(gts->avail_time_tables); + gts->avail_time_tables = NULL; + gts->num_avail_time_tables = 0; + } +} + +/** + * iio_gts_build_avail_tables - create tables of available scales and int times + * @gts: Gain time scale descriptor + * + * Build the tables which can represent the available scales and available + * integration times. Availability tables are built based on the originally + * given gain and given time tables. + * + * When both time and gain tables are + * given this results: + * 1. A set of sorted tables representing available scales for each supported + * integration time. + * 2. A single sorted table listing all the unique scales that any combination + * of supported gains and times can provide. + * 3. A sorted table of supported integration times + * + * After these tables are built one can use the iio_gts_all_avail_scales(), + * iio_gts_avail_scales_for_time() and iio_gts_avail_times() helpers to + * implement the read_avail operations. + * + * NOTE: Space allocated for the tables must be freed using + * iio_gts_purge_avail_tables() when the tables are no longer needed. + * + * Return: 0 on success. + */ +static int iio_gts_build_avail_tables(struct iio_gts *gts) +{ + int ret; + + ret = iio_gts_build_avail_scale_table(gts); + if (ret) + return ret; + + ret = iio_gts_build_avail_time_table(gts); + if (ret) + iio_gts_purge_avail_scale_table(gts); + + return ret; +} + +/** + * iio_gts_purge_avail_tables - free-up the availability tables + * @gts: Gain time scale descriptor + * + * Free the space reserved by iio_gts_build_avail_tables(). Frees both the + * integration time and scale tables. + */ +static void iio_gts_purge_avail_tables(struct iio_gts *gts) +{ + iio_gts_purge_avail_time_table(gts); + iio_gts_purge_avail_scale_table(gts); +} + +static void devm_iio_gts_avail_all_drop(void *res) +{ + iio_gts_purge_avail_tables(res); +} + +/** + * devm_iio_gts_build_avail_tables - manged add availability tables + * @dev: Pointer to the device whose lifetime tables are bound + * @gts: Gain time scale descriptor + * + * Build the tables which can represent the available scales and available + * integration times. Availability tables are built based on the originally + * given gain and given time tables. + * + * When both time and gain tables are given this results: + * 1. A set of sorted tables representing available scales for each supported + * integration time. + * 2. A single sorted table listing all the unique scales that any combination + * of supported gains and times can provide. + * 3. A sorted table of supported integration times + * + * After these tables are built one can use the iio_gts_all_avail_scales(), + * iio_gts_avail_scales_for_time() and iio_gts_avail_times() helpers to + * implement the read_avail operations. + * + * The tables are automatically released upon device detach. + * + * Return: 0 on success. + */ +static int devm_iio_gts_build_avail_tables(struct device *dev, + struct iio_gts *gts) +{ + int ret; + + ret = iio_gts_build_avail_tables(gts); + if (ret) + return ret; + + return devm_add_action_or_reset(dev, devm_iio_gts_avail_all_drop, gts); +} + +static int sanity_check_time(const struct iio_itime_sel_mul *t) +{ + if (t->sel < 0 || t->time_us < 0 || t->mul <= 0) + return -EINVAL; + + return 0; +} + +static int sanity_check_gain(const struct iio_gain_sel_pair *g) +{ + if (g->sel < 0 || g->gain <= 0) + return -EINVAL; + + return 0; +} + +static int iio_gts_sanity_check(struct iio_gts *gts) +{ + int g, t, ret; + + if (!gts->num_hwgain && !gts->num_itime) + return -EINVAL; + + for (t = 0; t < gts->num_itime; t++) { + ret = sanity_check_time(>s->itime_table[t]); + if (ret) + return ret; + } + + for (g = 0; g < gts->num_hwgain; g++) { + ret = sanity_check_gain(>s->hwgain_table[g]); + if (ret) + return ret; + } + + for (g = 0; g < gts->num_hwgain; g++) { + for (t = 0; t < gts->num_itime; t++) { + int gain, mul, res; + + gain = gts->hwgain_table[g].gain; + mul = gts->itime_table[t].mul; + + if (check_mul_overflow(gain, mul, &res)) + return -EOVERFLOW; + } + } + + return 0; +} + +static int iio_init_iio_gts(int max_scale_int, int max_scale_nano, + const struct iio_gain_sel_pair *gain_tbl, int num_gain, + const struct iio_itime_sel_mul *tim_tbl, int num_times, + struct iio_gts *gts) +{ + int ret; + + memset(gts, 0, sizeof(*gts)); + + ret = iio_gts_linearize(max_scale_int, max_scale_nano, NANO, + >s->max_scale); + if (ret) + return ret; + + gts->hwgain_table = gain_tbl; + gts->num_hwgain = num_gain; + gts->itime_table = tim_tbl; + gts->num_itime = num_times; + + return iio_gts_sanity_check(gts); +} + +/** + * devm_iio_init_iio_gts - Initialize the gain-time-scale helper + * @dev: Pointer to the device whose lifetime gts resources are + * bound + * @max_scale_int: integer part of the maximum scale value + * @max_scale_nano: fraction part of the maximum scale value + * @gain_tbl: table describing supported gains + * @num_gain: number of gains in the gain table + * @tim_tbl: table describing supported integration times. Provide + * the integration time table sorted so that the preferred + * integration time is in the first array index. The search + * functions like the + * iio_gts_find_time_and_gain_sel_for_scale() start search + * from first provided time. + * @num_times: number of times in the time table + * @gts: pointer to the helper struct + * + * Initialize the gain-time-scale helper for use. Note, gains, times, selectors + * and multipliers must be positive. Negative values are reserved for error + * checking. The total gain (maximum gain * maximum time multiplier) must not + * overflow int. The allocated resources will be released upon device detach. + * + * Return: 0 on success. + */ +int devm_iio_init_iio_gts(struct device *dev, int max_scale_int, int max_scale_nano, + const struct iio_gain_sel_pair *gain_tbl, int num_gain, + const struct iio_itime_sel_mul *tim_tbl, int num_times, + struct iio_gts *gts) +{ + int ret; + + ret = iio_init_iio_gts(max_scale_int, max_scale_nano, gain_tbl, + num_gain, tim_tbl, num_times, gts); + if (ret) + return ret; + + return devm_iio_gts_build_avail_tables(dev, gts); +} +EXPORT_SYMBOL_NS_GPL(devm_iio_init_iio_gts, IIO_GTS_HELPER); + +/** + * iio_gts_all_avail_scales - helper for listing all available scales + * @gts: Gain time scale descriptor + * @vals: Returned array of supported scales + * @type: Type of returned scale values + * @length: Amount of returned values in array + * + * Return: a value suitable to be returned from read_avail or a negative error. + */ +int iio_gts_all_avail_scales(struct iio_gts *gts, const int **vals, int *type, + int *length) +{ + if (!gts->num_avail_all_scales) + return -EINVAL; + + *vals = gts->avail_all_scales_table; + *type = IIO_VAL_INT_PLUS_NANO; + *length = gts->num_avail_all_scales * 2; + + return IIO_AVAIL_LIST; +} +EXPORT_SYMBOL_NS_GPL(iio_gts_all_avail_scales, IIO_GTS_HELPER); + +/** + * iio_gts_avail_scales_for_time - list scales for integration time + * @gts: Gain time scale descriptor + * @time: Integration time for which the scales are listed + * @vals: Returned array of supported scales + * @type: Type of returned scale values + * @length: Amount of returned values in array + * + * Drivers which do not allow scale setting to change integration time can + * use this helper to list only the scales which are valid for given integration + * time. + * + * Return: a value suitable to be returned from read_avail or a negative error. + */ +int iio_gts_avail_scales_for_time(struct iio_gts *gts, int time, + const int **vals, int *type, int *length) +{ + int i; + + for (i = 0; i < gts->num_itime; i++) + if (gts->itime_table[i].time_us == time) + break; + + if (i == gts->num_itime) + return -EINVAL; + + *vals = gts->per_time_avail_scale_tables[i]; + *type = IIO_VAL_INT_PLUS_NANO; + *length = gts->num_hwgain * 2; + + return IIO_AVAIL_LIST; +} +EXPORT_SYMBOL_NS_GPL(iio_gts_avail_scales_for_time, IIO_GTS_HELPER); + +/** + * iio_gts_avail_times - helper for listing available integration times + * @gts: Gain time scale descriptor + * @vals: Returned array of supported times + * @type: Type of returned scale values + * @length: Amount of returned values in array + * + * Return: a value suitable to be returned from read_avail or a negative error. + */ +int iio_gts_avail_times(struct iio_gts *gts, const int **vals, int *type, + int *length) +{ + if (!gts->num_avail_time_tables) + return -EINVAL; + + *vals = gts->avail_time_tables; + *type = IIO_VAL_INT; + *length = gts->num_avail_time_tables; + + return IIO_AVAIL_LIST; +} +EXPORT_SYMBOL_NS_GPL(iio_gts_avail_times, IIO_GTS_HELPER); + +/** + * iio_gts_find_sel_by_gain - find selector corresponding to a HW-gain + * @gts: Gain time scale descriptor + * @gain: HW-gain for which matching selector is searched for + * + * Return: a selector matching given HW-gain or -EINVAL if selector was + * not found. + */ +int iio_gts_find_sel_by_gain(struct iio_gts *gts, int gain) +{ + int i; + + for (i = 0; i < gts->num_hwgain; i++) + if (gts->hwgain_table[i].gain == gain) + return gts->hwgain_table[i].sel; + + return -EINVAL; +} +EXPORT_SYMBOL_NS_GPL(iio_gts_find_sel_by_gain, IIO_GTS_HELPER); + +/** + * iio_gts_find_gain_by_sel - find HW-gain corresponding to a selector + * @gts: Gain time scale descriptor + * @sel: selector for which matching HW-gain is searched for + * + * Return: a HW-gain matching given selector or -EINVAL if HW-gain was not + * found. + */ +int iio_gts_find_gain_by_sel(struct iio_gts *gts, int sel) +{ + int i; + + for (i = 0; i < gts->num_hwgain; i++) + if (gts->hwgain_table[i].sel == sel) + return gts->hwgain_table[i].gain; + + return -EINVAL; +} +EXPORT_SYMBOL_NS_GPL(iio_gts_find_gain_by_sel, IIO_GTS_HELPER); + +/** + * iio_gts_get_min_gain - find smallest valid HW-gain + * @gts: Gain time scale descriptor + * + * Return: The smallest HW-gain -EINVAL if no HW-gains were in the tables. + */ +int iio_gts_get_min_gain(struct iio_gts *gts) +{ + int i, min = -EINVAL; + + for (i = 0; i < gts->num_hwgain; i++) { + int gain = gts->hwgain_table[i].gain; + + if (min == -EINVAL) + min = gain; + else + min = min(min, gain); + } + + return min; +} +EXPORT_SYMBOL_NS_GPL(iio_gts_get_min_gain, IIO_GTS_HELPER); + +/** + * iio_find_closest_gain_low - Find the closest lower matching gain + * @gts: Gain time scale descriptor + * @gain: HW-gain for which the closest match is searched + * @in_range: indicate if the @gain was actually in the range of + * supported gains. + * + * Search for closest supported gain that is lower than or equal to the + * gain given as a parameter. This is usable for drivers which do not require + * user to request exact matching gain but rather for rounding to a supported + * gain value which is equal or lower (setting lower gain is typical for + * avoiding saturation) + * + * Return: The closest matching supported gain or -EINVAL if @gain + * was smaller than the smallest supported gain. + */ +int iio_find_closest_gain_low(struct iio_gts *gts, int gain, bool *in_range) +{ + int i, diff = 0; + int best = -1; + + *in_range = false; + + for (i = 0; i < gts->num_hwgain; i++) { + if (gain == gts->hwgain_table[i].gain) { + *in_range = true; + return gain; + } + + if (gain > gts->hwgain_table[i].gain) { + if (!diff) { + diff = gain - gts->hwgain_table[i].gain; + best = i; + } else { + int tmp = gain - gts->hwgain_table[i].gain; + + if (tmp < diff) { + diff = tmp; + best = i; + } + } + } else { + /* + * We found valid HW-gain which is greater than + * reference. So, unless we return a failure below we + * will have found an in-range gain + */ + *in_range = true; + } + } + /* The requested gain was smaller than anything we support */ + if (!diff) { + *in_range = false; + + return -EINVAL; + } + + return gts->hwgain_table[best].gain; +} +EXPORT_SYMBOL_NS_GPL(iio_find_closest_gain_low, IIO_GTS_HELPER); + +static int iio_gts_get_int_time_gain_multiplier_by_sel(struct iio_gts *gts, + int sel) +{ + const struct iio_itime_sel_mul *time; + + time = iio_gts_find_itime_by_sel(gts, sel); + if (!time) + return -EINVAL; + + return time->mul; +} + +/** + * iio_gts_find_gain_for_scale_using_time - Find gain by time and scale + * @gts: Gain time scale descriptor + * @time_sel: Integration time selector corresponding to the time gain is + * searched for + * @scale_int: Integral part of the scale (typically val1) + * @scale_nano: Fractional part of the scale (nano or ppb) + * @gain: Pointer to value where gain is stored. + * + * In some cases the light sensors may want to find a gain setting which + * corresponds given scale and integration time. Sensors which fill the + * gain and time tables may use this helper to retrieve the gain. + * + * Return: 0 on success. -EINVAL if gain matching the parameters is not + * found. + */ +static int iio_gts_find_gain_for_scale_using_time(struct iio_gts *gts, int time_sel, + int scale_int, int scale_nano, + int *gain) +{ + u64 scale_linear; + int ret, mul; + + ret = iio_gts_linearize(scale_int, scale_nano, NANO, &scale_linear); + if (ret) + return ret; + + ret = iio_gts_get_int_time_gain_multiplier_by_sel(gts, time_sel); + if (ret < 0) + return ret; + + mul = ret; + + ret = gain_get_scale_fraction(gts->max_scale, scale_linear, mul, gain); + if (ret) + return ret; + + if (!iio_gts_valid_gain(gts, *gain)) + return -EINVAL; + + return 0; +} + +/** + * iio_gts_find_gain_sel_for_scale_using_time - Fetch gain selector. + * @gts: Gain time scale descriptor + * @time_sel: Integration time selector corresponding to the time gain is + * searched for + * @scale_int: Integral part of the scale (typically val1) + * @scale_nano: Fractional part of the scale (nano or ppb) + * @gain_sel: Pointer to value where gain selector is stored. + * + * See iio_gts_find_gain_for_scale_using_time() for more information + */ +int iio_gts_find_gain_sel_for_scale_using_time(struct iio_gts *gts, int time_sel, + int scale_int, int scale_nano, + int *gain_sel) +{ + int gain, ret; + + ret = iio_gts_find_gain_for_scale_using_time(gts, time_sel, scale_int, + scale_nano, &gain); + if (ret) + return ret; + + ret = iio_gts_find_sel_by_gain(gts, gain); + if (ret < 0) + return ret; + + *gain_sel = ret; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(iio_gts_find_gain_sel_for_scale_using_time, IIO_GTS_HELPER); + +static int iio_gts_get_total_gain(struct iio_gts *gts, int gain, int time) +{ + const struct iio_itime_sel_mul *itime; + + if (!iio_gts_valid_gain(gts, gain)) + return -EINVAL; + + if (!gts->num_itime) + return gain; + + itime = iio_gts_find_itime_by_time(gts, time); + if (!itime) + return -EINVAL; + + return gain * itime->mul; +} + +static int iio_gts_get_scale_linear(struct iio_gts *gts, int gain, int time, + u64 *scale) +{ + int total_gain; + u64 tmp; + + total_gain = iio_gts_get_total_gain(gts, gain, time); + if (total_gain < 0) + return total_gain; + + tmp = gts->max_scale; + + do_div(tmp, total_gain); + + *scale = tmp; + + return 0; +} + +/** + * iio_gts_get_scale - get scale based on integration time and HW-gain + * @gts: Gain time scale descriptor + * @gain: HW-gain for which the scale is computed + * @time: Integration time for which the scale is computed + * @scale_int: Integral part of the scale (typically val1) + * @scale_nano: Fractional part of the scale (nano or ppb) + * + * Compute scale matching the integration time and HW-gain given as parameter. + * + * Return: 0 on success. + */ +int iio_gts_get_scale(struct iio_gts *gts, int gain, int time, int *scale_int, + int *scale_nano) +{ + u64 lin_scale; + int ret; + + ret = iio_gts_get_scale_linear(gts, gain, time, &lin_scale); + if (ret) + return ret; + + return iio_gts_delinearize(lin_scale, NANO, scale_int, scale_nano); +} +EXPORT_SYMBOL_NS_GPL(iio_gts_get_scale, IIO_GTS_HELPER); + +/** + * iio_gts_find_new_gain_sel_by_old_gain_time - compensate for time change + * @gts: Gain time scale descriptor + * @old_gain: Previously set gain + * @old_time_sel: Selector corresponding previously set time + * @new_time_sel: Selector corresponding new time to be set + * @new_gain: Pointer to value where new gain is to be written + * + * We may want to mitigate the scale change caused by setting a new integration + * time (for a light sensor) by also updating the (HW)gain. This helper computes + * new gain value to maintain the scale with new integration time. + * + * Return: 0 if an exactly matching supported new gain was found. When a + * non-zero value is returned, the @new_gain will be set to a negative or + * positive value. The negative value means that no gain could be computed. + * Positive value will be the "best possible new gain there could be". There + * can be two reasons why finding the "best possible" new gain is not deemed + * successful. 1) This new value cannot be supported by the hardware. 2) The new + * gain required to maintain the scale would not be an integer. In this case, + * the "best possible" new gain will be a floored optimal gain, which may or + * may not be supported by the hardware. + */ +int iio_gts_find_new_gain_sel_by_old_gain_time(struct iio_gts *gts, + int old_gain, int old_time_sel, + int new_time_sel, int *new_gain) +{ + const struct iio_itime_sel_mul *itime_old, *itime_new; + u64 scale; + int ret; + + *new_gain = -1; + + itime_old = iio_gts_find_itime_by_sel(gts, old_time_sel); + if (!itime_old) + return -EINVAL; + + itime_new = iio_gts_find_itime_by_sel(gts, new_time_sel); + if (!itime_new) + return -EINVAL; + + ret = iio_gts_get_scale_linear(gts, old_gain, itime_old->time_us, + &scale); + if (ret) + return ret; + + ret = gain_get_scale_fraction(gts->max_scale, scale, itime_new->mul, + new_gain); + if (ret) + return ret; + + if (!iio_gts_valid_gain(gts, *new_gain)) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(iio_gts_find_new_gain_sel_by_old_gain_time, IIO_GTS_HELPER); + +/** + * iio_gts_find_new_gain_by_old_gain_time - compensate for time change + * @gts: Gain time scale descriptor + * @old_gain: Previously set gain + * @old_time: Selector corresponding previously set time + * @new_time: Selector corresponding new time to be set + * @new_gain: Pointer to value where new gain is to be written + * + * We may want to mitigate the scale change caused by setting a new integration + * time (for a light sensor) by also updating the (HW)gain. This helper computes + * new gain value to maintain the scale with new integration time. + * + * Return: 0 if an exactly matching supported new gain was found. When a + * non-zero value is returned, the @new_gain will be set to a negative or + * positive value. The negative value means that no gain could be computed. + * Positive value will be the "best possible new gain there could be". There + * can be two reasons why finding the "best possible" new gain is not deemed + * successful. 1) This new value cannot be supported by the hardware. 2) The new + * gain required to maintain the scale would not be an integer. In this case, + * the "best possible" new gain will be a floored optimal gain, which may or + * may not be supported by the hardware. + */ +int iio_gts_find_new_gain_by_old_gain_time(struct iio_gts *gts, int old_gain, + int old_time, int new_time, + int *new_gain) +{ + const struct iio_itime_sel_mul *itime_new; + u64 scale; + int ret; + + *new_gain = -1; + + itime_new = iio_gts_find_itime_by_time(gts, new_time); + if (!itime_new) + return -EINVAL; + + ret = iio_gts_get_scale_linear(gts, old_gain, old_time, &scale); + if (ret) + return ret; + + ret = gain_get_scale_fraction(gts->max_scale, scale, itime_new->mul, + new_gain); + if (ret) + return ret; + + if (!iio_gts_valid_gain(gts, *new_gain)) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(iio_gts_find_new_gain_by_old_gain_time, IIO_GTS_HELPER); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Matti Vaittinen "); +MODULE_DESCRIPTION("IIO light sensor gain-time-scale helpers"); diff --git a/include/linux/iio/iio-gts-helper.h b/include/linux/iio/iio-gts-helper.h new file mode 100644 index 000000000000..dd64e544a3da --- /dev/null +++ b/include/linux/iio/iio-gts-helper.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* gain-time-scale conversion helpers for IIO light sensors + * + * Copyright (c) 2023 Matti Vaittinen + */ + +#ifndef __IIO_GTS_HELPER__ +#define __IIO_GTS_HELPER__ + +#include + +struct device; + +/** + * struct iio_gain_sel_pair - gain - selector values + * + * In many cases devices like light sensors allow setting signal amplification + * (gain) using a register interface. This structure describes amplification + * and corresponding selector (register value) + * + * @gain: Gain (multiplication) value. Gain must be positive, negative + * values are reserved for error handling. + * @sel: Selector (usually register value) used to indicate this gain. + * NOTE: Only selectors >= 0 supported. + */ +struct iio_gain_sel_pair { + int gain; + int sel; +}; + +/** + * struct iio_itime_sel_mul - integration time description + * + * In many cases devices like light sensors allow setting the duration of + * collecting data. Typically this duration has also an impact to the magnitude + * of measured values (gain). This structure describes the relation of + * integration time and amplification as well as corresponding selector + * (register value). + * + * An example could be a sensor allowing 50, 100, 200 and 400 mS times. The + * respective multiplication values could be 50 mS => 1, 100 mS => 2, + * 200 mS => 4 and 400 mS => 8 assuming the impact of integration time would be + * linear in a way that when collecting data for 50 mS caused value X, doubling + * the data collection time caused value 2X etc. + * + * @time_us: Integration time in microseconds. Time values must be positive, + * negative values are reserved for error handling. + * @sel: Selector (usually register value) used to indicate this time + * NOTE: Only selectors >= 0 supported. + * @mul: Multiplication to the values caused by this time. + * NOTE: Only multipliers > 0 supported. + */ +struct iio_itime_sel_mul { + int time_us; + int sel; + int mul; +}; + +struct iio_gts { + u64 max_scale; + const struct iio_gain_sel_pair *hwgain_table; + int num_hwgain; + const struct iio_itime_sel_mul *itime_table; + int num_itime; + int **per_time_avail_scale_tables; + int *avail_all_scales_table; + int num_avail_all_scales; + int *avail_time_tables; + int num_avail_time_tables; +}; + +#define GAIN_SCALE_GAIN(_gain, _sel) \ +{ \ + .gain = (_gain), \ + .sel = (_sel), \ +} + +#define GAIN_SCALE_ITIME_US(_itime, _sel, _mul) \ +{ \ + .time_us = (_itime), \ + .sel = (_sel), \ + .mul = (_mul), \ +} + +static inline const struct iio_itime_sel_mul * +iio_gts_find_itime_by_time(struct iio_gts *gts, int time) +{ + int i; + + if (!gts->num_itime) + return NULL; + + for (i = 0; i < gts->num_itime; i++) + if (gts->itime_table[i].time_us == time) + return >s->itime_table[i]; + + return NULL; +} + +static inline const struct iio_itime_sel_mul * +iio_gts_find_itime_by_sel(struct iio_gts *gts, int sel) +{ + int i; + + for (i = 0; i < gts->num_itime; i++) + if (gts->itime_table[i].sel == sel) + return >s->itime_table[i]; + + return NULL; +} + +int devm_iio_init_iio_gts(struct device *dev, int max_scale_int, int max_scale_nano, + const struct iio_gain_sel_pair *gain_tbl, int num_gain, + const struct iio_itime_sel_mul *tim_tbl, int num_times, + struct iio_gts *gts); +/** + * iio_gts_find_int_time_by_sel - find integration time matching a selector + * @gts: Gain time scale descriptor + * @sel: selector for which matching integration time is searched for + * + * Return: integration time matching given selector or -EINVAL if + * integration time was not found. + */ +static inline int iio_gts_find_int_time_by_sel(struct iio_gts *gts, int sel) +{ + const struct iio_itime_sel_mul *itime; + + itime = iio_gts_find_itime_by_sel(gts, sel); + if (!itime) + return -EINVAL; + + return itime->time_us; +} + +/** + * iio_gts_find_sel_by_int_time - find selector matching integration time + * @gts: Gain time scale descriptor + * @gain: HW-gain for which matching selector is searched for + * + * Return: a selector matching given integration time or -EINVAL if + * selector was not found. + */ +static inline int iio_gts_find_sel_by_int_time(struct iio_gts *gts, int time) +{ + const struct iio_itime_sel_mul *itime; + + itime = iio_gts_find_itime_by_time(gts, time); + if (!itime) + return -EINVAL; + + return itime->sel; +} + +/** + * iio_gts_valid_time - check if given integration time is valid + * @gts: Gain time scale descriptor + * @time_us: Integration time to check + * + * Return: True if given time is supported by device. False if not. + */ +static inline bool iio_gts_valid_time(struct iio_gts *gts, int time_us) +{ + return iio_gts_find_itime_by_time(gts, time_us) != NULL; +} + +int iio_gts_find_sel_by_gain(struct iio_gts *gts, int gain); + +/** + * iio_gts_valid_gain - check if given HW-gain is valid + * @gts: Gain time scale descriptor + * @gain: HW-gain to check + * + * Return: True if given time is supported by device. False if not. + */ +static inline bool iio_gts_valid_gain(struct iio_gts *gts, int gain) +{ + return iio_gts_find_sel_by_gain(gts, gain) >= 0; +} + +int iio_find_closest_gain_low(struct iio_gts *gts, int gain, bool *in_range); +int iio_gts_find_gain_by_sel(struct iio_gts *gts, int sel); +int iio_gts_get_min_gain(struct iio_gts *gts); +int iio_gts_find_int_time_by_sel(struct iio_gts *gts, int sel); +int iio_gts_find_sel_by_int_time(struct iio_gts *gts, int time); + +int iio_gts_total_gain_to_scale(struct iio_gts *gts, int total_gain, + int *scale_int, int *scale_nano); +int iio_gts_find_gain_sel_for_scale_using_time(struct iio_gts *gts, int time_sel, + int scale_int, int scale_nano, + int *gain_sel); +int iio_gts_get_scale(struct iio_gts *gts, int gain, int time, int *scale_int, + int *scale_nano); +int iio_gts_find_new_gain_sel_by_old_gain_time(struct iio_gts *gts, + int old_gain, int old_time_sel, + int new_time_sel, int *new_gain); +int iio_gts_find_new_gain_by_old_gain_time(struct iio_gts *gts, int old_gain, + int old_time, int new_time, + int *new_gain); +int iio_gts_avail_times(struct iio_gts *gts, const int **vals, int *type, + int *length); +int iio_gts_all_avail_scales(struct iio_gts *gts, const int **vals, int *type, + int *length); +int iio_gts_avail_scales_for_time(struct iio_gts *gts, int time, + const int **vals, int *type, int *length); + +#endif -- cgit v1.2.3 From 88a4d7bdeec97890cd543b58dd3588f1f879f51b Mon Sep 17 00:00:00 2001 From: Dave Wysochanski Date: Mon, 20 Feb 2023 08:43:05 -0500 Subject: NFS: Configure support for netfs when NFS fscache is configured As first steps for support of the netfs library when NFS_FSCACHE is configured, add NETFS_SUPPORT to Kconfig and add the required netfs_inode into struct nfs_inode. Using netfs requires we move the VFS inode structure to be stored inside struct netfs_inode, along with the fscache_cookie. Thus, if NFS_FSCACHE is configured, place netfs_inode inside an anonymous union so the vfs_inode memory is the same and we do not need to modify other non-fscache areas of NFS. In addition, inside the NFS fscache code, use the new helpers, netfs_inode() and netfs_i_cookie() helpers, and remove our own helper, nfs_i_fscache(). Later patches will convert NFS fscache to fully use netfs. Signed-off-by: Dave Wysochanski Tested-by: Daire Byrne Signed-off-by: Anna Schumaker --- fs/nfs/Kconfig | 1 + fs/nfs/fscache.c | 20 +++++++++----------- fs/nfs/fscache.h | 15 ++++++--------- include/linux/nfs_fs.h | 24 ++++++++++-------------- 4 files changed, 26 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig index c1c7ed2fd860..b6fc169be1b1 100644 --- a/fs/nfs/Kconfig +++ b/fs/nfs/Kconfig @@ -170,6 +170,7 @@ config ROOT_NFS config NFS_FSCACHE bool "Provide NFS client caching support" depends on NFS_FS=m && FSCACHE || NFS_FS=y && FSCACHE=y + select NETFS_SUPPORT help Say Y here if you want NFS data to be cached locally on disc through the general filesystem cache manager diff --git a/fs/nfs/fscache.c b/fs/nfs/fscache.c index ea5f2976dfab..3b4dafa771bd 100644 --- a/fs/nfs/fscache.c +++ b/fs/nfs/fscache.c @@ -163,13 +163,14 @@ void nfs_fscache_init_inode(struct inode *inode) struct nfs_server *nfss = NFS_SERVER(inode); struct nfs_inode *nfsi = NFS_I(inode); - nfsi->fscache = NULL; + netfs_inode(inode)->cache = NULL; if (!(nfss->fscache && S_ISREG(inode->i_mode))) return; nfs_fscache_update_auxdata(&auxdata, inode); - nfsi->fscache = fscache_acquire_cookie(NFS_SB(inode->i_sb)->fscache, + netfs_inode(inode)->cache = fscache_acquire_cookie( + nfss->fscache, 0, nfsi->fh.data, /* index_key */ nfsi->fh.size, @@ -183,11 +184,8 @@ void nfs_fscache_init_inode(struct inode *inode) */ void nfs_fscache_clear_inode(struct inode *inode) { - struct nfs_inode *nfsi = NFS_I(inode); - struct fscache_cookie *cookie = nfs_i_fscache(inode); - - fscache_relinquish_cookie(cookie, false); - nfsi->fscache = NULL; + fscache_relinquish_cookie(netfs_i_cookie(netfs_inode(inode)), false); + netfs_inode(inode)->cache = NULL; } /* @@ -212,7 +210,7 @@ void nfs_fscache_clear_inode(struct inode *inode) void nfs_fscache_open_file(struct inode *inode, struct file *filp) { struct nfs_fscache_inode_auxdata auxdata; - struct fscache_cookie *cookie = nfs_i_fscache(inode); + struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode)); bool open_for_write = inode_is_open_for_write(inode); if (!fscache_cookie_valid(cookie)) @@ -230,7 +228,7 @@ EXPORT_SYMBOL_GPL(nfs_fscache_open_file); void nfs_fscache_release_file(struct inode *inode, struct file *filp) { struct nfs_fscache_inode_auxdata auxdata; - struct fscache_cookie *cookie = nfs_i_fscache(inode); + struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode)); loff_t i_size = i_size_read(inode); nfs_fscache_update_auxdata(&auxdata, inode); @@ -243,7 +241,7 @@ void nfs_fscache_release_file(struct inode *inode, struct file *filp) static int fscache_fallback_read_page(struct inode *inode, struct page *page) { struct netfs_cache_resources cres; - struct fscache_cookie *cookie = nfs_i_fscache(inode); + struct fscache_cookie *cookie = netfs_i_cookie(&NFS_I(inode)->netfs); struct iov_iter iter; struct bio_vec bvec; int ret; @@ -269,7 +267,7 @@ static int fscache_fallback_write_page(struct inode *inode, struct page *page, bool no_space_allocated_yet) { struct netfs_cache_resources cres; - struct fscache_cookie *cookie = nfs_i_fscache(inode); + struct fscache_cookie *cookie = netfs_i_cookie(&NFS_I(inode)->netfs); struct iov_iter iter; struct bio_vec bvec; loff_t start = page_offset(page); diff --git a/fs/nfs/fscache.h b/fs/nfs/fscache.h index 2a37af880978..38614ed8f951 100644 --- a/fs/nfs/fscache.h +++ b/fs/nfs/fscache.h @@ -54,7 +54,7 @@ static inline bool nfs_fscache_release_folio(struct folio *folio, gfp_t gfp) if (current_is_kswapd() || !(gfp & __GFP_FS)) return false; folio_wait_fscache(folio); - fscache_note_page_release(nfs_i_fscache(folio->mapping->host)); + fscache_note_page_release(netfs_i_cookie(&NFS_I(folio->mapping->host)->netfs)); nfs_inc_fscache_stats(folio->mapping->host, NFSIOS_FSCACHE_PAGES_UNCACHED); } @@ -66,7 +66,7 @@ static inline bool nfs_fscache_release_folio(struct folio *folio, gfp_t gfp) */ static inline int nfs_fscache_read_page(struct inode *inode, struct page *page) { - if (nfs_i_fscache(inode)) + if (netfs_inode(inode)->cache) return __nfs_fscache_read_page(inode, page); return -ENOBUFS; } @@ -78,7 +78,7 @@ static inline int nfs_fscache_read_page(struct inode *inode, struct page *page) static inline void nfs_fscache_write_page(struct inode *inode, struct page *page) { - if (nfs_i_fscache(inode)) + if (netfs_inode(inode)->cache) __nfs_fscache_write_page(inode, page); } @@ -101,13 +101,10 @@ static inline void nfs_fscache_update_auxdata(struct nfs_fscache_inode_auxdata * static inline void nfs_fscache_invalidate(struct inode *inode, int flags) { struct nfs_fscache_inode_auxdata auxdata; - struct nfs_inode *nfsi = NFS_I(inode); + struct fscache_cookie *cookie = netfs_i_cookie(&NFS_I(inode)->netfs); - if (nfsi->fscache) { - nfs_fscache_update_auxdata(&auxdata, inode); - fscache_invalidate(nfsi->fscache, &auxdata, - i_size_read(inode), flags); - } + nfs_fscache_update_auxdata(&auxdata, inode); + fscache_invalidate(cookie, &auxdata, i_size_read(inode), flags); } /* diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index bf89fe6fc3ba..041e79a48f7a 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -31,6 +31,10 @@ #include #include +#ifdef CONFIG_NFS_FSCACHE +#include +#endif + #include #include #include @@ -204,14 +208,15 @@ struct nfs_inode { /* how many bytes have been written/read and how many bytes queued up */ __u64 write_io; __u64 read_io; -#ifdef CONFIG_NFS_FSCACHE - struct fscache_cookie *fscache; -#endif - struct inode vfs_inode; - #ifdef CONFIG_NFS_V4_2 struct nfs4_xattr_cache *xattr_cache; #endif + union { + struct inode vfs_inode; +#ifdef CONFIG_NFS_FSCACHE + struct netfs_inode netfs; /* netfs context and VFS inode */ +#endif + }; }; struct nfs4_copy_state { @@ -329,15 +334,6 @@ static inline int NFS_STALE(const struct inode *inode) return test_bit(NFS_INO_STALE, &NFS_I(inode)->flags); } -static inline struct fscache_cookie *nfs_i_fscache(struct inode *inode) -{ -#ifdef CONFIG_NFS_FSCACHE - return NFS_I(inode)->fscache; -#else - return NULL; -#endif -} - static inline __u64 NFS_FILEID(const struct inode *inode) { return NFS_I(inode)->fileid; -- cgit v1.2.3 From 000dbe0bec058cbf2ca9e156e4a5584f5158b0f9 Mon Sep 17 00:00:00 2001 From: Dave Wysochanski Date: Mon, 20 Feb 2023 08:43:06 -0500 Subject: NFS: Convert buffered read paths to use netfs when fscache is enabled Convert the NFS buffered read code paths to corresponding netfs APIs, but only when fscache is configured and enabled. The netfs API defines struct netfs_request_ops which must be filled in by the network filesystem. For NFS, we only need to define 5 of the functions, the main one being the issue_read() function. The issue_read() function is called by the netfs layer when a read cannot be fulfilled locally, and must be sent to the server (either the cache is not active, or it is active but the data is not available). Once the read from the server is complete, netfs requires a call to netfs_subreq_terminated() which conveys either how many bytes were read successfully, or an error. Note that issue_read() is called with a structure, netfs_io_subrequest, which defines the IO requested, and contains a start and a length (both in bytes), and assumes the underlying netfs will return a either an error on the whole region, or the number of bytes successfully read. The NFS IO path is page based and the main APIs are the pgio APIs defined in pagelist.c. For the pgio APIs, there is no way for the caller to know how many RPCs will be sent and how the pages will be broken up into underlying RPCs, each of which will have their own completion and return code. In contrast, netfs is subrequest based, a single subrequest may contain multiple pages, and a single subrequest is initiated with issue_read() and terminated with netfs_subreq_terminated(). Thus, to utilze the netfs APIs, NFS needs some way to accommodate the netfs API requirement on the single response to the whole subrequest, while also minimizing disruptive changes to the NFS pgio layer. The approach taken with this patch is to allocate a small structure for each nfs_netfs_issue_read() call, store the final error and number of bytes successfully transferred in the structure, and update these values as each RPC completes. The refcount on the structure is used as a marker for the last RPC completion, is incremented in nfs_netfs_read_initiate(), and decremented inside nfs_netfs_read_completion(), when a nfs_pgio_header contains a valid pointer to the data. On the final put (which signals the final outstanding RPC is complete) in nfs_netfs_read_completion(), call netfs_subreq_terminated() with either the final error value (if one or more READs complete with an error) or the number of bytes successfully transferred (if all RPCs complete successfully). Note that when all RPCs complete successfully, the number of bytes transferred is capped to the length of the subrequest. Capping the transferred length to the subrequest length prevents "Subreq overread" warnings from netfs. This is due to the "aligned_len" in nfs_pageio_add_page(), and the corner case where NFS requests a full page at the end of the file, even when i_size reflects only a partial page (NFS overread). Signed-off-by: Dave Wysochanski Tested-by: Daire Byrne Signed-off-by: Anna Schumaker --- fs/nfs/fscache.c | 222 ++++++++++++++++++++++++++++------------------- fs/nfs/fscache.h | 122 +++++++++++++++++++------- fs/nfs/inode.c | 2 + fs/nfs/internal.h | 9 ++ fs/nfs/pagelist.c | 4 + fs/nfs/read.c | 61 +++++++------ include/linux/nfs_page.h | 3 + include/linux/nfs_xdr.h | 3 + 8 files changed, 274 insertions(+), 152 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/fscache.c b/fs/nfs/fscache.c index 3b4dafa771bd..95c2b3056e2b 100644 --- a/fs/nfs/fscache.c +++ b/fs/nfs/fscache.c @@ -15,6 +15,9 @@ #include #include #include +#include +#include +#include #include "internal.h" #include "iostat.h" @@ -235,108 +238,153 @@ void nfs_fscache_release_file(struct inode *inode, struct file *filp) fscache_unuse_cookie(cookie, &auxdata, &i_size); } -/* - * Fallback page reading interface. - */ -static int fscache_fallback_read_page(struct inode *inode, struct page *page) +int nfs_netfs_read_folio(struct file *file, struct folio *folio) { - struct netfs_cache_resources cres; - struct fscache_cookie *cookie = netfs_i_cookie(&NFS_I(inode)->netfs); - struct iov_iter iter; - struct bio_vec bvec; - int ret; - - memset(&cres, 0, sizeof(cres)); - bvec_set_page(&bvec, page, PAGE_SIZE, 0); - iov_iter_bvec(&iter, ITER_DEST, &bvec, 1, PAGE_SIZE); - - ret = fscache_begin_read_operation(&cres, cookie); - if (ret < 0) - return ret; - - ret = fscache_read(&cres, page_offset(page), &iter, NETFS_READ_HOLE_FAIL, - NULL, NULL); - fscache_end_operation(&cres); - return ret; + if (!netfs_inode(folio_inode(folio))->cache) + return -ENOBUFS; + + return netfs_read_folio(file, folio); } -/* - * Fallback page writing interface. - */ -static int fscache_fallback_write_page(struct inode *inode, struct page *page, - bool no_space_allocated_yet) +int nfs_netfs_readahead(struct readahead_control *ractl) { - struct netfs_cache_resources cres; - struct fscache_cookie *cookie = netfs_i_cookie(&NFS_I(inode)->netfs); - struct iov_iter iter; - struct bio_vec bvec; - loff_t start = page_offset(page); - size_t len = PAGE_SIZE; - int ret; - - memset(&cres, 0, sizeof(cres)); - bvec_set_page(&bvec, page, PAGE_SIZE, 0); - iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, PAGE_SIZE); - - ret = fscache_begin_write_operation(&cres, cookie); - if (ret < 0) - return ret; - - ret = cres.ops->prepare_write(&cres, &start, &len, i_size_read(inode), - no_space_allocated_yet); - if (ret == 0) - ret = fscache_write(&cres, page_offset(page), &iter, NULL, NULL); - fscache_end_operation(&cres); - return ret; + struct inode *inode = ractl->mapping->host; + + if (!netfs_inode(inode)->cache) + return -ENOBUFS; + + netfs_readahead(ractl); + return 0; } -/* - * Retrieve a page from fscache - */ -int __nfs_fscache_read_page(struct inode *inode, struct page *page) +atomic_t nfs_netfs_debug_id; +static int nfs_netfs_init_request(struct netfs_io_request *rreq, struct file *file) { - int ret; + rreq->netfs_priv = get_nfs_open_context(nfs_file_open_context(file)); + rreq->debug_id = atomic_inc_return(&nfs_netfs_debug_id); - trace_nfs_fscache_read_page(inode, page); - if (PageChecked(page)) { - ClearPageChecked(page); - ret = 1; - goto out; - } + return 0; +} - ret = fscache_fallback_read_page(inode, page); - if (ret < 0) { - nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_FAIL); - SetPageChecked(page); - goto out; - } +static void nfs_netfs_free_request(struct netfs_io_request *rreq) +{ + put_nfs_open_context(rreq->netfs_priv); +} - /* Read completed synchronously */ - nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_READ_OK); - SetPageUptodate(page); - ret = 0; -out: - trace_nfs_fscache_read_page_exit(inode, page, ret); - return ret; +static inline int nfs_netfs_begin_cache_operation(struct netfs_io_request *rreq) +{ + return fscache_begin_read_operation(&rreq->cache_resources, + netfs_i_cookie(netfs_inode(rreq->inode))); } -/* - * Store a newly fetched page in fscache. We can be certain there's no page - * stored in the cache as yet otherwise we would've read it from there. - */ -void __nfs_fscache_write_page(struct inode *inode, struct page *page) +static struct nfs_netfs_io_data *nfs_netfs_alloc(struct netfs_io_subrequest *sreq) { - int ret; + struct nfs_netfs_io_data *netfs; + + netfs = kzalloc(sizeof(*netfs), GFP_KERNEL_ACCOUNT); + if (!netfs) + return NULL; + netfs->sreq = sreq; + refcount_set(&netfs->refcount, 1); + return netfs; +} - trace_nfs_fscache_write_page(inode, page); +static bool nfs_netfs_clamp_length(struct netfs_io_subrequest *sreq) +{ + size_t rsize = NFS_SB(sreq->rreq->inode->i_sb)->rsize; - ret = fscache_fallback_write_page(inode, page, true); + sreq->len = min(sreq->len, rsize); + return true; +} - if (ret != 0) { - nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_WRITTEN_FAIL); - nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_UNCACHED); - } else { - nfs_inc_fscache_stats(inode, NFSIOS_FSCACHE_PAGES_WRITTEN_OK); +static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq) +{ + struct nfs_netfs_io_data *netfs; + struct nfs_pageio_descriptor pgio; + struct inode *inode = sreq->rreq->inode; + struct nfs_open_context *ctx = sreq->rreq->netfs_priv; + struct page *page; + int err; + pgoff_t start = (sreq->start + sreq->transferred) >> PAGE_SHIFT; + pgoff_t last = ((sreq->start + sreq->len - + sreq->transferred - 1) >> PAGE_SHIFT); + XA_STATE(xas, &sreq->rreq->mapping->i_pages, start); + + nfs_pageio_init_read(&pgio, inode, false, + &nfs_async_read_completion_ops); + + netfs = nfs_netfs_alloc(sreq); + if (!netfs) + return netfs_subreq_terminated(sreq, -ENOMEM, false); + + pgio.pg_netfs = netfs; /* used in completion */ + + xas_lock(&xas); + xas_for_each(&xas, page, last) { + /* nfs_read_add_folio() may schedule() due to pNFS layout and other RPCs */ + xas_pause(&xas); + xas_unlock(&xas); + err = nfs_read_add_folio(&pgio, ctx, page_folio(page)); + if (err < 0) { + netfs->error = err; + goto out; + } + xas_lock(&xas); } - trace_nfs_fscache_write_page_exit(inode, page, ret); + xas_unlock(&xas); +out: + nfs_pageio_complete_read(&pgio); + nfs_netfs_put(netfs); } + +void nfs_netfs_initiate_read(struct nfs_pgio_header *hdr) +{ + struct nfs_netfs_io_data *netfs = hdr->netfs; + + if (!netfs) + return; + + nfs_netfs_get(netfs); +} + +int nfs_netfs_folio_unlock(struct folio *folio) +{ + struct inode *inode = folio_file_mapping(folio)->host; + + /* + * If fscache is enabled, netfs will unlock pages. + */ + if (netfs_inode(inode)->cache) + return 0; + + return 1; +} + +void nfs_netfs_read_completion(struct nfs_pgio_header *hdr) +{ + struct nfs_netfs_io_data *netfs = hdr->netfs; + struct netfs_io_subrequest *sreq; + + if (!netfs) + return; + + sreq = netfs->sreq; + if (test_bit(NFS_IOHDR_EOF, &hdr->flags)) + __set_bit(NETFS_SREQ_CLEAR_TAIL, &sreq->flags); + + if (hdr->error) + netfs->error = hdr->error; + else + atomic64_add(hdr->res.count, &netfs->transferred); + + nfs_netfs_put(netfs); + hdr->netfs = NULL; +} + +const struct netfs_request_ops nfs_netfs_ops = { + .init_request = nfs_netfs_init_request, + .free_request = nfs_netfs_free_request, + .begin_cache_operation = nfs_netfs_begin_cache_operation, + .issue_read = nfs_netfs_issue_read, + .clamp_length = nfs_netfs_clamp_length +}; diff --git a/fs/nfs/fscache.h b/fs/nfs/fscache.h index 38614ed8f951..e1706e736c64 100644 --- a/fs/nfs/fscache.h +++ b/fs/nfs/fscache.h @@ -34,6 +34,58 @@ struct nfs_fscache_inode_auxdata { u64 change_attr; }; +struct nfs_netfs_io_data { + /* + * NFS may split a netfs_io_subrequest into multiple RPCs, each + * with their own read completion. In netfs, we can only call + * netfs_subreq_terminated() once for each subrequest. Use the + * refcount here to double as a marker of the last RPC completion, + * and only call netfs via netfs_subreq_terminated() once. + */ + refcount_t refcount; + struct netfs_io_subrequest *sreq; + + /* + * Final disposition of the netfs_io_subrequest, sent in + * netfs_subreq_terminated() + */ + atomic64_t transferred; + int error; +}; + +static inline void nfs_netfs_get(struct nfs_netfs_io_data *netfs) +{ + refcount_inc(&netfs->refcount); +} + +static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs) +{ + ssize_t final_len; + + /* Only the last RPC completion should call netfs_subreq_terminated() */ + if (!refcount_dec_and_test(&netfs->refcount)) + return; + + /* + * The NFS pageio interface may read a complete page, even when netfs + * only asked for a partial page. Specifically, this may be seen when + * one thread is truncating a file while another one is reading the last + * page of the file. + * Correct the final length here to be no larger than the netfs subrequest + * length, and thus avoid netfs's "Subreq overread" warning message. + */ + final_len = min_t(s64, netfs->sreq->len, atomic64_read(&netfs->transferred)); + netfs_subreq_terminated(netfs->sreq, netfs->error ?: final_len, false); + kfree(netfs); +} +static inline void nfs_netfs_inode_init(struct nfs_inode *nfsi) +{ + netfs_inode_init(&nfsi->netfs, &nfs_netfs_ops); +} +extern void nfs_netfs_initiate_read(struct nfs_pgio_header *hdr); +extern void nfs_netfs_read_completion(struct nfs_pgio_header *hdr); +extern int nfs_netfs_folio_unlock(struct folio *folio); + /* * fscache.c */ @@ -44,9 +96,8 @@ extern void nfs_fscache_init_inode(struct inode *); extern void nfs_fscache_clear_inode(struct inode *); extern void nfs_fscache_open_file(struct inode *, struct file *); extern void nfs_fscache_release_file(struct inode *, struct file *); - -extern int __nfs_fscache_read_page(struct inode *, struct page *); -extern void __nfs_fscache_write_page(struct inode *, struct page *); +extern int nfs_netfs_readahead(struct readahead_control *ractl); +extern int nfs_netfs_read_folio(struct file *file, struct folio *folio); static inline bool nfs_fscache_release_folio(struct folio *folio, gfp_t gfp) { @@ -54,34 +105,11 @@ static inline bool nfs_fscache_release_folio(struct folio *folio, gfp_t gfp) if (current_is_kswapd() || !(gfp & __GFP_FS)) return false; folio_wait_fscache(folio); - fscache_note_page_release(netfs_i_cookie(&NFS_I(folio->mapping->host)->netfs)); - nfs_inc_fscache_stats(folio->mapping->host, - NFSIOS_FSCACHE_PAGES_UNCACHED); } + fscache_note_page_release(netfs_i_cookie(netfs_inode(folio->mapping->host))); return true; } -/* - * Retrieve a page from an inode data storage object. - */ -static inline int nfs_fscache_read_page(struct inode *inode, struct page *page) -{ - if (netfs_inode(inode)->cache) - return __nfs_fscache_read_page(inode, page); - return -ENOBUFS; -} - -/* - * Store a page newly fetched from the server in an inode data storage object - * in the cache. - */ -static inline void nfs_fscache_write_page(struct inode *inode, - struct page *page) -{ - if (netfs_inode(inode)->cache) - __nfs_fscache_write_page(inode, page); -} - static inline void nfs_fscache_update_auxdata(struct nfs_fscache_inode_auxdata *auxdata, struct inode *inode) { @@ -117,7 +145,28 @@ static inline const char *nfs_server_fscache_state(struct nfs_server *server) return "no "; } +static inline void nfs_netfs_set_pgio_header(struct nfs_pgio_header *hdr, + struct nfs_pageio_descriptor *desc) +{ + hdr->netfs = desc->pg_netfs; +} +static inline void nfs_netfs_set_pageio_descriptor(struct nfs_pageio_descriptor *desc, + struct nfs_pgio_header *hdr) +{ + desc->pg_netfs = hdr->netfs; +} +static inline void nfs_netfs_reset_pageio_descriptor(struct nfs_pageio_descriptor *desc) +{ + desc->pg_netfs = NULL; +} #else /* CONFIG_NFS_FSCACHE */ +static inline void nfs_netfs_inode_init(struct nfs_inode *nfsi) {} +static inline void nfs_netfs_initiate_read(struct nfs_pgio_header *hdr) {} +static inline void nfs_netfs_read_completion(struct nfs_pgio_header *hdr) {} +static inline int nfs_netfs_folio_unlock(struct folio *folio) +{ + return 1; +} static inline void nfs_fscache_release_super_cookie(struct super_block *sb) {} static inline void nfs_fscache_init_inode(struct inode *inode) {} @@ -125,22 +174,29 @@ static inline void nfs_fscache_clear_inode(struct inode *inode) {} static inline void nfs_fscache_open_file(struct inode *inode, struct file *filp) {} static inline void nfs_fscache_release_file(struct inode *inode, struct file *file) {} - -static inline bool nfs_fscache_release_folio(struct folio *folio, gfp_t gfp) +static inline int nfs_netfs_readahead(struct readahead_control *ractl) { - return true; /* may release folio */ + return -ENOBUFS; } -static inline int nfs_fscache_read_page(struct inode *inode, struct page *page) +static inline int nfs_netfs_read_folio(struct file *file, struct folio *folio) { return -ENOBUFS; } -static inline void nfs_fscache_write_page(struct inode *inode, struct page *page) {} + +static inline bool nfs_fscache_release_folio(struct folio *folio, gfp_t gfp) +{ + return true; /* may release folio */ +} static inline void nfs_fscache_invalidate(struct inode *inode, int flags) {} static inline const char *nfs_server_fscache_state(struct nfs_server *server) { return "no "; } - +static inline void nfs_netfs_set_pgio_header(struct nfs_pgio_header *hdr, + struct nfs_pageio_descriptor *desc) {} +static inline void nfs_netfs_set_pageio_descriptor(struct nfs_pageio_descriptor *desc, + struct nfs_pgio_header *hdr) {} +static inline void nfs_netfs_reset_pageio_descriptor(struct nfs_pageio_descriptor *desc) {} #endif /* CONFIG_NFS_FSCACHE */ #endif /* _NFS_FSCACHE_H */ diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 222a28320e1c..5c8027e3c961 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -2254,6 +2254,8 @@ struct inode *nfs_alloc_inode(struct super_block *sb) #ifdef CONFIG_NFS_V4_2 nfsi->xattr_cache = NULL; #endif + nfs_netfs_inode_init(nfsi); + return &nfsi->vfs_inode; } EXPORT_SYMBOL_GPL(nfs_alloc_inode); diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 2a65fe2a63ab..9ba07553eab8 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -452,6 +452,10 @@ extern void nfs_sb_deactive(struct super_block *sb); extern int nfs_client_for_each_server(struct nfs_client *clp, int (*fn)(struct nfs_server *, void *), void *data); +#ifdef CONFIG_NFS_FSCACHE +extern const struct netfs_request_ops nfs_netfs_ops; +#endif + /* io.c */ extern void nfs_start_io_read(struct inode *inode); extern void nfs_end_io_read(struct inode *inode); @@ -481,9 +485,14 @@ extern int nfs4_get_rootfh(struct nfs_server *server, struct nfs_fh *mntfh, bool struct nfs_pgio_completion_ops; /* read.c */ +extern const struct nfs_pgio_completion_ops nfs_async_read_completion_ops; extern void nfs_pageio_init_read(struct nfs_pageio_descriptor *pgio, struct inode *inode, bool force_mds, const struct nfs_pgio_completion_ops *compl_ops); +extern int nfs_read_add_folio(struct nfs_pageio_descriptor *pgio, + struct nfs_open_context *ctx, + struct folio *folio); +extern void nfs_pageio_complete_read(struct nfs_pageio_descriptor *pgio); extern void nfs_read_prepare(struct rpc_task *task, void *calldata); extern void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio); diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 64fa8de199de..6efb5068c116 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -26,6 +26,7 @@ #include "internal.h" #include "pnfs.h" #include "nfstrace.h" +#include "fscache.h" #define NFSDBG_FACILITY NFSDBG_PAGECACHE @@ -105,6 +106,7 @@ void nfs_pgheader_init(struct nfs_pageio_descriptor *desc, hdr->good_bytes = mirror->pg_count; hdr->io_completion = desc->pg_io_completion; hdr->dreq = desc->pg_dreq; + nfs_netfs_set_pgio_header(hdr, desc); hdr->release = release; hdr->completion_ops = desc->pg_completion_ops; if (hdr->completion_ops->init_hdr) @@ -941,6 +943,7 @@ void nfs_pageio_init(struct nfs_pageio_descriptor *desc, desc->pg_lseg = NULL; desc->pg_io_completion = NULL; desc->pg_dreq = NULL; + nfs_netfs_reset_pageio_descriptor(desc); desc->pg_bsize = bsize; desc->pg_mirror_count = 1; @@ -1477,6 +1480,7 @@ int nfs_pageio_resend(struct nfs_pageio_descriptor *desc, desc->pg_io_completion = hdr->io_completion; desc->pg_dreq = hdr->dreq; + nfs_netfs_set_pageio_descriptor(desc, hdr); list_splice_init(&hdr->pages, &pages); while (!list_empty(&pages)) { struct nfs_page *req = nfs_list_entry(pages.next); diff --git a/fs/nfs/read.c b/fs/nfs/read.c index 8cb9e8d3c51d..f71eeee67e20 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -31,7 +31,7 @@ #define NFSDBG_FACILITY NFSDBG_PAGECACHE -static const struct nfs_pgio_completion_ops nfs_async_read_completion_ops; +const struct nfs_pgio_completion_ops nfs_async_read_completion_ops; static const struct nfs_rw_ops nfs_rw_read_ops; static struct kmem_cache *nfs_rdata_cachep; @@ -74,7 +74,7 @@ void nfs_pageio_init_read(struct nfs_pageio_descriptor *pgio, } EXPORT_SYMBOL_GPL(nfs_pageio_init_read); -static void nfs_pageio_complete_read(struct nfs_pageio_descriptor *pgio) +void nfs_pageio_complete_read(struct nfs_pageio_descriptor *pgio) { struct nfs_pgio_mirror *pgm; unsigned long npages; @@ -110,20 +110,14 @@ EXPORT_SYMBOL_GPL(nfs_pageio_reset_read_mds); static void nfs_readpage_release(struct nfs_page *req, int error) { - struct inode *inode = d_inode(nfs_req_openctx(req)->dentry); struct folio *folio = nfs_page_to_folio(req); - dprintk("NFS: read done (%s/%llu %d@%lld)\n", inode->i_sb->s_id, - (unsigned long long)NFS_FILEID(inode), req->wb_bytes, - (long long)req_offset(req)); - if (nfs_error_is_fatal_on_server(error) && error != -ETIMEDOUT) folio_set_error(folio); - if (nfs_page_group_sync_on_bit(req, PG_UNLOCKPAGE)) { - if (folio_test_uptodate(folio)) - nfs_fscache_write_page(inode, &folio->page); - folio_unlock(folio); - } + if (nfs_page_group_sync_on_bit(req, PG_UNLOCKPAGE)) + if (nfs_netfs_folio_unlock(folio)) + folio_unlock(folio); + nfs_release_request(req); } @@ -177,6 +171,8 @@ static void nfs_read_completion(struct nfs_pgio_header *hdr) nfs_list_remove_request(req); nfs_readpage_release(req, error); } + nfs_netfs_read_completion(hdr); + out: hdr->release(hdr); } @@ -187,6 +183,7 @@ static void nfs_initiate_read(struct nfs_pgio_header *hdr, struct rpc_task_setup *task_setup_data, int how) { rpc_ops->read_setup(hdr, msg); + nfs_netfs_initiate_read(hdr); trace_nfs_initiate_read(hdr); } @@ -202,7 +199,7 @@ nfs_async_read_error(struct list_head *head, int error) } } -static const struct nfs_pgio_completion_ops nfs_async_read_completion_ops = { +const struct nfs_pgio_completion_ops nfs_async_read_completion_ops = { .error_cleanup = nfs_async_read_error, .completion = nfs_read_completion, }; @@ -277,9 +274,9 @@ static void nfs_readpage_result(struct rpc_task *task, nfs_readpage_retry(task, hdr); } -static int nfs_read_add_folio(struct nfs_pageio_descriptor *pgio, - struct nfs_open_context *ctx, - struct folio *folio) +int nfs_read_add_folio(struct nfs_pageio_descriptor *pgio, + struct nfs_open_context *ctx, + struct folio *folio) { struct inode *inode = folio_file_mapping(folio)->host; struct nfs_server *server = NFS_SERVER(inode); @@ -295,15 +292,11 @@ static int nfs_read_add_folio(struct nfs_pageio_descriptor *pgio, aligned_len = min_t(unsigned int, ALIGN(len, rsize), fsize); - if (!IS_SYNC(inode)) { - error = nfs_fscache_read_page(inode, &folio->page); - if (error == 0) - goto out_unlock; - } - new = nfs_page_create_from_folio(ctx, folio, 0, aligned_len); - if (IS_ERR(new)) - goto out_error; + if (IS_ERR(new)) { + error = PTR_ERR(new); + goto out; + } if (len < fsize) folio_zero_segment(folio, len, fsize); @@ -314,10 +307,6 @@ static int nfs_read_add_folio(struct nfs_pageio_descriptor *pgio, goto out; } return 0; -out_error: - error = PTR_ERR(new); -out_unlock: - folio_unlock(folio); out: return error; } @@ -356,6 +345,10 @@ int nfs_read_folio(struct file *file, struct folio *folio) if (NFS_STALE(inode)) goto out_unlock; + ret = nfs_netfs_read_folio(file, folio); + if (!ret) + goto out; + ctx = get_nfs_open_context(nfs_file_open_context(file)); xchg(&ctx->error, 0); @@ -364,7 +357,7 @@ int nfs_read_folio(struct file *file, struct folio *folio) ret = nfs_read_add_folio(&pgio, ctx, folio); if (ret) - goto out; + goto out_put; nfs_pageio_complete_read(&pgio); ret = pgio.pg_error < 0 ? pgio.pg_error : 0; @@ -373,14 +366,14 @@ int nfs_read_folio(struct file *file, struct folio *folio) if (!folio_test_uptodate(folio) && !ret) ret = xchg(&ctx->error, 0); } -out: +out_put: put_nfs_open_context(ctx); +out: trace_nfs_aop_readpage_done(inode, folio, ret); return ret; out_unlock: folio_unlock(folio); - trace_nfs_aop_readpage_done(inode, folio, ret); - return ret; + goto out; } void nfs_readahead(struct readahead_control *ractl) @@ -401,6 +394,10 @@ void nfs_readahead(struct readahead_control *ractl) if (NFS_STALE(inode)) goto out; + ret = nfs_netfs_readahead(ractl); + if (!ret) + goto out; + if (file == NULL) { ret = -EBADF; ctx = nfs_find_open_context(inode, NULL, FMODE_READ); diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index a2f1ca657623..aa9f4c6ebe26 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -105,6 +105,9 @@ struct nfs_pageio_descriptor { struct pnfs_layout_segment *pg_lseg; struct nfs_io_completion *pg_io_completion; struct nfs_direct_req *pg_dreq; +#ifdef CONFIG_NFS_FSCACHE + void *pg_netfs; +#endif unsigned int pg_bsize; /* default bsize for mirrors */ u32 pg_mirror_count; diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index e86cf6642d21..e196ef595908 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1619,6 +1619,9 @@ struct nfs_pgio_header { const struct nfs_rw_ops *rw_ops; struct nfs_io_completion *io_completion; struct nfs_direct_req *dreq; +#ifdef CONFIG_NFS_FSCACHE + void *netfs; +#endif int pnfs_error; int error; /* merge with pnfs_error */ -- cgit v1.2.3 From 0631d5e02a1c722b90c11a47dcb6d83ef88ab47b Mon Sep 17 00:00:00 2001 From: Dave Wysochanski Date: Mon, 20 Feb 2023 08:43:07 -0500 Subject: NFS: Remove all NFSIOS_FSCACHE counters due to conversion to netfs API The old NFSIOS_FSCACHE counters are no longer accurate or useful with the conversion to the new netfs API. The new API does not have a page based interface, and so the counters in nfs_stat_fscachecounters are no longer obtainable. The new netfs the API has extensive statistics inside /proc/fs/fscache/stats so we no longer need NFS specific fscache stats. Note this also removes the 'fsc:' line from /proc/self/mountstats so it will be a user-visible change. Signed-off-by: Dave Wysochanski Reviewed-by: Jeff Layton Tested-by: Daire Byrne Signed-off-by: Anna Schumaker --- fs/nfs/iostat.h | 17 ----------------- fs/nfs/super.c | 11 ----------- include/linux/nfs_iostat.h | 12 ------------ 3 files changed, 40 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/iostat.h b/fs/nfs/iostat.h index 2ddaab1ac653..5aa776b5a3e7 100644 --- a/fs/nfs/iostat.h +++ b/fs/nfs/iostat.h @@ -17,9 +17,6 @@ struct nfs_iostats { unsigned long long bytes[__NFSIOS_BYTESMAX]; -#ifdef CONFIG_NFS_FSCACHE - unsigned long long fscache[__NFSIOS_FSCACHEMAX]; -#endif unsigned long events[__NFSIOS_COUNTSMAX]; } ____cacheline_aligned; @@ -49,20 +46,6 @@ static inline void nfs_add_stats(const struct inode *inode, nfs_add_server_stats(NFS_SERVER(inode), stat, addend); } -#ifdef CONFIG_NFS_FSCACHE -static inline void nfs_add_fscache_stats(struct inode *inode, - enum nfs_stat_fscachecounters stat, - long addend) -{ - this_cpu_add(NFS_SERVER(inode)->io_stats->fscache[stat], addend); -} -static inline void nfs_inc_fscache_stats(struct inode *inode, - enum nfs_stat_fscachecounters stat) -{ - this_cpu_inc(NFS_SERVER(inode)->io_stats->fscache[stat]); -} -#endif - static inline struct nfs_iostats __percpu *nfs_alloc_iostats(void) { return alloc_percpu(struct nfs_iostats); diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 05ae23657527..90796db8c205 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -692,10 +692,6 @@ int nfs_show_stats(struct seq_file *m, struct dentry *root) totals.events[i] += stats->events[i]; for (i = 0; i < __NFSIOS_BYTESMAX; i++) totals.bytes[i] += stats->bytes[i]; -#ifdef CONFIG_NFS_FSCACHE - for (i = 0; i < __NFSIOS_FSCACHEMAX; i++) - totals.fscache[i] += stats->fscache[i]; -#endif preempt_enable(); } @@ -706,13 +702,6 @@ int nfs_show_stats(struct seq_file *m, struct dentry *root) seq_puts(m, "\n\tbytes:\t"); for (i = 0; i < __NFSIOS_BYTESMAX; i++) seq_printf(m, "%Lu ", totals.bytes[i]); -#ifdef CONFIG_NFS_FSCACHE - if (nfss->options & NFS_OPTION_FSCACHE) { - seq_puts(m, "\n\tfsc:\t"); - for (i = 0; i < __NFSIOS_FSCACHEMAX; i++) - seq_printf(m, "%Lu ", totals.fscache[i]); - } -#endif seq_putc(m, '\n'); rpc_clnt_show_stats(m, nfss->client); diff --git a/include/linux/nfs_iostat.h b/include/linux/nfs_iostat.h index 027874c36c88..8d946089d151 100644 --- a/include/linux/nfs_iostat.h +++ b/include/linux/nfs_iostat.h @@ -119,16 +119,4 @@ enum nfs_stat_eventcounters { __NFSIOS_COUNTSMAX, }; -/* - * NFS local caching servicing counters - */ -enum nfs_stat_fscachecounters { - NFSIOS_FSCACHE_PAGES_READ_OK, - NFSIOS_FSCACHE_PAGES_READ_FAIL, - NFSIOS_FSCACHE_PAGES_WRITTEN_OK, - NFSIOS_FSCACHE_PAGES_WRITTEN_FAIL, - NFSIOS_FSCACHE_PAGES_UNCACHED, - __NFSIOS_FSCACHEMAX, -}; - #endif /* _LINUX_NFS_IOSTAT */ -- cgit v1.2.3 From 03f5bd75a4c19714a929a0f4e7bbf36b49e874c1 Mon Sep 17 00:00:00 2001 From: Dave Wysochanski Date: Mon, 20 Feb 2023 08:43:08 -0500 Subject: NFS: Remove fscache specific trace points and NFS_INO_FSCACHE bit The NFS specific trace points are no longer needed as tracing is well covered by netfs and fscache. Signed-off-by: Dave Wysochanski Reviewed-by: Jeff Layton Tested-by: Daire Byrne Signed-off-by: Anna Schumaker --- fs/nfs/nfstrace.h | 91 -------------------------------------------------- include/linux/nfs_fs.h | 1 - 2 files changed, 92 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h index a778713343df..4e90ca531176 100644 --- a/fs/nfs/nfstrace.h +++ b/fs/nfs/nfstrace.h @@ -39,7 +39,6 @@ { BIT(NFS_INO_STALE), "STALE" }, \ { BIT(NFS_INO_ACL_LRU_SET), "ACL_LRU_SET" }, \ { BIT(NFS_INO_INVALIDATING), "INVALIDATING" }, \ - { BIT(NFS_INO_FSCACHE), "FSCACHE" }, \ { BIT(NFS_INO_LAYOUTCOMMIT), "NEED_LAYOUTCOMMIT" }, \ { BIT(NFS_INO_LAYOUTCOMMITTING), "LAYOUTCOMMIT" }, \ { BIT(NFS_INO_LAYOUTSTATS), "LAYOUTSTATS" }, \ @@ -1243,96 +1242,6 @@ TRACE_EVENT(nfs_readpage_short, ) ); -DECLARE_EVENT_CLASS(nfs_fscache_page_event, - TP_PROTO( - const struct inode *inode, - struct page *page - ), - - TP_ARGS(inode, page), - - TP_STRUCT__entry( - __field(dev_t, dev) - __field(u32, fhandle) - __field(u64, fileid) - __field(loff_t, offset) - ), - - TP_fast_assign( - const struct nfs_inode *nfsi = NFS_I(inode); - const struct nfs_fh *fh = &nfsi->fh; - - __entry->offset = page_index(page) << PAGE_SHIFT; - __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; - __entry->fhandle = nfs_fhandle_hash(fh); - ), - - TP_printk( - "fileid=%02x:%02x:%llu fhandle=0x%08x " - "offset=%lld", - MAJOR(__entry->dev), MINOR(__entry->dev), - (unsigned long long)__entry->fileid, - __entry->fhandle, - (long long)__entry->offset - ) -); -DECLARE_EVENT_CLASS(nfs_fscache_page_event_done, - TP_PROTO( - const struct inode *inode, - struct page *page, - int error - ), - - TP_ARGS(inode, page, error), - - TP_STRUCT__entry( - __field(int, error) - __field(dev_t, dev) - __field(u32, fhandle) - __field(u64, fileid) - __field(loff_t, offset) - ), - - TP_fast_assign( - const struct nfs_inode *nfsi = NFS_I(inode); - const struct nfs_fh *fh = &nfsi->fh; - - __entry->offset = page_index(page) << PAGE_SHIFT; - __entry->dev = inode->i_sb->s_dev; - __entry->fileid = nfsi->fileid; - __entry->fhandle = nfs_fhandle_hash(fh); - __entry->error = error; - ), - - TP_printk( - "fileid=%02x:%02x:%llu fhandle=0x%08x " - "offset=%lld error=%d", - MAJOR(__entry->dev), MINOR(__entry->dev), - (unsigned long long)__entry->fileid, - __entry->fhandle, - (long long)__entry->offset, __entry->error - ) -); -#define DEFINE_NFS_FSCACHE_PAGE_EVENT(name) \ - DEFINE_EVENT(nfs_fscache_page_event, name, \ - TP_PROTO( \ - const struct inode *inode, \ - struct page *page \ - ), \ - TP_ARGS(inode, page)) -#define DEFINE_NFS_FSCACHE_PAGE_EVENT_DONE(name) \ - DEFINE_EVENT(nfs_fscache_page_event_done, name, \ - TP_PROTO( \ - const struct inode *inode, \ - struct page *page, \ - int error \ - ), \ - TP_ARGS(inode, page, error)) -DEFINE_NFS_FSCACHE_PAGE_EVENT(nfs_fscache_read_page); -DEFINE_NFS_FSCACHE_PAGE_EVENT_DONE(nfs_fscache_read_page_exit); -DEFINE_NFS_FSCACHE_PAGE_EVENT(nfs_fscache_write_page); -DEFINE_NFS_FSCACHE_PAGE_EVENT_DONE(nfs_fscache_write_page_exit); TRACE_EVENT(nfs_pgio_error, TP_PROTO( diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 041e79a48f7a..12bb868f9a18 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -281,7 +281,6 @@ struct nfs4_copy_state { #define NFS_INO_ACL_LRU_SET (2) /* Inode is on the LRU list */ #define NFS_INO_INVALIDATING (3) /* inode is being invalidated */ #define NFS_INO_PRESERVE_UNLINKED (4) /* preserve file if removed while open */ -#define NFS_INO_FSCACHE (5) /* inode can be cached by FS-Cache */ #define NFS_INO_LAYOUTCOMMIT (9) /* layoutcommit required */ #define NFS_INO_LAYOUTCOMMITTING (10) /* layoutcommit inflight */ #define NFS_INO_LAYOUTSTATS (11) /* layoutstats inflight */ -- cgit v1.2.3 From 3db63daabe210af32a09533fe7d8d47c711a103c Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Wed, 22 Mar 2023 09:27:04 +1100 Subject: NFSv3: handle out-of-order write replies. NFSv3 includes pre/post wcc attributes which allow the client to determine if all changes to the file have been made by the client itself, or if any might have been made by some other client. If there are gaps in the pre/post ctime sequence it must be assumed that some other client changed the file in that gap and the local cache must be suspect. The next time the file is opened the cache should be invalidated. Since Commit 1c341b777501 ("NFS: Add deferred cache invalidation for close-to-open consistency violations") in linux 5.3 the Linux client has been triggering this invalidation. The chunk in nfs_update_inode() in particularly triggers. Unfortunately Linux NFS assumes that all replies will be processed in the order sent, and will arrive in the order processed. This is not true in general. Consequently Linux NFS might ignore the wcc info in a WRITE reply because the reply is in response to a WRITE that was sent before some other request for which a reply has already been seen. This is detected by Linux using the gencount tests in nfs_inode_attr_cmp(). Also, when the gencount tests pass it is still possible that the request were processed on the server in a different order, and a gap seen in the ctime sequence might be filled in by a subsequent reply, so gaps should not immediately trigger delayed invalidation. The net result is that writing to a server and then reading the file back can result in going to the server for the read rather than serving it from cache - all because a couple of replies arrived out-of-order. This is a performance regression over kernels before 5.3, though the change in 5.3 is a correctness improvement. This has been seen with Linux writing to a Netapp server which occasionally re-orders requests. In testing the majority of requests were in-order, but a few (maybe 2 or three at a time) could be re-ordered. This patch addresses the problem by recording any gaps seen in the pre/post ctime sequence and not triggering invalidation until either there are too many gaps to fit in the table, or until there are no more active writes and the remaining gaps cannot be resolved. We allocate a table of 16 gaps on demand. If the allocation fails we revert to current behaviour which is of little cost as we are unlikely to be able to cache the writes anyway. In the table we store "start->end" pair when iversion is updated and "end<-start" pairs pre/post pairs reported by the server. Usually these exactly cancel out and so nothing is stored. When there are out-of-order replies we do store gaps and these will eventually be cancelled against later replies when this client is the only writer. If the final write is out-of-order there may be one gap remaining when the file is closed. This will be noticed and if there is precisely on gap and if the iversion can be advanced to match it, then we do so. This patch makes no attempt to handle directories correctly. The same problem potentially exists in the out-of-order replies to create/unlink requests can cause future lookup requires to be sent to the server unnecessarily. A similar scheme using the same primitives could be used to notice and handle out-of-order replies. Signed-off-by: NeilBrown Signed-off-by: Anna Schumaker --- fs/nfs/inode.c | 112 ++++++++++++++++++++++++++++++++++++++++++------- include/linux/nfs_fs.h | 47 +++++++++++++++++++++ 2 files changed, 144 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 5c8027e3c961..eb8af1e404d9 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -208,11 +208,12 @@ void nfs_set_cache_invalid(struct inode *inode, unsigned long flags) nfsi->cache_validity |= flags; - if (inode->i_mapping->nrpages == 0) - nfsi->cache_validity &= ~(NFS_INO_INVALID_DATA | - NFS_INO_DATA_INVAL_DEFER); - else if (nfsi->cache_validity & NFS_INO_INVALID_DATA) - nfsi->cache_validity &= ~NFS_INO_DATA_INVAL_DEFER; + if (inode->i_mapping->nrpages == 0) { + nfsi->cache_validity &= ~NFS_INO_INVALID_DATA; + nfs_ooo_clear(nfsi); + } else if (nfsi->cache_validity & NFS_INO_INVALID_DATA) { + nfs_ooo_clear(nfsi); + } trace_nfs_set_cache_invalid(inode, 0); } EXPORT_SYMBOL_GPL(nfs_set_cache_invalid); @@ -677,9 +678,10 @@ static int nfs_vmtruncate(struct inode * inode, loff_t offset) trace_nfs_size_truncate(inode, offset); i_size_write(inode, offset); /* Optimisation */ - if (offset == 0) - NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_DATA | - NFS_INO_DATA_INVAL_DEFER); + if (offset == 0) { + NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_DATA; + nfs_ooo_clear(NFS_I(inode)); + } NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE; spin_unlock(&inode->i_lock); @@ -1109,7 +1111,7 @@ void nfs_inode_attach_open_context(struct nfs_open_context *ctx) spin_lock(&inode->i_lock); if (list_empty(&nfsi->open_files) && - (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER)) + nfs_ooo_test(nfsi)) nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA | NFS_INO_REVAL_FORCED); list_add_tail_rcu(&ctx->list, &nfsi->open_files); @@ -1353,8 +1355,8 @@ int nfs_clear_invalid_mapping(struct address_space *mapping) set_bit(NFS_INO_INVALIDATING, bitlock); smp_wmb(); - nfsi->cache_validity &= - ~(NFS_INO_INVALID_DATA | NFS_INO_DATA_INVAL_DEFER); + nfsi->cache_validity &= ~NFS_INO_INVALID_DATA; + nfs_ooo_clear(nfsi); spin_unlock(&inode->i_lock); trace_nfs_invalidate_mapping_enter(inode); ret = nfs_invalidate_mapping(inode, mapping); @@ -1816,6 +1818,66 @@ static int nfs_inode_finish_partial_attr_update(const struct nfs_fattr *fattr, return 0; } +static void nfs_ooo_merge(struct nfs_inode *nfsi, + u64 start, u64 end) +{ + int i, cnt; + + if (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER) + /* No point merging anything */ + return; + + if (!nfsi->ooo) { + nfsi->ooo = kmalloc(sizeof(*nfsi->ooo), GFP_ATOMIC); + if (!nfsi->ooo) { + nfsi->cache_validity |= NFS_INO_DATA_INVAL_DEFER; + return; + } + nfsi->ooo->cnt = 0; + } + + /* add this range, merging if possible */ + cnt = nfsi->ooo->cnt; + for (i = 0; i < cnt; i++) { + if (end == nfsi->ooo->gap[i].start) + end = nfsi->ooo->gap[i].end; + else if (start == nfsi->ooo->gap[i].end) + start = nfsi->ooo->gap[i].start; + else + continue; + /* Remove 'i' from table and loop to insert the new range */ + cnt -= 1; + nfsi->ooo->gap[i] = nfsi->ooo->gap[cnt]; + i = -1; + } + if (start != end) { + if (cnt >= ARRAY_SIZE(nfsi->ooo->gap)) { + nfsi->cache_validity |= NFS_INO_DATA_INVAL_DEFER; + kfree(nfsi->ooo); + nfsi->ooo = NULL; + return; + } + nfsi->ooo->gap[cnt].start = start; + nfsi->ooo->gap[cnt].end = end; + cnt += 1; + } + nfsi->ooo->cnt = cnt; +} + +static void nfs_ooo_record(struct nfs_inode *nfsi, + struct nfs_fattr *fattr) +{ + /* This reply was out-of-order, so record in the + * pre/post change id, possibly cancelling + * gaps created when iversion was jumpped forward. + */ + if ((fattr->valid & NFS_ATTR_FATTR_CHANGE) && + (fattr->valid & NFS_ATTR_FATTR_PRECHANGE)) + nfs_ooo_merge(nfsi, + fattr->change_attr, + fattr->pre_change_attr); +} + static int nfs_refresh_inode_locked(struct inode *inode, struct nfs_fattr *fattr) { @@ -1826,8 +1888,12 @@ static int nfs_refresh_inode_locked(struct inode *inode, if (attr_cmp > 0 || nfs_inode_finish_partial_attr_update(fattr, inode)) ret = nfs_update_inode(inode, fattr); - else if (attr_cmp == 0) - ret = nfs_check_inode_attributes(inode, fattr); + else { + nfs_ooo_record(NFS_I(inode), fattr); + + if (attr_cmp == 0) + ret = nfs_check_inode_attributes(inode, fattr); + } trace_nfs_refresh_inode_exit(inode, ret); return ret; @@ -1918,6 +1984,8 @@ int nfs_post_op_update_inode_force_wcc_locked(struct inode *inode, struct nfs_fa if (attr_cmp < 0) return 0; if ((fattr->valid & NFS_ATTR_FATTR) == 0 || !attr_cmp) { + /* Record the pre/post change info before clearing PRECHANGE */ + nfs_ooo_record(NFS_I(inode), fattr); fattr->valid &= ~(NFS_ATTR_FATTR_PRECHANGE | NFS_ATTR_FATTR_PRESIZE | NFS_ATTR_FATTR_PREMTIME @@ -2072,6 +2140,15 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) /* More cache consistency checks */ if (fattr->valid & NFS_ATTR_FATTR_CHANGE) { + if (!have_writers && nfsi->ooo && nfsi->ooo->cnt == 1 && + nfsi->ooo->gap[0].end == inode_peek_iversion_raw(inode)) { + /* There is one remaining gap that hasn't been + * merged into iversion - do that now. + */ + inode_set_iversion_raw(inode, nfsi->ooo->gap[0].start); + kfree(nfsi->ooo); + nfsi->ooo = NULL; + } if (!inode_eq_iversion_raw(inode, fattr->change_attr)) { /* Could it be a race with writeback? */ if (!(have_writers || have_delegation)) { @@ -2093,8 +2170,11 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) dprintk("NFS: change_attr change on server for file %s/%ld\n", inode->i_sb->s_id, inode->i_ino); - } else if (!have_delegation) - nfsi->cache_validity |= NFS_INO_DATA_INVAL_DEFER; + } else if (!have_delegation) { + nfs_ooo_record(nfsi, fattr); + nfs_ooo_merge(nfsi, inode_peek_iversion_raw(inode), + fattr->change_attr); + } inode_set_iversion_raw(inode, fattr->change_attr); } } else { @@ -2248,6 +2328,7 @@ struct inode *nfs_alloc_inode(struct super_block *sb) return NULL; nfsi->flags = 0UL; nfsi->cache_validity = 0UL; + nfsi->ooo = NULL; #if IS_ENABLED(CONFIG_NFS_V4) nfsi->nfs4_acl = NULL; #endif /* CONFIG_NFS_V4 */ @@ -2262,6 +2343,7 @@ EXPORT_SYMBOL_GPL(nfs_alloc_inode); void nfs_free_inode(struct inode *inode) { + kfree(NFS_I(inode)->ooo); kmem_cache_free(nfs_inode_cachep, NFS_I(inode)); } EXPORT_SYMBOL_GPL(nfs_free_inode); diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 12bb868f9a18..279262057a92 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -195,6 +195,39 @@ struct nfs_inode { /* Open contexts for shared mmap writes */ struct list_head open_files; + /* Keep track of out-of-order replies. + * The ooo array contains start/end pairs of + * numbers from the changeid sequence when + * the inode's iversion has been updated. + * It also contains end/start pair (i.e. reverse order) + * of sections of the changeid sequence that have + * been seen in replies from the server. + * Normally these should match and when both + * A:B and B:A are found in ooo, they are both removed. + * And if a reply with A:B causes an iversion update + * of A:B, then neither are added. + * When a reply has pre_change that doesn't match + * iversion, then the changeid pair and any consequent + * change in iversion ARE added. Later replies + * might fill in the gaps, or possibly a gap is caused + * by a change from another client. + * When a file or directory is opened, if the ooo table + * is not empty, then we assume the gaps were due to + * another client and we invalidate the cached data. + * + * We can only track a limited number of concurrent gaps. + * Currently that limit is 16. + * We allocate the table on demand. If there is insufficient + * memory, then we probably cannot cache the file anyway + * so there is no loss. + */ + struct { + int cnt; + struct { + u64 start, end; + } gap[16]; + } *ooo; + #if IS_ENABLED(CONFIG_NFS_V4) struct nfs4_cached_acl *nfs4_acl; /* NFSv4 state */ @@ -612,6 +645,20 @@ nfs_fileid_to_ino_t(u64 fileid) return ino; } +static inline void nfs_ooo_clear(struct nfs_inode *nfsi) +{ + nfsi->cache_validity &= ~NFS_INO_DATA_INVAL_DEFER; + kfree(nfsi->ooo); + nfsi->ooo = NULL; +} + +static inline bool nfs_ooo_test(struct nfs_inode *nfsi) +{ + return (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER) || + (nfsi->ooo && nfsi->ooo->cnt > 0); + +} + #define NFS_JUKEBOX_RETRY_TIME (5 * HZ) /* We need to block new opens while a file is being unlinked. -- cgit v1.2.3 From d8f48fbdfd9af268e92b33462472559d2840228c Mon Sep 17 00:00:00 2001 From: Vijendar Mukunda Date: Tue, 21 Mar 2023 10:38:55 +0530 Subject: soundwire: amd: Add support for AMD Manager driver AMD ACP(v6.x) IP block has two SoundWire manager devices. Add support for - Manager driver probe & remove sequence - Helper functions to enable/disable interrupts, Initialize sdw manager, enable sdw pads - Manager driver sdw_master_ops & port_ops callbacks Signed-off-by: Vijendar Mukunda Reviewed-by: Pierre-Louis Bossart Link: https://lore.kernel.org/lkml/20230310162554.699766-3-Vijendar.Mukunda@amd.com Link: https://lore.kernel.org/r/20230321050901.115439-3-Vijendar.Mukunda@amd.com Signed-off-by: Vinod Koul --- drivers/soundwire/amd_manager.c | 670 ++++++++++++++++++++++++++++++++++++++ drivers/soundwire/amd_manager.h | 235 +++++++++++++ include/linux/soundwire/sdw_amd.h | 67 ++++ 3 files changed, 972 insertions(+) create mode 100644 drivers/soundwire/amd_manager.c create mode 100644 drivers/soundwire/amd_manager.h create mode 100644 include/linux/soundwire/sdw_amd.h (limited to 'include/linux') diff --git a/drivers/soundwire/amd_manager.c b/drivers/soundwire/amd_manager.c new file mode 100644 index 000000000000..f02522d11417 --- /dev/null +++ b/drivers/soundwire/amd_manager.c @@ -0,0 +1,670 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * SoundWire AMD Manager driver + * + * Copyright 2023 Advanced Micro Devices, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "bus.h" +#include "amd_manager.h" + +#define DRV_NAME "amd_sdw_manager" + +#define to_amd_sdw(b) container_of(b, struct amd_sdw_manager, bus) + +static void amd_enable_sdw_pads(struct amd_sdw_manager *amd_manager) +{ + u32 sw_pad_pulldown_val; + u32 val; + + mutex_lock(amd_manager->acp_sdw_lock); + val = readl(amd_manager->acp_mmio + ACP_SW_PAD_KEEPER_EN); + val |= amd_manager->reg_mask->sw_pad_enable_mask; + writel(val, amd_manager->acp_mmio + ACP_SW_PAD_KEEPER_EN); + usleep_range(1000, 1500); + + sw_pad_pulldown_val = readl(amd_manager->acp_mmio + ACP_PAD_PULLDOWN_CTRL); + sw_pad_pulldown_val &= amd_manager->reg_mask->sw_pad_pulldown_mask; + writel(sw_pad_pulldown_val, amd_manager->acp_mmio + ACP_PAD_PULLDOWN_CTRL); + mutex_unlock(amd_manager->acp_sdw_lock); +} + +static int amd_init_sdw_manager(struct amd_sdw_manager *amd_manager) +{ + u32 val; + int ret; + + writel(AMD_SDW_ENABLE, amd_manager->mmio + ACP_SW_EN); + ret = readl_poll_timeout(amd_manager->mmio + ACP_SW_EN_STATUS, val, val, ACP_DELAY_US, + AMD_SDW_TIMEOUT); + if (ret) + return ret; + + /* SoundWire manager bus reset */ + writel(AMD_SDW_BUS_RESET_REQ, amd_manager->mmio + ACP_SW_BUS_RESET_CTRL); + ret = readl_poll_timeout(amd_manager->mmio + ACP_SW_BUS_RESET_CTRL, val, + (val & AMD_SDW_BUS_RESET_DONE), ACP_DELAY_US, AMD_SDW_TIMEOUT); + if (ret) + return ret; + + writel(AMD_SDW_BUS_RESET_CLEAR_REQ, amd_manager->mmio + ACP_SW_BUS_RESET_CTRL); + ret = readl_poll_timeout(amd_manager->mmio + ACP_SW_BUS_RESET_CTRL, val, !val, + ACP_DELAY_US, AMD_SDW_TIMEOUT); + if (ret) { + dev_err(amd_manager->dev, "Failed to reset SoundWire manager instance%d\n", + amd_manager->instance); + return ret; + } + + writel(AMD_SDW_DISABLE, amd_manager->mmio + ACP_SW_EN); + return readl_poll_timeout(amd_manager->mmio + ACP_SW_EN_STATUS, val, !val, ACP_DELAY_US, + AMD_SDW_TIMEOUT); +} + +static int amd_enable_sdw_manager(struct amd_sdw_manager *amd_manager) +{ + u32 val; + + writel(AMD_SDW_ENABLE, amd_manager->mmio + ACP_SW_EN); + return readl_poll_timeout(amd_manager->mmio + ACP_SW_EN_STATUS, val, val, ACP_DELAY_US, + AMD_SDW_TIMEOUT); +} + +static int amd_disable_sdw_manager(struct amd_sdw_manager *amd_manager) +{ + u32 val; + + writel(AMD_SDW_DISABLE, amd_manager->mmio + ACP_SW_EN); + /* + * After invoking manager disable sequence, check whether + * manager has executed clock stop sequence. In this case, + * manager should ignore checking enable status register. + */ + val = readl(amd_manager->mmio + ACP_SW_CLK_RESUME_CTRL); + if (val) + return 0; + return readl_poll_timeout(amd_manager->mmio + ACP_SW_EN_STATUS, val, !val, ACP_DELAY_US, + AMD_SDW_TIMEOUT); +} + +static void amd_enable_sdw_interrupts(struct amd_sdw_manager *amd_manager) +{ + struct sdw_manager_reg_mask *reg_mask = amd_manager->reg_mask; + u32 val; + + mutex_lock(amd_manager->acp_sdw_lock); + val = readl(amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); + val |= reg_mask->acp_sdw_intr_mask; + writel(val, amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); + mutex_unlock(amd_manager->acp_sdw_lock); + + writel(AMD_SDW_IRQ_MASK_0TO7, amd_manager->mmio + + ACP_SW_STATE_CHANGE_STATUS_MASK_0TO7); + writel(AMD_SDW_IRQ_MASK_8TO11, amd_manager->mmio + + ACP_SW_STATE_CHANGE_STATUS_MASK_8TO11); + writel(AMD_SDW_IRQ_ERROR_MASK, amd_manager->mmio + ACP_SW_ERROR_INTR_MASK); +} + +static void amd_disable_sdw_interrupts(struct amd_sdw_manager *amd_manager) +{ + struct sdw_manager_reg_mask *reg_mask = amd_manager->reg_mask; + u32 val; + + mutex_lock(amd_manager->acp_sdw_lock); + val = readl(amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); + val &= ~reg_mask->acp_sdw_intr_mask; + writel(val, amd_manager->acp_mmio + ACP_EXTERNAL_INTR_CNTL(amd_manager->instance)); + mutex_unlock(amd_manager->acp_sdw_lock); + + writel(0x00, amd_manager->mmio + ACP_SW_STATE_CHANGE_STATUS_MASK_0TO7); + writel(0x00, amd_manager->mmio + ACP_SW_STATE_CHANGE_STATUS_MASK_8TO11); + writel(0x00, amd_manager->mmio + ACP_SW_ERROR_INTR_MASK); +} + +static void amd_sdw_set_frameshape(struct amd_sdw_manager *amd_manager) +{ + u32 frame_size; + + frame_size = (amd_manager->rows_index << 3) | amd_manager->cols_index; + writel(frame_size, amd_manager->mmio + ACP_SW_FRAMESIZE); +} + +static void amd_sdw_ctl_word_prep(u32 *lower_word, u32 *upper_word, struct sdw_msg *msg, + int cmd_offset) +{ + u32 upper_data; + u32 lower_data = 0; + u16 addr; + u8 upper_addr, lower_addr; + u8 data = 0; + + addr = msg->addr + cmd_offset; + upper_addr = (addr & 0xFF00) >> 8; + lower_addr = addr & 0xFF; + + if (msg->flags == SDW_MSG_FLAG_WRITE) + data = msg->buf[cmd_offset]; + + upper_data = FIELD_PREP(AMD_SDW_MCP_CMD_DEV_ADDR, msg->dev_num); + upper_data |= FIELD_PREP(AMD_SDW_MCP_CMD_COMMAND, msg->flags + 2); + upper_data |= FIELD_PREP(AMD_SDW_MCP_CMD_REG_ADDR_HIGH, upper_addr); + lower_data |= FIELD_PREP(AMD_SDW_MCP_CMD_REG_ADDR_LOW, lower_addr); + lower_data |= FIELD_PREP(AMD_SDW_MCP_CMD_REG_DATA, data); + + *upper_word = upper_data; + *lower_word = lower_data; +} + +static u64 amd_sdw_send_cmd_get_resp(struct amd_sdw_manager *amd_manager, u32 lower_data, + u32 upper_data) +{ + u64 resp; + u32 lower_resp, upper_resp; + u32 sts; + int ret; + + ret = readl_poll_timeout(amd_manager->mmio + ACP_SW_IMM_CMD_STS, sts, + !(sts & AMD_SDW_IMM_CMD_BUSY), ACP_DELAY_US, AMD_SDW_TIMEOUT); + if (ret) { + dev_err(amd_manager->dev, "SDW%x previous cmd status clear failed\n", + amd_manager->instance); + return ret; + } + + if (sts & AMD_SDW_IMM_RES_VALID) { + dev_err(amd_manager->dev, "SDW%x manager is in bad state\n", amd_manager->instance); + writel(0x00, amd_manager->mmio + ACP_SW_IMM_CMD_STS); + } + writel(upper_data, amd_manager->mmio + ACP_SW_IMM_CMD_UPPER_WORD); + writel(lower_data, amd_manager->mmio + ACP_SW_IMM_CMD_LOWER_QWORD); + + ret = readl_poll_timeout(amd_manager->mmio + ACP_SW_IMM_CMD_STS, sts, + (sts & AMD_SDW_IMM_RES_VALID), ACP_DELAY_US, AMD_SDW_TIMEOUT); + if (ret) { + dev_err(amd_manager->dev, "SDW%x cmd response timeout occurred\n", + amd_manager->instance); + return ret; + } + upper_resp = readl(amd_manager->mmio + ACP_SW_IMM_RESP_UPPER_WORD); + lower_resp = readl(amd_manager->mmio + ACP_SW_IMM_RESP_LOWER_QWORD); + + writel(AMD_SDW_IMM_RES_VALID, amd_manager->mmio + ACP_SW_IMM_CMD_STS); + ret = readl_poll_timeout(amd_manager->mmio + ACP_SW_IMM_CMD_STS, sts, + !(sts & AMD_SDW_IMM_RES_VALID), ACP_DELAY_US, AMD_SDW_TIMEOUT); + if (ret) { + dev_err(amd_manager->dev, "SDW%x cmd status retry failed\n", + amd_manager->instance); + return ret; + } + resp = upper_resp; + resp = (resp << 32) | lower_resp; + return resp; +} + +static enum sdw_command_response +amd_program_scp_addr(struct amd_sdw_manager *amd_manager, struct sdw_msg *msg) +{ + struct sdw_msg scp_msg = {0}; + u64 response_buf[2] = {0}; + u32 upper_data = 0, lower_data = 0; + int index; + + scp_msg.dev_num = msg->dev_num; + scp_msg.addr = SDW_SCP_ADDRPAGE1; + scp_msg.buf = &msg->addr_page1; + scp_msg.flags = SDW_MSG_FLAG_WRITE; + amd_sdw_ctl_word_prep(&lower_data, &upper_data, &scp_msg, 0); + response_buf[0] = amd_sdw_send_cmd_get_resp(amd_manager, lower_data, upper_data); + scp_msg.addr = SDW_SCP_ADDRPAGE2; + scp_msg.buf = &msg->addr_page2; + amd_sdw_ctl_word_prep(&lower_data, &upper_data, &scp_msg, 0); + response_buf[1] = amd_sdw_send_cmd_get_resp(amd_manager, lower_data, upper_data); + + for (index = 0; index < 2; index++) { + if (response_buf[index] == -ETIMEDOUT) { + dev_err_ratelimited(amd_manager->dev, + "SCP_addrpage command timeout for Slave %d\n", + msg->dev_num); + return SDW_CMD_TIMEOUT; + } else if (!(response_buf[index] & AMD_SDW_MCP_RESP_ACK)) { + if (response_buf[index] & AMD_SDW_MCP_RESP_NACK) { + dev_err_ratelimited(amd_manager->dev, + "SCP_addrpage NACKed for Slave %d\n", + msg->dev_num); + return SDW_CMD_FAIL; + } + dev_dbg_ratelimited(amd_manager->dev, "SCP_addrpage ignored for Slave %d\n", + msg->dev_num); + return SDW_CMD_IGNORED; + } + } + return SDW_CMD_OK; +} + +static int amd_prep_msg(struct amd_sdw_manager *amd_manager, struct sdw_msg *msg) +{ + int ret; + + if (msg->page) { + ret = amd_program_scp_addr(amd_manager, msg); + if (ret) { + msg->len = 0; + return ret; + } + } + switch (msg->flags) { + case SDW_MSG_FLAG_READ: + case SDW_MSG_FLAG_WRITE: + break; + default: + dev_err(amd_manager->dev, "Invalid msg cmd: %d\n", msg->flags); + return -EINVAL; + } + return 0; +} + +static enum sdw_command_response amd_sdw_fill_msg_resp(struct amd_sdw_manager *amd_manager, + struct sdw_msg *msg, u64 response, + int offset) +{ + if (response & AMD_SDW_MCP_RESP_ACK) { + if (msg->flags == SDW_MSG_FLAG_READ) + msg->buf[offset] = FIELD_GET(AMD_SDW_MCP_RESP_RDATA, response); + } else { + if (response == -ETIMEDOUT) { + dev_err_ratelimited(amd_manager->dev, "command timeout for Slave %d\n", + msg->dev_num); + return SDW_CMD_TIMEOUT; + } else if (response & AMD_SDW_MCP_RESP_NACK) { + dev_err_ratelimited(amd_manager->dev, + "command response NACK received for Slave %d\n", + msg->dev_num); + return SDW_CMD_FAIL; + } + dev_err_ratelimited(amd_manager->dev, "command is ignored for Slave %d\n", + msg->dev_num); + return SDW_CMD_IGNORED; + } + return SDW_CMD_OK; +} + +static unsigned int _amd_sdw_xfer_msg(struct amd_sdw_manager *amd_manager, struct sdw_msg *msg, + int cmd_offset) +{ + u64 response; + u32 upper_data = 0, lower_data = 0; + + amd_sdw_ctl_word_prep(&lower_data, &upper_data, msg, cmd_offset); + response = amd_sdw_send_cmd_get_resp(amd_manager, lower_data, upper_data); + return amd_sdw_fill_msg_resp(amd_manager, msg, response, cmd_offset); +} + +static enum sdw_command_response amd_sdw_xfer_msg(struct sdw_bus *bus, struct sdw_msg *msg) +{ + struct amd_sdw_manager *amd_manager = to_amd_sdw(bus); + int ret, i; + + ret = amd_prep_msg(amd_manager, msg); + if (ret) + return SDW_CMD_FAIL_OTHER; + for (i = 0; i < msg->len; i++) { + ret = _amd_sdw_xfer_msg(amd_manager, msg, i); + if (ret) + return ret; + } + return SDW_CMD_OK; +} + +static u32 amd_sdw_read_ping_status(struct sdw_bus *bus) +{ + struct amd_sdw_manager *amd_manager = to_amd_sdw(bus); + u64 response; + u32 slave_stat; + + response = amd_sdw_send_cmd_get_resp(amd_manager, 0, 0); + /* slave status from ping response */ + slave_stat = FIELD_GET(AMD_SDW_MCP_SLAVE_STAT_0_3, response); + slave_stat |= FIELD_GET(AMD_SDW_MCP_SLAVE_STAT_4_11, response) << 8; + dev_dbg(amd_manager->dev, "slave_stat:0x%x\n", slave_stat); + return slave_stat; +} + +static int amd_sdw_compute_params(struct sdw_bus *bus) +{ + struct sdw_transport_data t_data = {0}; + struct sdw_master_runtime *m_rt; + struct sdw_port_runtime *p_rt; + struct sdw_bus_params *b_params = &bus->params; + int port_bo, hstart, hstop, sample_int; + unsigned int rate, bps; + + port_bo = 0; + hstart = 1; + hstop = bus->params.col - 1; + t_data.hstop = hstop; + t_data.hstart = hstart; + + list_for_each_entry(m_rt, &bus->m_rt_list, bus_node) { + rate = m_rt->stream->params.rate; + bps = m_rt->stream->params.bps; + sample_int = (bus->params.curr_dr_freq / rate); + list_for_each_entry(p_rt, &m_rt->port_list, port_node) { + port_bo = (p_rt->num * 64) + 1; + dev_dbg(bus->dev, "p_rt->num=%d hstart=%d hstop=%d port_bo=%d\n", + p_rt->num, hstart, hstop, port_bo); + sdw_fill_xport_params(&p_rt->transport_params, p_rt->num, + false, SDW_BLK_GRP_CNT_1, sample_int, + port_bo, port_bo >> 8, hstart, hstop, + SDW_BLK_PKG_PER_PORT, 0x0); + + sdw_fill_port_params(&p_rt->port_params, + p_rt->num, bps, + SDW_PORT_FLOW_MODE_ISOCH, + b_params->m_data_mode); + t_data.hstart = hstart; + t_data.hstop = hstop; + t_data.block_offset = port_bo; + t_data.sub_block_offset = 0; + } + sdw_compute_slave_ports(m_rt, &t_data); + } + return 0; +} + +static int amd_sdw_port_params(struct sdw_bus *bus, struct sdw_port_params *p_params, + unsigned int bank) +{ + struct amd_sdw_manager *amd_manager = to_amd_sdw(bus); + u32 frame_fmt_reg, dpn_frame_fmt; + + dev_dbg(amd_manager->dev, "p_params->num:0x%x\n", p_params->num); + switch (amd_manager->instance) { + case ACP_SDW0: + frame_fmt_reg = sdw0_manager_dp_reg[p_params->num].frame_fmt_reg; + break; + case ACP_SDW1: + frame_fmt_reg = sdw1_manager_dp_reg[p_params->num].frame_fmt_reg; + break; + default: + return -EINVAL; + } + + dpn_frame_fmt = readl(amd_manager->mmio + frame_fmt_reg); + u32p_replace_bits(&dpn_frame_fmt, p_params->flow_mode, AMD_DPN_FRAME_FMT_PFM); + u32p_replace_bits(&dpn_frame_fmt, p_params->data_mode, AMD_DPN_FRAME_FMT_PDM); + u32p_replace_bits(&dpn_frame_fmt, p_params->bps - 1, AMD_DPN_FRAME_FMT_WORD_LEN); + writel(dpn_frame_fmt, amd_manager->mmio + frame_fmt_reg); + return 0; +} + +static int amd_sdw_transport_params(struct sdw_bus *bus, + struct sdw_transport_params *params, + enum sdw_reg_bank bank) +{ + struct amd_sdw_manager *amd_manager = to_amd_sdw(bus); + u32 dpn_frame_fmt; + u32 dpn_sampleinterval; + u32 dpn_hctrl; + u32 dpn_offsetctrl; + u32 dpn_lanectrl; + u32 frame_fmt_reg, sample_int_reg, hctrl_dp0_reg; + u32 offset_reg, lane_ctrl_ch_en_reg; + + switch (amd_manager->instance) { + case ACP_SDW0: + frame_fmt_reg = sdw0_manager_dp_reg[params->port_num].frame_fmt_reg; + sample_int_reg = sdw0_manager_dp_reg[params->port_num].sample_int_reg; + hctrl_dp0_reg = sdw0_manager_dp_reg[params->port_num].hctrl_dp0_reg; + offset_reg = sdw0_manager_dp_reg[params->port_num].offset_reg; + lane_ctrl_ch_en_reg = sdw0_manager_dp_reg[params->port_num].lane_ctrl_ch_en_reg; + break; + case ACP_SDW1: + frame_fmt_reg = sdw1_manager_dp_reg[params->port_num].frame_fmt_reg; + sample_int_reg = sdw1_manager_dp_reg[params->port_num].sample_int_reg; + hctrl_dp0_reg = sdw1_manager_dp_reg[params->port_num].hctrl_dp0_reg; + offset_reg = sdw1_manager_dp_reg[params->port_num].offset_reg; + lane_ctrl_ch_en_reg = sdw1_manager_dp_reg[params->port_num].lane_ctrl_ch_en_reg; + break; + default: + return -EINVAL; + } + writel(AMD_SDW_SSP_COUNTER_VAL, amd_manager->mmio + ACP_SW_SSP_COUNTER); + + dpn_frame_fmt = readl(amd_manager->mmio + frame_fmt_reg); + u32p_replace_bits(&dpn_frame_fmt, params->blk_pkg_mode, AMD_DPN_FRAME_FMT_BLK_PKG_MODE); + u32p_replace_bits(&dpn_frame_fmt, params->blk_grp_ctrl, AMD_DPN_FRAME_FMT_BLK_GRP_CTRL); + u32p_replace_bits(&dpn_frame_fmt, SDW_STREAM_PCM, AMD_DPN_FRAME_FMT_PCM_OR_PDM); + writel(dpn_frame_fmt, amd_manager->mmio + frame_fmt_reg); + + dpn_sampleinterval = params->sample_interval - 1; + writel(dpn_sampleinterval, amd_manager->mmio + sample_int_reg); + + dpn_hctrl = FIELD_PREP(AMD_DPN_HCTRL_HSTOP, params->hstop); + dpn_hctrl |= FIELD_PREP(AMD_DPN_HCTRL_HSTART, params->hstart); + writel(dpn_hctrl, amd_manager->mmio + hctrl_dp0_reg); + + dpn_offsetctrl = FIELD_PREP(AMD_DPN_OFFSET_CTRL_1, params->offset1); + dpn_offsetctrl |= FIELD_PREP(AMD_DPN_OFFSET_CTRL_2, params->offset2); + writel(dpn_offsetctrl, amd_manager->mmio + offset_reg); + + /* + * lane_ctrl_ch_en_reg will be used to program lane_ctrl and ch_mask + * parameters. + */ + dpn_lanectrl = readl(amd_manager->mmio + lane_ctrl_ch_en_reg); + u32p_replace_bits(&dpn_lanectrl, params->lane_ctrl, AMD_DPN_CH_EN_LCTRL); + writel(dpn_lanectrl, amd_manager->mmio + lane_ctrl_ch_en_reg); + return 0; +} + +static int amd_sdw_port_enable(struct sdw_bus *bus, + struct sdw_enable_ch *enable_ch, + unsigned int bank) +{ + struct amd_sdw_manager *amd_manager = to_amd_sdw(bus); + u32 dpn_ch_enable; + u32 lane_ctrl_ch_en_reg; + + switch (amd_manager->instance) { + case ACP_SDW0: + lane_ctrl_ch_en_reg = sdw0_manager_dp_reg[enable_ch->port_num].lane_ctrl_ch_en_reg; + break; + case ACP_SDW1: + lane_ctrl_ch_en_reg = sdw1_manager_dp_reg[enable_ch->port_num].lane_ctrl_ch_en_reg; + break; + default: + return -EINVAL; + } + + /* + * lane_ctrl_ch_en_reg will be used to program lane_ctrl and ch_mask + * parameters. + */ + dpn_ch_enable = readl(amd_manager->mmio + lane_ctrl_ch_en_reg); + u32p_replace_bits(&dpn_ch_enable, enable_ch->ch_mask, AMD_DPN_CH_EN_CHMASK); + if (enable_ch->enable) + writel(dpn_ch_enable, amd_manager->mmio + lane_ctrl_ch_en_reg); + else + writel(0, amd_manager->mmio + lane_ctrl_ch_en_reg); + return 0; +} + +static int sdw_master_read_amd_prop(struct sdw_bus *bus) +{ + struct amd_sdw_manager *amd_manager = to_amd_sdw(bus); + struct fwnode_handle *link; + struct sdw_master_prop *prop; + u32 quirk_mask = 0; + u32 wake_en_mask = 0; + u32 power_mode_mask = 0; + char name[32]; + + prop = &bus->prop; + /* Find manager handle */ + snprintf(name, sizeof(name), "mipi-sdw-link-%d-subproperties", bus->link_id); + link = device_get_named_child_node(bus->dev, name); + if (!link) { + dev_err(bus->dev, "Manager node %s not found\n", name); + return -EIO; + } + fwnode_property_read_u32(link, "amd-sdw-enable", &quirk_mask); + if (!(quirk_mask & AMD_SDW_QUIRK_MASK_BUS_ENABLE)) + prop->hw_disabled = true; + prop->quirks = SDW_MASTER_QUIRKS_CLEAR_INITIAL_CLASH | + SDW_MASTER_QUIRKS_CLEAR_INITIAL_PARITY; + + fwnode_property_read_u32(link, "amd-sdw-wakeup-enable", &wake_en_mask); + amd_manager->wake_en_mask = wake_en_mask; + fwnode_property_read_u32(link, "amd-sdw-power-mode", &power_mode_mask); + amd_manager->power_mode_mask = power_mode_mask; + return 0; +} + +static int amd_prop_read(struct sdw_bus *bus) +{ + sdw_master_read_prop(bus); + sdw_master_read_amd_prop(bus); + return 0; +} + +static const struct sdw_master_port_ops amd_sdw_port_ops = { + .dpn_set_port_params = amd_sdw_port_params, + .dpn_set_port_transport_params = amd_sdw_transport_params, + .dpn_port_enable_ch = amd_sdw_port_enable, +}; + +static const struct sdw_master_ops amd_sdw_ops = { + .read_prop = amd_prop_read, + .xfer_msg = amd_sdw_xfer_msg, + .read_ping_status = amd_sdw_read_ping_status, +}; + +static void amd_sdw_probe_work(struct work_struct *work) +{ + struct amd_sdw_manager *amd_manager = container_of(work, struct amd_sdw_manager, + probe_work); + struct sdw_master_prop *prop; + int ret; + + prop = &amd_manager->bus.prop; + if (!prop->hw_disabled) { + amd_enable_sdw_pads(amd_manager); + ret = amd_init_sdw_manager(amd_manager); + if (ret) + return; + amd_enable_sdw_interrupts(amd_manager); + ret = amd_enable_sdw_manager(amd_manager); + if (ret) + return; + amd_sdw_set_frameshape(amd_manager); + } +} + +static int amd_sdw_manager_probe(struct platform_device *pdev) +{ + const struct acp_sdw_pdata *pdata = pdev->dev.platform_data; + struct resource *res; + struct device *dev = &pdev->dev; + struct sdw_master_prop *prop; + struct sdw_bus_params *params; + struct amd_sdw_manager *amd_manager; + int ret; + + amd_manager = devm_kzalloc(dev, sizeof(struct amd_sdw_manager), GFP_KERNEL); + if (!amd_manager) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -ENOMEM; + + amd_manager->acp_mmio = devm_ioremap(dev, res->start, resource_size(res)); + if (IS_ERR(amd_manager->mmio)) { + dev_err(dev, "mmio not found\n"); + return PTR_ERR(amd_manager->mmio); + } + amd_manager->instance = pdata->instance; + amd_manager->mmio = amd_manager->acp_mmio + + (amd_manager->instance * SDW_MANAGER_REG_OFFSET); + amd_manager->acp_sdw_lock = pdata->acp_sdw_lock; + amd_manager->cols_index = sdw_find_col_index(AMD_SDW_DEFAULT_COLUMNS); + amd_manager->rows_index = sdw_find_row_index(AMD_SDW_DEFAULT_ROWS); + amd_manager->dev = dev; + amd_manager->bus.ops = &amd_sdw_ops; + amd_manager->bus.port_ops = &amd_sdw_port_ops; + amd_manager->bus.compute_params = &amd_sdw_compute_params; + amd_manager->bus.clk_stop_timeout = 200; + amd_manager->bus.link_id = amd_manager->instance; + + switch (amd_manager->instance) { + case ACP_SDW0: + amd_manager->num_dout_ports = AMD_SDW0_MAX_TX_PORTS; + amd_manager->num_din_ports = AMD_SDW0_MAX_RX_PORTS; + break; + case ACP_SDW1: + amd_manager->num_dout_ports = AMD_SDW1_MAX_TX_PORTS; + amd_manager->num_din_ports = AMD_SDW1_MAX_RX_PORTS; + break; + default: + return -EINVAL; + } + + amd_manager->reg_mask = &sdw_manager_reg_mask_array[amd_manager->instance]; + params = &amd_manager->bus.params; + params->max_dr_freq = AMD_SDW_DEFAULT_CLK_FREQ * 2; + params->curr_dr_freq = AMD_SDW_DEFAULT_CLK_FREQ * 2; + params->col = AMD_SDW_DEFAULT_COLUMNS; + params->row = AMD_SDW_DEFAULT_ROWS; + prop = &amd_manager->bus.prop; + prop->clk_freq = &amd_sdw_freq_tbl[0]; + prop->mclk_freq = AMD_SDW_BUS_BASE_FREQ; + + ret = sdw_bus_master_add(&amd_manager->bus, dev, dev->fwnode); + if (ret) { + dev_err(dev, "Failed to register SoundWire manager(%d)\n", ret); + return ret; + } + dev_set_drvdata(dev, amd_manager); + INIT_WORK(&amd_manager->probe_work, amd_sdw_probe_work); + /* + * Instead of having lengthy probe sequence, use deferred probe. + */ + schedule_work(&amd_manager->probe_work); + return 0; +} + +static int amd_sdw_manager_remove(struct platform_device *pdev) +{ + struct amd_sdw_manager *amd_manager = dev_get_drvdata(&pdev->dev); + + cancel_work_sync(&amd_manager->probe_work); + amd_disable_sdw_interrupts(amd_manager); + sdw_bus_master_delete(&amd_manager->bus); + return amd_disable_sdw_manager(amd_manager); +} + +static struct platform_driver amd_sdw_driver = { + .probe = &amd_sdw_manager_probe, + .remove = &amd_sdw_manager_remove, + .driver = { + .name = "amd_sdw_manager", + } +}; +module_platform_driver(amd_sdw_driver); + +MODULE_AUTHOR("Vijendar.Mukunda@amd.com"); +MODULE_DESCRIPTION("AMD SoundWire driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/soundwire/amd_manager.h b/drivers/soundwire/amd_manager.h new file mode 100644 index 000000000000..f7106521b500 --- /dev/null +++ b/drivers/soundwire/amd_manager.h @@ -0,0 +1,235 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved. + */ + +#ifndef __AMD_MANAGER_H +#define __AMD_MANAGER_H + +#include + +#define SDW_MANAGER_REG_OFFSET 0xc00 +#define AMD_SDW_DEFAULT_ROWS 50 +#define AMD_SDW_DEFAULT_COLUMNS 10 +#define ACP_PAD_PULLDOWN_CTRL 0x0001448 +#define ACP_SW_PAD_KEEPER_EN 0x0001454 +#define ACP_SW0_WAKE_EN 0x0001458 +#define ACP_EXTERNAL_INTR_CNTL0 0x0001a04 +#define ACP_EXTERNAL_INTR_STAT0 0x0001a0c +#define ACP_EXTERNAL_INTR_CNTL(i) (ACP_EXTERNAL_INTR_CNTL0 + ((i) * 4)) +#define ACP_EXTERNAL_INTR_STAT(i) (ACP_EXTERNAL_INTR_STAT0 + ((i) * 4)) +#define ACP_SW_WAKE_EN(i) (ACP_SW0_WAKE_EN + ((i) * 8)) + +#define ACP_SW_EN 0x0003000 +#define ACP_SW_EN_STATUS 0x0003004 +#define ACP_SW_FRAMESIZE 0x0003008 +#define ACP_SW_SSP_COUNTER 0x000300c +#define ACP_SW_AUDIO0_TX_EN 0x0003010 +#define ACP_SW_AUDIO0_TX_EN_STATUS 0x0003014 +#define ACP_SW_AUDIO0_TX_FRAME_FORMAT 0x0003018 +#define ACP_SW_AUDIO0_TX_SAMPLEINTERVAL 0x000301c +#define ACP_SW_AUDIO0_TX_HCTRL_DP0 0x0003020 +#define ACP_SW_AUDIO0_TX_HCTRL_DP1 0x0003024 +#define ACP_SW_AUDIO0_TX_HCTRL_DP2 0x0003028 +#define ACP_SW_AUDIO0_TX_HCTRL_DP3 0x000302c +#define ACP_SW_AUDIO0_TX_OFFSET_DP0 0x0003030 +#define ACP_SW_AUDIO0_TX_OFFSET_DP1 0x0003034 +#define ACP_SW_AUDIO0_TX_OFFSET_DP2 0x0003038 +#define ACP_SW_AUDIO0_TX_OFFSET_DP3 0x000303c +#define ACP_SW_AUDIO0_TX_CHANNEL_ENABLE_DP0 0x0003040 +#define ACP_SW_AUDIO0_TX_CHANNEL_ENABLE_DP1 0x0003044 +#define ACP_SW_AUDIO0_TX_CHANNEL_ENABLE_DP2 0x0003048 +#define ACP_SW_AUDIO0_TX_CHANNEL_ENABLE_DP3 0x000304c +#define ACP_SW_AUDIO1_TX_EN 0x0003050 +#define ACP_SW_AUDIO1_TX_EN_STATUS 0x0003054 +#define ACP_SW_AUDIO1_TX_FRAME_FORMAT 0x0003058 +#define ACP_SW_AUDIO1_TX_SAMPLEINTERVAL 0x000305c +#define ACP_SW_AUDIO1_TX_HCTRL 0x0003060 +#define ACP_SW_AUDIO1_TX_OFFSET 0x0003064 +#define ACP_SW_AUDIO1_TX_CHANNEL_ENABLE_DP0 0x0003068 +#define ACP_SW_AUDIO2_TX_EN 0x000306c +#define ACP_SW_AUDIO2_TX_EN_STATUS 0x0003070 +#define ACP_SW_AUDIO2_TX_FRAME_FORMAT 0x0003074 +#define ACP_SW_AUDIO2_TX_SAMPLEINTERVAL 0x0003078 +#define ACP_SW_AUDIO2_TX_HCTRL 0x000307c +#define ACP_SW_AUDIO2_TX_OFFSET 0x0003080 +#define ACP_SW_AUDIO2_TX_CHANNEL_ENABLE_DP0 0x0003084 +#define ACP_SW_AUDIO0_RX_EN 0x0003088 +#define ACP_SW_AUDIO0_RX_EN_STATUS 0x000308c +#define ACP_SW_AUDIO0_RX_FRAME_FORMAT 0x0003090 +#define ACP_SW_AUDIO0_RX_SAMPLEINTERVAL 0x0003094 +#define ACP_SW_AUDIO0_RX_HCTRL_DP0 0x0003098 +#define ACP_SW_AUDIO0_RX_HCTRL_DP1 0x000309c +#define ACP_SW_AUDIO0_RX_HCTRL_DP2 0x0003100 +#define ACP_SW_AUDIO0_RX_HCTRL_DP3 0x0003104 +#define ACP_SW_AUDIO0_RX_OFFSET_DP0 0x0003108 +#define ACP_SW_AUDIO0_RX_OFFSET_DP1 0x000310c +#define ACP_SW_AUDIO0_RX_OFFSET_DP2 0x0003110 +#define ACP_SW_AUDIO0_RX_OFFSET_DP3 0x0003114 +#define ACP_SW_AUDIO0_RX_CHANNEL_ENABLE_DP0 0x0003118 +#define ACP_SW_AUDIO0_RX_CHANNEL_ENABLE_DP1 0x000311c +#define ACP_SW_AUDIO0_RX_CHANNEL_ENABLE_DP2 0x0003120 +#define ACP_SW_AUDIO0_RX_CHANNEL_ENABLE_DP3 0x0003124 +#define ACP_SW_AUDIO1_RX_EN 0x0003128 +#define ACP_SW_AUDIO1_RX_EN_STATUS 0x000312c +#define ACP_SW_AUDIO1_RX_FRAME_FORMAT 0x0003130 +#define ACP_SW_AUDIO1_RX_SAMPLEINTERVAL 0x0003134 +#define ACP_SW_AUDIO1_RX_HCTRL 0x0003138 +#define ACP_SW_AUDIO1_RX_OFFSET 0x000313c +#define ACP_SW_AUDIO1_RX_CHANNEL_ENABLE_DP0 0x0003140 +#define ACP_SW_AUDIO2_RX_EN 0x0003144 +#define ACP_SW_AUDIO2_RX_EN_STATUS 0x0003148 +#define ACP_SW_AUDIO2_RX_FRAME_FORMAT 0x000314c +#define ACP_SW_AUDIO2_RX_SAMPLEINTERVAL 0x0003150 +#define ACP_SW_AUDIO2_RX_HCTRL 0x0003154 +#define ACP_SW_AUDIO2_RX_OFFSET 0x0003158 +#define ACP_SW_AUDIO2_RX_CHANNEL_ENABLE_DP0 0x000315c +#define ACP_SW_BPT_PORT_EN 0x0003160 +#define ACP_SW_BPT_PORT_EN_STATUS 0x0003164 +#define ACP_SW_BPT_PORT_FRAME_FORMAT 0x0003168 +#define ACP_SW_BPT_PORT_SAMPLEINTERVAL 0x000316c +#define ACP_SW_BPT_PORT_HCTRL 0x0003170 +#define ACP_SW_BPT_PORT_OFFSET 0x0003174 +#define ACP_SW_BPT_PORT_CHANNEL_ENABLE 0x0003178 +#define ACP_SW_BPT_PORT_FIRST_BYTE_ADDR 0x000317c +#define ACP_SW_CLK_RESUME_CTRL 0x0003180 +#define ACP_SW_CLK_RESUME_DELAY_CNTR 0x0003184 +#define ACP_SW_BUS_RESET_CTRL 0x0003188 +#define ACP_SW_PRBS_ERR_STATUS 0x000318c +#define ACP_SW_IMM_CMD_UPPER_WORD 0x0003230 +#define ACP_SW_IMM_CMD_LOWER_QWORD 0x0003234 +#define ACP_SW_IMM_RESP_UPPER_WORD 0x0003238 +#define ACP_SW_IMM_RESP_LOWER_QWORD 0x000323c +#define ACP_SW_IMM_CMD_STS 0x0003240 +#define ACP_SW_BRA_BASE_ADDRESS 0x0003244 +#define ACP_SW_BRA_TRANSFER_SIZE 0x0003248 +#define ACP_SW_BRA_DMA_BUSY 0x000324c +#define ACP_SW_BRA_RESP 0x0003250 +#define ACP_SW_BRA_RESP_FRAME_ADDR 0x0003254 +#define ACP_SW_BRA_CURRENT_TRANSFER_SIZE 0x0003258 +#define ACP_SW_STATE_CHANGE_STATUS_0TO7 0x000325c +#define ACP_SW_STATE_CHANGE_STATUS_8TO11 0x0003260 +#define ACP_SW_STATE_CHANGE_STATUS_MASK_0TO7 0x0003264 +#define ACP_SW_STATE_CHANGE_STATUS_MASK_8TO11 0x0003268 +#define ACP_SW_CLK_FREQUENCY_CTRL 0x000326c +#define ACP_SW_ERROR_INTR_MASK 0x0003270 +#define ACP_SW_PHY_TEST_MODE_DATA_OFF 0x0003274 + +#define ACP_DELAY_US 10 +#define AMD_SDW_TIMEOUT 1000 +#define AMD_SDW_DEFAULT_CLK_FREQ 12000000 + +#define AMD_SDW_MCP_RESP_ACK BIT(0) +#define AMD_SDW_MCP_RESP_NACK BIT(1) +#define AMD_SDW_MCP_RESP_RDATA GENMASK(14, 7) + +#define AMD_SDW_MCP_CMD_SSP_TAG BIT(31) +#define AMD_SDW_MCP_CMD_COMMAND GENMASK(14, 12) +#define AMD_SDW_MCP_CMD_DEV_ADDR GENMASK(11, 8) +#define AMD_SDW_MCP_CMD_REG_ADDR_HIGH GENMASK(7, 0) +#define AMD_SDW_MCP_CMD_REG_ADDR_LOW GENMASK(31, 24) +#define AMD_SDW_MCP_CMD_REG_DATA GENMASK(14, 7) +#define AMD_SDW_MCP_SLAVE_STAT_0_3 GENMASK(14, 7) +#define AMD_SDW_MCP_SLAVE_STAT_4_11 GENMASK_ULL(39, 24) +#define AMD_SDW_MCP_SLAVE_STATUS_MASK GENMASK(1, 0) +#define AMD_SDW_MCP_SLAVE_STATUS_BITS GENMASK(3, 2) +#define AMD_SDW_MCP_SLAVE_STATUS_8TO_11 GENMASK_ULL(15, 0) +#define AMD_SDW_MCP_SLAVE_STATUS_VALID_MASK(x) BIT(((x) * 4)) +#define AMD_SDW_MCP_SLAVE_STAT_SHIFT_MASK(x) (((x) * 4) + 1) + +#define AMD_SDW_MASTER_SUSPEND_DELAY_MS 2000 +#define AMD_SDW_QUIRK_MASK_BUS_ENABLE BIT(0) + +#define AMD_SDW_IMM_RES_VALID 1 +#define AMD_SDW_IMM_CMD_BUSY 2 +#define AMD_SDW_ENABLE 1 +#define AMD_SDW_DISABLE 0 +#define AMD_SDW_BUS_RESET_CLEAR_REQ 0 +#define AMD_SDW_BUS_RESET_REQ 1 +#define AMD_SDW_BUS_RESET_DONE 2 +#define AMD_SDW_BUS_BASE_FREQ 24000000 + +#define AMD_SDW0_EXT_INTR_MASK 0x200000 +#define AMD_SDW1_EXT_INTR_MASK 4 +#define AMD_SDW_IRQ_MASK_0TO7 0x77777777 +#define AMD_SDW_IRQ_MASK_8TO11 0x000d7777 +#define AMD_SDW_IRQ_ERROR_MASK 0xff +#define AMD_SDW_MAX_FREQ_NUM 1 +#define AMD_SDW0_MAX_TX_PORTS 3 +#define AMD_SDW0_MAX_RX_PORTS 3 +#define AMD_SDW1_MAX_TX_PORTS 1 +#define AMD_SDW1_MAX_RX_PORTS 1 +#define AMD_SDW0_MAX_DAI 6 +#define AMD_SDW1_MAX_DAI 2 +#define AMD_SDW_SLAVE_0_ATTACHED 5 +#define AMD_SDW_SSP_COUNTER_VAL 3 + +#define AMD_DPN_FRAME_FMT_PFM GENMASK(1, 0) +#define AMD_DPN_FRAME_FMT_PDM GENMASK(3, 2) +#define AMD_DPN_FRAME_FMT_BLK_PKG_MODE BIT(4) +#define AMD_DPN_FRAME_FMT_BLK_GRP_CTRL GENMASK(6, 5) +#define AMD_DPN_FRAME_FMT_WORD_LEN GENMASK(12, 7) +#define AMD_DPN_FRAME_FMT_PCM_OR_PDM BIT(13) +#define AMD_DPN_HCTRL_HSTOP GENMASK(3, 0) +#define AMD_DPN_HCTRL_HSTART GENMASK(7, 4) +#define AMD_DPN_OFFSET_CTRL_1 GENMASK(7, 0) +#define AMD_DPN_OFFSET_CTRL_2 GENMASK(15, 8) +#define AMD_DPN_CH_EN_LCTRL GENMASK(2, 0) +#define AMD_DPN_CH_EN_CHMASK GENMASK(10, 3) +#define AMD_SDW_STAT_MAX_RETRY_COUNT 100 +#define AMD_SDW0_PAD_PULLDOWN_CTRL_ENABLE_MASK 0x7f9f +#define AMD_SDW1_PAD_PULLDOWN_CTRL_ENABLE_MASK 0x7ffa +#define AMD_SDW0_PAD_PULLDOWN_CTRL_DISABLE_MASK 0x60 +#define AMD_SDW1_PAD_PULLDOWN_CTRL_DISABLE_MASK 5 +#define AMD_SDW0_PAD_KEEPER_EN_MASK 1 +#define AMD_SDW1_PAD_KEEPER_EN_MASK 0x10 +#define AMD_SDW0_PAD_KEEPER_DISABLE_MASK 0x1e +#define AMD_SDW1_PAD_KEEPER_DISABLE_MASK 0xf + +static u32 amd_sdw_freq_tbl[AMD_SDW_MAX_FREQ_NUM] = { + AMD_SDW_DEFAULT_CLK_FREQ, +}; + +struct sdw_manager_dp_reg { + u32 frame_fmt_reg; + u32 sample_int_reg; + u32 hctrl_dp0_reg; + u32 offset_reg; + u32 lane_ctrl_ch_en_reg; +}; + +static struct sdw_manager_dp_reg sdw0_manager_dp_reg[AMD_SDW0_MAX_DAI] = { + {ACP_SW_AUDIO0_TX_FRAME_FORMAT, ACP_SW_AUDIO0_TX_SAMPLEINTERVAL, ACP_SW_AUDIO0_TX_HCTRL_DP0, + ACP_SW_AUDIO0_TX_OFFSET_DP0, ACP_SW_AUDIO0_TX_CHANNEL_ENABLE_DP0}, + {ACP_SW_AUDIO1_TX_FRAME_FORMAT, ACP_SW_AUDIO1_TX_SAMPLEINTERVAL, ACP_SW_AUDIO1_TX_HCTRL, + ACP_SW_AUDIO1_TX_OFFSET, ACP_SW_AUDIO1_TX_CHANNEL_ENABLE_DP0}, + {ACP_SW_AUDIO2_TX_FRAME_FORMAT, ACP_SW_AUDIO2_TX_SAMPLEINTERVAL, ACP_SW_AUDIO2_TX_HCTRL, + ACP_SW_AUDIO2_TX_OFFSET, ACP_SW_AUDIO2_TX_CHANNEL_ENABLE_DP0}, + {ACP_SW_AUDIO0_RX_FRAME_FORMAT, ACP_SW_AUDIO0_RX_SAMPLEINTERVAL, ACP_SW_AUDIO0_RX_HCTRL_DP0, + ACP_SW_AUDIO0_RX_OFFSET_DP0, ACP_SW_AUDIO0_RX_CHANNEL_ENABLE_DP0}, + {ACP_SW_AUDIO1_RX_FRAME_FORMAT, ACP_SW_AUDIO1_RX_SAMPLEINTERVAL, ACP_SW_AUDIO1_RX_HCTRL, + ACP_SW_AUDIO1_RX_OFFSET, ACP_SW_AUDIO1_RX_CHANNEL_ENABLE_DP0}, + {ACP_SW_AUDIO2_RX_FRAME_FORMAT, ACP_SW_AUDIO2_RX_SAMPLEINTERVAL, ACP_SW_AUDIO2_RX_HCTRL, + ACP_SW_AUDIO2_RX_OFFSET, ACP_SW_AUDIO2_RX_CHANNEL_ENABLE_DP0}, +}; + +static struct sdw_manager_dp_reg sdw1_manager_dp_reg[AMD_SDW1_MAX_DAI] = { + {ACP_SW_AUDIO1_TX_FRAME_FORMAT, ACP_SW_AUDIO1_TX_SAMPLEINTERVAL, ACP_SW_AUDIO1_TX_HCTRL, + ACP_SW_AUDIO1_TX_OFFSET, ACP_SW_AUDIO1_TX_CHANNEL_ENABLE_DP0}, + {ACP_SW_AUDIO1_RX_FRAME_FORMAT, ACP_SW_AUDIO1_RX_SAMPLEINTERVAL, ACP_SW_AUDIO1_RX_HCTRL, + ACP_SW_AUDIO1_RX_OFFSET, ACP_SW_AUDIO1_RX_CHANNEL_ENABLE_DP0} +}; + +static struct sdw_manager_reg_mask sdw_manager_reg_mask_array[2] = { + { + AMD_SDW0_PAD_KEEPER_EN_MASK, + AMD_SDW0_PAD_PULLDOWN_CTRL_ENABLE_MASK, + AMD_SDW0_EXT_INTR_MASK + }, + { + AMD_SDW1_PAD_KEEPER_EN_MASK, + AMD_SDW1_PAD_PULLDOWN_CTRL_ENABLE_MASK, + AMD_SDW1_EXT_INTR_MASK + } +}; +#endif diff --git a/include/linux/soundwire/sdw_amd.h b/include/linux/soundwire/sdw_amd.h new file mode 100644 index 000000000000..c14a291a40e8 --- /dev/null +++ b/include/linux/soundwire/sdw_amd.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved. + */ + +#ifndef __SDW_AMD_H +#define __SDW_AMD_H + +#include + +#define ACP_SDW0 0 +#define ACP_SDW1 1 + +struct acp_sdw_pdata { + u16 instance; + /* mutex to protect acp common register access */ + struct mutex *acp_sdw_lock; +}; + +struct sdw_manager_reg_mask { + u32 sw_pad_enable_mask; + u32 sw_pad_pulldown_mask; + u32 acp_sdw_intr_mask; +}; + +/** + * struct amd_sdw_manager - amd manager driver context + * @bus: bus handle + * @dev: linux device + * @mmio: SoundWire registers mmio base + * @acp_mmio: acp registers mmio base + * @reg_mask: register mask structure per manager instance + * @probe_work: SoundWire manager probe workqueue + * @acp_sdw_lock: mutex to protect acp share register access + * @num_din_ports: number of input ports + * @num_dout_ports: number of output ports + * @cols_index: Column index in frame shape + * @rows_index: Rows index in frame shape + * @instance: SoundWire manager instance + * @quirks: SoundWire manager quirks + * @wake_en_mask: wake enable mask per SoundWire manager + * @power_mode_mask: flag interprets amd SoundWire manager power mode + */ +struct amd_sdw_manager { + struct sdw_bus bus; + struct device *dev; + + void __iomem *mmio; + void __iomem *acp_mmio; + + struct sdw_manager_reg_mask *reg_mask; + struct work_struct probe_work; + /* mutex to protect acp common register access */ + struct mutex *acp_sdw_lock; + + int num_din_ports; + int num_dout_ports; + + int cols_index; + int rows_index; + + u32 instance; + u32 quirks; + u32 wake_en_mask; + u32 power_mode_mask; +}; +#endif -- cgit v1.2.3 From 2b13596f7c9c03b1653b21a440ce672c3d7e5233 Mon Sep 17 00:00:00 2001 From: Vijendar Mukunda Date: Tue, 21 Mar 2023 10:38:56 +0530 Subject: soundwire: amd: register SoundWire manager dai ops Register dai ops for SoundWire manager instances. Signed-off-by: Vijendar Mukunda Reviewed-by: Pierre-Louis Bossart Link: https://lore.kernel.org/lkml/20230227154801.50319-4-Vijendar.Mukunda@amd.com Link: https://lore.kernel.org/r/20230321050901.115439-4-Vijendar.Mukunda@amd.com Signed-off-by: Vinod Koul --- drivers/soundwire/amd_manager.c | 178 ++++++++++++++++++++++++++++++++++++++ drivers/soundwire/amd_manager.h | 18 ++++ include/linux/soundwire/sdw_amd.h | 18 ++++ 3 files changed, 214 insertions(+) (limited to 'include/linux') diff --git a/drivers/soundwire/amd_manager.c b/drivers/soundwire/amd_manager.c index f02522d11417..f12dad6c3226 100644 --- a/drivers/soundwire/amd_manager.c +++ b/drivers/soundwire/amd_manager.c @@ -551,6 +551,178 @@ static const struct sdw_master_ops amd_sdw_ops = { .read_ping_status = amd_sdw_read_ping_status, }; +static int amd_sdw_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct amd_sdw_manager *amd_manager = snd_soc_dai_get_drvdata(dai); + struct sdw_amd_dai_runtime *dai_runtime; + struct sdw_stream_config sconfig; + struct sdw_port_config *pconfig; + int ch, dir; + int ret; + + dai_runtime = amd_manager->dai_runtime_array[dai->id]; + if (!dai_runtime) + return -EIO; + + ch = params_channels(params); + if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) + dir = SDW_DATA_DIR_RX; + else + dir = SDW_DATA_DIR_TX; + dev_dbg(amd_manager->dev, "dir:%d dai->id:0x%x\n", dir, dai->id); + + sconfig.direction = dir; + sconfig.ch_count = ch; + sconfig.frame_rate = params_rate(params); + sconfig.type = dai_runtime->stream_type; + + sconfig.bps = snd_pcm_format_width(params_format(params)); + + /* Port configuration */ + pconfig = kzalloc(sizeof(*pconfig), GFP_KERNEL); + if (!pconfig) { + ret = -ENOMEM; + goto error; + } + + pconfig->num = dai->id; + pconfig->ch_mask = (1 << ch) - 1; + ret = sdw_stream_add_master(&amd_manager->bus, &sconfig, + pconfig, 1, dai_runtime->stream); + if (ret) + dev_err(amd_manager->dev, "add manager to stream failed:%d\n", ret); + + kfree(pconfig); +error: + return ret; +} + +static int amd_sdw_hw_free(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) +{ + struct amd_sdw_manager *amd_manager = snd_soc_dai_get_drvdata(dai); + struct sdw_amd_dai_runtime *dai_runtime; + int ret; + + dai_runtime = amd_manager->dai_runtime_array[dai->id]; + if (!dai_runtime) + return -EIO; + + ret = sdw_stream_remove_master(&amd_manager->bus, dai_runtime->stream); + if (ret < 0) + dev_err(dai->dev, "remove manager from stream %s failed: %d\n", + dai_runtime->stream->name, ret); + return ret; +} + +static int amd_set_sdw_stream(struct snd_soc_dai *dai, void *stream, int direction) +{ + struct amd_sdw_manager *amd_manager = snd_soc_dai_get_drvdata(dai); + struct sdw_amd_dai_runtime *dai_runtime; + + dai_runtime = amd_manager->dai_runtime_array[dai->id]; + if (stream) { + /* first paranoia check */ + if (dai_runtime) { + dev_err(dai->dev, "dai_runtime already allocated for dai %s\n", dai->name); + return -EINVAL; + } + + /* allocate and set dai_runtime info */ + dai_runtime = kzalloc(sizeof(*dai_runtime), GFP_KERNEL); + if (!dai_runtime) + return -ENOMEM; + + dai_runtime->stream_type = SDW_STREAM_PCM; + dai_runtime->bus = &amd_manager->bus; + dai_runtime->stream = stream; + amd_manager->dai_runtime_array[dai->id] = dai_runtime; + } else { + /* second paranoia check */ + if (!dai_runtime) { + dev_err(dai->dev, "dai_runtime not allocated for dai %s\n", dai->name); + return -EINVAL; + } + + /* for NULL stream we release allocated dai_runtime */ + kfree(dai_runtime); + amd_manager->dai_runtime_array[dai->id] = NULL; + } + return 0; +} + +static int amd_pcm_set_sdw_stream(struct snd_soc_dai *dai, void *stream, int direction) +{ + return amd_set_sdw_stream(dai, stream, direction); +} + +static void *amd_get_sdw_stream(struct snd_soc_dai *dai, int direction) +{ + struct amd_sdw_manager *amd_manager = snd_soc_dai_get_drvdata(dai); + struct sdw_amd_dai_runtime *dai_runtime; + + dai_runtime = amd_manager->dai_runtime_array[dai->id]; + if (!dai_runtime) + return ERR_PTR(-EINVAL); + + return dai_runtime->stream; +} + +static const struct snd_soc_dai_ops amd_sdw_dai_ops = { + .hw_params = amd_sdw_hw_params, + .hw_free = amd_sdw_hw_free, + .set_stream = amd_pcm_set_sdw_stream, + .get_stream = amd_get_sdw_stream, +}; + +static const struct snd_soc_component_driver amd_sdw_dai_component = { + .name = "soundwire", +}; + +static int amd_sdw_register_dais(struct amd_sdw_manager *amd_manager) +{ + struct sdw_amd_dai_runtime **dai_runtime_array; + struct snd_soc_dai_driver *dais; + struct snd_soc_pcm_stream *stream; + struct device *dev; + int i, num_dais; + + dev = amd_manager->dev; + num_dais = amd_manager->num_dout_ports + amd_manager->num_din_ports; + dais = devm_kcalloc(dev, num_dais, sizeof(*dais), GFP_KERNEL); + if (!dais) + return -ENOMEM; + + dai_runtime_array = devm_kcalloc(dev, num_dais, + sizeof(struct sdw_amd_dai_runtime *), + GFP_KERNEL); + if (!dai_runtime_array) + return -ENOMEM; + amd_manager->dai_runtime_array = dai_runtime_array; + for (i = 0; i < num_dais; i++) { + dais[i].name = devm_kasprintf(dev, GFP_KERNEL, "SDW%d Pin%d", amd_manager->instance, + i); + if (!dais[i].name) + return -ENOMEM; + if (i < amd_manager->num_dout_ports) + stream = &dais[i].playback; + else + stream = &dais[i].capture; + + stream->channels_min = 2; + stream->channels_max = 2; + stream->rates = SNDRV_PCM_RATE_48000; + stream->formats = SNDRV_PCM_FMTBIT_S16_LE; + + dais[i].ops = &amd_sdw_dai_ops; + dais[i].id = i; + } + + return devm_snd_soc_register_component(dev, &amd_sdw_dai_component, + dais, num_dais); +} + static void amd_sdw_probe_work(struct work_struct *work) { struct amd_sdw_manager *amd_manager = container_of(work, struct amd_sdw_manager, @@ -636,6 +808,12 @@ static int amd_sdw_manager_probe(struct platform_device *pdev) dev_err(dev, "Failed to register SoundWire manager(%d)\n", ret); return ret; } + ret = amd_sdw_register_dais(amd_manager); + if (ret) { + dev_err(dev, "CPU DAI registration failed\n"); + sdw_bus_master_delete(&amd_manager->bus); + return ret; + } dev_set_drvdata(dev, amd_manager); INIT_WORK(&amd_manager->probe_work, amd_sdw_probe_work); /* diff --git a/drivers/soundwire/amd_manager.h b/drivers/soundwire/amd_manager.h index f7106521b500..c92e0dee2cb1 100644 --- a/drivers/soundwire/amd_manager.h +++ b/drivers/soundwire/amd_manager.h @@ -198,6 +198,24 @@ struct sdw_manager_dp_reg { u32 lane_ctrl_ch_en_reg; }; +/* + * SDW0 Manager instance registers 6 CPU DAI (3 TX & 3 RX Ports) + * whereas SDW1 Manager Instance registers 2 CPU DAI (one TX & one RX port) + * Below is the CPU DAI <->Manager port number mapping + * i.e SDW0 Pin0 -> port number 0 -> AUDIO0 TX + * SDW0 Pin1 -> Port number 1 -> AUDIO1 TX + * SDW0 Pin2 -> Port number 2 -> AUDIO2 TX + * SDW0 Pin3 -> port number 3 -> AUDIO0 RX + * SDW0 Pin4 -> Port number 4 -> AUDIO1 RX + * SDW0 Pin5 -> Port number 5 -> AUDIO2 RX + * Whereas for SDW1 instance + * SDW1 Pin0 -> port number 0 -> AUDIO1 TX + * SDW1 Pin1 -> Port number 1 -> AUDIO1 RX + * Same mapping should be used for programming DMA controller registers in SoundWire DMA driver. + * i.e if AUDIO0 TX channel is selected then we need to use AUDIO0 TX registers for DMA programming + * in SoundWire DMA driver. + */ + static struct sdw_manager_dp_reg sdw0_manager_dp_reg[AMD_SDW0_MAX_DAI] = { {ACP_SW_AUDIO0_TX_FRAME_FORMAT, ACP_SW_AUDIO0_TX_SAMPLEINTERVAL, ACP_SW_AUDIO0_TX_HCTRL_DP0, ACP_SW_AUDIO0_TX_OFFSET_DP0, ACP_SW_AUDIO0_TX_CHANNEL_ENABLE_DP0}, diff --git a/include/linux/soundwire/sdw_amd.h b/include/linux/soundwire/sdw_amd.h index c14a291a40e8..ac537419301d 100644 --- a/include/linux/soundwire/sdw_amd.h +++ b/include/linux/soundwire/sdw_amd.h @@ -23,6 +23,21 @@ struct sdw_manager_reg_mask { u32 acp_sdw_intr_mask; }; +/** + * struct sdw_amd_dai_runtime: AMD sdw dai runtime data + * + * @name: SoundWire stream name + * @stream: stream runtime + * @bus: Bus handle + * @stream_type: Stream type + */ +struct sdw_amd_dai_runtime { + char *name; + struct sdw_stream_runtime *stream; + struct sdw_bus *bus; + enum sdw_stream_type stream_type; +}; + /** * struct amd_sdw_manager - amd manager driver context * @bus: bus handle @@ -40,6 +55,7 @@ struct sdw_manager_reg_mask { * @quirks: SoundWire manager quirks * @wake_en_mask: wake enable mask per SoundWire manager * @power_mode_mask: flag interprets amd SoundWire manager power mode + * @dai_runtime_array: dai runtime array */ struct amd_sdw_manager { struct sdw_bus bus; @@ -63,5 +79,7 @@ struct amd_sdw_manager { u32 quirks; u32 wake_en_mask; u32 power_mode_mask; + + struct sdw_amd_dai_runtime **dai_runtime_array; }; #endif -- cgit v1.2.3 From 65f93e4096a07abd41acf0d240715bd8c7ef7eeb Mon Sep 17 00:00:00 2001 From: Vijendar Mukunda Date: Tue, 21 Mar 2023 10:38:58 +0530 Subject: soundwire: amd: add SoundWire manager interrupt handling Add support for handling SoundWire manager interrupts. Signed-off-by: Vijendar Mukunda Signed-off-by: Mastan Katragadda Reviewed-by: Pierre-Louis Bossart Link: https://lore.kernel.org/lkml/20230227154801.50319-6-Vijendar.Mukunda@amd.com Link: https://lore.kernel.org/r/20230321050901.115439-6-Vijendar.Mukunda@amd.com Signed-off-by: Vinod Koul --- drivers/soundwire/amd_manager.c | 126 ++++++++++++++++++++++++++++++++++++++ drivers/soundwire/amd_manager.h | 1 + include/linux/soundwire/sdw_amd.h | 7 +++ 3 files changed, 134 insertions(+) (limited to 'include/linux') diff --git a/drivers/soundwire/amd_manager.c b/drivers/soundwire/amd_manager.c index f12dad6c3226..49b7133cb41f 100644 --- a/drivers/soundwire/amd_manager.c +++ b/drivers/soundwire/amd_manager.c @@ -327,6 +327,47 @@ static enum sdw_command_response amd_sdw_xfer_msg(struct sdw_bus *bus, struct sd return SDW_CMD_OK; } +static void amd_sdw_fill_slave_status(struct amd_sdw_manager *amd_manager, u16 index, u32 status) +{ + switch (status) { + case SDW_SLAVE_ATTACHED: + case SDW_SLAVE_UNATTACHED: + case SDW_SLAVE_ALERT: + amd_manager->status[index] = status; + break; + default: + amd_manager->status[index] = SDW_SLAVE_RESERVED; + break; + } +} + +static void amd_sdw_process_ping_status(u64 response, struct amd_sdw_manager *amd_manager) +{ + u64 slave_stat; + u32 val; + u16 dev_index; + + /* slave status response */ + slave_stat = FIELD_GET(AMD_SDW_MCP_SLAVE_STAT_0_3, response); + slave_stat |= FIELD_GET(AMD_SDW_MCP_SLAVE_STAT_4_11, response) << 8; + dev_dbg(amd_manager->dev, "slave_stat:0x%llx\n", slave_stat); + for (dev_index = 0; dev_index <= SDW_MAX_DEVICES; ++dev_index) { + val = (slave_stat >> (dev_index * 2)) & AMD_SDW_MCP_SLAVE_STATUS_MASK; + dev_dbg(amd_manager->dev, "val:0x%x\n", val); + amd_sdw_fill_slave_status(amd_manager, dev_index, val); + } +} + +static void amd_sdw_read_and_process_ping_status(struct amd_sdw_manager *amd_manager) +{ + u64 response; + + mutex_lock(&amd_manager->bus.msg_lock); + response = amd_sdw_send_cmd_get_resp(amd_manager, 0, 0); + mutex_unlock(&amd_manager->bus.msg_lock); + amd_sdw_process_ping_status(response, amd_manager); +} + static u32 amd_sdw_read_ping_status(struct sdw_bus *bus) { struct amd_sdw_manager *amd_manager = to_amd_sdw(bus); @@ -723,6 +764,89 @@ static int amd_sdw_register_dais(struct amd_sdw_manager *amd_manager) dais, num_dais); } +static void amd_sdw_update_slave_status_work(struct work_struct *work) +{ + struct amd_sdw_manager *amd_manager = + container_of(work, struct amd_sdw_manager, amd_sdw_work); + int retry_count = 0; + + if (amd_manager->status[0] == SDW_SLAVE_ATTACHED) { + writel(0, amd_manager->mmio + ACP_SW_STATE_CHANGE_STATUS_MASK_0TO7); + writel(0, amd_manager->mmio + ACP_SW_STATE_CHANGE_STATUS_MASK_8TO11); + } + +update_status: + sdw_handle_slave_status(&amd_manager->bus, amd_manager->status); + /* + * During the peripheral enumeration sequence, the SoundWire manager interrupts + * are masked. Once the device number programming is done for all peripherals, + * interrupts will be unmasked. Read the peripheral device status from ping command + * and process the response. This sequence will ensure all peripheral devices enumerated + * and initialized properly. + */ + if (amd_manager->status[0] == SDW_SLAVE_ATTACHED) { + if (retry_count++ < SDW_MAX_DEVICES) { + writel(AMD_SDW_IRQ_MASK_0TO7, amd_manager->mmio + + ACP_SW_STATE_CHANGE_STATUS_MASK_0TO7); + writel(AMD_SDW_IRQ_MASK_8TO11, amd_manager->mmio + + ACP_SW_STATE_CHANGE_STATUS_MASK_8TO11); + amd_sdw_read_and_process_ping_status(amd_manager); + goto update_status; + } else { + dev_err_ratelimited(amd_manager->dev, + "Device0 detected after %d iterations\n", + retry_count); + } + } +} + +static void amd_sdw_update_slave_status(u32 status_change_0to7, u32 status_change_8to11, + struct amd_sdw_manager *amd_manager) +{ + u64 slave_stat; + u32 val; + int dev_index; + + if (status_change_0to7 == AMD_SDW_SLAVE_0_ATTACHED) + memset(amd_manager->status, 0, sizeof(amd_manager->status)); + slave_stat = status_change_0to7; + slave_stat |= FIELD_GET(AMD_SDW_MCP_SLAVE_STATUS_8TO_11, status_change_8to11) << 32; + dev_dbg(amd_manager->dev, "status_change_0to7:0x%x status_change_8to11:0x%x\n", + status_change_0to7, status_change_8to11); + if (slave_stat) { + for (dev_index = 0; dev_index <= SDW_MAX_DEVICES; ++dev_index) { + if (slave_stat & AMD_SDW_MCP_SLAVE_STATUS_VALID_MASK(dev_index)) { + val = (slave_stat >> AMD_SDW_MCP_SLAVE_STAT_SHIFT_MASK(dev_index)) & + AMD_SDW_MCP_SLAVE_STATUS_MASK; + amd_sdw_fill_slave_status(amd_manager, dev_index, val); + } + } + } +} + +static void amd_sdw_irq_thread(struct work_struct *work) +{ + struct amd_sdw_manager *amd_manager = + container_of(work, struct amd_sdw_manager, amd_sdw_irq_thread); + u32 status_change_8to11; + u32 status_change_0to7; + + status_change_8to11 = readl(amd_manager->mmio + ACP_SW_STATE_CHANGE_STATUS_8TO11); + status_change_0to7 = readl(amd_manager->mmio + ACP_SW_STATE_CHANGE_STATUS_0TO7); + dev_dbg(amd_manager->dev, "[SDW%d] SDW INT: 0to7=0x%x, 8to11=0x%x\n", + amd_manager->instance, status_change_0to7, status_change_8to11); + if (status_change_8to11 & AMD_SDW_PREQ_INTR_STAT) { + amd_sdw_read_and_process_ping_status(amd_manager); + } else { + /* Check for the updated status on peripheral device */ + amd_sdw_update_slave_status(status_change_0to7, status_change_8to11, amd_manager); + } + if (status_change_8to11 || status_change_0to7) + schedule_work(&amd_manager->amd_sdw_work); + writel(0x00, amd_manager->mmio + ACP_SW_STATE_CHANGE_STATUS_8TO11); + writel(0x00, amd_manager->mmio + ACP_SW_STATE_CHANGE_STATUS_0TO7); +} + static void amd_sdw_probe_work(struct work_struct *work) { struct amd_sdw_manager *amd_manager = container_of(work, struct amd_sdw_manager, @@ -815,6 +939,8 @@ static int amd_sdw_manager_probe(struct platform_device *pdev) return ret; } dev_set_drvdata(dev, amd_manager); + INIT_WORK(&amd_manager->amd_sdw_irq_thread, amd_sdw_irq_thread); + INIT_WORK(&amd_manager->amd_sdw_work, amd_sdw_update_slave_status_work); INIT_WORK(&amd_manager->probe_work, amd_sdw_probe_work); /* * Instead of having lengthy probe sequence, use deferred probe. diff --git a/drivers/soundwire/amd_manager.h b/drivers/soundwire/amd_manager.h index c92e0dee2cb1..fca7d792bf5f 100644 --- a/drivers/soundwire/amd_manager.h +++ b/drivers/soundwire/amd_manager.h @@ -185,6 +185,7 @@ #define AMD_SDW1_PAD_KEEPER_EN_MASK 0x10 #define AMD_SDW0_PAD_KEEPER_DISABLE_MASK 0x1e #define AMD_SDW1_PAD_KEEPER_DISABLE_MASK 0xf +#define AMD_SDW_PREQ_INTR_STAT BIT(19) static u32 amd_sdw_freq_tbl[AMD_SDW_MAX_FREQ_NUM] = { AMD_SDW_DEFAULT_CLK_FREQ, diff --git a/include/linux/soundwire/sdw_amd.h b/include/linux/soundwire/sdw_amd.h index ac537419301d..df60bc0de6fc 100644 --- a/include/linux/soundwire/sdw_amd.h +++ b/include/linux/soundwire/sdw_amd.h @@ -45,8 +45,11 @@ struct sdw_amd_dai_runtime { * @mmio: SoundWire registers mmio base * @acp_mmio: acp registers mmio base * @reg_mask: register mask structure per manager instance + * @amd_sdw_irq_thread: SoundWire manager irq workqueue + * @amd_sdw_work: peripheral status work queue * @probe_work: SoundWire manager probe workqueue * @acp_sdw_lock: mutex to protect acp share register access + * @status: peripheral devices status array * @num_din_ports: number of input ports * @num_dout_ports: number of output ports * @cols_index: Column index in frame shape @@ -65,10 +68,14 @@ struct amd_sdw_manager { void __iomem *acp_mmio; struct sdw_manager_reg_mask *reg_mask; + struct work_struct amd_sdw_irq_thread; + struct work_struct amd_sdw_work; struct work_struct probe_work; /* mutex to protect acp common register access */ struct mutex *acp_sdw_lock; + enum sdw_slave_status status[SDW_MAX_DEVICES + 1]; + int num_din_ports; int num_dout_ports; -- cgit v1.2.3 From 81ff58ff71ad9dcddf5caffcf912cde6589d07bd Mon Sep 17 00:00:00 2001 From: Vijendar Mukunda Date: Tue, 21 Mar 2023 10:38:59 +0530 Subject: soundwire: amd: add runtime pm ops for AMD SoundWire manager driver Add support for runtime pm ops for AMD SoundWire manager driver. Signed-off-by: Vijendar Mukunda Signed-off-by: Mastan Katragadda Reviewed-by: Pierre-Louis Bossart Link: https://lore.kernel.org/lkml/20230227154801.50319-7-Vijendar.Mukunda@amd.com Link: https://lore.kernel.org/r/20230321050901.115439-7-Vijendar.Mukunda@amd.com Signed-off-by: Vinod Koul --- drivers/soundwire/amd_manager.c | 140 ++++++++++++++++++++++++++++++++++++++ drivers/soundwire/amd_manager.h | 3 + include/linux/soundwire/sdw_amd.h | 17 +++++ 3 files changed, 160 insertions(+) (limited to 'include/linux') diff --git a/drivers/soundwire/amd_manager.c b/drivers/soundwire/amd_manager.c index 49b7133cb41f..6172b3cb84d4 100644 --- a/drivers/soundwire/amd_manager.c +++ b/drivers/soundwire/amd_manager.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -133,6 +134,12 @@ static void amd_disable_sdw_interrupts(struct amd_sdw_manager *amd_manager) writel(0x00, amd_manager->mmio + ACP_SW_ERROR_INTR_MASK); } +static int amd_deinit_sdw_manager(struct amd_sdw_manager *amd_manager) +{ + amd_disable_sdw_interrupts(amd_manager); + return amd_disable_sdw_manager(amd_manager); +} + static void amd_sdw_set_frameshape(struct amd_sdw_manager *amd_manager) { u32 frame_size; @@ -866,6 +873,12 @@ static void amd_sdw_probe_work(struct work_struct *work) return; amd_sdw_set_frameshape(amd_manager); } + /* Enable runtime PM */ + pm_runtime_set_autosuspend_delay(amd_manager->dev, AMD_SDW_MASTER_SUSPEND_DELAY_MS); + pm_runtime_use_autosuspend(amd_manager->dev); + pm_runtime_mark_last_busy(amd_manager->dev); + pm_runtime_set_active(amd_manager->dev); + pm_runtime_enable(amd_manager->dev); } static int amd_sdw_manager_probe(struct platform_device *pdev) @@ -953,17 +966,144 @@ static int amd_sdw_manager_remove(struct platform_device *pdev) { struct amd_sdw_manager *amd_manager = dev_get_drvdata(&pdev->dev); + pm_runtime_disable(&pdev->dev); cancel_work_sync(&amd_manager->probe_work); amd_disable_sdw_interrupts(amd_manager); sdw_bus_master_delete(&amd_manager->bus); return amd_disable_sdw_manager(amd_manager); } +static int amd_sdw_clock_stop(struct amd_sdw_manager *amd_manager) +{ + u32 val; + int ret; + + ret = sdw_bus_prep_clk_stop(&amd_manager->bus); + if (ret < 0 && ret != -ENODATA) { + dev_err(amd_manager->dev, "prepare clock stop failed %d", ret); + return 0; + } + ret = sdw_bus_clk_stop(&amd_manager->bus); + if (ret < 0 && ret != -ENODATA) { + dev_err(amd_manager->dev, "bus clock stop failed %d", ret); + return 0; + } + + ret = readl_poll_timeout(amd_manager->mmio + ACP_SW_CLK_RESUME_CTRL, val, + (val & AMD_SDW_CLK_STOP_DONE), ACP_DELAY_US, AMD_SDW_TIMEOUT); + if (ret) { + dev_err(amd_manager->dev, "SDW%x clock stop failed\n", amd_manager->instance); + return 0; + } + + amd_manager->clk_stopped = true; + if (amd_manager->wake_en_mask) + writel(0x01, amd_manager->acp_mmio + ACP_SW_WAKE_EN(amd_manager->instance)); + + dev_dbg(amd_manager->dev, "SDW%x clock stop successful\n", amd_manager->instance); + return 0; +} + +static int amd_sdw_clock_stop_exit(struct amd_sdw_manager *amd_manager) +{ + int ret; + u32 val; + + if (amd_manager->clk_stopped) { + val = readl(amd_manager->mmio + ACP_SW_CLK_RESUME_CTRL); + val |= AMD_SDW_CLK_RESUME_REQ; + writel(val, amd_manager->mmio + ACP_SW_CLK_RESUME_CTRL); + ret = readl_poll_timeout(amd_manager->mmio + ACP_SW_CLK_RESUME_CTRL, val, + (val & AMD_SDW_CLK_RESUME_DONE), ACP_DELAY_US, + AMD_SDW_TIMEOUT); + if (val & AMD_SDW_CLK_RESUME_DONE) { + writel(0, amd_manager->mmio + ACP_SW_CLK_RESUME_CTRL); + ret = sdw_bus_exit_clk_stop(&amd_manager->bus); + if (ret < 0) + dev_err(amd_manager->dev, "bus failed to exit clock stop %d\n", + ret); + amd_manager->clk_stopped = false; + } + } + if (amd_manager->clk_stopped) { + dev_err(amd_manager->dev, "SDW%x clock stop exit failed\n", amd_manager->instance); + return 0; + } + dev_dbg(amd_manager->dev, "SDW%x clock stop exit successful\n", amd_manager->instance); + return 0; +} + +static int __maybe_unused amd_suspend_runtime(struct device *dev) +{ + struct amd_sdw_manager *amd_manager = dev_get_drvdata(dev); + struct sdw_bus *bus = &amd_manager->bus; + int ret; + + if (bus->prop.hw_disabled) { + dev_dbg(bus->dev, "SoundWire manager %d is disabled,\n", + bus->link_id); + return 0; + } + if (amd_manager->power_mode_mask & AMD_SDW_CLK_STOP_MODE) { + return amd_sdw_clock_stop(amd_manager); + } else if (amd_manager->power_mode_mask & AMD_SDW_POWER_OFF_MODE) { + ret = amd_sdw_clock_stop(amd_manager); + if (ret) + return ret; + return amd_deinit_sdw_manager(amd_manager); + } + return 0; +} + +static int __maybe_unused amd_resume_runtime(struct device *dev) +{ + struct amd_sdw_manager *amd_manager = dev_get_drvdata(dev); + struct sdw_bus *bus = &amd_manager->bus; + int ret; + u32 val; + + if (bus->prop.hw_disabled) { + dev_dbg(bus->dev, "SoundWire manager %d is disabled, ignoring\n", + bus->link_id); + return 0; + } + + if (amd_manager->power_mode_mask & AMD_SDW_CLK_STOP_MODE) { + return amd_sdw_clock_stop_exit(amd_manager); + } else if (amd_manager->power_mode_mask & AMD_SDW_POWER_OFF_MODE) { + val = readl(amd_manager->mmio + ACP_SW_CLK_RESUME_CTRL); + if (val) { + val |= AMD_SDW_CLK_RESUME_REQ; + writel(val, amd_manager->mmio + ACP_SW_CLK_RESUME_CTRL); + ret = readl_poll_timeout(amd_manager->mmio + ACP_SW_CLK_RESUME_CTRL, val, + (val & AMD_SDW_CLK_RESUME_DONE), ACP_DELAY_US, + AMD_SDW_TIMEOUT); + if (val & AMD_SDW_CLK_RESUME_DONE) { + writel(0, amd_manager->mmio + ACP_SW_CLK_RESUME_CTRL); + amd_manager->clk_stopped = false; + } + } + sdw_clear_slave_status(bus, SDW_UNATTACH_REQUEST_MASTER_RESET); + amd_init_sdw_manager(amd_manager); + amd_enable_sdw_interrupts(amd_manager); + ret = amd_enable_sdw_manager(amd_manager); + if (ret) + return ret; + amd_sdw_set_frameshape(amd_manager); + } + return 0; +} + +static const struct dev_pm_ops amd_pm = { + SET_RUNTIME_PM_OPS(amd_suspend_runtime, amd_resume_runtime, NULL) +}; + static struct platform_driver amd_sdw_driver = { .probe = &amd_sdw_manager_probe, .remove = &amd_sdw_manager_remove, .driver = { .name = "amd_sdw_manager", + .pm = &amd_pm, } }; module_platform_driver(amd_sdw_driver); diff --git a/drivers/soundwire/amd_manager.h b/drivers/soundwire/amd_manager.h index fca7d792bf5f..b101f4536230 100644 --- a/drivers/soundwire/amd_manager.h +++ b/drivers/soundwire/amd_manager.h @@ -186,6 +186,9 @@ #define AMD_SDW0_PAD_KEEPER_DISABLE_MASK 0x1e #define AMD_SDW1_PAD_KEEPER_DISABLE_MASK 0xf #define AMD_SDW_PREQ_INTR_STAT BIT(19) +#define AMD_SDW_CLK_STOP_DONE 1 +#define AMD_SDW_CLK_RESUME_REQ 2 +#define AMD_SDW_CLK_RESUME_DONE 3 static u32 amd_sdw_freq_tbl[AMD_SDW_MAX_FREQ_NUM] = { AMD_SDW_DEFAULT_CLK_FREQ, diff --git a/include/linux/soundwire/sdw_amd.h b/include/linux/soundwire/sdw_amd.h index df60bc0de6fc..ceecad74aef9 100644 --- a/include/linux/soundwire/sdw_amd.h +++ b/include/linux/soundwire/sdw_amd.h @@ -8,6 +8,21 @@ #include +/* AMD pm_runtime quirk definitions */ + +/* + * Force the clock to stop(ClockStopMode0) when suspend callback + * is invoked. + */ +#define AMD_SDW_CLK_STOP_MODE 1 + +/* + * Stop the bus when runtime suspend/system level suspend callback + * is invoked. If set, a complete bus reset and re-enumeration will + * be performed when the bus restarts. In-band wake interrupts are + * not supported in this mode. + */ +#define AMD_SDW_POWER_OFF_MODE 2 #define ACP_SDW0 0 #define ACP_SDW1 1 @@ -57,6 +72,7 @@ struct sdw_amd_dai_runtime { * @instance: SoundWire manager instance * @quirks: SoundWire manager quirks * @wake_en_mask: wake enable mask per SoundWire manager + * @clk_stopped: flag set to true when clock is stopped * @power_mode_mask: flag interprets amd SoundWire manager power mode * @dai_runtime_array: dai runtime array */ @@ -86,6 +102,7 @@ struct amd_sdw_manager { u32 quirks; u32 wake_en_mask; u32 power_mode_mask; + bool clk_stopped; struct sdw_amd_dai_runtime **dai_runtime_array; }; -- cgit v1.2.3 From 6539cffa94957241c096099a57d05fa4d8c7db8a Mon Sep 17 00:00:00 2001 From: Radu Rendec Date: Wed, 12 Apr 2023 14:57:57 -0400 Subject: cacheinfo: Add arch specific early level initializer This patch gives architecture specific code the ability to initialize the cache level and allocate cacheinfo memory early, when cache level initialization runs on the primary CPU for all possible CPUs. This is part of a patch series that attempts to further the work in commit 5944ce092b97 ("arch_topology: Build cacheinfo from primary CPU"). Previously, in the absence of any DT/ACPI cache info, architecture specific cache detection and info allocation for secondary CPUs would happen in non-preemptible context during early CPU initialization and trigger a "BUG: sleeping function called from invalid context" splat on an RT kernel. More specifically, this patch adds the early_cache_level() function, which is called by fetch_cache_info() as a fallback when the number of cache leaves cannot be extracted from DT/ACPI. In the default generic (weak) implementation, this new function returns -ENOENT, which preserves the original behavior for architectures that do not implement the function. Since early detection can get the number of cache leaves wrong in some cases*, additional logic is added to still call init_cache_level() later on the secondary CPU, therefore giving the architecture specific code an opportunity to go back and fix the initial guess. Again, the original behavior is preserved for architectures that do not implement the new function. * For example, on arm64, CLIDR_EL1 detection works only when it runs on the current CPU. In other words, a CPU cannot detect the cache depth for any other CPU than itself. Signed-off-by: Radu Rendec Reviewed-by: Pierre Gondois Link: https://lore.kernel.org/r/20230412185759.755408-2-rrendec@redhat.com Signed-off-by: Sudeep Holla --- drivers/base/cacheinfo.c | 75 +++++++++++++++++++++++++++++++++-------------- include/linux/cacheinfo.h | 2 ++ 2 files changed, 55 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index f6573c335f4c..6e0a44bad305 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -398,6 +398,11 @@ static void free_cache_attributes(unsigned int cpu) cache_shared_cpu_map_remove(cpu); } +int __weak early_cache_level(unsigned int cpu) +{ + return -ENOENT; +} + int __weak init_cache_level(unsigned int cpu) { return -ENOENT; @@ -423,56 +428,82 @@ int allocate_cache_info(int cpu) int fetch_cache_info(unsigned int cpu) { - struct cpu_cacheinfo *this_cpu_ci; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); unsigned int levels = 0, split_levels = 0; int ret; if (acpi_disabled) { ret = init_of_cache_level(cpu); - if (ret < 0) - return ret; } else { ret = acpi_get_cache_info(cpu, &levels, &split_levels); - if (ret < 0) + if (!ret) { + this_cpu_ci->num_levels = levels; + /* + * This assumes that: + * - there cannot be any split caches (data/instruction) + * above a unified cache + * - data/instruction caches come by pair + */ + this_cpu_ci->num_leaves = levels + split_levels; + } + } + + if (ret || !cache_leaves(cpu)) { + ret = early_cache_level(cpu); + if (ret) return ret; - this_cpu_ci = get_cpu_cacheinfo(cpu); - this_cpu_ci->num_levels = levels; - /* - * This assumes that: - * - there cannot be any split caches (data/instruction) - * above a unified cache - * - data/instruction caches come by pair - */ - this_cpu_ci->num_leaves = levels + split_levels; + if (!cache_leaves(cpu)) + return -ENOENT; + + this_cpu_ci->early_ci_levels = true; } - if (!cache_leaves(cpu)) - return -ENOENT; return allocate_cache_info(cpu); } -int detect_cache_attributes(unsigned int cpu) +static inline int init_level_allocate_ci(unsigned int cpu) { - int ret; + unsigned int early_leaves = cache_leaves(cpu); /* Since early initialization/allocation of the cacheinfo is allowed * via fetch_cache_info() and this also gets called as CPU hotplug * callbacks via cacheinfo_cpu_online, the init/alloc can be skipped * as it will happen only once (the cacheinfo memory is never freed). - * Just populate the cacheinfo. + * Just populate the cacheinfo. However, if the cacheinfo has been + * allocated early through the arch-specific early_cache_level() call, + * there is a chance the info is wrong (this can happen on arm64). In + * that case, call init_cache_level() anyway to give the arch-specific + * code a chance to make things right. */ - if (per_cpu_cacheinfo(cpu)) - goto populate_leaves; + if (per_cpu_cacheinfo(cpu) && !ci_cacheinfo(cpu)->early_ci_levels) + return 0; if (init_cache_level(cpu) || !cache_leaves(cpu)) return -ENOENT; - ret = allocate_cache_info(cpu); + /* + * Now that we have properly initialized the cache level info, make + * sure we don't try to do that again the next time we are called + * (e.g. as CPU hotplug callbacks). + */ + ci_cacheinfo(cpu)->early_ci_levels = false; + + if (cache_leaves(cpu) <= early_leaves) + return 0; + + kfree(per_cpu_cacheinfo(cpu)); + return allocate_cache_info(cpu); +} + +int detect_cache_attributes(unsigned int cpu) +{ + int ret; + + ret = init_level_allocate_ci(cpu); if (ret) return ret; -populate_leaves: /* * populate_cache_leaves() may completely setup the cache leaves and * shared_cpu_map or it may leave it partially setup. diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h index 908e19d17f49..6147b2672555 100644 --- a/include/linux/cacheinfo.h +++ b/include/linux/cacheinfo.h @@ -76,9 +76,11 @@ struct cpu_cacheinfo { unsigned int num_levels; unsigned int num_leaves; bool cpu_map_populated; + bool early_ci_levels; }; struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu); +int early_cache_level(unsigned int cpu); int init_cache_level(unsigned int cpu); int init_of_cache_level(unsigned int cpu); int populate_cache_leaves(unsigned int cpu); -- cgit v1.2.3 From 79d9622d622d49d6c14c51edec2059b8591e7975 Mon Sep 17 00:00:00 2001 From: Patrik Dahlström Date: Sat, 8 Apr 2023 13:48:19 +0200 Subject: iio: adc: palmas: remove adc_wakeupX_data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It does not seem to be used by anyone and later patches in this series are made simpler by first removing this. There is now a lot of dead code that cannot be reached, until later patches revive it. Arguably, this is preferred over removing the code only to add it again. Signed-off-by: Patrik Dahlström Link: https://lore.kernel.org/r/20230408114825.824505-4-risca@dalakolonin.se Signed-off-by: Jonathan Cameron --- drivers/iio/adc/palmas_gpadc.c | 50 +++++------------------------------------- include/linux/mfd/palmas.h | 8 ------- 2 files changed, 6 insertions(+), 52 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/adc/palmas_gpadc.c b/drivers/iio/adc/palmas_gpadc.c index 2921186458e0..03af6cd73ec8 100644 --- a/drivers/iio/adc/palmas_gpadc.c +++ b/drivers/iio/adc/palmas_gpadc.c @@ -76,6 +76,12 @@ static struct palmas_gpadc_info palmas_gpadc_info[] = { PALMAS_ADC_INFO(IN15, 0, 0, 0, 0, INVALID, INVALID, true), }; +struct palmas_adc_wakeup_property { + int adc_channel_number; + int adc_high_threshold; + int adc_low_threshold; +}; + /* * struct palmas_gpadc - the palmas_gpadc structure * @ch0_current: channel 0 current source setting @@ -493,11 +499,6 @@ static int palmas_gpadc_get_adc_dt_data(struct platform_device *pdev, return 0; } -static void palmas_disable_wakeup(void *dev) -{ - device_wakeup_disable(dev); -} - static int palmas_gpadc_probe(struct platform_device *pdev) { struct palmas_gpadc *adc; @@ -548,36 +549,6 @@ static int palmas_gpadc_probe(struct platform_device *pdev) return dev_err_probe(adc->dev, ret, "request irq %d failed\n", adc->irq); - if (gpadc_pdata->adc_wakeup1_data) { - memcpy(&adc->wakeup1_data, gpadc_pdata->adc_wakeup1_data, - sizeof(adc->wakeup1_data)); - adc->wakeup1_enable = true; - adc->irq_auto_0 = platform_get_irq(pdev, 1); - ret = devm_request_threaded_irq(&pdev->dev, adc->irq_auto_0, - NULL, palmas_gpadc_irq_auto, - IRQF_ONESHOT, - "palmas-adc-auto-0", adc); - if (ret < 0) - return dev_err_probe(adc->dev, ret, - "request auto0 irq %d failed\n", - adc->irq_auto_0); - } - - if (gpadc_pdata->adc_wakeup2_data) { - memcpy(&adc->wakeup2_data, gpadc_pdata->adc_wakeup2_data, - sizeof(adc->wakeup2_data)); - adc->wakeup2_enable = true; - adc->irq_auto_1 = platform_get_irq(pdev, 2); - ret = devm_request_threaded_irq(&pdev->dev, adc->irq_auto_1, - NULL, palmas_gpadc_irq_auto, - IRQF_ONESHOT, - "palmas-adc-auto-1", adc); - if (ret < 0) - return dev_err_probe(adc->dev, ret, - "request auto1 irq %d failed\n", - adc->irq_auto_1); - } - /* set the current source 0 (value 0/5/15/20 uA => 0..3) */ if (gpadc_pdata->ch0_current <= 1) adc->ch0_current = PALMAS_ADC_CH0_CURRENT_SRC_0; @@ -617,15 +588,6 @@ static int palmas_gpadc_probe(struct platform_device *pdev) palmas_gpadc_calibrate(adc, i); } - if (adc->wakeup1_enable || adc->wakeup2_enable) { - device_wakeup_enable(&pdev->dev); - ret = devm_add_action_or_reset(&pdev->dev, - palmas_disable_wakeup, - &pdev->dev); - if (ret) - return ret; - } - return 0; } diff --git a/include/linux/mfd/palmas.h b/include/linux/mfd/palmas.h index 117d02708439..eda1ffd99c1a 100644 --- a/include/linux/mfd/palmas.h +++ b/include/linux/mfd/palmas.h @@ -128,12 +128,6 @@ struct palmas_pmic_driver_data { struct regulator_config config); }; -struct palmas_adc_wakeup_property { - int adc_channel_number; - int adc_high_threshold; - int adc_low_threshold; -}; - struct palmas_gpadc_platform_data { /* Channel 3 current source is only enabled during conversion */ int ch3_current; /* 0: off; 1: 10uA; 2: 400uA; 3: 800 uA */ @@ -152,8 +146,6 @@ struct palmas_gpadc_platform_data { int start_polarity; int auto_conversion_period_ms; - struct palmas_adc_wakeup_property *adc_wakeup1_data; - struct palmas_adc_wakeup_property *adc_wakeup2_data; }; struct palmas_reg_init { -- cgit v1.2.3 From 0af4d704ba8e5ee632b6e65015ffe4d229c1a9a9 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Wed, 12 Apr 2023 13:56:36 +0200 Subject: pwm: Delete deprecated functions pwm_request() and pwm_free() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit 5a7fbe452ad9 ("backlight: pwm_bl: Drop support for legacy PWM probing") the last user of pwm_request() and pwm_free() is gone. So remove these functions that were deprecated over 10 years ago in commit 8138d2ddbcca ("pwm: Add table-based lookup for static mappings"). Signed-off-by: Uwe Kleine-König [thierry.reding@gmail.com: clean up a bit after removal] Signed-off-by: Thierry Reding --- Documentation/driver-api/pwm.rst | 13 ++++----- drivers/pwm/core.c | 58 ++-------------------------------------- include/linux/pwm.h | 13 --------- 3 files changed, 7 insertions(+), 77 deletions(-) (limited to 'include/linux') diff --git a/Documentation/driver-api/pwm.rst b/Documentation/driver-api/pwm.rst index 8c71a2055d27..3fdc95f7a1d1 100644 --- a/Documentation/driver-api/pwm.rst +++ b/Documentation/driver-api/pwm.rst @@ -35,12 +35,9 @@ consumers to providers, as given in the following example:: Using PWMs ---------- -Legacy users can request a PWM device using pwm_request() and free it -after usage with pwm_free(). - -New users should use the pwm_get() function and pass to it the consumer -device or a consumer name. pwm_put() is used to free the PWM device. Managed -variants of the getter, devm_pwm_get() and devm_fwnode_pwm_get(), also exist. +Consumers use the pwm_get() function and pass to it the consumer device or a +consumer name. pwm_put() is used to free the PWM device. Managed variants of +the getter, devm_pwm_get() and devm_fwnode_pwm_get(), also exist. After being requested, a PWM has to be configured using:: @@ -165,8 +162,8 @@ consumers should implement it as described in the "Using PWMs" section. Locking ------- -The PWM core list manipulations are protected by a mutex, so pwm_request() -and pwm_free() may not be called from an atomic context. Currently the +The PWM core list manipulations are protected by a mutex, so pwm_get() +and pwm_put() may not be called from an atomic context. Currently the PWM core does not enforce any locking to pwm_enable(), pwm_disable() and pwm_config(), so the calling context is currently driver specific. This is an issue derived from the former barebone API and should be fixed soon. diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index 474725714a05..9ce85c6157e4 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -35,11 +35,6 @@ static LIST_HEAD(pwm_chips); static DECLARE_BITMAP(allocated_pwms, MAX_PWMS); static RADIX_TREE(pwm_tree, GFP_KERNEL); -static struct pwm_device *pwm_to_device(unsigned int pwm) -{ - return radix_tree_lookup(&pwm_tree, pwm); -} - /* Called with pwm_lock held */ static int alloc_pwms(unsigned int count) { @@ -369,43 +364,6 @@ int devm_pwmchip_add(struct device *dev, struct pwm_chip *chip) } EXPORT_SYMBOL_GPL(devm_pwmchip_add); -/** - * pwm_request() - request a PWM device - * @pwm: global PWM device index - * @label: PWM device label - * - * This function is deprecated, use pwm_get() instead. - * - * Returns: A pointer to a PWM device or an ERR_PTR()-encoded error code on - * failure. - */ -struct pwm_device *pwm_request(int pwm, const char *label) -{ - struct pwm_device *dev; - int err; - - if (pwm < 0 || pwm >= MAX_PWMS) - return ERR_PTR(-EINVAL); - - mutex_lock(&pwm_lock); - - dev = pwm_to_device(pwm); - if (!dev) { - dev = ERR_PTR(-EPROBE_DEFER); - goto out; - } - - err = pwm_device_request(dev, label); - if (err < 0) - dev = ERR_PTR(err); - -out: - mutex_unlock(&pwm_lock); - - return dev; -} -EXPORT_SYMBOL_GPL(pwm_request); - /** * pwm_request_from_chip() - request a PWM device relative to a PWM chip * @chip: PWM chip @@ -438,18 +396,6 @@ struct pwm_device *pwm_request_from_chip(struct pwm_chip *chip, } EXPORT_SYMBOL_GPL(pwm_request_from_chip); -/** - * pwm_free() - free a PWM device - * @pwm: PWM device - * - * This function is deprecated, use pwm_put() instead. - */ -void pwm_free(struct pwm_device *pwm) -{ - pwm_put(pwm); -} -EXPORT_SYMBOL_GPL(pwm_free); - static void pwm_apply_state_debug(struct pwm_device *pwm, const struct pwm_state *state) { @@ -790,7 +736,7 @@ static struct pwm_device *of_pwm_get(struct device *dev, struct device_node *np, dl = pwm_device_link_add(dev, pwm); if (IS_ERR(dl)) { /* of_xlate ended up calling pwm_request_from_chip() */ - pwm_free(pwm); + pwm_put(pwm); pwm = ERR_CAST(dl); goto put; } @@ -1014,7 +960,7 @@ struct pwm_device *pwm_get(struct device *dev, const char *con_id) dl = pwm_device_link_add(dev, pwm); if (IS_ERR(dl)) { - pwm_free(pwm); + pwm_put(pwm); return ERR_CAST(dl); } diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 7b7b93b6fb81..04ae1d9073a7 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -309,8 +309,6 @@ struct pwm_chip { #if IS_ENABLED(CONFIG_PWM) /* PWM user APIs */ -struct pwm_device *pwm_request(int pwm_id, const char *label); -void pwm_free(struct pwm_device *pwm); int pwm_apply_state(struct pwm_device *pwm, const struct pwm_state *state); int pwm_adjust_config(struct pwm_device *pwm); @@ -410,17 +408,6 @@ struct pwm_device *devm_fwnode_pwm_get(struct device *dev, struct fwnode_handle *fwnode, const char *con_id); #else -static inline struct pwm_device *pwm_request(int pwm_id, const char *label) -{ - might_sleep(); - return ERR_PTR(-ENODEV); -} - -static inline void pwm_free(struct pwm_device *pwm) -{ - might_sleep(); -} - static inline int pwm_apply_state(struct pwm_device *pwm, const struct pwm_state *state) { -- cgit v1.2.3 From 8cbc82f3ec0d58961cf9c1e5d99e56741f4bf134 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Mon, 20 Mar 2023 15:40:10 +0800 Subject: mm: memory-failure: Move memory failure sysctls to its own file The sysctl_memory_failure_early_kill and memory_failure_recovery are only used in memory-failure.c, move them to its own file. Acked-by: Naoya Horiguchi Signed-off-by: Kefeng Wang [mcgrof: fix by adding empty ctl entry, this caused a crash] Signed-off-by: Luis Chamberlain --- include/linux/mm.h | 2 -- kernel/sysctl.c | 20 -------------------- mm/memory-failure.c | 36 ++++++++++++++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 1f79667824eb..98da268b834a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3442,8 +3442,6 @@ int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, extern int memory_failure(unsigned long pfn, int flags); extern void memory_failure_queue_kick(int cpu); extern int unpoison_memory(unsigned long pfn); -extern int sysctl_memory_failure_early_kill; -extern int sysctl_memory_failure_recovery; extern void shake_page(struct page *p); extern atomic_long_t num_poisoned_pages __read_mostly; extern int soft_offline_page(unsigned long pfn, int flags); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index ce0297acf97c..0a8a3c9c82e4 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2350,26 +2350,6 @@ static struct ctl_table vm_table[] = { .proc_handler = proc_dointvec, .extra1 = SYSCTL_ZERO, }, -#endif -#ifdef CONFIG_MEMORY_FAILURE - { - .procname = "memory_failure_early_kill", - .data = &sysctl_memory_failure_early_kill, - .maxlen = sizeof(sysctl_memory_failure_early_kill), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, - { - .procname = "memory_failure_recovery", - .data = &sysctl_memory_failure_recovery, - .maxlen = sizeof(sysctl_memory_failure_recovery), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, #endif { .procname = "user_reserve_kbytes", diff --git a/mm/memory-failure.c b/mm/memory-failure.c index fae9baf3be16..10e60b6b2447 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -62,13 +62,14 @@ #include #include #include +#include #include "swap.h" #include "internal.h" #include "ras/ras_event.h" -int sysctl_memory_failure_early_kill __read_mostly = 0; +static int sysctl_memory_failure_early_kill __read_mostly; -int sysctl_memory_failure_recovery __read_mostly = 1; +static int sysctl_memory_failure_recovery __read_mostly = 1; atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); @@ -122,6 +123,37 @@ const struct attribute_group memory_failure_attr_group = { .attrs = memory_failure_attr, }; +#ifdef CONFIG_SYSCTL +static struct ctl_table memory_failure_table[] = { + { + .procname = "memory_failure_early_kill", + .data = &sysctl_memory_failure_early_kill, + .maxlen = sizeof(sysctl_memory_failure_early_kill), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "memory_failure_recovery", + .data = &sysctl_memory_failure_recovery, + .maxlen = sizeof(sysctl_memory_failure_recovery), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { } +}; + +static int __init memory_failure_sysctl_init(void) +{ + register_sysctl_init("vm", memory_failure_table); + return 0; +} +late_initcall(memory_failure_sysctl_init); +#endif /* CONFIG_SYSCTL */ + /* * Return values: * 1: the page is dissolved (if needed) and taken off from buddy, -- cgit v1.2.3 From 48fe8ab8d5a39c7bc49cb41d0ad92c75f48a9550 Mon Sep 17 00:00:00 2001 From: Minghao Chi Date: Tue, 28 Mar 2023 14:46:28 +0800 Subject: mm: compaction: move compaction sysctl to its own file This moves all compaction sysctls to its own file. Move sysctl to where the functionality truly belongs to improve readability, reduce merge conflicts, and facilitate maintenance. I use x86_defconfig and linux-next-20230327 branch $ make defconfig;make all -jn CONFIG_COMPACTION=y add/remove: 1/0 grow/shrink: 1/1 up/down: 350/-256 (94) Function old new delta vm_compaction - 320 +320 kcompactd_init 180 210 +30 vm_table 2112 1856 -256 Total: Before=21119987, After=21120081, chg +0.00% Despite the addition of 94 bytes the patch still seems a worthwile cleanup. Link: https://lore.kernel.org/lkml/067f7347-ba10-5405-920c-0f5f985c84f4@suse.cz/ Signed-off-by: Minghao Chi Acked-by: Vlastimil Babka Signed-off-by: Luis Chamberlain --- include/linux/compaction.h | 7 ---- kernel/sysctl.c | 59 -------------------------------- mm/compaction.c | 84 +++++++++++++++++++++++++++++++++++++++------- 3 files changed, 72 insertions(+), 78 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compaction.h b/include/linux/compaction.h index 52a9ff65faee..a6e512cfb670 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -81,13 +81,6 @@ static inline unsigned long compact_gap(unsigned int order) } #ifdef CONFIG_COMPACTION -extern unsigned int sysctl_compaction_proactiveness; -extern int sysctl_compaction_handler(struct ctl_table *table, int write, - void *buffer, size_t *length, loff_t *ppos); -extern int compaction_proactiveness_sysctl_handler(struct ctl_table *table, - int write, void *buffer, size_t *length, loff_t *ppos); -extern int sysctl_extfrag_threshold; -extern int sysctl_compact_unevictable_allowed; extern unsigned int extfrag_for_order(struct zone *zone, unsigned int order); extern int fragmentation_index(struct zone *zone, unsigned int order); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 0a8a3c9c82e4..bfe53e835524 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -42,7 +42,6 @@ #include #include #include -#include #include #include #include @@ -746,27 +745,6 @@ int proc_dointvec(struct ctl_table *table, int write, void *buffer, return do_proc_dointvec(table, write, buffer, lenp, ppos, NULL, NULL); } -#ifdef CONFIG_COMPACTION -static int proc_dointvec_minmax_warn_RT_change(struct ctl_table *table, - int write, void *buffer, size_t *lenp, loff_t *ppos) -{ - int ret, old; - - if (!IS_ENABLED(CONFIG_PREEMPT_RT) || !write) - return proc_dointvec_minmax(table, write, buffer, lenp, ppos); - - old = *(int *)table->data; - ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); - if (ret) - return ret; - if (old != *(int *)table->data) - pr_warn_once("sysctl attribute %s changed by %s[%d]\n", - table->procname, current->comm, - task_pid_nr(current)); - return ret; -} -#endif - /** * proc_douintvec - read a vector of unsigned integers * @table: the sysctl table @@ -2157,43 +2135,6 @@ static struct ctl_table vm_table[] = { .extra1 = SYSCTL_ONE, .extra2 = SYSCTL_FOUR, }, -#ifdef CONFIG_COMPACTION - { - .procname = "compact_memory", - .data = NULL, - .maxlen = sizeof(int), - .mode = 0200, - .proc_handler = sysctl_compaction_handler, - }, - { - .procname = "compaction_proactiveness", - .data = &sysctl_compaction_proactiveness, - .maxlen = sizeof(sysctl_compaction_proactiveness), - .mode = 0644, - .proc_handler = compaction_proactiveness_sysctl_handler, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE_HUNDRED, - }, - { - .procname = "extfrag_threshold", - .data = &sysctl_extfrag_threshold, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE_THOUSAND, - }, - { - .procname = "compact_unevictable_allowed", - .data = &sysctl_compact_unevictable_allowed, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax_warn_RT_change, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, - -#endif /* CONFIG_COMPACTION */ { .procname = "min_free_kbytes", .data = &min_free_kbytes, diff --git a/mm/compaction.c b/mm/compaction.c index 5a9501e0ae01..f2ddfb1140e2 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1716,7 +1716,14 @@ typedef enum { * Allow userspace to control policy on scanning the unevictable LRU for * compactable pages. */ -int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT; +static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT; +/* + * Tunable for proactive compaction. It determines how + * aggressively the kernel should compact memory in the + * background. It takes values in the range [0, 100]. + */ +static unsigned int __read_mostly sysctl_compaction_proactiveness = 20; +static int sysctl_extfrag_threshold = 500; static inline void update_fast_start_pfn(struct compact_control *cc, unsigned long pfn) @@ -2572,8 +2579,6 @@ static enum compact_result compact_zone_order(struct zone *zone, int order, return ret; } -int sysctl_extfrag_threshold = 500; - /** * try_to_compact_pages - Direct compact to satisfy a high-order allocation * @gfp_mask: The GFP mask of the current allocation @@ -2730,14 +2735,7 @@ static void compact_nodes(void) compact_node(nid); } -/* - * Tunable for proactive compaction. It determines how - * aggressively the kernel should compact memory in the - * background. It takes values in the range [0, 100]. - */ -unsigned int __read_mostly sysctl_compaction_proactiveness = 20; - -int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write, +static int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write, void *buffer, size_t *length, loff_t *ppos) { int rc, nid; @@ -2767,7 +2765,7 @@ int compaction_proactiveness_sysctl_handler(struct ctl_table *table, int write, * This is the entry point for compacting all nodes via * /proc/sys/vm/compact_memory */ -int sysctl_compaction_handler(struct ctl_table *table, int write, +static int sysctl_compaction_handler(struct ctl_table *table, int write, void *buffer, size_t *length, loff_t *ppos) { if (write) @@ -3063,6 +3061,65 @@ static int kcompactd_cpu_online(unsigned int cpu) return 0; } +static int proc_dointvec_minmax_warn_RT_change(struct ctl_table *table, + int write, void *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, old; + + if (!IS_ENABLED(CONFIG_PREEMPT_RT) || !write) + return proc_dointvec_minmax(table, write, buffer, lenp, ppos); + + old = *(int *)table->data; + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + return ret; + if (old != *(int *)table->data) + pr_warn_once("sysctl attribute %s changed by %s[%d]\n", + table->procname, current->comm, + task_pid_nr(current)); + return ret; +} + +#ifdef CONFIG_SYSCTL +static struct ctl_table vm_compaction[] = { + { + .procname = "compact_memory", + .data = NULL, + .maxlen = sizeof(int), + .mode = 0200, + .proc_handler = sysctl_compaction_handler, + }, + { + .procname = "compaction_proactiveness", + .data = &sysctl_compaction_proactiveness, + .maxlen = sizeof(sysctl_compaction_proactiveness), + .mode = 0644, + .proc_handler = compaction_proactiveness_sysctl_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, + }, + { + .procname = "extfrag_threshold", + .data = &sysctl_extfrag_threshold, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_THOUSAND, + }, + { + .procname = "compact_unevictable_allowed", + .data = &sysctl_compact_unevictable_allowed, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax_warn_RT_change, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { } +}; +#endif + static int __init kcompactd_init(void) { int nid; @@ -3078,6 +3135,9 @@ static int __init kcompactd_init(void) for_each_node_state(nid, N_MEMORY) kcompactd_run(nid); +#ifdef CONFIG_SYSCTL + register_sysctl_init("vm", vm_compaction); +#endif return 0; } subsys_initcall(kcompactd_init) -- cgit v1.2.3 From 987d2e0aaa55de40938435be760aa96428470fd6 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Fri, 31 Mar 2023 17:15:52 +0800 Subject: module: Move is_arm_mapping_symbol() to module_symbol.h In order to avoid duplicated code, move is_arm_mapping_symbol() to include/linux/module_symbol.h, then remove is_arm_mapping_symbol() in the other places. Signed-off-by: Tiezhu Yang Signed-off-by: Luis Chamberlain --- include/linux/module_symbol.h | 15 +++++++++++++++ kernel/module/kallsyms.c | 14 +------------- scripts/mod/modpost.c | 10 +--------- 3 files changed, 17 insertions(+), 22 deletions(-) create mode 100644 include/linux/module_symbol.h (limited to 'include/linux') diff --git a/include/linux/module_symbol.h b/include/linux/module_symbol.h new file mode 100644 index 000000000000..9fa4173d0197 --- /dev/null +++ b/include/linux/module_symbol.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MODULE_SYMBOL_H +#define _LINUX_MODULE_SYMBOL_H + +/* This ignores the intensely annoying "mapping symbols" found in ELF files. */ +static inline int is_arm_mapping_symbol(const char *str) +{ + if (str[0] == '.' && str[1] == 'L') + return true; + return str[0] == '$' && + (str[1] == 'a' || str[1] == 'd' || str[1] == 't' || str[1] == 'x') + && (str[2] == '\0' || str[2] == '.'); +} + +#endif /* _LINUX_MODULE_SYMBOL_H */ diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index a9045fe55bcd..5de3207f18af 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -243,19 +244,6 @@ void init_build_id(struct module *mod, const struct load_info *info) } #endif -/* - * This ignores the intensely annoying "mapping symbols" found - * in ARM ELF files: $a, $t and $d. - */ -static inline int is_arm_mapping_symbol(const char *str) -{ - if (str[0] == '.' && str[1] == 'L') - return true; - return str[0] == '$' && - (str[1] == 'a' || str[1] == 'd' || str[1] == 't' || str[1] == 'x') - && (str[2] == '\0' || str[2] == '.'); -} - static const char *kallsyms_symbol_name(struct mod_kallsyms *kallsyms, unsigned int symnum) { return kallsyms->strtab + kallsyms->symtab[symnum].st_name; diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 79a27cc5f0b1..7241db81ffde 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -22,6 +22,7 @@ #include #include "modpost.h" #include "../../include/linux/license.h" +#include "../../include/linux/module_symbol.h" /* Are we using CONFIG_MODVERSIONS? */ static bool modversions; @@ -1112,15 +1113,6 @@ static int secref_whitelist(const struct sectioncheck *mismatch, return 1; } -static inline int is_arm_mapping_symbol(const char *str) -{ - if (str[0] == '.' && str[1] == 'L') - return true; - return str[0] == '$' && - (str[1] == 'a' || str[1] == 'd' || str[1] == 't' || str[1] == 'x') - && (str[2] == '\0' || str[2] == '.'); -} - /* * If there's no name there, ignore it; likewise, ignore it if it's * one of the magic symbols emitted used by current ARM tools. -- cgit v1.2.3 From 0a3bf86092c38f7b72c56c6901c78dd302411307 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Fri, 31 Mar 2023 17:15:53 +0800 Subject: module: Ignore L0 and rename is_arm_mapping_symbol() The L0 symbol is generated when build module on LoongArch, ignore it in modpost and when looking at module symbols, otherwise we can not see the expected call trace. Now is_arm_mapping_symbol() is not only for ARM, in order to reflect the reality, rename is_arm_mapping_symbol() to is_mapping_symbol(). This is related with commit c17a2538704f ("mksysmap: Fix the mismatch of 'L0' symbols in System.map"). (1) Simple test case [loongson@linux hello]$ cat hello.c #include #include #include static void test_func(void) { pr_info("This is a test\n"); dump_stack(); } static int __init hello_init(void) { pr_warn("Hello, world\n"); test_func(); return 0; } static void __exit hello_exit(void) { pr_warn("Goodbye\n"); } module_init(hello_init); module_exit(hello_exit); MODULE_LICENSE("GPL"); [loongson@linux hello]$ cat Makefile obj-m:=hello.o ccflags-y += -g -Og all: make -C /lib/modules/$(shell uname -r)/build/ M=$(PWD) modules clean: make -C /lib/modules/$(shell uname -r)/build/ M=$(PWD) clean (2) Test environment system: LoongArch CLFS 5.5 https://github.com/sunhaiyong1978/CLFS-for-LoongArch/releases/tag/5.0 It needs to update grub to avoid booting error "invalid magic number". kernel: 6.3-rc1 with loongson3_defconfig + CONFIG_DYNAMIC_FTRACE=y (3) Test result Without this patch: [root@linux hello]# insmod hello.ko [root@linux hello]# dmesg ... Hello, world This is a test ... Call Trace: [<9000000000223728>] show_stack+0x68/0x18c [<90000000013374cc>] dump_stack_lvl+0x60/0x88 [] L0\x01+0x20/0x2c [hello] [] L0\x01+0x20/0x30 [hello] [<900000000022097c>] do_one_initcall+0x88/0x288 [<90000000002df890>] do_init_module+0x54/0x200 [<90000000002e1e18>] __do_sys_finit_module+0xc4/0x114 [<90000000013382e8>] do_syscall+0x7c/0x94 [<9000000000221e3c>] handle_syscall+0xbc/0x158 With this patch: [root@linux hello]# insmod hello.ko [root@linux hello]# dmesg ... Hello, world This is a test ... Call Trace: [<9000000000223728>] show_stack+0x68/0x18c [<90000000013374cc>] dump_stack_lvl+0x60/0x88 [] test_func+0x28/0x34 [hello] [] hello_init+0x28/0x38 [hello] [<900000000022097c>] do_one_initcall+0x88/0x288 [<90000000002df890>] do_init_module+0x54/0x200 [<90000000002e1e18>] __do_sys_finit_module+0xc4/0x114 [<90000000013382e8>] do_syscall+0x7c/0x94 [<9000000000221e3c>] handle_syscall+0xbc/0x158 Signed-off-by: Tiezhu Yang Tested-by: Youling Tang # for LoongArch Signed-off-by: Luis Chamberlain --- include/linux/module_symbol.h | 4 +++- kernel/module/kallsyms.c | 2 +- scripts/mod/modpost.c | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/module_symbol.h b/include/linux/module_symbol.h index 9fa4173d0197..7ace7ba30203 100644 --- a/include/linux/module_symbol.h +++ b/include/linux/module_symbol.h @@ -3,10 +3,12 @@ #define _LINUX_MODULE_SYMBOL_H /* This ignores the intensely annoying "mapping symbols" found in ELF files. */ -static inline int is_arm_mapping_symbol(const char *str) +static inline int is_mapping_symbol(const char *str) { if (str[0] == '.' && str[1] == 'L') return true; + if (str[0] == 'L' && str[1] == '0') + return true; return str[0] == '$' && (str[1] == 'a' || str[1] == 'd' || str[1] == 't' || str[1] == 'x') && (str[2] == '\0' || str[2] == '.'); diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index 5de3207f18af..d8e426a9a0cd 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -289,7 +289,7 @@ static const char *find_kallsyms_symbol(struct module *mod, * and inserted at a whim. */ if (*kallsyms_symbol_name(kallsyms, i) == '\0' || - is_arm_mapping_symbol(kallsyms_symbol_name(kallsyms, i))) + is_mapping_symbol(kallsyms_symbol_name(kallsyms, i))) continue; if (thisval <= addr && thisval > bestval) { diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 7241db81ffde..5cddf7623945 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -1115,7 +1115,7 @@ static int secref_whitelist(const struct sectioncheck *mismatch, /* * If there's no name there, ignore it; likewise, ignore it if it's - * one of the magic symbols emitted used by current ARM tools. + * one of the magic symbols emitted used by current tools. * * Otherwise if find_symbols_between() returns those symbols, they'll * fail the whitelist tests and cause lots of false alarms ... fixable @@ -1128,7 +1128,7 @@ static inline int is_valid_name(struct elf_info *elf, Elf_Sym *sym) if (!name || !strlen(name)) return 0; - return !is_arm_mapping_symbol(name); + return !is_mapping_symbol(name); } /** -- cgit v1.2.3 From ef9f643a9f8b62bcbcc51f0e0af8599adc2e17ed Mon Sep 17 00:00:00 2001 From: Pierre Gondois Date: Fri, 14 Apr 2023 10:14:52 +0200 Subject: cacheinfo: Add use_arch[|_cache]_info field/function The cache information can be extracted from either a Device Tree (DT), the PPTT ACPI table, or arch registers (clidr_el1 for arm64). The clidr_el1 register is used only if DT/ACPI information is not available. It does not states how caches are shared among CPUs. Add a use_arch_cache_info field/function to identify when the DT/ACPI doesn't provide cache information. Use this information to assume L1 caches are privates and L2 and higher are shared among all CPUs. Signed-off-by: Pierre Gondois Link: https://lore.kernel.org/r/20230414081453.244787-5-pierre.gondois@arm.com Signed-off-by: Sudeep Holla --- drivers/base/cacheinfo.c | 12 ++++++++++-- include/linux/cacheinfo.h | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index f16e5a82f0f3..45e36721bc24 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -28,6 +28,9 @@ static DEFINE_PER_CPU(struct cpu_cacheinfo, ci_cpu_cacheinfo); #define per_cpu_cacheinfo_idx(cpu, idx) \ (per_cpu_cacheinfo(cpu) + (idx)) +/* Set if no cache information is found in DT/ACPI. */ +static bool use_arch_info; + struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu) { return ci_cacheinfo(cpu); @@ -40,7 +43,8 @@ static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf, * For non DT/ACPI systems, assume unique level 1 caches, * system-wide shared caches for all other levels. */ - if (!(IS_ENABLED(CONFIG_OF) || IS_ENABLED(CONFIG_ACPI))) + if (!(IS_ENABLED(CONFIG_OF) || IS_ENABLED(CONFIG_ACPI)) || + use_arch_info) return (this_leaf->level != 1) && (sib_leaf->level != 1); if ((sib_leaf->attributes & CACHE_ID) && @@ -343,6 +347,10 @@ static int cache_setup_properties(unsigned int cpu) else if (!acpi_disabled) ret = cache_setup_acpi(cpu); + // Assume there is no cache information available in DT/ACPI from now. + if (ret && use_arch_cache_info()) + use_arch_info = true; + return ret; } @@ -361,7 +369,7 @@ static int cache_shared_cpu_map_setup(unsigned int cpu) * to update the shared cpu_map if the cache attributes were * populated early before all the cpus are brought online */ - if (!last_level_cache_is_valid(cpu)) { + if (!last_level_cache_is_valid(cpu) && !use_arch_info) { ret = cache_setup_properties(cpu); if (ret) return ret; diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h index 6147b2672555..a5cfd44fab45 100644 --- a/include/linux/cacheinfo.h +++ b/include/linux/cacheinfo.h @@ -131,4 +131,10 @@ static inline int get_cpu_cacheinfo_id(int cpu, int level) return -1; } +#ifdef CONFIG_ARM64 +#define use_arch_cache_info() (true) +#else +#define use_arch_cache_info() (false) +#endif + #endif /* _LINUX_CACHEINFO_H */ -- cgit v1.2.3 From e223864f8257afde5e23eca4c006a0d69581a7a2 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 12 Apr 2023 11:10:45 -0300 Subject: iommu: Make iommu_release_device() static This is not called outside the core code, and indeed cannot be called correctly outside the bus notifier. Make it static. Signed-off-by: Jason Gunthorpe Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/0-v1-c3da18124d2d+56-rm_iommu_release_jgg@nvidia.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 3 ++- include/linux/iommu.h | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 7abee83610b6..435fc902df19 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -87,6 +87,7 @@ static const char * const iommu_group_resv_type_string[] = { static int iommu_bus_notifier(struct notifier_block *nb, unsigned long action, void *data); +static void iommu_release_device(struct device *dev); static int iommu_alloc_default_domain(struct iommu_group *group, struct device *dev); static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, @@ -493,7 +494,7 @@ static void __iommu_group_release_device(struct iommu_group *group, kobject_put(group->devices_kobj); } -void iommu_release_device(struct device *dev) +static void iommu_release_device(struct device *dev) { struct iommu_group *group = dev->iommu_group; struct group_device *device; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 54f535ff9868..c892e06f8357 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -699,7 +699,6 @@ static inline void dev_iommu_priv_set(struct device *dev, void *priv) } int iommu_probe_device(struct device *dev); -void iommu_release_device(struct device *dev); int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f); int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f); -- cgit v1.2.3 From f7f9c054a227ad4922070d748b1f4fc4b5657329 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 12 Apr 2023 11:11:58 -0300 Subject: iommu: Remove iommu_group_get_by_id() This is never called. Signed-off-by: Jason Gunthorpe Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/0-v1-60bbc66d7e92+24-rm_iommu_get_by_id_jgg@nvidia.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 29 ----------------------------- include/linux/iommu.h | 6 ------ 2 files changed, 35 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 435fc902df19..f35058f1d68e 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -882,35 +882,6 @@ struct iommu_group *iommu_group_alloc(void) } EXPORT_SYMBOL_GPL(iommu_group_alloc); -struct iommu_group *iommu_group_get_by_id(int id) -{ - struct kobject *group_kobj; - struct iommu_group *group; - const char *name; - - if (!iommu_group_kset) - return NULL; - - name = kasprintf(GFP_KERNEL, "%d", id); - if (!name) - return NULL; - - group_kobj = kset_find_obj(iommu_group_kset, name); - kfree(name); - - if (!group_kobj) - return NULL; - - group = container_of(group_kobj, struct iommu_group, kobj); - BUG_ON(group->id != id); - - kobject_get(group->devices_kobj); - kobject_put(&group->kobj); - - return group; -} -EXPORT_SYMBOL_GPL(iommu_group_get_by_id); - /** * iommu_group_get_iommudata - retrieve iommu_data registered for a group * @group: the group diff --git a/include/linux/iommu.h b/include/linux/iommu.h index c892e06f8357..7dbdd13d7ce0 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -460,7 +460,6 @@ extern bool iommu_present(struct bus_type *bus); extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap); extern bool iommu_group_has_isolated_msi(struct iommu_group *group); extern struct iommu_domain *iommu_domain_alloc(struct bus_type *bus); -extern struct iommu_group *iommu_group_get_by_id(int id); extern void iommu_domain_free(struct iommu_domain *domain); extern int iommu_attach_device(struct iommu_domain *domain, struct device *dev); @@ -746,11 +745,6 @@ static inline struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) return NULL; } -static inline struct iommu_group *iommu_group_get_by_id(int id) -{ - return NULL; -} - static inline void iommu_domain_free(struct iommu_domain *domain) { } -- cgit v1.2.3 From e0b081d17a9f4e5c0cbb0e5fbeb1abe3de0f7e4e Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 10:24:07 -0700 Subject: sched: Fix KCSAN noinstr violation With KCSAN enabled, end_of_stack() can get out-of-lined. Force it inline. Fixes the following warnings: vmlinux.o: warning: objtool: check_stackleak_irqoff+0x2b: call to end_of_stack() leaves .noinstr.text section Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/cc1b4d73d3a428a00d206242a68fdf99a934ca7b.1681320026.git.jpoimboe@kernel.org --- include/linux/sched/task_stack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sched/task_stack.h b/include/linux/sched/task_stack.h index 5e799a47431e..f158b025c175 100644 --- a/include/linux/sched/task_stack.h +++ b/include/linux/sched/task_stack.h @@ -23,7 +23,7 @@ static __always_inline void *task_stack_page(const struct task_struct *task) #define setup_thread_stack(new,old) do { } while(0) -static inline unsigned long *end_of_stack(const struct task_struct *task) +static __always_inline unsigned long *end_of_stack(const struct task_struct *task) { #ifdef CONFIG_STACK_GROWSUP return (unsigned long *)((unsigned long)task->stack + THREAD_SIZE) - 1; -- cgit v1.2.3 From e8deb00c0c4808654d1bf96a8f79cf1deb59b631 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 10:24:06 -0700 Subject: context_tracking: Fix KCSAN noinstr violation With KCSAN enabled, even empty inline stubs can be out-of-lined. Force the context_tracking_guest_exit() stub inline. Fixes the following warnings: vmlinux.o: warning: objtool: vmx_vcpu_enter_exit+0x1be: call to context_tracking_guest_exit() leaves .noinstr.text section vmlinux.o: warning: objtool: svm_vcpu_enter_exit+0x85: call to context_tracking_guest_exit() leaves .noinstr.text section Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/dc93f45abdec90c171108b4b590b7fff5790963c.1681320026.git.jpoimboe@kernel.org --- include/linux/context_tracking.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index d4afa8508a80..5ae3abd767b4 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -97,7 +97,7 @@ static inline int exception_enter(void) { return 0; } static inline void exception_exit(enum ctx_state prev_ctx) { } static inline int ct_state(void) { return -1; } static __always_inline bool context_tracking_guest_enter(void) { return false; } -static inline void context_tracking_guest_exit(void) { } +static __always_inline void context_tracking_guest_exit(void) { } #define CT_WARN_ON(cond) do { } while (0) #endif /* !CONFIG_CONTEXT_TRACKING_USER */ -- cgit v1.2.3 From 9ea7e6b62c2bd2f7bbfc3f10099df803002dd33b Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:31 -0700 Subject: init: Mark [arch_call_]rest_init() __noreturn In preparation for improving objtool's handling of weak noreturn functions, mark start_kernel(), arch_call_rest_init(), and rest_init() __noreturn. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Nick Desaulniers Link: https://lore.kernel.org/r/7194ed8a989a85b98d92e62df660f4a90435a723.1681342859.git.jpoimboe@kernel.org --- arch/s390/kernel/setup.c | 2 +- include/linux/start_kernel.h | 4 ++-- init/main.c | 4 ++-- tools/objtool/check.c | 2 ++ 4 files changed, 7 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 8ec5cdf9dadc..4259b6c50516 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -396,7 +396,7 @@ int __init arch_early_irq_init(void) return 0; } -void __init arch_call_rest_init(void) +void __init __noreturn arch_call_rest_init(void) { unsigned long stack; diff --git a/include/linux/start_kernel.h b/include/linux/start_kernel.h index 8b369a41c03c..864921e54c92 100644 --- a/include/linux/start_kernel.h +++ b/include/linux/start_kernel.h @@ -9,7 +9,7 @@ up something else. */ extern asmlinkage void __init start_kernel(void); -extern void __init arch_call_rest_init(void); -extern void __ref rest_init(void); +extern void __init __noreturn arch_call_rest_init(void); +extern void __ref __noreturn rest_init(void); #endif /* _LINUX_START_KERNEL_H */ diff --git a/init/main.c b/init/main.c index 4425d1783d5c..161ed956d738 100644 --- a/init/main.c +++ b/init/main.c @@ -683,7 +683,7 @@ static void __init setup_command_line(char *command_line) static __initdata DECLARE_COMPLETION(kthreadd_done); -noinline void __ref rest_init(void) +noinline void __ref __noreturn rest_init(void) { struct task_struct *tsk; int pid; @@ -889,7 +889,7 @@ static int __init early_randomize_kstack_offset(char *buf) early_param("randomize_kstack_offset", early_randomize_kstack_offset); #endif -void __init __weak arch_call_rest_init(void) +void __init __weak __noreturn arch_call_rest_init(void) { rest_init(); } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index df634dafefc4..3d7227f0ea2a 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -202,6 +202,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "__reiserfs_panic", "__stack_chk_fail", "__ubsan_handle_builtin_unreachable", + "arch_call_rest_init", "arch_cpu_idle_dead", "cpu_bringup_and_idle", "cpu_startup_entry", @@ -217,6 +218,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "machine_real_restart", "make_task_dead", "panic", + "rest_init", "rewind_stack_and_make_dead", "sev_es_terminate", "snp_abort", -- cgit v1.2.3 From 25a6917ca63ad4470bf88535c56f0dec72b570fe Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:32 -0700 Subject: init: Mark start_kernel() __noreturn Now that arch_call_rest_init() is __noreturn, mark its caller start_kernel() __noreturn. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/7069acf026a195f26a88061227fba5a3b0337b9a.1681342859.git.jpoimboe@kernel.org --- include/linux/start_kernel.h | 2 +- init/main.c | 2 +- tools/objtool/check.c | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/start_kernel.h b/include/linux/start_kernel.h index 864921e54c92..a9806a44a605 100644 --- a/include/linux/start_kernel.h +++ b/include/linux/start_kernel.h @@ -8,7 +8,7 @@ /* Define the prototype for start_kernel here, rather than cluttering up something else. */ -extern asmlinkage void __init start_kernel(void); +extern asmlinkage void __init __noreturn start_kernel(void); extern void __init __noreturn arch_call_rest_init(void); extern void __ref __noreturn rest_init(void); diff --git a/init/main.c b/init/main.c index 161ed956d738..65aab4e9bb49 100644 --- a/init/main.c +++ b/init/main.c @@ -937,7 +937,7 @@ static void __init print_unknown_bootoptions(void) memblock_free(unknown_options, len); } -asmlinkage __visible void __init __no_sanitize_address start_kernel(void) +asmlinkage __visible void __init __no_sanitize_address __noreturn start_kernel(void) { char *command_line; char *after_dashes; diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 3d7227f0ea2a..9aaad9dc6c20 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -222,6 +222,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "rewind_stack_and_make_dead", "sev_es_terminate", "snp_abort", + "start_kernel", "stop_this_cpu", "usercopy_abort", "xen_cpu_bringup_again", -- cgit v1.2.3 From 7412a60decec2a6744cf773280ff17a0f89e8395 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:35 -0700 Subject: cpu: Mark panic_smp_self_stop() __noreturn In preparation for improving objtool's handling of weak noreturn functions, mark panic_smp_self_stop() __noreturn. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/92d76ab5c8bf660f04fdcd3da1084519212de248.1681342859.git.jpoimboe@kernel.org --- arch/arm/kernel/smp.c | 2 +- arch/arm64/include/asm/smp.h | 1 - arch/arm64/kernel/smp.c | 2 +- arch/powerpc/kernel/setup_64.c | 2 +- include/linux/smp.h | 2 +- kernel/panic.c | 2 +- tools/objtool/check.c | 1 + 7 files changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index d6be4507d22d..f4a4ac028b6b 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -779,7 +779,7 @@ void smp_send_stop(void) * kdump fails. So split out the panic_smp_self_stop() and add * set_cpu_online(smp_processor_id(), false). */ -void panic_smp_self_stop(void) +void __noreturn panic_smp_self_stop(void) { pr_debug("CPU %u will stop doing anything useful since another CPU has paniced\n", smp_processor_id()); diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 07f4ea1490f4..f2d26235bfb4 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -143,7 +143,6 @@ bool cpus_are_stuck_in_kernel(void); extern void crash_smp_send_stop(void); extern bool smp_crash_stop_failed(void); -extern void panic_smp_self_stop(void); #endif /* ifndef __ASSEMBLY__ */ diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 07d156fddb5f..05fe797e4203 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -830,7 +830,7 @@ static void __noreturn local_cpu_stop(void) * that cpu_online_mask gets correctly updated and smp_send_stop() can skip * CPUs that have already stopped themselves. */ -void panic_smp_self_stop(void) +void __noreturn panic_smp_self_stop(void) { local_cpu_stop(); } diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index b2e0d3ce4261..246201d0d879 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -480,7 +480,7 @@ void early_setup_secondary(void) #endif /* CONFIG_SMP */ -void panic_smp_self_stop(void) +void __noreturn panic_smp_self_stop(void) { hard_irq_disable(); spin_begin(); diff --git a/include/linux/smp.h b/include/linux/smp.h index a80ab58ae3f1..2a737b39cf0a 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -59,7 +59,7 @@ int smp_call_function_single_async(int cpu, struct __call_single_data *csd); * Cpus stopping functions in panic. All have default weak definitions. * Architecture-dependent code may override them. */ -void panic_smp_self_stop(void); +void __noreturn panic_smp_self_stop(void); void nmi_panic_self_stop(struct pt_regs *regs); void crash_smp_send_stop(void); diff --git a/kernel/panic.c b/kernel/panic.c index 5cfea8302d23..5e4982db8dc9 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -141,7 +141,7 @@ EXPORT_SYMBOL(panic_blink); /* * Stop ourself in panic -- architecture code may override this */ -void __weak panic_smp_self_stop(void) +void __weak __noreturn panic_smp_self_stop(void) { while (1) cpu_relax(); diff --git a/tools/objtool/check.c b/tools/objtool/check.c index a436bc1e63d9..4b52ed6bff66 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -218,6 +218,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "machine_real_restart", "make_task_dead", "panic", + "panic_smp_self_stop", "rest_init", "rewind_stack_and_make_dead", "sev_es_terminate", -- cgit v1.2.3 From 27dea14c7f05106f39850a9239874cd38703b405 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2023 16:49:36 -0700 Subject: cpu: Mark nmi_panic_self_stop() __noreturn In preparation for improving objtool's handling of weak noreturn functions, mark nmi_panic_self_stop() __noreturn. Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/316fc6dfab5a8c4e024c7185484a1ee5fb0afb79.1681342859.git.jpoimboe@kernel.org --- arch/x86/include/asm/reboot.h | 1 - arch/x86/kernel/reboot.c | 2 +- include/linux/smp.h | 2 +- kernel/panic.c | 2 +- tools/objtool/check.c | 1 + 5 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/reboot.h b/arch/x86/include/asm/reboot.h index bc5b4d788c08..9177b4354c3f 100644 --- a/arch/x86/include/asm/reboot.h +++ b/arch/x86/include/asm/reboot.h @@ -28,7 +28,6 @@ void __noreturn machine_real_restart(unsigned int type); void cpu_emergency_disable_virtualization(void); typedef void (*nmi_shootdown_cb)(int, struct pt_regs*); -void nmi_panic_self_stop(struct pt_regs *regs); void nmi_shootdown_cpus(nmi_shootdown_cb callback); void run_crash_ipi_callback(struct pt_regs *regs); diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index d03c551defcc..3adbe97015c1 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -920,7 +920,7 @@ void run_crash_ipi_callback(struct pt_regs *regs) } /* Override the weak function in kernel/panic.c */ -void nmi_panic_self_stop(struct pt_regs *regs) +void __noreturn nmi_panic_self_stop(struct pt_regs *regs) { while (1) { /* If no CPU is preparing crash dump, we simply loop here. */ diff --git a/include/linux/smp.h b/include/linux/smp.h index 2a737b39cf0a..7b93504eed26 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -60,7 +60,7 @@ int smp_call_function_single_async(int cpu, struct __call_single_data *csd); * Architecture-dependent code may override them. */ void __noreturn panic_smp_self_stop(void); -void nmi_panic_self_stop(struct pt_regs *regs); +void __noreturn nmi_panic_self_stop(struct pt_regs *regs); void crash_smp_send_stop(void); /* diff --git a/kernel/panic.c b/kernel/panic.c index 5e4982db8dc9..886d2ebd0a0d 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -151,7 +151,7 @@ void __weak __noreturn panic_smp_self_stop(void) * Stop ourselves in NMI context if another CPU has already panicked. Arch code * may override this to prepare for crash dumping, e.g. save regs info. */ -void __weak nmi_panic_self_stop(struct pt_regs *regs) +void __weak __noreturn nmi_panic_self_stop(struct pt_regs *regs) { panic_smp_self_stop(); } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 4b52ed6bff66..8d073bfd7d7f 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -217,6 +217,7 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "lbug_with_loc", "machine_real_restart", "make_task_dead", + "nmi_panic_self_stop", "panic", "panic_smp_self_stop", "rest_init", -- cgit v1.2.3 From 38e4614c27210e63aafad1bb2e4a427515e46d71 Mon Sep 17 00:00:00 2001 From: Simon Horman Date: Wed, 29 Mar 2023 14:06:03 +0200 Subject: vfio: correct kdoc for ops structures Address minor omissions from kdoc for ops structures flagged by check-kdoc: ./scripts/kernel-doc -Werror -none include/linux/vfio.h include/linux/vfio.h:114: warning: Function parameter or member 'name' not described in 'vfio_device_ops' include/linux/vfio.h:143: warning: Cannot understand * @migration_set_state: Optional callback to change the migration state for on line 143 - I thought it was a doc line include/linux/vfio.h:168: warning: Cannot understand * @log_start: Optional callback to ask the device start DMA logging. on line 168 - I thought it was a doc line Signed-off-by: Simon Horman Link: https://lore.kernel.org/r/20230329120603.468031-1-horms@kernel.org Signed-off-by: Alex Williamson --- include/linux/vfio.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 93134b023968..cb46050045c0 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -68,6 +68,7 @@ struct vfio_device { /** * struct vfio_device_ops - VFIO bus driver device callbacks * + * @name: Name of the device driver. * @init: initialize private fields in device structure * @release: Reclaim private fields in device structure * @bind_iommufd: Called when binding the device to an iommufd @@ -140,6 +141,8 @@ int vfio_iommufd_emulated_attach_ioas(struct vfio_device *vdev, u32 *pt_id); #endif /** + * struct vfio_migration_ops - VFIO bus device driver migration callbacks + * * @migration_set_state: Optional callback to change the migration state for * devices that support migration. It's mandatory for * VFIO_DEVICE_FEATURE_MIGRATION migration support. @@ -165,6 +168,8 @@ struct vfio_migration_ops { }; /** + * struct vfio_log_ops - VFIO bus device driver logging callbacks + * * @log_start: Optional callback to ask the device start DMA logging. * @log_stop: Optional callback to ask the device stop DMA logging. * @log_read_and_clear: Optional callback to ask the device read -- cgit v1.2.3 From 8b0977ecc8b30a30966e76fcb64cef5041626b02 Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Thu, 13 Apr 2023 10:57:37 -0700 Subject: swiotlb: track and report io_tlb_used high water marks in debugfs swiotlb currently reports the total number of slabs and the instantaneous in-use slabs in debugfs. But with increased usage of swiotlb for all I/O in Confidential Computing (coco) VMs, it has become difficult to know how much memory to allocate for swiotlb bounce buffers, either via the automatic algorithm in the kernel or by specifying a value on the kernel boot line. The current automatic algorithm generously allocates swiotlb bounce buffer memory, and may be wasting significant memory in many use cases. To support better understanding of swiotlb usage, add tracking of the the high water mark for usage of the default swiotlb bounce buffer memory pool and any reserved memory pools. Report these high water marks in debugfs along with the other swiotlb pool metrics. Allow the high water marks to be reset to zero at runtime by writing to them. Signed-off-by: Michael Kelley Signed-off-by: Christoph Hellwig --- include/linux/swiotlb.h | 7 ++++++ kernel/dma/swiotlb.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) (limited to 'include/linux') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index bcef10e20ea4..6dc4598d2260 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -87,6 +87,11 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, * @for_alloc: %true if the pool is used for memory allocation * @nareas: The area number in the pool. * @area_nslabs: The slot number in the area. + * @total_used: The total number of slots in the pool that are currently used + * across all areas. Used only for calculating used_hiwater in + * debugfs. + * @used_hiwater: The high water mark for total_used. Used only for reporting + * in debugfs. */ struct io_tlb_mem { phys_addr_t start; @@ -102,6 +107,8 @@ struct io_tlb_mem { unsigned int area_nslabs; struct io_tlb_area *areas; struct io_tlb_slot *slots; + atomic_long_t total_used; + atomic_long_t used_hiwater; }; extern struct io_tlb_mem io_tlb_default_mem; diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 938c959ab19e..9bbc2802a444 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -608,6 +608,40 @@ static unsigned int wrap_area_index(struct io_tlb_mem *mem, unsigned int index) return index; } +/* + * Track the total used slots with a global atomic value in order to have + * correct information to determine the high water mark. The mem_used() + * function gives imprecise results because there's no locking across + * multiple areas. + */ +#ifdef CONFIG_DEBUG_FS +static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots) +{ + unsigned long old_hiwater, new_used; + + new_used = atomic_long_add_return(nslots, &mem->total_used); + old_hiwater = atomic_long_read(&mem->used_hiwater); + do { + if (new_used <= old_hiwater) + break; + } while (!atomic_long_try_cmpxchg(&mem->used_hiwater, + &old_hiwater, new_used)); +} + +static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) +{ + atomic_long_sub(nslots, &mem->total_used); +} + +#else /* !CONFIG_DEBUG_FS */ +static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots) +{ +} +static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) +{ +} +#endif /* CONFIG_DEBUG_FS */ + /* * Find a suitable number of IO TLB entries size that will fit this request and * allocate a buffer from that IO TLB pool. @@ -702,6 +736,8 @@ found: area->index = wrap_area_index(mem, index + nslots); area->used += nslots; spin_unlock_irqrestore(&area->lock, flags); + + inc_used_and_hiwater(mem, nslots); return slot_index; } @@ -834,6 +870,8 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr) mem->slots[i].list = ++count; area->used -= nslots; spin_unlock_irqrestore(&area->lock, flags); + + dec_used(mem, nslots); } /* @@ -935,11 +973,37 @@ static int io_tlb_used_get(void *data, u64 *val) *val = mem_used(mem); return 0; } + +static int io_tlb_hiwater_get(void *data, u64 *val) +{ + struct io_tlb_mem *mem = data; + + *val = atomic_long_read(&mem->used_hiwater); + return 0; +} + +static int io_tlb_hiwater_set(void *data, u64 val) +{ + struct io_tlb_mem *mem = data; + + /* Only allow setting to zero */ + if (val != 0) + return -EINVAL; + + atomic_long_set(&mem->used_hiwater, val); + return 0; +} + DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get, + io_tlb_hiwater_set, "%llu\n"); static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, const char *dirname) { + atomic_long_set(&mem->total_used, 0); + atomic_long_set(&mem->used_hiwater, 0); + mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs); if (!mem->nslabs) return; @@ -947,6 +1011,8 @@ static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs); debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem, &fops_io_tlb_used); + debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem, + &fops_io_tlb_hiwater); } static int __init __maybe_unused swiotlb_create_default_debugfs(void) -- cgit v1.2.3 From ccbbfe0682f2fff1e157413c30092dd27c50e20e Mon Sep 17 00:00:00 2001 From: Avihai Horon Date: Mon, 10 Apr 2023 16:07:52 +0300 Subject: net/mlx5: Update relaxed ordering read HCA capabilities Rename existing HCA capability relaxed_ordering_read to relaxed_ordering_read_pci_enabled. This is in accordance with recent PRM change to better describe the capability, as it's set only if both the device supports relaxed ordering (RO) read and RO is enabled in PCI config space. In addition, add new HCA capability relaxed_ordering_read which is set if the device supports RO read, regardless of RO in PCI config space. This will be used in the following patch to allow RO in VFs and VMs. Signed-off-by: Avihai Horon Reviewed-by: Shay Drory Link: https://lore.kernel.org/r/caa0002fd8135086357dfcc368e2f5cc73b08480.1681131553.git.leon@kernel.org Reviewed-by: Jacob Keller Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/mr.c | 5 +++-- drivers/infiniband/hw/mlx5/umr.h | 2 +- drivers/net/ethernet/mellanox/mlx5/core/en_common.c | 2 +- include/linux/mlx5/mlx5_ifc.h | 5 +++-- 4 files changed, 8 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index bd0a818ba1cd..6a3a8e00bfaa 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -70,7 +70,8 @@ static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr, if (acc & IB_ACCESS_RELAXED_ORDERING) { if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) MLX5_SET(mkc, mkc, relaxed_ordering_write, 1); - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) && + if (MLX5_CAP_GEN(dev->mdev, + relaxed_ordering_read_pci_enabled) && pcie_relaxed_ordering_enabled(dev->mdev->pdev)) MLX5_SET(mkc, mkc, relaxed_ordering_read, 1); } @@ -791,7 +792,7 @@ static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev, ret |= IB_ACCESS_RELAXED_ORDERING; if ((access_flags & IB_ACCESS_RELAXED_ORDERING) && - MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) && + MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_pci_enabled) && !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) ret |= IB_ACCESS_RELAXED_ORDERING; diff --git a/drivers/infiniband/hw/mlx5/umr.h b/drivers/infiniband/hw/mlx5/umr.h index c9d0021381a2..e12ecd7e079c 100644 --- a/drivers/infiniband/hw/mlx5/umr.h +++ b/drivers/infiniband/hw/mlx5/umr.h @@ -62,7 +62,7 @@ static inline bool mlx5r_umr_can_reconfig(struct mlx5_ib_dev *dev, return false; if ((diffs & IB_ACCESS_RELAXED_ORDERING) && - MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) && + MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_pci_enabled) && !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) return false; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c index 993af4c12d90..3c765a1f91a5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c @@ -41,7 +41,7 @@ void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc) { bool ro_pci_enable = pcie_relaxed_ordering_enabled(mdev->pdev); bool ro_write = MLX5_CAP_GEN(mdev, relaxed_ordering_write); - bool ro_read = MLX5_CAP_GEN(mdev, relaxed_ordering_read); + bool ro_read = MLX5_CAP_GEN(mdev, relaxed_ordering_read_pci_enabled); MLX5_SET(mkc, mkc, relaxed_ordering_read, ro_pci_enable && ro_read); MLX5_SET(mkc, mkc, relaxed_ordering_write, ro_write); diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index e4306cd87cd7..b54339a1b1c6 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1511,7 +1511,7 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 log_max_eq_sz[0x8]; u8 relaxed_ordering_write[0x1]; - u8 relaxed_ordering_read[0x1]; + u8 relaxed_ordering_read_pci_enabled[0x1]; u8 log_max_mkey[0x6]; u8 reserved_at_f0[0x6]; u8 terminate_scatter_list_mkey[0x1]; @@ -1727,7 +1727,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_320[0x3]; u8 log_max_transport_domain[0x5]; - u8 reserved_at_328[0x3]; + u8 reserved_at_328[0x2]; + u8 relaxed_ordering_read[0x1]; u8 log_max_pd[0x5]; u8 reserved_at_330[0x9]; u8 q_counter_aggregation[0x1]; -- cgit v1.2.3 From 1f6277bf716cc5ba0e3fa0c3e0af1adb4160fb5d Mon Sep 17 00:00:00 2001 From: Saurabh Sengar Date: Mon, 20 Mar 2023 00:47:37 -0700 Subject: ACPI: bus: Add stub acpi_sleep_state_supported() in non-ACPI cases acpi_sleep_state_supported() is defined only when CONFIG_ACPI=y. The function is in acpi_bus.h, and acpi_bus.h can only be used in CONFIG_ACPI=y cases. Add the stub function to linux/acpi.h to make compilation successful for !CONFIG_ACPI cases. Signed-off-by: Saurabh Sengar Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/1679298460-11855-3-git-send-email-ssengar@linux.microsoft.com Signed-off-by: Wei Liu --- include/linux/acpi.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/acpi.h b/include/linux/acpi.h index efff750f326d..d331f76b0c19 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1075,6 +1075,11 @@ static inline u32 acpi_osc_ctx_get_cxl_control(struct acpi_osc_context *context) return 0; } +static inline bool acpi_sleep_state_supported(u8 sleep_state) +{ + return false; +} + #endif /* !CONFIG_ACPI */ #ifdef CONFIG_ACPI_HOTPLUG_IOAPIC -- cgit v1.2.3 From 0459ff4873739986dccafbb417cfc69e71bdacf4 Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Sun, 26 Mar 2023 06:52:02 -0700 Subject: swiotlb: Remove bounce buffer remapping for Hyper-V With changes to how Hyper-V guest VMs flip memory between private (encrypted) and shared (decrypted), creating a second kernel virtual mapping for shared memory is no longer necessary. Everything needed for the transition to shared is handled by set_memory_decrypted(). As such, remove swiotlb_unencrypted_base and the associated code. Signed-off-by: Michael Kelley Acked-by: Christoph Hellwig Acked-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/1679838727-87310-8-git-send-email-mikelley@microsoft.com Signed-off-by: Wei Liu --- arch/x86/kernel/cpu/mshyperv.c | 7 +------ include/linux/swiotlb.h | 2 -- kernel/dma/swiotlb.c | 45 +----------------------------------------- 3 files changed, 2 insertions(+), 52 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index 315fc358e584..ac630ecde795 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -408,12 +407,8 @@ static void __init ms_hyperv_init_platform(void) pr_info("Hyper-V: Isolation Config: Group A 0x%x, Group B 0x%x\n", ms_hyperv.isolation_config_a, ms_hyperv.isolation_config_b); - if (hv_get_isolation_type() == HV_ISOLATION_TYPE_SNP) { + if (hv_get_isolation_type() == HV_ISOLATION_TYPE_SNP) static_branch_enable(&isolation_type_snp); -#ifdef CONFIG_SWIOTLB - swiotlb_unencrypted_base = ms_hyperv.shared_gpa_boundary; -#endif - } } if (hv_max_functions_eax >= HYPERV_CPUID_NESTED_FEATURES) { diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index bcef10e20ea4..2ef25e6fa1b4 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -180,6 +180,4 @@ static inline bool is_swiotlb_for_alloc(struct device *dev) } #endif /* CONFIG_DMA_RESTRICTED_POOL */ -extern phys_addr_t swiotlb_unencrypted_base; - #endif /* __LINUX_SWIOTLB_H */ diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index dac42a2ad588..0fb16bdbbcae 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -73,8 +73,6 @@ static bool swiotlb_force_disable; struct io_tlb_mem io_tlb_default_mem; -phys_addr_t swiotlb_unencrypted_base; - static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT; static unsigned long default_nareas; @@ -201,34 +199,6 @@ static inline unsigned long nr_slots(u64 val) return DIV_ROUND_UP(val, IO_TLB_SIZE); } -/* - * Remap swioltb memory in the unencrypted physical address space - * when swiotlb_unencrypted_base is set. (e.g. for Hyper-V AMD SEV-SNP - * Isolation VMs). - */ -#ifdef CONFIG_HAS_IOMEM -static void *swiotlb_mem_remap(struct io_tlb_mem *mem, unsigned long bytes) -{ - void *vaddr = NULL; - - if (swiotlb_unencrypted_base) { - phys_addr_t paddr = mem->start + swiotlb_unencrypted_base; - - vaddr = memremap(paddr, bytes, MEMREMAP_WB); - if (!vaddr) - pr_err("Failed to map the unencrypted memory %pa size %lx.\n", - &paddr, bytes); - } - - return vaddr; -} -#else -static void *swiotlb_mem_remap(struct io_tlb_mem *mem, unsigned long bytes) -{ - return NULL; -} -#endif - /* * Early SWIOTLB allocation may be too early to allow an architecture to * perform the desired operations. This function allows the architecture to @@ -238,18 +208,12 @@ static void *swiotlb_mem_remap(struct io_tlb_mem *mem, unsigned long bytes) void __init swiotlb_update_mem_attributes(void) { struct io_tlb_mem *mem = &io_tlb_default_mem; - void *vaddr; unsigned long bytes; if (!mem->nslabs || mem->late_alloc) return; - vaddr = phys_to_virt(mem->start); bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); - set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT); - - mem->vaddr = swiotlb_mem_remap(mem, bytes); - if (!mem->vaddr) - mem->vaddr = vaddr; + set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT); } static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start, @@ -280,13 +244,6 @@ static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start, mem->slots[i].alloc_size = 0; } - /* - * If swiotlb_unencrypted_base is set, the bounce buffer memory will - * be remapped and cleared in swiotlb_update_mem_attributes. - */ - if (swiotlb_unencrypted_base) - return; - memset(vaddr, 0, bytes); mem->vaddr = vaddr; return; -- cgit v1.2.3 From 3e358ea8614ddfbc59ca7a3f5dff5dde2b350b2c Mon Sep 17 00:00:00 2001 From: Mark Bloch Date: Thu, 13 Apr 2023 12:23:09 +0300 Subject: RDMA/mlx5: Fix flow counter query via DEVX Commit cited in "fixes" tag added bulk support for flow counters but it didn't account that's also possible to query a counter using a non-base id if the counter was allocated as bulk. When a user performs a query, validate the flow counter id given in the mailbox is inside the valid range taking bulk value into account. Fixes: 208d70f562e5 ("IB/mlx5: Support flow counters offset for bulk counters") Signed-off-by: Mark Bloch Reviewed-by: Maor Gottlieb Link: https://lore.kernel.org/r/79d7fbe291690128e44672418934256254d93115.1681377114.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/devx.c | 31 ++++++++++++++++++++++++++----- include/linux/mlx5/mlx5_ifc.h | 3 ++- 2 files changed, 28 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 07037b829c7e..db5fb196c728 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -666,7 +666,21 @@ static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs, obj_id; case MLX5_IB_OBJECT_DEVX_OBJ: - return ((struct devx_obj *)uobj->object)->obj_id == obj_id; + { + u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); + struct devx_obj *devx_uobj = uobj->object; + + if (opcode == MLX5_CMD_OP_QUERY_FLOW_COUNTER && + devx_uobj->flow_counter_bulk_size) { + u64 end; + + end = devx_uobj->obj_id + + devx_uobj->flow_counter_bulk_size; + return devx_uobj->obj_id <= obj_id && end > obj_id; + } + + return devx_uobj->obj_id == obj_id; + } default: return false; @@ -1517,10 +1531,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( goto obj_free; if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) { - u8 bulk = MLX5_GET(alloc_flow_counter_in, - cmd_in, - flow_counter_bulk); - obj->flow_counter_bulk_size = 128UL * bulk; + u32 bulk = MLX5_GET(alloc_flow_counter_in, + cmd_in, + flow_counter_bulk_log_size); + + if (bulk) + bulk = 1 << bulk; + else + bulk = 128UL * MLX5_GET(alloc_flow_counter_in, + cmd_in, + flow_counter_bulk); + obj->flow_counter_bulk_size = bulk; } uobj->object = obj; diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index b54339a1b1c6..3976e6266bcc 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -9283,7 +9283,8 @@ struct mlx5_ifc_alloc_flow_counter_in_bits { u8 reserved_at_20[0x10]; u8 op_mod[0x10]; - u8 reserved_at_40[0x38]; + u8 reserved_at_40[0x33]; + u8 flow_counter_bulk_log_size[0x5]; u8 flow_counter_bulk[0x8]; }; -- cgit v1.2.3 From 85a953806557dbf25d16e8c132b5b9b100d16496 Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Mon, 10 Apr 2023 09:16:52 -0700 Subject: mailbox: Allow direct registration to a channel Support virtual mailbox controllers and clients which are not platform devices or come from the devicetree by allowing them to match client to channel via some other mechanism. Tested-by: Sudeep Holla (pcc) Signed-off-by: Elliot Berman Signed-off-by: Jassi Brar --- drivers/mailbox/mailbox.c | 96 ++++++++++++++++++++++++++++++------------ include/linux/mailbox_client.h | 1 + 2 files changed, 69 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c index 4229b9b5da98..adf36c05fa43 100644 --- a/drivers/mailbox/mailbox.c +++ b/drivers/mailbox/mailbox.c @@ -317,6 +317,71 @@ int mbox_flush(struct mbox_chan *chan, unsigned long timeout) } EXPORT_SYMBOL_GPL(mbox_flush); +static int __mbox_bind_client(struct mbox_chan *chan, struct mbox_client *cl) +{ + struct device *dev = cl->dev; + unsigned long flags; + int ret; + + if (chan->cl || !try_module_get(chan->mbox->dev->driver->owner)) { + dev_dbg(dev, "%s: mailbox not free\n", __func__); + return -EBUSY; + } + + spin_lock_irqsave(&chan->lock, flags); + chan->msg_free = 0; + chan->msg_count = 0; + chan->active_req = NULL; + chan->cl = cl; + init_completion(&chan->tx_complete); + + if (chan->txdone_method == TXDONE_BY_POLL && cl->knows_txdone) + chan->txdone_method = TXDONE_BY_ACK; + + spin_unlock_irqrestore(&chan->lock, flags); + + if (chan->mbox->ops->startup) { + ret = chan->mbox->ops->startup(chan); + + if (ret) { + dev_err(dev, "Unable to startup the chan (%d)\n", ret); + mbox_free_channel(chan); + return ret; + } + } + + return 0; +} + +/** + * mbox_bind_client - Request a mailbox channel. + * @chan: The mailbox channel to bind the client to. + * @cl: Identity of the client requesting the channel. + * + * The Client specifies its requirements and capabilities while asking for + * a mailbox channel. It can't be called from atomic context. + * The channel is exclusively allocated and can't be used by another + * client before the owner calls mbox_free_channel. + * After assignment, any packet received on this channel will be + * handed over to the client via the 'rx_callback'. + * The framework holds reference to the client, so the mbox_client + * structure shouldn't be modified until the mbox_free_channel returns. + * + * Return: 0 if the channel was assigned to the client successfully. + * <0 for request failure. + */ +int mbox_bind_client(struct mbox_chan *chan, struct mbox_client *cl) +{ + int ret; + + mutex_lock(&con_mutex); + ret = __mbox_bind_client(chan, cl); + mutex_unlock(&con_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(mbox_bind_client); + /** * mbox_request_channel - Request a mailbox channel. * @cl: Identity of the client requesting the channel. @@ -340,7 +405,6 @@ struct mbox_chan *mbox_request_channel(struct mbox_client *cl, int index) struct mbox_controller *mbox; struct of_phandle_args spec; struct mbox_chan *chan; - unsigned long flags; int ret; if (!dev || !dev->of_node) { @@ -372,33 +436,9 @@ struct mbox_chan *mbox_request_channel(struct mbox_client *cl, int index) return chan; } - if (chan->cl || !try_module_get(mbox->dev->driver->owner)) { - dev_dbg(dev, "%s: mailbox not free\n", __func__); - mutex_unlock(&con_mutex); - return ERR_PTR(-EBUSY); - } - - spin_lock_irqsave(&chan->lock, flags); - chan->msg_free = 0; - chan->msg_count = 0; - chan->active_req = NULL; - chan->cl = cl; - init_completion(&chan->tx_complete); - - if (chan->txdone_method == TXDONE_BY_POLL && cl->knows_txdone) - chan->txdone_method = TXDONE_BY_ACK; - - spin_unlock_irqrestore(&chan->lock, flags); - - if (chan->mbox->ops->startup) { - ret = chan->mbox->ops->startup(chan); - - if (ret) { - dev_err(dev, "Unable to startup the chan (%d)\n", ret); - mbox_free_channel(chan); - chan = ERR_PTR(ret); - } - } + ret = __mbox_bind_client(chan, cl); + if (ret) + chan = ERR_PTR(ret); mutex_unlock(&con_mutex); return chan; diff --git a/include/linux/mailbox_client.h b/include/linux/mailbox_client.h index 65229a45590f..734694912ef7 100644 --- a/include/linux/mailbox_client.h +++ b/include/linux/mailbox_client.h @@ -37,6 +37,7 @@ struct mbox_client { void (*tx_done)(struct mbox_client *cl, void *mssg, int r); }; +int mbox_bind_client(struct mbox_chan *chan, struct mbox_client *cl); struct mbox_chan *mbox_request_channel_byname(struct mbox_client *cl, const char *name); struct mbox_chan *mbox_request_channel(struct mbox_client *cl, int index); -- cgit v1.2.3 From 62e8b17ffc2ff0b0e29d5e05a18570c3e70b35ff Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Sat, 11 Mar 2023 15:40:07 +0100 Subject: PCI/DOE: Provide synchronous API and use it internally The DOE API only allows asynchronous exchanges and forces callers to provide a completion callback. Yet all existing callers only perform synchronous exchanges. Upcoming commits for CMA (Component Measurement and Authentication, PCIe r6.0 sec 6.31) likewise require only synchronous DOE exchanges. Provide a synchronous pci_doe() API call which builds on the internal asynchronous machinery. Convert the internal pci_doe_discovery() to the new call. The new API allows submission of const-declared requests, necessitating the addition of a const qualifier in struct pci_doe_task. Tested-by: Ira Weiny Signed-off-by: Lukas Wunner Reviewed-by: Ming Li Reviewed-by: Ira Weiny Reviewed-by: Davidlohr Bueso Reviewed-by: Dan Williams Reviewed-by: Jonathan Cameron Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/0f444206da9615c56301fbaff459c0f45d27f122.1678543498.git.lukas@wunner.de Signed-off-by: Dan Williams --- drivers/pci/doe.c | 69 ++++++++++++++++++++++++++++++++++++++----------- include/linux/pci-doe.h | 6 ++++- 2 files changed, 59 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c index e5e9b287b976..adde8c66499c 100644 --- a/drivers/pci/doe.c +++ b/drivers/pci/doe.c @@ -321,26 +321,15 @@ static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 *index, u16 *vid, __le32 request_pl_le = cpu_to_le32(request_pl); __le32 response_pl_le; u32 response_pl; - DECLARE_COMPLETION_ONSTACK(c); - struct pci_doe_task task = { - .prot.vid = PCI_VENDOR_ID_PCI_SIG, - .prot.type = PCI_DOE_PROTOCOL_DISCOVERY, - .request_pl = &request_pl_le, - .request_pl_sz = sizeof(request_pl), - .response_pl = &response_pl_le, - .response_pl_sz = sizeof(response_pl), - .complete = pci_doe_task_complete, - .private = &c, - }; int rc; - rc = pci_doe_submit_task(doe_mb, &task); + rc = pci_doe(doe_mb, PCI_VENDOR_ID_PCI_SIG, PCI_DOE_PROTOCOL_DISCOVERY, + &request_pl_le, sizeof(request_pl_le), + &response_pl_le, sizeof(response_pl_le)); if (rc < 0) return rc; - wait_for_completion(&c); - - if (task.rv != sizeof(response_pl)) + if (rc != sizeof(response_pl_le)) return -EIO; response_pl = le32_to_cpu(response_pl_le); @@ -552,3 +541,53 @@ int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task) return 0; } EXPORT_SYMBOL_GPL(pci_doe_submit_task); + +/** + * pci_doe() - Perform Data Object Exchange + * + * @doe_mb: DOE Mailbox + * @vendor: Vendor ID + * @type: Data Object Type + * @request: Request payload + * @request_sz: Size of request payload (bytes) + * @response: Response payload + * @response_sz: Size of response payload (bytes) + * + * Submit @request to @doe_mb and store the @response. + * The DOE exchange is performed synchronously and may therefore sleep. + * + * Payloads are treated as opaque byte streams which are transmitted verbatim, + * without byte-swapping. If payloads contain little-endian register values, + * the caller is responsible for conversion with cpu_to_le32() / le32_to_cpu(). + * + * RETURNS: Length of received response or negative errno. + * Received data in excess of @response_sz is discarded. + * The length may be smaller than @response_sz and the caller + * is responsible for checking that. + */ +int pci_doe(struct pci_doe_mb *doe_mb, u16 vendor, u8 type, + const void *request, size_t request_sz, + void *response, size_t response_sz) +{ + DECLARE_COMPLETION_ONSTACK(c); + struct pci_doe_task task = { + .prot.vid = vendor, + .prot.type = type, + .request_pl = request, + .request_pl_sz = request_sz, + .response_pl = response, + .response_pl_sz = response_sz, + .complete = pci_doe_task_complete, + .private = &c, + }; + int rc; + + rc = pci_doe_submit_task(doe_mb, &task); + if (rc) + return rc; + + wait_for_completion(&c); + + return task.rv; +} +EXPORT_SYMBOL_GPL(pci_doe); diff --git a/include/linux/pci-doe.h b/include/linux/pci-doe.h index 43765eaf2342..5dcd54f892e5 100644 --- a/include/linux/pci-doe.h +++ b/include/linux/pci-doe.h @@ -49,7 +49,7 @@ struct pci_doe_mb; */ struct pci_doe_task { struct pci_doe_protocol prot; - __le32 *request_pl; + const __le32 *request_pl; size_t request_pl_sz; __le32 *response_pl; size_t response_pl_sz; @@ -78,4 +78,8 @@ struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset); bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type); int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task); +int pci_doe(struct pci_doe_mb *doe_mb, u16 vendor, u8 type, + const void *request, size_t request_sz, + void *response, size_t response_sz); + #endif -- cgit v1.2.3 From 0821ff8ed059d38dfc9bec2106c5cc53bcaa15b1 Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Sat, 11 Mar 2023 15:40:09 +0100 Subject: PCI/DOE: Make asynchronous API private A synchronous API for DOE has just been introduced. CXL (the only in-tree DOE user so far) was converted to use it instead of the asynchronous API. Consequently, pci_doe_submit_task() as well as the pci_doe_task struct are only used internally, so make them private. Tested-by: Ira Weiny Signed-off-by: Lukas Wunner Reviewed-by: Ming Li Reviewed-by: Ira Weiny Reviewed-by: Jonathan Cameron Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/cc19544068483681e91dfe27545c2180cd09f931.1678543498.git.lukas@wunner.de Signed-off-by: Dan Williams --- drivers/pci/doe.c | 45 +++++++++++++++++++++++++++++++++++++++++++-- include/linux/pci-doe.h | 48 ------------------------------------------------ 2 files changed, 43 insertions(+), 50 deletions(-) (limited to 'include/linux') diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c index adde8c66499c..dfdec73f6abc 100644 --- a/drivers/pci/doe.c +++ b/drivers/pci/doe.c @@ -56,6 +56,47 @@ struct pci_doe_mb { unsigned long flags; }; +struct pci_doe_protocol { + u16 vid; + u8 type; +}; + +/** + * struct pci_doe_task - represents a single query/response + * + * @prot: DOE Protocol + * @request_pl: The request payload + * @request_pl_sz: Size of the request payload (bytes) + * @response_pl: The response payload + * @response_pl_sz: Size of the response payload (bytes) + * @rv: Return value. Length of received response or error (bytes) + * @complete: Called when task is complete + * @private: Private data for the consumer + * @work: Used internally by the mailbox + * @doe_mb: Used internally by the mailbox + * + * The payload sizes and rv are specified in bytes with the following + * restrictions concerning the protocol. + * + * 1) The request_pl_sz must be a multiple of double words (4 bytes) + * 2) The response_pl_sz must be >= a single double word (4 bytes) + * 3) rv is returned as bytes but it will be a multiple of double words + */ +struct pci_doe_task { + struct pci_doe_protocol prot; + const __le32 *request_pl; + size_t request_pl_sz; + __le32 *response_pl; + size_t response_pl_sz; + int rv; + void (*complete)(struct pci_doe_task *task); + void *private; + + /* initialized by pci_doe_submit_task() */ + struct work_struct work; + struct pci_doe_mb *doe_mb; +}; + static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout) { if (wait_event_timeout(doe_mb->wq, @@ -519,7 +560,8 @@ EXPORT_SYMBOL_GPL(pci_doe_supports_prot); * * RETURNS: 0 when task has been successfully queued, -ERRNO on error */ -int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task) +static int pci_doe_submit_task(struct pci_doe_mb *doe_mb, + struct pci_doe_task *task) { if (!pci_doe_supports_prot(doe_mb, task->prot.vid, task->prot.type)) return -EINVAL; @@ -540,7 +582,6 @@ int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task) queue_work(doe_mb->work_queue, &task->work); return 0; } -EXPORT_SYMBOL_GPL(pci_doe_submit_task); /** * pci_doe() - Perform Data Object Exchange diff --git a/include/linux/pci-doe.h b/include/linux/pci-doe.h index 5dcd54f892e5..7f16749c6aa3 100644 --- a/include/linux/pci-doe.h +++ b/include/linux/pci-doe.h @@ -13,55 +13,8 @@ #ifndef LINUX_PCI_DOE_H #define LINUX_PCI_DOE_H -struct pci_doe_protocol { - u16 vid; - u8 type; -}; - struct pci_doe_mb; -/** - * struct pci_doe_task - represents a single query/response - * - * @prot: DOE Protocol - * @request_pl: The request payload - * @request_pl_sz: Size of the request payload (bytes) - * @response_pl: The response payload - * @response_pl_sz: Size of the response payload (bytes) - * @rv: Return value. Length of received response or error (bytes) - * @complete: Called when task is complete - * @private: Private data for the consumer - * @work: Used internally by the mailbox - * @doe_mb: Used internally by the mailbox - * - * Payloads are treated as opaque byte streams which are transmitted verbatim, - * without byte-swapping. If payloads contain little-endian register values, - * the caller is responsible for conversion with cpu_to_le32() / le32_to_cpu(). - * - * The payload sizes and rv are specified in bytes with the following - * restrictions concerning the protocol. - * - * 1) The request_pl_sz must be a multiple of double words (4 bytes) - * 2) The response_pl_sz must be >= a single double word (4 bytes) - * 3) rv is returned as bytes but it will be a multiple of double words - * - * NOTE there is no need for the caller to initialize work or doe_mb. - */ -struct pci_doe_task { - struct pci_doe_protocol prot; - const __le32 *request_pl; - size_t request_pl_sz; - __le32 *response_pl; - size_t response_pl_sz; - int rv; - void (*complete)(struct pci_doe_task *task); - void *private; - - /* No need for the user to initialize these fields */ - struct work_struct work; - struct pci_doe_mb *doe_mb; -}; - /** * pci_doe_for_each_off - Iterate each DOE capability * @pdev: struct pci_dev to iterate @@ -76,7 +29,6 @@ struct pci_doe_task { struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset); bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type); -int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task); int pci_doe(struct pci_doe_mb *doe_mb, u16 vendor, u8 type, const void *request, size_t request_sz, -- cgit v1.2.3 From ac04840350e2c21a17d867b262a1586603b87a92 Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Sat, 11 Mar 2023 15:40:12 +0100 Subject: PCI/DOE: Create mailboxes on device enumeration Currently a DOE instance cannot be shared by multiple drivers because each driver creates its own pci_doe_mb struct for a given DOE instance. For the same reason a DOE instance cannot be shared between the PCI core and a driver. Moreover, finding out which protocols a DOE instance supports requires creating a pci_doe_mb for it. If a device has multiple DOE instances, a driver looking for a specific protocol may need to create a pci_doe_mb for each of the device's DOE instances and then destroy those which do not support the desired protocol. That's obviously an inefficient way to do things. Overcome these issues by creating mailboxes in the PCI core on device enumeration. Provide a pci_find_doe_mailbox() API call to allow drivers to get a pci_doe_mb for a given (pci_dev, vendor, protocol) triple. This API is modeled after pci_find_capability() and can later be amended with a pci_find_next_doe_mailbox() call to iterate over all mailboxes of a given pci_dev which support a specific protocol. On removal, destroy the mailboxes in pci_destroy_dev(), after the driver is unbound. This allows drivers to use DOE in their ->remove() hook. On surprise removal, cancel ongoing DOE exchanges and prevent new ones from being scheduled. Thereby ensure that a hot-removed device doesn't needlessly wait for a running exchange to time out. Tested-by: Ira Weiny Signed-off-by: Lukas Wunner Reviewed-by: Ming Li Reviewed-by: Ira Weiny Reviewed-by: Jonathan Cameron Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/40a6f973f72ef283d79dd55e7e6fddc7481199af.1678543498.git.lukas@wunner.de Signed-off-by: Dan Williams --- drivers/pci/doe.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++++ drivers/pci/pci.h | 11 ++++++++ drivers/pci/probe.c | 1 + drivers/pci/remove.c | 1 + include/linux/pci-doe.h | 2 ++ include/linux/pci.h | 3 ++ 6 files changed, 91 insertions(+) (limited to 'include/linux') diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c index 7539e72db5cb..9c577f5b3878 100644 --- a/drivers/pci/doe.c +++ b/drivers/pci/doe.c @@ -20,6 +20,8 @@ #include #include +#include "pci.h" + #define PCI_DOE_PROTOCOL_DISCOVERY 0 /* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */ @@ -658,3 +660,74 @@ int pci_doe(struct pci_doe_mb *doe_mb, u16 vendor, u8 type, return task.rv; } EXPORT_SYMBOL_GPL(pci_doe); + +/** + * pci_find_doe_mailbox() - Find Data Object Exchange mailbox + * + * @pdev: PCI device + * @vendor: Vendor ID + * @type: Data Object Type + * + * Find first DOE mailbox of a PCI device which supports the given protocol. + * + * RETURNS: Pointer to the DOE mailbox or NULL if none was found. + */ +struct pci_doe_mb *pci_find_doe_mailbox(struct pci_dev *pdev, u16 vendor, + u8 type) +{ + struct pci_doe_mb *doe_mb; + unsigned long index; + + xa_for_each(&pdev->doe_mbs, index, doe_mb) + if (pci_doe_supports_prot(doe_mb, vendor, type)) + return doe_mb; + + return NULL; +} +EXPORT_SYMBOL_GPL(pci_find_doe_mailbox); + +void pci_doe_init(struct pci_dev *pdev) +{ + struct pci_doe_mb *doe_mb; + u16 offset = 0; + int rc; + + xa_init(&pdev->doe_mbs); + + while ((offset = pci_find_next_ext_capability(pdev, offset, + PCI_EXT_CAP_ID_DOE))) { + doe_mb = pci_doe_create_mb(pdev, offset); + if (IS_ERR(doe_mb)) { + pci_err(pdev, "[%x] failed to create mailbox: %ld\n", + offset, PTR_ERR(doe_mb)); + continue; + } + + rc = xa_insert(&pdev->doe_mbs, offset, doe_mb, GFP_KERNEL); + if (rc) { + pci_err(pdev, "[%x] failed to insert mailbox: %d\n", + offset, rc); + pci_doe_destroy_mb(doe_mb); + } + } +} + +void pci_doe_destroy(struct pci_dev *pdev) +{ + struct pci_doe_mb *doe_mb; + unsigned long index; + + xa_for_each(&pdev->doe_mbs, index, doe_mb) + pci_doe_destroy_mb(doe_mb); + + xa_destroy(&pdev->doe_mbs); +} + +void pci_doe_disconnected(struct pci_dev *pdev) +{ + struct pci_doe_mb *doe_mb; + unsigned long index; + + xa_for_each(&pdev->doe_mbs, index, doe_mb) + pci_doe_cancel_tasks(doe_mb); +} diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index d2c08670a20e..815a4d2a41da 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -318,6 +318,16 @@ struct pci_sriov { bool drivers_autoprobe; /* Auto probing of VFs by driver */ }; +#ifdef CONFIG_PCI_DOE +void pci_doe_init(struct pci_dev *pdev); +void pci_doe_destroy(struct pci_dev *pdev); +void pci_doe_disconnected(struct pci_dev *pdev); +#else +static inline void pci_doe_init(struct pci_dev *pdev) { } +static inline void pci_doe_destroy(struct pci_dev *pdev) { } +static inline void pci_doe_disconnected(struct pci_dev *pdev) { } +#endif + /** * pci_dev_set_io_state - Set the new error state if possible. * @@ -354,6 +364,7 @@ static inline bool pci_dev_set_io_state(struct pci_dev *dev, static inline int pci_dev_set_disconnected(struct pci_dev *dev, void *unused) { pci_dev_set_io_state(dev, pci_channel_io_perm_failure); + pci_doe_disconnected(dev); return 0; } diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index a3f68b6ba6ac..02d2bf80eedb 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -2479,6 +2479,7 @@ static void pci_init_capabilities(struct pci_dev *dev) pci_aer_init(dev); /* Advanced Error Reporting */ pci_dpc_init(dev); /* Downstream Port Containment */ pci_rcec_init(dev); /* Root Complex Event Collector */ + pci_doe_init(dev); /* Data Object Exchange */ pcie_report_downtraining(dev); pci_init_reset_methods(dev); diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index 0145aef1b930..f25acf50879f 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -39,6 +39,7 @@ static void pci_destroy_dev(struct pci_dev *dev) list_del(&dev->bus_list); up_write(&pci_bus_sem); + pci_doe_destroy(dev); pcie_aspm_exit_link_state(dev); pci_bridge_d3_update(dev); pci_free_resources(dev); diff --git a/include/linux/pci-doe.h b/include/linux/pci-doe.h index 7f16749c6aa3..d6192ee0ac07 100644 --- a/include/linux/pci-doe.h +++ b/include/linux/pci-doe.h @@ -29,6 +29,8 @@ struct pci_doe_mb; struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset); bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type); +struct pci_doe_mb *pci_find_doe_mailbox(struct pci_dev *pdev, u16 vendor, + u8 type); int pci_doe(struct pci_doe_mb *doe_mb, u16 vendor, u8 type, const void *request, size_t request_sz, diff --git a/include/linux/pci.h b/include/linux/pci.h index b50e5c79f7e3..29d99530ec17 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -511,6 +511,9 @@ struct pci_dev { #endif #ifdef CONFIG_PCI_P2PDMA struct pci_p2pdma __rcu *p2pdma; +#endif +#ifdef CONFIG_PCI_DOE + struct xarray doe_mbs; /* Data Object Exchange mailboxes */ #endif u16 acs_cap; /* ACS Capability offset */ phys_addr_t rom; /* Physical address if not from BAR */ -- cgit v1.2.3 From 74e491e5d1bcc35a699291df720191760ff4130e Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Sat, 11 Mar 2023 15:40:14 +0100 Subject: PCI/DOE: Make mailbox creation API private The PCI core has just been amended to create a pci_doe_mb struct for every DOE instance on device enumeration. CXL (the only in-tree DOE user so far) has been migrated to use those mailboxes instead of creating its own. That leaves pcim_doe_create_mb() and pci_doe_for_each_off() without any callers, so drop them. pci_doe_supports_prot() is now only used internally, so declare it static. pci_doe_destroy_mb() is no longer used as callback for devm_add_action(), so refactor it to accept a struct pci_doe_mb pointer instead of a generic void pointer. Because pci_doe_create_mb() is only called on device enumeration, i.e. before driver binding, the workqueue name never contains a driver name. So replace dev_driver_string() with dev_bus_name() when generating the workqueue name. Tested-by: Ira Weiny Signed-off-by: Lukas Wunner Reviewed-by: Ming Li Reviewed-by: Ira Weiny Reviewed-by: Jonathan Cameron Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/64f614b6584982986c55d2c6229b4ee2b276dd59.1678543498.git.lukas@wunner.de Signed-off-by: Dan Williams --- .clang-format | 1 - drivers/pci/doe.c | 41 ++++------------------------------------- include/linux/pci-doe.h | 14 -------------- 3 files changed, 4 insertions(+), 52 deletions(-) (limited to 'include/linux') diff --git a/.clang-format b/.clang-format index d988e9fa9b26..1c85f6ddc71a 100644 --- a/.clang-format +++ b/.clang-format @@ -520,7 +520,6 @@ ForEachMacros: - 'of_property_for_each_string' - 'of_property_for_each_u32' - 'pci_bus_for_each_resource' - - 'pci_doe_for_each_off' - 'pcl_for_each_chunk' - 'pcl_for_each_segment' - 'pcm_for_each_format' diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c index 9c577f5b3878..db9a6b0c33c7 100644 --- a/drivers/pci/doe.c +++ b/drivers/pci/doe.c @@ -455,7 +455,7 @@ static struct pci_doe_mb *pci_doe_create_mb(struct pci_dev *pdev, xa_init(&doe_mb->prots); doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0, - dev_driver_string(&pdev->dev), + dev_bus_name(&pdev->dev), pci_name(pdev), doe_mb->cap_offset); if (!doe_mb->work_queue) { @@ -499,50 +499,18 @@ err_free: /** * pci_doe_destroy_mb() - Destroy a DOE mailbox object * - * @ptr: Pointer to DOE mailbox + * @doe_mb: DOE mailbox * * Destroy all internal data structures created for the DOE mailbox. */ -static void pci_doe_destroy_mb(void *ptr) +static void pci_doe_destroy_mb(struct pci_doe_mb *doe_mb) { - struct pci_doe_mb *doe_mb = ptr; - pci_doe_cancel_tasks(doe_mb); xa_destroy(&doe_mb->prots); destroy_workqueue(doe_mb->work_queue); kfree(doe_mb); } -/** - * pcim_doe_create_mb() - Create a DOE mailbox object - * - * @pdev: PCI device to create the DOE mailbox for - * @cap_offset: Offset of the DOE mailbox - * - * Create a single mailbox object to manage the mailbox protocol at the - * cap_offset specified. The mailbox will automatically be destroyed on - * driver unbinding from @pdev. - * - * RETURNS: created mailbox object on success - * ERR_PTR(-errno) on failure - */ -struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset) -{ - struct pci_doe_mb *doe_mb; - int rc; - - doe_mb = pci_doe_create_mb(pdev, cap_offset); - if (IS_ERR(doe_mb)) - return doe_mb; - - rc = devm_add_action_or_reset(&pdev->dev, pci_doe_destroy_mb, doe_mb); - if (rc) - return ERR_PTR(rc); - - return doe_mb; -} -EXPORT_SYMBOL_GPL(pcim_doe_create_mb); - /** * pci_doe_supports_prot() - Return if the DOE instance supports the given * protocol @@ -552,7 +520,7 @@ EXPORT_SYMBOL_GPL(pcim_doe_create_mb); * * RETURNS: True if the DOE mailbox supports the protocol specified */ -bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type) +static bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type) { unsigned long index; void *entry; @@ -567,7 +535,6 @@ bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type) return false; } -EXPORT_SYMBOL_GPL(pci_doe_supports_prot); /** * pci_doe_submit_task() - Submit a task to be processed by the state machine diff --git a/include/linux/pci-doe.h b/include/linux/pci-doe.h index d6192ee0ac07..1f14aed4354b 100644 --- a/include/linux/pci-doe.h +++ b/include/linux/pci-doe.h @@ -15,20 +15,6 @@ struct pci_doe_mb; -/** - * pci_doe_for_each_off - Iterate each DOE capability - * @pdev: struct pci_dev to iterate - * @off: u16 of config space offset of each mailbox capability found - */ -#define pci_doe_for_each_off(pdev, off) \ - for (off = pci_find_next_ext_capability(pdev, off, \ - PCI_EXT_CAP_ID_DOE); \ - off > 0; \ - off = pci_find_next_ext_capability(pdev, off, \ - PCI_EXT_CAP_ID_DOE)) - -struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset); -bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type); struct pci_doe_mb *pci_find_doe_mailbox(struct pci_dev *pdev, u16 vendor, u8 type); -- cgit v1.2.3 From 48380368dec14859723b9e3fbd43e042638d9a76 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 29 Mar 2023 12:14:42 +0200 Subject: Change DEFINE_SEMAPHORE() to take a number argument Fundamentally semaphores are a counted primitive, but DEFINE_SEMAPHORE() does not expose this and explicitly creates a binary semaphore. Change DEFINE_SEMAPHORE() to take a number argument and use that in the few places that open-coded it using __SEMAPHORE_INITIALIZER(). Signed-off-by: Peter Zijlstra (Intel) [mcgrof: add some tribal knowledge about why some folks prefer binary sempahores over mutexes] Reviewed-by: Sergey Senozhatsky Reviewed-by: Davidlohr Bueso Signed-off-by: Luis Chamberlain --- arch/mips/cavium-octeon/setup.c | 2 +- arch/x86/kernel/cpu/intel.c | 2 +- drivers/firmware/efi/runtime-wrappers.c | 2 +- drivers/firmware/efi/vars.c | 2 +- drivers/macintosh/adb.c | 2 +- drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c | 2 +- drivers/platform/x86/intel/ifs/sysfs.c | 2 +- drivers/scsi/esas2r/esas2r_ioctl.c | 2 +- drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c | 2 +- include/linux/semaphore.h | 10 ++++++++-- kernel/printk/printk.c | 2 +- net/rxrpc/call_object.c | 6 ++---- 12 files changed, 20 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/arch/mips/cavium-octeon/setup.c b/arch/mips/cavium-octeon/setup.c index a71727f7a608..c5561016f577 100644 --- a/arch/mips/cavium-octeon/setup.c +++ b/arch/mips/cavium-octeon/setup.c @@ -72,7 +72,7 @@ extern void pci_console_init(const char *arg); static unsigned long long max_memory = ULLONG_MAX; static unsigned long long reserve_low_mem; -DEFINE_SEMAPHORE(octeon_bootbus_sem); +DEFINE_SEMAPHORE(octeon_bootbus_sem, 1); EXPORT_SYMBOL(octeon_bootbus_sem); static struct octeon_boot_descriptor *octeon_boot_desc_ptr; diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 291d4167fab8..12bad63822f0 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -1177,7 +1177,7 @@ static const struct { static struct ratelimit_state bld_ratelimit; static unsigned int sysctl_sld_mitigate = 1; -static DEFINE_SEMAPHORE(buslock_sem); +static DEFINE_SEMAPHORE(buslock_sem, 1); #ifdef CONFIG_PROC_SYSCTL static struct ctl_table sld_sysctls[] = { diff --git a/drivers/firmware/efi/runtime-wrappers.c b/drivers/firmware/efi/runtime-wrappers.c index 1fba4e09cdcf..a400c4312c82 100644 --- a/drivers/firmware/efi/runtime-wrappers.c +++ b/drivers/firmware/efi/runtime-wrappers.c @@ -158,7 +158,7 @@ void efi_call_virt_check_flags(unsigned long flags, const char *call) * none of the remaining functions are actually ever called at runtime. * So let's just use a single lock to serialize all Runtime Services calls. */ -static DEFINE_SEMAPHORE(efi_runtime_lock); +static DEFINE_SEMAPHORE(efi_runtime_lock, 1); /* * Expose the EFI runtime lock to the UV platform diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c index bd75b87f5fc1..bfc5fa6aa47b 100644 --- a/drivers/firmware/efi/vars.c +++ b/drivers/firmware/efi/vars.c @@ -21,7 +21,7 @@ /* Private pointer to registered efivars */ static struct efivars *__efivars; -static DEFINE_SEMAPHORE(efivars_lock); +static DEFINE_SEMAPHORE(efivars_lock, 1); static efi_status_t check_var_size(bool nonblocking, u32 attributes, unsigned long size) diff --git a/drivers/macintosh/adb.c b/drivers/macintosh/adb.c index 23bd0c77ac1a..56599515d51a 100644 --- a/drivers/macintosh/adb.c +++ b/drivers/macintosh/adb.c @@ -80,7 +80,7 @@ static struct adb_driver *adb_controller; BLOCKING_NOTIFIER_HEAD(adb_client_list); static int adb_got_sleep; static int adb_inited; -static DEFINE_SEMAPHORE(adb_probe_mutex); +static DEFINE_SEMAPHORE(adb_probe_mutex, 1); static int sleepy_trackpad; static int autopoll_devs; int __adb_probe_sync; diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index 5d1e4fe335aa..5a105bab4387 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -298,7 +298,7 @@ const u32 dmae_reg_go_c[] = { /* Global resources for unloading a previously loaded device */ #define BNX2X_PREV_WAIT_NEEDED 1 -static DEFINE_SEMAPHORE(bnx2x_prev_sem); +static DEFINE_SEMAPHORE(bnx2x_prev_sem, 1); static LIST_HEAD(bnx2x_prev_list); /* Forward declaration */ diff --git a/drivers/platform/x86/intel/ifs/sysfs.c b/drivers/platform/x86/intel/ifs/sysfs.c index ee636a76b083..4c3c642ee19a 100644 --- a/drivers/platform/x86/intel/ifs/sysfs.c +++ b/drivers/platform/x86/intel/ifs/sysfs.c @@ -13,7 +13,7 @@ * Protects against simultaneous tests on multiple cores, or * reloading can file while a test is in progress */ -static DEFINE_SEMAPHORE(ifs_sem); +static DEFINE_SEMAPHORE(ifs_sem, 1); /* * The sysfs interface to check additional details of last test diff --git a/drivers/scsi/esas2r/esas2r_ioctl.c b/drivers/scsi/esas2r/esas2r_ioctl.c index e003d923acbf..055d2e87a2c8 100644 --- a/drivers/scsi/esas2r/esas2r_ioctl.c +++ b/drivers/scsi/esas2r/esas2r_ioctl.c @@ -56,7 +56,7 @@ dma_addr_t esas2r_buffered_ioctl_addr; u32 esas2r_buffered_ioctl_size; struct pci_dev *esas2r_buffered_ioctl_pcid; -static DEFINE_SEMAPHORE(buffered_ioctl_semaphore); +static DEFINE_SEMAPHORE(buffered_ioctl_semaphore, 1); typedef int (*BUFFERED_IOCTL_CALLBACK)(struct esas2r_adapter *, struct esas2r_request *, struct esas2r_sg_context *, diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c index cddcd3c596c9..1a656fdc9445 100644 --- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c +++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c @@ -149,7 +149,7 @@ static char *g_fragments_base; static char *g_free_fragments; static struct semaphore g_free_fragments_sema; -static DEFINE_SEMAPHORE(g_free_fragments_mutex); +static DEFINE_SEMAPHORE(g_free_fragments_mutex, 1); static int vchiq_blocking_bulk_transfer(struct vchiq_instance *instance, unsigned int handle, void *data, diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h index 6694d0019a68..04655faadc2d 100644 --- a/include/linux/semaphore.h +++ b/include/linux/semaphore.h @@ -25,8 +25,14 @@ struct semaphore { .wait_list = LIST_HEAD_INIT((name).wait_list), \ } -#define DEFINE_SEMAPHORE(name) \ - struct semaphore name = __SEMAPHORE_INITIALIZER(name, 1) +/* + * Unlike mutexes, binary semaphores do not have an owner, so up() can + * be called in a different thread from the one which called down(). + * It is also safe to call down_trylock() and up() from interrupt + * context. + */ +#define DEFINE_SEMAPHORE(_name, _n) \ + struct semaphore _name = __SEMAPHORE_INITIALIZER(_name, _n) static inline void sema_init(struct semaphore *sem, int val) { diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index fd0c9f913940..76987aaa5a45 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -89,7 +89,7 @@ static DEFINE_MUTEX(console_mutex); * console_sem protects updates to console->seq and console_suspended, * and also provides serialization for console printing. */ -static DEFINE_SEMAPHORE(console_sem); +static DEFINE_SEMAPHORE(console_sem, 1); HLIST_HEAD(console_list); EXPORT_SYMBOL_GPL(console_list); DEFINE_STATIC_SRCU(console_srcu); diff --git a/net/rxrpc/call_object.c b/net/rxrpc/call_object.c index e9f1f49d18c2..3e5cc70884dd 100644 --- a/net/rxrpc/call_object.c +++ b/net/rxrpc/call_object.c @@ -40,10 +40,8 @@ const char *const rxrpc_call_completions[NR__RXRPC_CALL_COMPLETIONS] = { struct kmem_cache *rxrpc_call_jar; -static struct semaphore rxrpc_call_limiter = - __SEMAPHORE_INITIALIZER(rxrpc_call_limiter, 1000); -static struct semaphore rxrpc_kernel_call_limiter = - __SEMAPHORE_INITIALIZER(rxrpc_kernel_call_limiter, 1000); +static DEFINE_SEMAPHORE(rxrpc_call_limiter, 1000); +static DEFINE_SEMAPHORE(rxrpc_kernel_call_limiter, 1000); void rxrpc_poke_call(struct rxrpc_call *call, enum rxrpc_call_poke_trace what) { -- cgit v1.2.3 From 5f300fd59a2ae90b8a7fb5ed3d5fd43768236c38 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 14 Apr 2023 10:03:53 +0200 Subject: mm: make arch_has_descending_max_zone_pfns() static clang produces a build failure on x86 for some randconfig builds after a change that moves around code to mm/mm_init.c: Cannot find symbol for section 2: .text. mm/mm_init.o: failed I have not been able to figure out why this happens, but the __weak annotation on arch_has_descending_max_zone_pfns() is the trigger here. Removing the weak function in favor of an open-coded Kconfig option check avoids the problem and becomes clearer as well as better to optimize by the compiler. [arnd@arndb.de: fix logic bug] Link: https://lkml.kernel.org/r/20230415081904.969049-1-arnd@kernel.org Link: https://lkml.kernel.org/r/20230414080418.110236-1-arnd@kernel.org Fixes: 9420f89db2dd ("mm: move most of core MM initialization to mm/mm_init.c") Signed-off-by: Arnd Bergmann Acked-by: Vlastimil Babka Tested-by: SeongJae Park Tested-by: Geert Uytterhoeven Acked-by: Mike Rapoport (IBM) Cc: kernel test robot Signed-off-by: Andrew Morton --- arch/arc/mm/init.c | 5 ----- include/linux/mm.h | 1 - mm/mm_init.c | 4 ++-- 3 files changed, 2 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/arch/arc/mm/init.c b/arch/arc/mm/init.c index ce4e939a7f07..2b89b6c53801 100644 --- a/arch/arc/mm/init.c +++ b/arch/arc/mm/init.c @@ -74,11 +74,6 @@ void __init early_init_dt_add_memory_arch(u64 base, u64 size) base, TO_MB(size), !in_use ? "Not used":""); } -bool arch_has_descending_max_zone_pfns(void) -{ - return !IS_ENABLED(CONFIG_ARC_HAS_PAE40); -} - /* * First memory setup routine called from setup_arch() * 1. setup swapper's mm @init_mm diff --git a/include/linux/mm.h b/include/linux/mm.h index e249208f8fbe..5e5ef6ee4003 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3035,7 +3035,6 @@ extern void setup_per_cpu_pageset(void); extern int min_free_kbytes; extern int watermark_boost_factor; extern int watermark_scale_factor; -extern bool arch_has_descending_max_zone_pfns(void); /* nommu.c */ extern atomic_long_t mmap_pages_allocated; diff --git a/mm/mm_init.c b/mm/mm_init.c index dd3a6ed9663f..a0ec3b3acb5e 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1752,9 +1752,9 @@ static void __init free_area_init_memoryless_node(int nid) * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For * such cases we allow max_zone_pfn sorted in the descending order */ -bool __weak arch_has_descending_max_zone_pfns(void) +static bool arch_has_descending_max_zone_pfns(void) { - return false; + return IS_ENABLED(CONFIG_ARC) && !IS_ENABLED(CONFIG_ARC_HAS_PAE40); } /** -- cgit v1.2.3 From a85c2257a8ac353af16dbcbf32c50d3380860bc5 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 17 Mar 2023 14:44:47 +0100 Subject: sched/isolation: add cpu_is_isolated() API Patch series "memcg, cpuisol: do not interfere pcp cache charges draining with cpuisol workloads". Leonardo has reported [1] that pcp memcg charge draining can interfere with cpu isolated workloads. The said draining is done from a WQ context with a pcp worker scheduled on each CPU which holds any cached charges for a specific memcg hierarchy. Operation is not really a common operation [2]. It can be triggered from the userspace though so some care is definitely due. Leonardo has tried to address the issue by allowing remote charge draining [3]. This approach requires an additional locking to synchronize pcp caches sync from a remote cpu from local pcp consumers. Even though the proposed lock was per-cpu there is still potential for contention and less predictable behavior. This patchset addresses the issue from a different angle. Rather than dealing with a potential synchronization, cpus which are isolated are simply never scheduled to be drained. This means that a small amount of charges could be laying around and waiting for a later use or they are flushed when a different memcg is charged from the same cpu. More details are in patch 2. The first patch from Frederic is implementing an abstraction to tell whether a specific cpu has been isolated and therefore require a special treatment. This patch (of 2): Provide this new API to check if a CPU has been isolated either through isolcpus= or nohz_full= kernel parameter. It aims at avoiding kernel load deemed to be safely spared on CPUs running sensitive workload that can't bear any disturbance, such as pcp cache draining. Link: https://lkml.kernel.org/r/20230317134448.11082-1-mhocko@kernel.org Link: https://lkml.kernel.org/r/20230317134448.11082-2-mhocko@kernel.org Signed-off-by: Frederic Weisbecker Signed-off-by: Michal Hocko Suggested-by: Michal Hocko Cc: Johannes Weiner Cc: Marcelo Tosatti Cc: Muchun Song Cc: Peter Zijlstra Cc: Roman Gushchin Cc: Shakeel Butt Cc: Thomas Gleixner Cc: Leonardo Bras Cc: Ingo Molnar Signed-off-by: Andrew Morton --- include/linux/sched/isolation.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h index 8c15abd67aed..fe1a46f30d24 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -46,6 +46,12 @@ static inline bool housekeeping_enabled(enum hk_type type) static inline void housekeeping_affine(struct task_struct *t, enum hk_type type) { } + +static inline bool housekeeping_test_cpu(int cpu, enum hk_type type) +{ + return true; +} + static inline void housekeeping_init(void) { } #endif /* CONFIG_CPU_ISOLATION */ @@ -58,4 +64,10 @@ static inline bool housekeeping_cpu(int cpu, enum hk_type type) return true; } +static inline bool cpu_is_isolated(int cpu) +{ + return !housekeeping_test_cpu(cpu, HK_TYPE_DOMAIN) || + !housekeeping_test_cpu(cpu, HK_TYPE_TICK); +} + #endif /* _LINUX_SCHED_ISOLATION_H */ -- cgit v1.2.3 From 957ebbdf434013ee01f29f6c9174eced995ebbe7 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 27 Mar 2023 16:10:50 +0100 Subject: hugetlb: remove PageHeadHuge() Sidhartha Kumar removed the last caller of PageHeadHuge(), so we can now remove it and make folio_test_hugetlb() the real implementation. Add kernel-doc for folio_test_hugetlb(). Link: https://lkml.kernel.org/r/20230327151050.1787744-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Sidhartha Kumar Reviewed-by: Mike Kravetz Reviewed-by: David Hildenbrand Reviewed-by: Muchun Song Signed-off-by: Andrew Morton --- include/linux/page-flags.h | 7 +------ mm/hugetlb.c | 18 +++++++++++------- 2 files changed, 12 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index dcda20c47b8f..efc42fae3d96 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -815,14 +815,9 @@ static inline void ClearPageCompound(struct page *page) #ifdef CONFIG_HUGETLB_PAGE int PageHuge(struct page *page); -int PageHeadHuge(struct page *page); -static inline bool folio_test_hugetlb(struct folio *folio) -{ - return PageHeadHuge(&folio->page); -} +bool folio_test_hugetlb(struct folio *folio); #else TESTPAGEFLAG_FALSE(Huge, hugetlb) -TESTPAGEFLAG_FALSE(HeadHuge, headhuge) #endif #ifdef CONFIG_TRANSPARENT_HUGEPAGE diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a58b3739ed4b..7e4a80769c9e 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2050,19 +2050,23 @@ int PageHuge(struct page *page) } EXPORT_SYMBOL_GPL(PageHuge); -/* - * PageHeadHuge() only returns true for hugetlbfs head page, but not for - * normal or transparent huge pages. +/** + * folio_test_hugetlb - Determine if the folio belongs to hugetlbfs + * @folio: The folio to test. + * + * Context: Any context. Caller should have a reference on the folio to + * prevent it from being turned into a tail page. + * Return: True for hugetlbfs folios, false for anon folios or folios + * belonging to other filesystems. */ -int PageHeadHuge(struct page *page_head) +bool folio_test_hugetlb(struct folio *folio) { - struct folio *folio = (struct folio *)page_head; if (!folio_test_large(folio)) - return 0; + return false; return folio->_folio_dtor == HUGETLB_PAGE_DTOR; } -EXPORT_SYMBOL_GPL(PageHeadHuge); +EXPORT_SYMBOL_GPL(folio_test_hugetlb); /* * Find and lock address space (mapping) in write mode. -- cgit v1.2.3 From 62f31bd4dcedffe3c919deb76ed65bf62c3cf80b Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (IBM)" Date: Sun, 26 Mar 2023 19:02:15 +0300 Subject: mm: move free_area_empty() to mm/internal.h The free_area_empty() helper is only used inside mm/ so move it there to reduce noise in include/linux/mmzone.h Link: https://lkml.kernel.org/r/20230326160215.2674531-1-rppt@kernel.org Signed-off-by: Mike Rapoport (IBM) Suggested-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 5 ----- mm/internal.h | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 2d22e47dc1eb..337956748976 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -110,11 +110,6 @@ struct free_area { unsigned long nr_free; }; -static inline bool free_area_empty(struct free_area *area, int migratetype) -{ - return list_empty(&area->free_list[migratetype]); -} - struct pglist_data; #ifdef CONFIG_NUMA diff --git a/mm/internal.h b/mm/internal.h index 73b167b59cc5..92ddd3a05b74 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -517,6 +517,11 @@ void init_cma_reserved_pageblock(struct page *page); int find_suitable_fallback(struct free_area *area, unsigned int order, int migratetype, bool only_stealable, bool *can_steal); +static inline bool free_area_empty(struct free_area *area, int migratetype) +{ + return list_empty(&area->free_list[migratetype]); +} + /* * These three helpers classifies VMAs for virtual memory accounting. */ -- cgit v1.2.3 From 8bff9a04ca33476213ea6155850505787c540c25 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Thu, 30 Mar 2023 19:17:54 +0000 Subject: cgroup: rename cgroup_rstat_flush_"irqsafe" to "atomic" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "memcg: avoid flushing stats atomically where possible", v3. rstat flushing is an expensive operation that scales with the number of cpus and the number of cgroups in the system. The purpose of this series is to minimize the contexts where we flush stats atomically. Patches 1 and 2 are cleanups requested during reviews of prior versions of this series. Patch 3 makes sure we never try to flush from within an irq context. Patches 4 to 7 introduce separate variants of mem_cgroup_flush_stats() for atomic and non-atomic flushing, and make sure we only flush the stats atomically when necessary. Patch 8 is a slightly tangential optimization that limits the work done by rstat flushing in some scenarios. This patch (of 8): cgroup_rstat_flush_irqsafe() can be a confusing name. It may read as "irqs are disabled throughout", which is what the current implementation does (currently under discussion [1]), but is not the intention. The intention is that this function is safe to call from atomic contexts. Name it as such. Link: https://lkml.kernel.org/r/20230330191801.1967435-1-yosryahmed@google.com Link: https://lkml.kernel.org/r/20230330191801.1967435-2-yosryahmed@google.com Signed-off-by: Yosry Ahmed Suggested-by: Johannes Weiner Acked-by: Shakeel Butt Acked-by: Johannes Weiner Cc: Jens Axboe Cc: Josef Bacik Cc: Michal Hocko Cc: Michal Koutný Cc: Muchun Song Cc: Roman Gushchin Cc: Tejun Heo Cc: Vasily Averin Cc: Zefan Li Cc: Michal Hocko Signed-off-by: Andrew Morton --- include/linux/cgroup.h | 2 +- kernel/cgroup/rstat.c | 4 ++-- mm/memcontrol.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 3410aecffdb4..885f5395fcd0 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -692,7 +692,7 @@ static inline void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen) */ void cgroup_rstat_updated(struct cgroup *cgrp, int cpu); void cgroup_rstat_flush(struct cgroup *cgrp); -void cgroup_rstat_flush_irqsafe(struct cgroup *cgrp); +void cgroup_rstat_flush_atomic(struct cgroup *cgrp); void cgroup_rstat_flush_hold(struct cgroup *cgrp); void cgroup_rstat_flush_release(void); diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c index 831f1f472bb8..d3252b0416b6 100644 --- a/kernel/cgroup/rstat.c +++ b/kernel/cgroup/rstat.c @@ -241,12 +241,12 @@ __bpf_kfunc void cgroup_rstat_flush(struct cgroup *cgrp) } /** - * cgroup_rstat_flush_irqsafe - irqsafe version of cgroup_rstat_flush() + * cgroup_rstat_flush_atomic- atomic version of cgroup_rstat_flush() * @cgrp: target cgroup * * This function can be called from any context. */ -void cgroup_rstat_flush_irqsafe(struct cgroup *cgrp) +void cgroup_rstat_flush_atomic(struct cgroup *cgrp) { unsigned long flags; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 681e7528a714..b7c3eab6b4fe 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -643,7 +643,7 @@ static void __mem_cgroup_flush_stats(void) return; flush_next_time = jiffies_64 + 2*FLUSH_TIME; - cgroup_rstat_flush_irqsafe(root_mem_cgroup->css.cgroup); + cgroup_rstat_flush_atomic(root_mem_cgroup->css.cgroup); atomic_set(&stats_flush_threshold, 0); spin_unlock_irqrestore(&stats_flush_lock, flag); } -- cgit v1.2.3 From 92fbbc7202ac4fb16250dc91c88211814ae2190b Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Thu, 30 Mar 2023 19:17:55 +0000 Subject: memcg: rename mem_cgroup_flush_stats_"delayed" to "ratelimited" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mem_cgroup_flush_stats_delayed() suggests his is using a delayed_work, but this is actually sometimes flushing directly from the callsite. What it's doing is ratelimited calls. A better name would be mem_cgroup_flush_stats_ratelimited(). Link: https://lkml.kernel.org/r/20230330191801.1967435-3-yosryahmed@google.com Signed-off-by: Yosry Ahmed Suggested-by: Johannes Weiner Acked-by: Shakeel Butt Acked-by: Johannes Weiner Acked-by: Michal Hocko Cc: Jens Axboe Cc: Josef Bacik Cc: Michal Hocko Cc: Michal Koutný Cc: Muchun Song Cc: Roman Gushchin Cc: Tejun Heo Cc: Vasily Averin Cc: Zefan Li Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 4 ++-- mm/memcontrol.c | 2 +- mm/workingset.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index aa69ea98e2d8..00a88cf947e1 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1038,7 +1038,7 @@ static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, } void mem_cgroup_flush_stats(void); -void mem_cgroup_flush_stats_delayed(void); +void mem_cgroup_flush_stats_ratelimited(void); void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, int val); @@ -1536,7 +1536,7 @@ static inline void mem_cgroup_flush_stats(void) { } -static inline void mem_cgroup_flush_stats_delayed(void) +static inline void mem_cgroup_flush_stats_ratelimited(void) { } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b7c3eab6b4fe..a01f062e7007 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -654,7 +654,7 @@ void mem_cgroup_flush_stats(void) __mem_cgroup_flush_stats(); } -void mem_cgroup_flush_stats_delayed(void) +void mem_cgroup_flush_stats_ratelimited(void) { if (time_after64(jiffies_64, flush_next_time)) mem_cgroup_flush_stats(); diff --git a/mm/workingset.c b/mm/workingset.c index 00c6f4d9d9be..af862c6738c3 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -462,7 +462,7 @@ void workingset_refault(struct folio *folio, void *shadow) mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr); - mem_cgroup_flush_stats_delayed(); + mem_cgroup_flush_stats_ratelimited(); /* * Compare the distance to the existing workingset size. We * don't activate pages that couldn't stay resident even if -- cgit v1.2.3 From 9fad9aee1f267a8ad1f86b87ae70b2c4d6796164 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Thu, 30 Mar 2023 19:17:58 +0000 Subject: memcg: sleep during flushing stats in safe contexts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, all contexts that flush memcg stats do so with sleeping not allowed. Some of these contexts are perfectly safe to sleep in, such as reading cgroup files from userspace or the background periodic flusher. Flushing is an expensive operation that scales with the number of cpus and the number of cgroups in the system, so avoid doing it atomically where possible. Refactor the code to make mem_cgroup_flush_stats() non-atomic (aka sleepable), and provide a separate atomic version. The atomic version is used in reclaim, refault, writeback, and in mem_cgroup_usage(). All other code paths are left to use the non-atomic version. This includes callbacks for userspace reads and the periodic flusher. Since refault is the only caller of mem_cgroup_flush_stats_ratelimited(), change it to mem_cgroup_flush_stats_atomic_ratelimited(). Reclaim and refault code paths are modified to do non-atomic flushing in separate later patches -- so it will eventually be changed back to mem_cgroup_flush_stats_ratelimited(). Link: https://lkml.kernel.org/r/20230330191801.1967435-6-yosryahmed@google.com Signed-off-by: Yosry Ahmed Acked-by: Shakeel Butt Acked-by: Michal Hocko Acked-by: Johannes Weiner Cc: Jens Axboe Cc: Josef Bacik Cc: Michal Hocko Cc: Michal Koutný Cc: Muchun Song Cc: Roman Gushchin Cc: Tejun Heo Cc: Vasily Averin Cc: Zefan Li Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 9 +++++++-- mm/memcontrol.c | 45 ++++++++++++++++++++++++++++++++++++--------- mm/vmscan.c | 2 +- mm/workingset.c | 2 +- 4 files changed, 45 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 00a88cf947e1..3db355e6677f 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1038,7 +1038,8 @@ static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, } void mem_cgroup_flush_stats(void); -void mem_cgroup_flush_stats_ratelimited(void); +void mem_cgroup_flush_stats_atomic(void); +void mem_cgroup_flush_stats_atomic_ratelimited(void); void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, int val); @@ -1536,7 +1537,11 @@ static inline void mem_cgroup_flush_stats(void) { } -static inline void mem_cgroup_flush_stats_ratelimited(void) +static inline void mem_cgroup_flush_stats_atomic(void) +{ +} + +static inline void mem_cgroup_flush_stats_atomic_ratelimited(void) { } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index a9f18456a16b..06786dea50b8 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -635,7 +635,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) } } -static void __mem_cgroup_flush_stats(void) +static void do_flush_stats(bool atomic) { /* * We always flush the entire tree, so concurrent flushers can just @@ -647,26 +647,46 @@ static void __mem_cgroup_flush_stats(void) return; WRITE_ONCE(flush_next_time, jiffies_64 + 2*FLUSH_TIME); - cgroup_rstat_flush_atomic(root_mem_cgroup->css.cgroup); + + if (atomic) + cgroup_rstat_flush_atomic(root_mem_cgroup->css.cgroup); + else + cgroup_rstat_flush(root_mem_cgroup->css.cgroup); + atomic_set(&stats_flush_threshold, 0); atomic_set(&stats_flush_ongoing, 0); } +static bool should_flush_stats(void) +{ + return atomic_read(&stats_flush_threshold) > num_online_cpus(); +} + void mem_cgroup_flush_stats(void) { - if (atomic_read(&stats_flush_threshold) > num_online_cpus()) - __mem_cgroup_flush_stats(); + if (should_flush_stats()) + do_flush_stats(false); } -void mem_cgroup_flush_stats_ratelimited(void) +void mem_cgroup_flush_stats_atomic(void) +{ + if (should_flush_stats()) + do_flush_stats(true); +} + +void mem_cgroup_flush_stats_atomic_ratelimited(void) { if (time_after64(jiffies_64, READ_ONCE(flush_next_time))) - mem_cgroup_flush_stats(); + mem_cgroup_flush_stats_atomic(); } static void flush_memcg_stats_dwork(struct work_struct *w) { - __mem_cgroup_flush_stats(); + /* + * Always flush here so that flushing in latency-sensitive paths is + * as cheap as possible. + */ + do_flush_stats(false); queue_delayed_work(system_unbound_wq, &stats_flush_dwork, FLUSH_TIME); } @@ -3686,9 +3706,12 @@ static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) * done from irq context; use stale stats in this case. * Arguably, usage threshold events are not reliable on the root * memcg anyway since its usage is ill-defined. + * + * Additionally, other call paths through memcg_check_events() + * disable irqs, so make sure we are flushing stats atomically. */ if (in_task()) - mem_cgroup_flush_stats(); + mem_cgroup_flush_stats_atomic(); val = memcg_page_state(memcg, NR_FILE_PAGES) + memcg_page_state(memcg, NR_ANON_MAPPED); if (swap) @@ -4611,7 +4634,11 @@ void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css); struct mem_cgroup *parent; - mem_cgroup_flush_stats(); + /* + * wb_writeback() takes a spinlock and calls + * wb_over_bg_thresh()->mem_cgroup_wb_stats(). Do not sleep. + */ + mem_cgroup_flush_stats_atomic(); *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY); *pwriteback = memcg_page_state(memcg, NR_WRITEBACK); diff --git a/mm/vmscan.c b/mm/vmscan.c index 98719e72b5e2..c68db50ec109 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2861,7 +2861,7 @@ static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) * Flush the memory cgroup stats, so that we read accurate per-memcg * lruvec stats for heuristics. */ - mem_cgroup_flush_stats(); + mem_cgroup_flush_stats_atomic(); /* * Determine the scan balance between anon and file LRUs. diff --git a/mm/workingset.c b/mm/workingset.c index af862c6738c3..dab0c362b9e3 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -462,7 +462,7 @@ void workingset_refault(struct folio *folio, void *shadow) mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr); - mem_cgroup_flush_stats_ratelimited(); + mem_cgroup_flush_stats_atomic_ratelimited(); /* * Compare the distance to the existing workingset size. We * don't activate pages that couldn't stay resident even if -- cgit v1.2.3 From 4009b2f1887036d30637bc06dd0ade7e18408bb3 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Thu, 30 Mar 2023 19:17:59 +0000 Subject: workingset: memcg: sleep when flushing stats in workingset_refault() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In workingset_refault(), we call mem_cgroup_flush_stats_atomic_ratelimited() to read accurate stats within an RCU read section and with sleeping disallowed. Move the call above the RCU read section to make it non-atomic. Flushing is an expensive operation that scales with the number of cpus and the number of cgroups in the system, so avoid doing it atomically where possible. Since workingset_refault() is the only caller of mem_cgroup_flush_stats_atomic_ratelimited(), just make it non-atomic, and rename it to mem_cgroup_flush_stats_ratelimited(). Link: https://lkml.kernel.org/r/20230330191801.1967435-7-yosryahmed@google.com Signed-off-by: Yosry Ahmed Acked-by: Shakeel Butt Acked-by: Johannes Weiner Acked-by: Michal Hocko Cc: Jens Axboe Cc: Josef Bacik Cc: Michal Hocko Cc: Michal Koutný Cc: Muchun Song Cc: Roman Gushchin Cc: Tejun Heo Cc: Vasily Averin Cc: Zefan Li Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 4 ++-- mm/memcontrol.c | 4 ++-- mm/workingset.c | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 3db355e6677f..222d7370134c 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1039,7 +1039,7 @@ static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, void mem_cgroup_flush_stats(void); void mem_cgroup_flush_stats_atomic(void); -void mem_cgroup_flush_stats_atomic_ratelimited(void); +void mem_cgroup_flush_stats_ratelimited(void); void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, int val); @@ -1541,7 +1541,7 @@ static inline void mem_cgroup_flush_stats_atomic(void) { } -static inline void mem_cgroup_flush_stats_atomic_ratelimited(void) +static inline void mem_cgroup_flush_stats_ratelimited(void) { } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 06786dea50b8..e9e79ceb9a11 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -674,10 +674,10 @@ void mem_cgroup_flush_stats_atomic(void) do_flush_stats(true); } -void mem_cgroup_flush_stats_atomic_ratelimited(void) +void mem_cgroup_flush_stats_ratelimited(void) { if (time_after64(jiffies_64, READ_ONCE(flush_next_time))) - mem_cgroup_flush_stats_atomic(); + mem_cgroup_flush_stats(); } static void flush_memcg_stats_dwork(struct work_struct *w) diff --git a/mm/workingset.c b/mm/workingset.c index dab0c362b9e3..3025beee9b34 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -406,6 +406,9 @@ void workingset_refault(struct folio *folio, void *shadow) unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset); eviction <<= bucket_order; + /* Flush stats (and potentially sleep) before holding RCU read lock */ + mem_cgroup_flush_stats_ratelimited(); + rcu_read_lock(); /* * Look up the memcg associated with the stored ID. It might @@ -461,8 +464,6 @@ void workingset_refault(struct folio *folio, void *shadow) lruvec = mem_cgroup_lruvec(memcg, pgdat); mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr); - - mem_cgroup_flush_stats_atomic_ratelimited(); /* * Compare the distance to the existing workingset size. We * don't activate pages that couldn't stay resident even if -- cgit v1.2.3 From 6efc7afb5cc98488410d44695685d003d832534d Mon Sep 17 00:00:00 2001 From: Jiaqi Yan Date: Wed, 29 Mar 2023 08:11:20 -0700 Subject: mm/hwpoison: introduce copy_mc_highpage Similar to how copy_mc_user_highpage is implemented for copy_user_highpage on #MC supported architecture, introduce the #MC handled version of copy_highpage. This helper has immediate usage when khugepaged wants to copy file-backed memory pages and tolerate #MC. Link: https://lkml.kernel.org/r/20230329151121.949896-3-jiaqiyan@google.com Signed-off-by: Jiaqi Yan Reviewed-by: Yang Shi Cc: David Stevens Cc: Hugh Dickins Cc: Kefeng Wang Cc: Kirill A. Shutemov Cc: "Kirill A. Shutemov" Cc: Miaohe Lin Cc: Naoya Horiguchi Cc: Oscar Salvador Cc: Tong Tiangen Cc: Tony Luck Signed-off-by: Andrew Morton --- include/linux/highmem.h | 54 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 9c7cdaa3de8c..4de1dbcd3ef6 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -315,7 +315,29 @@ static inline void copy_user_highpage(struct page *to, struct page *from, #endif +#ifndef __HAVE_ARCH_COPY_HIGHPAGE + +static inline void copy_highpage(struct page *to, struct page *from) +{ + char *vfrom, *vto; + + vfrom = kmap_local_page(from); + vto = kmap_local_page(to); + copy_page(vto, vfrom); + kmsan_copy_page_meta(to, from); + kunmap_local(vto); + kunmap_local(vfrom); +} + +#endif + #ifdef copy_mc_to_kernel +/* + * If architecture supports machine check exception handling, define the + * #MC versions of copy_user_highpage and copy_highpage. They copy a memory + * page with #MC in source page (@from) handled, and return the number + * of bytes not copied if there was a #MC, otherwise 0 for success. + */ static inline int copy_mc_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { @@ -332,29 +354,35 @@ static inline int copy_mc_user_highpage(struct page *to, struct page *from, return ret; } -#else -static inline int copy_mc_user_highpage(struct page *to, struct page *from, - unsigned long vaddr, struct vm_area_struct *vma) -{ - copy_user_highpage(to, from, vaddr, vma); - return 0; -} -#endif -#ifndef __HAVE_ARCH_COPY_HIGHPAGE - -static inline void copy_highpage(struct page *to, struct page *from) +static inline int copy_mc_highpage(struct page *to, struct page *from) { + unsigned long ret; char *vfrom, *vto; vfrom = kmap_local_page(from); vto = kmap_local_page(to); - copy_page(vto, vfrom); - kmsan_copy_page_meta(to, from); + ret = copy_mc_to_kernel(vto, vfrom, PAGE_SIZE); + if (!ret) + kmsan_copy_page_meta(to, from); kunmap_local(vto); kunmap_local(vfrom); + + return ret; +} +#else +static inline int copy_mc_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma) +{ + copy_user_highpage(to, from, vaddr, vma); + return 0; } +static inline int copy_mc_highpage(struct page *to, struct page *from) +{ + copy_highpage(to, from); + return 0; +} #endif static inline void memcpy_page(struct page *dst_page, size_t dst_off, -- cgit v1.2.3 From b4aca54792e76556bb9f8661bab07b4bf2eb4e5f Mon Sep 17 00:00:00 2001 From: Steven Price Date: Wed, 5 Apr 2023 11:38:19 +0100 Subject: smaps: fix defined but not used smaps_shmem_walk_ops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When !CONFIG_SHMEM smaps_shmem_walk_ops is defined but not used, triggering a compiler warning. To avoid the warning remove the #ifdef around the usage. This has no effect because shmem_mapping() is a stub returning false when !CONFIG_SHMEM so the code will be compiled out, however we now need to also provide a stub for shmem_swap_usage(). Link: https://lkml.kernel.org/r/20230405103819.151246-1-steven.price@arm.com Fixes: 7b86ac3371b7 ("pagewalk: separate function pointers from iterator data") Signed-off-by: Steven Price Reported-by: kernel test robot Link: https://lore.kernel.org/oe-kbuild-all/202304031749.UiyJpxzF-lkp@intel.com/ Reviewed-by: Jason Gunthorpe Cc: Christoph Hellwig Cc: Alexey Dobriyan Cc: Thomas Hellström Signed-off-by: Andrew Morton --- fs/proc/task_mmu.c | 3 +-- include/linux/shmem_fs.h | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 6a96e1713fd5..cb49479acd2e 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -782,7 +782,6 @@ static void smap_gather_stats(struct vm_area_struct *vma, if (start >= vma->vm_end) return; -#ifdef CONFIG_SHMEM if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) { /* * For shared or readonly shmem mappings we know that all @@ -803,7 +802,7 @@ static void smap_gather_stats(struct vm_area_struct *vma, ops = &smaps_shmem_walk_ops; } } -#endif + /* mmap_lock is held in m_start */ if (!start) walk_page_vma(vma, ops, mss); diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 3bb8d21edbb3..b9516a9802bf 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -96,7 +96,14 @@ int shmem_unuse(unsigned int type); extern bool shmem_is_huge(struct inode *inode, pgoff_t index, bool shmem_huge_force, struct mm_struct *mm, unsigned long vm_flags); +#ifdef CONFIG_SHMEM extern unsigned long shmem_swap_usage(struct vm_area_struct *vma); +#else +static inline unsigned long shmem_swap_usage(struct vm_area_struct *vma) +{ + return 0; +} +#endif extern unsigned long shmem_partial_swap_usage(struct address_space *mapping, pgoff_t start, pgoff_t end); -- cgit v1.2.3 From e87340ca5c9cecc8a11daf1a2dcabf23f06a4e10 Mon Sep 17 00:00:00 2001 From: ZhangPeng Date: Mon, 10 Apr 2023 21:39:29 +0800 Subject: userfaultfd: convert copy_huge_page_from_user() to copy_folio_from_user() Replace copy_huge_page_from_user() with copy_folio_from_user(). copy_folio_from_user() does the same as copy_huge_page_from_user(), but takes in a folio instead of a page. Convert page_kaddr to kaddr in copy_folio_from_user() to do indenting cleanup. Link: https://lkml.kernel.org/r/20230410133932.32288-4-zhangpeng362@huawei.com Signed-off-by: ZhangPeng Reviewed-by: Sidhartha Kumar Reviewed-by: Mike Kravetz Cc: Kefeng Wang Cc: Matthew Wilcox Cc: Muchun Song Cc: Nanyong Sun Cc: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 7 +++---- mm/hugetlb.c | 5 ++--- mm/memory.c | 23 +++++++++++------------ mm/userfaultfd.c | 6 ++---- 4 files changed, 18 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 5e5ef6ee4003..01a009efc2cb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3681,10 +3681,9 @@ extern void copy_user_huge_page(struct page *dst, struct page *src, unsigned long addr_hint, struct vm_area_struct *vma, unsigned int pages_per_huge_page); -extern long copy_huge_page_from_user(struct page *dst_page, - const void __user *usr_src, - unsigned int pages_per_huge_page, - bool allow_pagefault); +long copy_folio_from_user(struct folio *dst_folio, + const void __user *usr_src, + bool allow_pagefault); /** * vma_is_special_huge - Are transhuge page-table entries considered special? diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 7e4a80769c9e..aade1b513474 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6217,9 +6217,8 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, goto out; } - ret = copy_huge_page_from_user(&folio->page, - (const void __user *) src_addr, - pages_per_huge_page(h), false); + ret = copy_folio_from_user(folio, (const void __user *) src_addr, + false); /* fallback to copy_from_user outside mmap_lock */ if (unlikely(ret)) { diff --git a/mm/memory.c b/mm/memory.c index 808f354bce65..021cab989703 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5868,26 +5868,25 @@ void copy_user_huge_page(struct page *dst, struct page *src, process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg); } -long copy_huge_page_from_user(struct page *dst_page, - const void __user *usr_src, - unsigned int pages_per_huge_page, - bool allow_pagefault) +long copy_folio_from_user(struct folio *dst_folio, + const void __user *usr_src, + bool allow_pagefault) { - void *page_kaddr; + void *kaddr; unsigned long i, rc = 0; - unsigned long ret_val = pages_per_huge_page * PAGE_SIZE; + unsigned int nr_pages = folio_nr_pages(dst_folio); + unsigned long ret_val = nr_pages * PAGE_SIZE; struct page *subpage; - for (i = 0; i < pages_per_huge_page; i++) { - subpage = nth_page(dst_page, i); - page_kaddr = kmap_local_page(subpage); + for (i = 0; i < nr_pages; i++) { + subpage = folio_page(dst_folio, i); + kaddr = kmap_local_page(subpage); if (!allow_pagefault) pagefault_disable(); - rc = copy_from_user(page_kaddr, - usr_src + i * PAGE_SIZE, PAGE_SIZE); + rc = copy_from_user(kaddr, usr_src + i * PAGE_SIZE, PAGE_SIZE); if (!allow_pagefault) pagefault_enable(); - kunmap_local(page_kaddr); + kunmap_local(kaddr); ret_val -= (PAGE_SIZE - rc); if (rc) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 313bc683c2b6..1e7dba6c4c5f 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -421,10 +421,8 @@ retry: mmap_read_unlock(dst_mm); BUG_ON(!page); - err = copy_huge_page_from_user(page, - (const void __user *)src_addr, - vma_hpagesize / PAGE_SIZE, - true); + err = copy_folio_from_user(page_folio(page), + (const void __user *)src_addr, true); if (unlikely(err)) { err = -EFAULT; goto out; -- cgit v1.2.3 From 0169fd518a8934d8d723659752b07589ecc9f692 Mon Sep 17 00:00:00 2001 From: ZhangPeng Date: Mon, 10 Apr 2023 21:39:30 +0800 Subject: userfaultfd: convert mfill_atomic_hugetlb() to use a folio Convert hugetlb_mfill_atomic_pte() to take in a folio pointer instead of a page pointer. Convert mfill_atomic_hugetlb() to use a folio. Link: https://lkml.kernel.org/r/20230410133932.32288-5-zhangpeng362@huawei.com Signed-off-by: ZhangPeng Reviewed-by: Sidhartha Kumar Reviewed-by: Mike Kravetz Cc: Kefeng Wang Cc: Matthew Wilcox Cc: Muchun Song Cc: Nanyong Sun Cc: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 4 ++-- mm/hugetlb.c | 26 +++++++++++++------------- mm/userfaultfd.c | 16 ++++++++-------- 3 files changed, 23 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 2a758bcd6719..28703fe22386 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -163,7 +163,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, unsigned long dst_addr, unsigned long src_addr, uffd_flags_t flags, - struct page **pagep); + struct folio **foliop); #endif /* CONFIG_USERFAULTFD */ bool hugetlb_reserve_pages(struct inode *inode, long from, long to, struct vm_area_struct *vma, @@ -397,7 +397,7 @@ static inline int hugetlb_mfill_atomic_pte(pte_t *dst_pte, unsigned long dst_addr, unsigned long src_addr, uffd_flags_t flags, - struct page **pagep) + struct folio **foliop) { BUG(); return 0; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index aade1b513474..c88f856ec2e2 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6178,7 +6178,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, unsigned long dst_addr, unsigned long src_addr, uffd_flags_t flags, - struct page **pagep) + struct folio **foliop) { struct mm_struct *dst_mm = dst_vma->vm_mm; bool is_continue = uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE); @@ -6201,8 +6201,8 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, if (IS_ERR(folio)) goto out; folio_in_pagecache = true; - } else if (!*pagep) { - /* If a page already exists, then it's UFFDIO_COPY for + } else if (!*foliop) { + /* If a folio already exists, then it's UFFDIO_COPY for * a non-missing case. Return -EEXIST. */ if (vm_shared && @@ -6237,33 +6237,33 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, ret = -ENOMEM; goto out; } - *pagep = &folio->page; - /* Set the outparam pagep and return to the caller to + *foliop = folio; + /* Set the outparam foliop and return to the caller to * copy the contents outside the lock. Don't free the - * page. + * folio. */ goto out; } } else { if (vm_shared && hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) { - put_page(*pagep); + folio_put(*foliop); ret = -EEXIST; - *pagep = NULL; + *foliop = NULL; goto out; } folio = alloc_hugetlb_folio(dst_vma, dst_addr, 0); if (IS_ERR(folio)) { - put_page(*pagep); + folio_put(*foliop); ret = -ENOMEM; - *pagep = NULL; + *foliop = NULL; goto out; } - copy_user_huge_page(&folio->page, *pagep, dst_addr, dst_vma, + copy_user_huge_page(&folio->page, &(*foliop)->page, dst_addr, dst_vma, pages_per_huge_page(h)); - put_page(*pagep); - *pagep = NULL; + folio_put(*foliop); + *foliop = NULL; } /* diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 1e7dba6c4c5f..2f263afb823d 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -321,7 +321,7 @@ static __always_inline ssize_t mfill_atomic_hugetlb( pte_t *dst_pte; unsigned long src_addr, dst_addr; long copied; - struct page *page; + struct folio *folio; unsigned long vma_hpagesize; pgoff_t idx; u32 hash; @@ -341,7 +341,7 @@ static __always_inline ssize_t mfill_atomic_hugetlb( src_addr = src_start; dst_addr = dst_start; copied = 0; - page = NULL; + folio = NULL; vma_hpagesize = vma_kernel_pagesize(dst_vma); /* @@ -410,7 +410,7 @@ retry: } err = hugetlb_mfill_atomic_pte(dst_pte, dst_vma, dst_addr, - src_addr, flags, &page); + src_addr, flags, &folio); hugetlb_vma_unlock_read(dst_vma); mutex_unlock(&hugetlb_fault_mutex_table[hash]); @@ -419,9 +419,9 @@ retry: if (unlikely(err == -ENOENT)) { mmap_read_unlock(dst_mm); - BUG_ON(!page); + BUG_ON(!folio); - err = copy_folio_from_user(page_folio(page), + err = copy_folio_from_user(folio, (const void __user *)src_addr, true); if (unlikely(err)) { err = -EFAULT; @@ -432,7 +432,7 @@ retry: dst_vma = NULL; goto retry; } else - BUG_ON(page); + BUG_ON(folio); if (!err) { dst_addr += vma_hpagesize; @@ -449,8 +449,8 @@ retry: out_unlock: mmap_read_unlock(dst_mm); out: - if (page) - put_page(page); + if (folio) + folio_put(folio); BUG_ON(copied < 0); BUG_ON(err > 0); BUG_ON(!copied && !err); -- cgit v1.2.3 From c0e8150e144b62ae467520d0b51c4707c09e897b Mon Sep 17 00:00:00 2001 From: ZhangPeng Date: Mon, 10 Apr 2023 21:39:31 +0800 Subject: mm: convert copy_user_huge_page() to copy_user_large_folio() Replace copy_user_huge_page() with copy_user_large_folio(). copy_user_large_folio() does the same as copy_user_huge_page(), but takes in folios instead of pages. Remove pages_per_huge_page from copy_user_large_folio(), because we can get that from folio_nr_pages(dst). Convert copy_user_gigantic_page() to take in folios. Link: https://lkml.kernel.org/r/20230410133932.32288-6-zhangpeng362@huawei.com Signed-off-by: ZhangPeng Cc: Kefeng Wang Cc: Matthew Wilcox Cc: Mike Kravetz Cc: Muchun Song Cc: Nanyong Sun Cc: Sidhartha Kumar Cc: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- include/linux/mm.h | 7 +++---- mm/hugetlb.c | 11 +++++------ mm/memory.c | 28 ++++++++++++++-------------- 3 files changed, 22 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 01a009efc2cb..5a3eaa9a1f8c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3677,10 +3677,9 @@ extern const struct attribute_group memory_failure_attr_group; extern void clear_huge_page(struct page *page, unsigned long addr_hint, unsigned int pages_per_huge_page); -extern void copy_user_huge_page(struct page *dst, struct page *src, - unsigned long addr_hint, - struct vm_area_struct *vma, - unsigned int pages_per_huge_page); +void copy_user_large_folio(struct folio *dst, struct folio *src, + unsigned long addr_hint, + struct vm_area_struct *vma); long copy_folio_from_user(struct folio *dst_folio, const void __user *usr_src, bool allow_pagefault); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index c88f856ec2e2..f16b25b1a6b9 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5097,8 +5097,9 @@ again: ret = PTR_ERR(new_folio); break; } - copy_user_huge_page(&new_folio->page, ptepage, addr, dst_vma, - npages); + copy_user_large_folio(new_folio, + page_folio(ptepage), + addr, dst_vma); put_page(ptepage); /* Install the new hugetlb folio if src pte stable */ @@ -5616,8 +5617,7 @@ retry_avoidcopy: goto out_release_all; } - copy_user_huge_page(&new_folio->page, old_page, address, vma, - pages_per_huge_page(h)); + copy_user_large_folio(new_folio, page_folio(old_page), address, vma); __folio_mark_uptodate(new_folio); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, haddr, @@ -6260,8 +6260,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, *foliop = NULL; goto out; } - copy_user_huge_page(&folio->page, &(*foliop)->page, dst_addr, dst_vma, - pages_per_huge_page(h)); + copy_user_large_folio(folio, *foliop, dst_addr, dst_vma); folio_put(*foliop); *foliop = NULL; } diff --git a/mm/memory.c b/mm/memory.c index 021cab989703..f315c2198098 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5815,21 +5815,21 @@ void clear_huge_page(struct page *page, process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page); } -static void copy_user_gigantic_page(struct page *dst, struct page *src, - unsigned long addr, - struct vm_area_struct *vma, - unsigned int pages_per_huge_page) +static void copy_user_gigantic_page(struct folio *dst, struct folio *src, + unsigned long addr, + struct vm_area_struct *vma, + unsigned int pages_per_huge_page) { int i; - struct page *dst_base = dst; - struct page *src_base = src; + struct page *dst_page; + struct page *src_page; for (i = 0; i < pages_per_huge_page; i++) { - dst = nth_page(dst_base, i); - src = nth_page(src_base, i); + dst_page = folio_page(dst, i); + src_page = folio_page(src, i); cond_resched(); - copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma); + copy_user_highpage(dst_page, src_page, addr + i*PAGE_SIZE, vma); } } @@ -5847,15 +5847,15 @@ static void copy_subpage(unsigned long addr, int idx, void *arg) addr, copy_arg->vma); } -void copy_user_huge_page(struct page *dst, struct page *src, - unsigned long addr_hint, struct vm_area_struct *vma, - unsigned int pages_per_huge_page) +void copy_user_large_folio(struct folio *dst, struct folio *src, + unsigned long addr_hint, struct vm_area_struct *vma) { + unsigned int pages_per_huge_page = folio_nr_pages(dst); unsigned long addr = addr_hint & ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1); struct copy_subpage_arg arg = { - .dst = dst, - .src = src, + .dst = &dst->page, + .src = &src->page, .vma = vma, }; -- cgit v1.2.3 From d7be6d7eee1bbf98671d7a2c95654322241e2ae4 Mon Sep 17 00:00:00 2001 From: ZhangPeng Date: Mon, 10 Apr 2023 21:39:32 +0800 Subject: userfaultfd: convert mfill_atomic() to use a folio Convert mfill_atomic_pte_copy(), shmem_mfill_atomic_pte() and mfill_atomic_pte() to take in a folio pointer. Convert mfill_atomic() to use a folio. Convert page_kaddr to kaddr in mfill_atomic(). Link: https://lkml.kernel.org/r/20230410133932.32288-7-zhangpeng362@huawei.com Signed-off-by: ZhangPeng Reviewed-by: Mike Kravetz Cc: Kefeng Wang Cc: Matthew Wilcox Cc: Muchun Song Cc: Nanyong Sun Cc: Sidhartha Kumar Cc: Vishal Moola (Oracle) Signed-off-by: Andrew Morton --- include/linux/shmem_fs.h | 4 ++-- mm/shmem.c | 16 ++++++++-------- mm/userfaultfd.c | 40 ++++++++++++++++++++-------------------- 3 files changed, 30 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index b9516a9802bf..9029abd29b1c 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -165,10 +165,10 @@ extern int shmem_mfill_atomic_pte(pmd_t *dst_pmd, unsigned long dst_addr, unsigned long src_addr, uffd_flags_t flags, - struct page **pagep); + struct folio **foliop); #else /* !CONFIG_SHMEM */ #define shmem_mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, \ - src_addr, flags, pagep) ({ BUG(); 0; }) + src_addr, flags, foliop) ({ BUG(); 0; }) #endif /* CONFIG_SHMEM */ #endif /* CONFIG_USERFAULTFD */ diff --git a/mm/shmem.c b/mm/shmem.c index b185c1db3009..3531ebb3494e 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2433,7 +2433,7 @@ int shmem_mfill_atomic_pte(pmd_t *dst_pmd, unsigned long dst_addr, unsigned long src_addr, uffd_flags_t flags, - struct page **pagep) + struct folio **foliop) { struct inode *inode = file_inode(dst_vma->vm_file); struct shmem_inode_info *info = SHMEM_I(inode); @@ -2451,14 +2451,14 @@ int shmem_mfill_atomic_pte(pmd_t *dst_pmd, * and now we find ourselves with -ENOMEM. Release the page, to * avoid a BUG_ON in our caller. */ - if (unlikely(*pagep)) { - put_page(*pagep); - *pagep = NULL; + if (unlikely(*foliop)) { + folio_put(*foliop); + *foliop = NULL; } return -ENOMEM; } - if (!*pagep) { + if (!*foliop) { ret = -ENOMEM; folio = shmem_alloc_folio(gfp, info, pgoff); if (!folio) @@ -2490,7 +2490,7 @@ int shmem_mfill_atomic_pte(pmd_t *dst_pmd, /* fallback to copy_from_user outside mmap_lock */ if (unlikely(ret)) { - *pagep = &folio->page; + *foliop = folio; ret = -ENOENT; /* don't free the page */ goto out_unacct_blocks; @@ -2501,9 +2501,9 @@ int shmem_mfill_atomic_pte(pmd_t *dst_pmd, clear_user_highpage(&folio->page, dst_addr); } } else { - folio = page_folio(*pagep); + folio = *foliop; VM_BUG_ON_FOLIO(folio_test_large(folio), folio); - *pagep = NULL; + *foliop = NULL; } VM_BUG_ON(folio_test_locked(folio)); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 2f263afb823d..11cfd82c6726 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -133,13 +133,13 @@ static int mfill_atomic_pte_copy(pmd_t *dst_pmd, unsigned long dst_addr, unsigned long src_addr, uffd_flags_t flags, - struct page **pagep) + struct folio **foliop) { void *kaddr; int ret; struct folio *folio; - if (!*pagep) { + if (!*foliop) { ret = -ENOMEM; folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, dst_vma, dst_addr, false); @@ -171,15 +171,15 @@ static int mfill_atomic_pte_copy(pmd_t *dst_pmd, /* fallback to copy_from_user outside mmap_lock */ if (unlikely(ret)) { ret = -ENOENT; - *pagep = &folio->page; + *foliop = folio; /* don't free the page */ goto out; } flush_dcache_folio(folio); } else { - folio = page_folio(*pagep); - *pagep = NULL; + folio = *foliop; + *foliop = NULL; } /* @@ -470,7 +470,7 @@ static __always_inline ssize_t mfill_atomic_pte(pmd_t *dst_pmd, unsigned long dst_addr, unsigned long src_addr, uffd_flags_t flags, - struct page **pagep) + struct folio **foliop) { ssize_t err; @@ -493,14 +493,14 @@ static __always_inline ssize_t mfill_atomic_pte(pmd_t *dst_pmd, if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) err = mfill_atomic_pte_copy(dst_pmd, dst_vma, dst_addr, src_addr, - flags, pagep); + flags, foliop); else err = mfill_atomic_pte_zeropage(dst_pmd, dst_vma, dst_addr); } else { err = shmem_mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, src_addr, - flags, pagep); + flags, foliop); } return err; @@ -518,7 +518,7 @@ static __always_inline ssize_t mfill_atomic(struct mm_struct *dst_mm, pmd_t *dst_pmd; unsigned long src_addr, dst_addr; long copied; - struct page *page; + struct folio *folio; /* * Sanitize the command parameters: @@ -533,7 +533,7 @@ static __always_inline ssize_t mfill_atomic(struct mm_struct *dst_mm, src_addr = src_start; dst_addr = dst_start; copied = 0; - page = NULL; + folio = NULL; retry: mmap_read_lock(dst_mm); @@ -629,28 +629,28 @@ retry: BUG_ON(pmd_trans_huge(*dst_pmd)); err = mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, - src_addr, flags, &page); + src_addr, flags, &folio); cond_resched(); if (unlikely(err == -ENOENT)) { - void *page_kaddr; + void *kaddr; mmap_read_unlock(dst_mm); - BUG_ON(!page); + BUG_ON(!folio); - page_kaddr = kmap_local_page(page); - err = copy_from_user(page_kaddr, + kaddr = kmap_local_folio(folio, 0); + err = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE); - kunmap_local(page_kaddr); + kunmap_local(kaddr); if (unlikely(err)) { err = -EFAULT; goto out; } - flush_dcache_page(page); + flush_dcache_folio(folio); goto retry; } else - BUG_ON(page); + BUG_ON(folio); if (!err) { dst_addr += PAGE_SIZE; @@ -667,8 +667,8 @@ retry: out_unlock: mmap_read_unlock(dst_mm); out: - if (page) - put_page(page); + if (folio) + folio_put(folio); BUG_ON(copied < 0); BUG_ON(err > 0); BUG_ON(!copied && !err); -- cgit v1.2.3 From 87a7ae75d7383afa998f57656d1d14e2a730cc47 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Tue, 11 Apr 2023 19:52:13 +0530 Subject: mm/vmemmap/devdax: fix kernel crash when probing devdax devices commit 4917f55b4ef9 ("mm/sparse-vmemmap: improve memory savings for compound devmaps") added support for using optimized vmmemap for devdax devices. But how vmemmap mappings are created are architecture specific. For example, powerpc with hash translation doesn't have vmemmap mappings in init_mm page table instead they are bolted table entries in the hardware page table vmemmap_populate_compound_pages() used by vmemmap optimization code is not aware of these architecture-specific mapping. Hence allow architecture to opt for this feature. I selected architectures supporting HUGETLB_PAGE_OPTIMIZE_VMEMMAP option as also supporting this feature. This patch fixes the below crash on ppc64. BUG: Unable to handle kernel data access on write at 0xc00c000100400038 Faulting instruction address: 0xc000000001269d90 Oops: Kernel access of bad area, sig: 11 [#1] LE PAGE_SIZE=64K MMU=Hash SMP NR_CPUS=2048 NUMA pSeries Modules linked in: CPU: 7 PID: 1 Comm: swapper/0 Not tainted 6.3.0-rc5-150500.34-default+ #2 5c90a668b6bbd142599890245c2fb5de19d7d28a Hardware name: IBM,9009-42G POWER9 (raw) 0x4e0202 0xf000005 of:IBM,FW950.40 (VL950_099) hv:phyp pSeries NIP: c000000001269d90 LR: c0000000004c57d4 CTR: 0000000000000000 REGS: c000000003632c30 TRAP: 0300 Not tainted (6.3.0-rc5-150500.34-default+) MSR: 8000000000009033 CR: 24842228 XER: 00000000 CFAR: c0000000004c57d0 DAR: c00c000100400038 DSISR: 42000000 IRQMASK: 0 .... NIP [c000000001269d90] __init_single_page.isra.74+0x14/0x4c LR [c0000000004c57d4] __init_zone_device_page+0x44/0xd0 Call Trace: [c000000003632ed0] [c000000003632f60] 0xc000000003632f60 (unreliable) [c000000003632f10] [c0000000004c5ca0] memmap_init_zone_device+0x170/0x250 [c000000003632fe0] [c0000000005575f8] memremap_pages+0x2c8/0x7f0 [c0000000036330c0] [c000000000557b5c] devm_memremap_pages+0x3c/0xa0 [c000000003633100] [c000000000d458a8] dev_dax_probe+0x108/0x3e0 [c0000000036331a0] [c000000000d41430] dax_bus_probe+0xb0/0x140 [c0000000036331d0] [c000000000cef27c] really_probe+0x19c/0x520 [c000000003633260] [c000000000cef6b4] __driver_probe_device+0xb4/0x230 [c0000000036332e0] [c000000000cef888] driver_probe_device+0x58/0x120 [c000000003633320] [c000000000cefa6c] __device_attach_driver+0x11c/0x1e0 [c0000000036333a0] [c000000000cebc58] bus_for_each_drv+0xa8/0x130 [c000000003633400] [c000000000ceefcc] __device_attach+0x15c/0x250 [c0000000036334a0] [c000000000ced458] bus_probe_device+0x108/0x110 [c0000000036334f0] [c000000000ce92dc] device_add+0x7fc/0xa10 [c0000000036335b0] [c000000000d447c8] devm_create_dev_dax+0x1d8/0x530 [c000000003633640] [c000000000d46b60] __dax_pmem_probe+0x200/0x270 [c0000000036337b0] [c000000000d46bf0] dax_pmem_probe+0x20/0x70 [c0000000036337d0] [c000000000d2279c] nvdimm_bus_probe+0xac/0x2b0 [c000000003633860] [c000000000cef27c] really_probe+0x19c/0x520 [c0000000036338f0] [c000000000cef6b4] __driver_probe_device+0xb4/0x230 [c000000003633970] [c000000000cef888] driver_probe_device+0x58/0x120 [c0000000036339b0] [c000000000cefd08] __driver_attach+0x1d8/0x240 [c000000003633a30] [c000000000cebb04] bus_for_each_dev+0xb4/0x130 [c000000003633a90] [c000000000cee564] driver_attach+0x34/0x50 [c000000003633ab0] [c000000000ced878] bus_add_driver+0x218/0x300 [c000000003633b40] [c000000000cf1144] driver_register+0xa4/0x1b0 [c000000003633bb0] [c000000000d21a0c] __nd_driver_register+0x5c/0x100 [c000000003633c10] [c00000000206a2e8] dax_pmem_init+0x34/0x48 [c000000003633c30] [c0000000000132d0] do_one_initcall+0x60/0x320 [c000000003633d00] [c0000000020051b0] kernel_init_freeable+0x360/0x400 [c000000003633de0] [c000000000013764] kernel_init+0x34/0x1d0 [c000000003633e50] [c00000000000de14] ret_from_kernel_thread+0x5c/0x64 Link: https://lkml.kernel.org/r/20230411142214.64464-1-aneesh.kumar@linux.ibm.com Fixes: 4917f55b4ef9 ("mm/sparse-vmemmap: improve memory savings for compound devmaps") Signed-off-by: Aneesh Kumar K.V Reported-by: Tarun Sahu Reviewed-by: Joao Martins Cc: Muchun Song Cc: Dan Williams Cc: Mike Kravetz Cc: Signed-off-by: Andrew Morton --- include/linux/mm.h | 16 ++++++++++++++++ mm/mm_init.c | 10 ++++++---- mm/sparse-vmemmap.c | 3 +-- 3 files changed, 23 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 5a3eaa9a1f8c..21a7e2460084 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3560,6 +3560,22 @@ void vmemmap_populate_print_last(void); void vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap); #endif + +#ifdef CONFIG_ARCH_WANT_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +static inline bool vmemmap_can_optimize(struct vmem_altmap *altmap, + struct dev_pagemap *pgmap) +{ + return is_power_of_2(sizeof(struct page)) && + pgmap && (pgmap_vmemmap_nr(pgmap) > 1) && !altmap; +} +#else +static inline bool vmemmap_can_optimize(struct vmem_altmap *altmap, + struct dev_pagemap *pgmap) +{ + return false; +} +#endif + void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, unsigned long nr_pages); diff --git a/mm/mm_init.c b/mm/mm_init.c index a0ec3b3acb5e..7f7f9c677854 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1015,10 +1015,12 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, * of an altmap. See vmemmap_populate_compound_pages(). */ static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, - unsigned long nr_pages) + struct dev_pagemap *pgmap) { - return is_power_of_2(sizeof(struct page)) && - !altmap ? 2 * (PAGE_SIZE / sizeof(struct page)) : nr_pages; + if (!vmemmap_can_optimize(altmap, pgmap)) + return pgmap_vmemmap_nr(pgmap); + + return 2 * (PAGE_SIZE / sizeof(struct page)); } static void __ref memmap_init_compound(struct page *head, @@ -1083,7 +1085,7 @@ void __ref memmap_init_zone_device(struct zone *zone, continue; memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(altmap, pfns_per_compound)); + compound_nr_pages(altmap, pgmap)); } pr_debug("%s initialised %lu pages in %ums\n", __func__, diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index c5398a5960d0..10d73a0dfcec 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -458,8 +458,7 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn, !IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION))) return NULL; - if (is_power_of_2(sizeof(struct page)) && - pgmap && pgmap_vmemmap_nr(pgmap) > 1 && !altmap) + if (vmemmap_can_optimize(altmap, pgmap)) r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap); else r = vmemmap_populate(start, end, nid, altmap); -- cgit v1.2.3 From 0b376f1e0ff555435597fa16823ae0f30b2883e3 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Wed, 12 Apr 2023 10:30:25 +0530 Subject: mm/hugetlb_vmemmap: rename ARCH_WANT_HUGETLB_PAGE_OPTIMIZE_VMEMMAP Now we use ARCH_WANT_HUGETLB_PAGE_OPTIMIZE_VMEMMAP config option to indicate devdax and hugetlb vmemmap optimization support. Hence rename that to a generic ARCH_WANT_OPTIMIZE_VMEMMAP Link: https://lkml.kernel.org/r/20230412050025.84346-2-aneesh.kumar@linux.ibm.com Signed-off-by: Aneesh Kumar K.V Reviewed-by: Muchun Song Cc: Joao Martins Cc: Dan Williams Cc: Mike Kravetz Cc: Tarun Sahu Signed-off-by: Andrew Morton --- arch/loongarch/Kconfig | 2 +- arch/s390/Kconfig | 2 +- arch/x86/Kconfig | 2 +- fs/Kconfig | 9 +-------- include/linux/mm.h | 2 +- mm/Kconfig | 6 ++++++ 6 files changed, 11 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index e1e3a3828962..5abd13093c1a 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -53,8 +53,8 @@ config LOONGARCH select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT - select ARCH_WANT_HUGETLB_PAGE_OPTIMIZE_VMEMMAP select ARCH_WANT_LD_ORPHAN_WARN + select ARCH_WANT_OPTIMIZE_VMEMMAP select ARCH_WANTS_NO_INSTR select BUILDTIME_TABLE_SORT select COMMON_CLK diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 548b5b587003..61d778397720 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -126,8 +126,8 @@ config S390 select ARCH_WANTS_DYNAMIC_TASK_STRUCT select ARCH_WANTS_NO_INSTR select ARCH_WANT_DEFAULT_BPF_JIT - select ARCH_WANT_HUGETLB_PAGE_OPTIMIZE_VMEMMAP select ARCH_WANT_IPC_PARSE_VERSION + select ARCH_WANT_OPTIMIZE_VMEMMAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 select DMA_OPS if PCI diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index df21fba77db1..4c123315c440 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -126,8 +126,8 @@ config X86 select ARCH_WANTS_NO_INSTR select ARCH_WANT_GENERAL_HUGETLB select ARCH_WANT_HUGE_PMD_SHARE - select ARCH_WANT_HUGETLB_PAGE_OPTIMIZE_VMEMMAP if X86_64 select ARCH_WANT_LD_ORPHAN_WARN + select ARCH_WANT_OPTIMIZE_VMEMMAP if X86_64 select ARCH_WANTS_THP_SWAP if X86_64 select ARCH_HAS_PARANOID_L1D_FLUSH select BUILDTIME_TABLE_SORT diff --git a/fs/Kconfig b/fs/Kconfig index e99830c65033..cc07a0cd3172 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -250,16 +250,9 @@ config HUGETLBFS config HUGETLB_PAGE def_bool HUGETLBFS -# -# Select this config option from the architecture Kconfig, if it is preferred -# to enable the feature of HugeTLB Vmemmap Optimization (HVO). -# -config ARCH_WANT_HUGETLB_PAGE_OPTIMIZE_VMEMMAP - bool - config HUGETLB_PAGE_OPTIMIZE_VMEMMAP def_bool HUGETLB_PAGE - depends on ARCH_WANT_HUGETLB_PAGE_OPTIMIZE_VMEMMAP + depends on ARCH_WANT_OPTIMIZE_VMEMMAP depends on SPARSEMEM_VMEMMAP config HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON diff --git a/include/linux/mm.h b/include/linux/mm.h index 21a7e2460084..0b514de43143 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3561,7 +3561,7 @@ void vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap); #endif -#ifdef CONFIG_ARCH_WANT_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +#ifdef CONFIG_ARCH_WANT_OPTIMIZE_VMEMMAP static inline bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { diff --git a/mm/Kconfig b/mm/Kconfig index 6ee3b48ed298..5ca8fcfae243 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -479,6 +479,12 @@ config SPARSEMEM_VMEMMAP SPARSEMEM_VMEMMAP uses a virtually mapped memmap to optimise pfn_to_page and page_to_pfn operations. This is the most efficient option when sufficient kernel resources are available. +# +# Select this config option from the architecture Kconfig, if it is preferred +# to enable the feature of HugeTLB/dev_dax vmemmap optimization. +# +config ARCH_WANT_OPTIMIZE_VMEMMAP + bool config HAVE_MEMBLOCK_PHYS_MAP bool -- cgit v1.2.3 From 1cb9dc4b475c7418f925ab0c97b6750007d9f52e Mon Sep 17 00:00:00 2001 From: Liu Shixin Date: Thu, 13 Apr 2023 21:13:49 +0800 Subject: mm: hwpoison: support recovery from HugePage copy-on-write faults copy-on-write of hugetlb user pages with uncorrectable errors will result in a kernel crash. This is because the copy is performed in kernel mode and in general we can not handle accessing memory with such errors while in kernel mode. Commit a873dfe1032a ("mm, hwpoison: try to recover from copy-on write faults") introduced the routine copy_user_highpage_mc() to gracefully handle copying of user pages with uncorrectable errors. However, the separate hugetlb copy-on-write code paths were not modified as part of commit a873dfe1032a. Modify hugetlb copy-on-write code paths to use copy_mc_user_highpage() so that they can also gracefully handle uncorrectable errors in user pages. This involves changing the hugetlb specific routine copy_user_large_folio() from type void to int so that it can return an error. Modify the hugetlb userfaultfd code in the same way so that it can return -EHWPOISON if it encounters an uncorrectable error. Link: https://lkml.kernel.org/r/20230413131349.2524210-1-liushixin2@huawei.com Signed-off-by: Liu Shixin Acked-by: Mike Kravetz Reviewed-by: Naoya Horiguchi Cc: Miaohe Lin Cc: Muchun Song Cc: Tony Luck Signed-off-by: Andrew Morton --- include/linux/mm.h | 6 +++--- mm/hugetlb.c | 17 +++++++++++++--- mm/memory.c | 59 +++++++++++++++++++++++++++++++++++------------------- 3 files changed, 55 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 0b514de43143..2772a1800cf3 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3693,9 +3693,9 @@ extern const struct attribute_group memory_failure_attr_group; extern void clear_huge_page(struct page *page, unsigned long addr_hint, unsigned int pages_per_huge_page); -void copy_user_large_folio(struct folio *dst, struct folio *src, - unsigned long addr_hint, - struct vm_area_struct *vma); +int copy_user_large_folio(struct folio *dst, struct folio *src, + unsigned long addr_hint, + struct vm_area_struct *vma); long copy_folio_from_user(struct folio *dst_folio, const void __user *usr_src, bool allow_pagefault); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index f16b25b1a6b9..a08fb47fb200 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5097,10 +5097,14 @@ again: ret = PTR_ERR(new_folio); break; } - copy_user_large_folio(new_folio, + ret = copy_user_large_folio(new_folio, page_folio(ptepage), addr, dst_vma); put_page(ptepage); + if (ret) { + folio_put(new_folio); + break; + } /* Install the new hugetlb folio if src pte stable */ dst_ptl = huge_pte_lock(h, dst, dst_pte); @@ -5617,7 +5621,10 @@ retry_avoidcopy: goto out_release_all; } - copy_user_large_folio(new_folio, page_folio(old_page), address, vma); + if (copy_user_large_folio(new_folio, page_folio(old_page), address, vma)) { + ret = VM_FAULT_HWPOISON_LARGE; + goto out_release_all; + } __folio_mark_uptodate(new_folio); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, haddr, @@ -6260,9 +6267,13 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, *foliop = NULL; goto out; } - copy_user_large_folio(folio, *foliop, dst_addr, dst_vma); + ret = copy_user_large_folio(folio, *foliop, dst_addr, dst_vma); folio_put(*foliop); *foliop = NULL; + if (ret) { + folio_put(folio); + goto out; + } } /* diff --git a/mm/memory.c b/mm/memory.c index f315c2198098..f7510a06a2d0 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5733,12 +5733,12 @@ EXPORT_SYMBOL(__might_fault); * operation. The target subpage will be processed last to keep its * cache lines hot. */ -static inline void process_huge_page( +static inline int process_huge_page( unsigned long addr_hint, unsigned int pages_per_huge_page, - void (*process_subpage)(unsigned long addr, int idx, void *arg), + int (*process_subpage)(unsigned long addr, int idx, void *arg), void *arg) { - int i, n, base, l; + int i, n, base, l, ret; unsigned long addr = addr_hint & ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1); @@ -5752,7 +5752,9 @@ static inline void process_huge_page( /* Process subpages at the end of huge page */ for (i = pages_per_huge_page - 1; i >= 2 * n; i--) { cond_resched(); - process_subpage(addr + i * PAGE_SIZE, i, arg); + ret = process_subpage(addr + i * PAGE_SIZE, i, arg); + if (ret) + return ret; } } else { /* If target subpage in second half of huge page */ @@ -5761,7 +5763,9 @@ static inline void process_huge_page( /* Process subpages at the begin of huge page */ for (i = 0; i < base; i++) { cond_resched(); - process_subpage(addr + i * PAGE_SIZE, i, arg); + ret = process_subpage(addr + i * PAGE_SIZE, i, arg); + if (ret) + return ret; } } /* @@ -5773,10 +5777,15 @@ static inline void process_huge_page( int right_idx = base + 2 * l - 1 - i; cond_resched(); - process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg); + ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg); + if (ret) + return ret; cond_resched(); - process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg); + ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg); + if (ret) + return ret; } + return 0; } static void clear_gigantic_page(struct page *page, @@ -5794,11 +5803,12 @@ static void clear_gigantic_page(struct page *page, } } -static void clear_subpage(unsigned long addr, int idx, void *arg) +static int clear_subpage(unsigned long addr, int idx, void *arg) { struct page *page = arg; clear_user_highpage(page + idx, addr); + return 0; } void clear_huge_page(struct page *page, @@ -5815,7 +5825,7 @@ void clear_huge_page(struct page *page, process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page); } -static void copy_user_gigantic_page(struct folio *dst, struct folio *src, +static int copy_user_gigantic_page(struct folio *dst, struct folio *src, unsigned long addr, struct vm_area_struct *vma, unsigned int pages_per_huge_page) @@ -5829,8 +5839,13 @@ static void copy_user_gigantic_page(struct folio *dst, struct folio *src, src_page = folio_page(src, i); cond_resched(); - copy_user_highpage(dst_page, src_page, addr + i*PAGE_SIZE, vma); + if (copy_mc_user_highpage(dst_page, src_page, + addr + i*PAGE_SIZE, vma)) { + memory_failure_queue(page_to_pfn(src_page), 0); + return -EHWPOISON; + } } + return 0; } struct copy_subpage_arg { @@ -5839,16 +5854,20 @@ struct copy_subpage_arg { struct vm_area_struct *vma; }; -static void copy_subpage(unsigned long addr, int idx, void *arg) +static int copy_subpage(unsigned long addr, int idx, void *arg) { struct copy_subpage_arg *copy_arg = arg; - copy_user_highpage(copy_arg->dst + idx, copy_arg->src + idx, - addr, copy_arg->vma); + if (copy_mc_user_highpage(copy_arg->dst + idx, copy_arg->src + idx, + addr, copy_arg->vma)) { + memory_failure_queue(page_to_pfn(copy_arg->src + idx), 0); + return -EHWPOISON; + } + return 0; } -void copy_user_large_folio(struct folio *dst, struct folio *src, - unsigned long addr_hint, struct vm_area_struct *vma) +int copy_user_large_folio(struct folio *dst, struct folio *src, + unsigned long addr_hint, struct vm_area_struct *vma) { unsigned int pages_per_huge_page = folio_nr_pages(dst); unsigned long addr = addr_hint & @@ -5859,13 +5878,11 @@ void copy_user_large_folio(struct folio *dst, struct folio *src, .vma = vma, }; - if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { - copy_user_gigantic_page(dst, src, addr, vma, - pages_per_huge_page); - return; - } + if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) + return copy_user_gigantic_page(dst, src, addr, vma, + pages_per_huge_page); - process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg); + return process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg); } long copy_folio_from_user(struct folio *dst_folio, -- cgit v1.2.3 From bb1508c24c9c361e6344308c8de2cb81d7f228ba Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Thu, 13 Apr 2023 15:12:22 +0200 Subject: mm: kmsan: apply __must_check to non-void functions Non-void KMSAN hooks may return error codes that indicate that KMSAN failed to reflect the changed memory state in the metadata (e.g. it could not create the necessary memory mappings). In such cases the callers should handle the errors to prevent the tool from using the inconsistent metadata in the future. We mark non-void hooks with __must_check so that error handling is not skipped. Link: https://lkml.kernel.org/r/20230413131223.4135168-3-glider@google.com Signed-off-by: Alexander Potapenko Reviewed-by: Marco Elver Cc: Christoph Hellwig Cc: Dipanjan Das Cc: Dmitry Vyukov Cc: Uladzislau Rezki (Sony) Signed-off-by: Andrew Morton --- include/linux/kmsan.h | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kmsan.h b/include/linux/kmsan.h index 30b17647ce3c..e0c23a32cdf0 100644 --- a/include/linux/kmsan.h +++ b/include/linux/kmsan.h @@ -54,7 +54,8 @@ void __init kmsan_init_runtime(void); * Freed pages are either returned to buddy allocator or held back to be used * as metadata pages. */ -bool __init kmsan_memblock_free_pages(struct page *page, unsigned int order); +bool __init __must_check kmsan_memblock_free_pages(struct page *page, + unsigned int order); /** * kmsan_alloc_page() - Notify KMSAN about an alloc_pages() call. @@ -137,9 +138,11 @@ void kmsan_kfree_large(const void *ptr); * vmalloc metadata address range. Returns 0 on success, callers must check * for non-zero return value. */ -int kmsan_vmap_pages_range_noflush(unsigned long start, unsigned long end, - pgprot_t prot, struct page **pages, - unsigned int page_shift); +int __must_check kmsan_vmap_pages_range_noflush(unsigned long start, + unsigned long end, + pgprot_t prot, + struct page **pages, + unsigned int page_shift); /** * kmsan_vunmap_kernel_range_noflush() - Notify KMSAN about a vunmap. @@ -163,9 +166,9 @@ void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end); * virtual memory. Returns 0 on success, callers must check for non-zero return * value. */ -int kmsan_ioremap_page_range(unsigned long addr, unsigned long end, - phys_addr_t phys_addr, pgprot_t prot, - unsigned int page_shift); +int __must_check kmsan_ioremap_page_range(unsigned long addr, unsigned long end, + phys_addr_t phys_addr, pgprot_t prot, + unsigned int page_shift); /** * kmsan_iounmap_page_range() - Notify KMSAN about a iounmap_page_range() call. @@ -237,8 +240,8 @@ static inline void kmsan_init_runtime(void) { } -static inline bool kmsan_memblock_free_pages(struct page *page, - unsigned int order) +static inline bool __must_check kmsan_memblock_free_pages(struct page *page, + unsigned int order) { return true; } @@ -251,10 +254,9 @@ static inline void kmsan_task_exit(struct task_struct *task) { } -static inline int kmsan_alloc_page(struct page *page, unsigned int order, - gfp_t flags) +static inline void kmsan_alloc_page(struct page *page, unsigned int order, + gfp_t flags) { - return 0; } static inline void kmsan_free_page(struct page *page, unsigned int order) @@ -283,11 +285,9 @@ static inline void kmsan_kfree_large(const void *ptr) { } -static inline int kmsan_vmap_pages_range_noflush(unsigned long start, - unsigned long end, - pgprot_t prot, - struct page **pages, - unsigned int page_shift) +static inline int __must_check kmsan_vmap_pages_range_noflush( + unsigned long start, unsigned long end, pgprot_t prot, + struct page **pages, unsigned int page_shift) { return 0; } @@ -297,10 +297,11 @@ static inline void kmsan_vunmap_range_noflush(unsigned long start, { } -static inline int kmsan_ioremap_page_range(unsigned long start, - unsigned long end, - phys_addr_t phys_addr, pgprot_t prot, - unsigned int page_shift) +static inline int __must_check kmsan_ioremap_page_range(unsigned long start, + unsigned long end, + phys_addr_t phys_addr, + pgprot_t prot, + unsigned int page_shift) { return 0; } -- cgit v1.2.3 From c7b23b68e2aa93f86a206222d23ccd9a21f5982a Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Thu, 13 Apr 2023 10:40:34 +0000 Subject: mm: vmscan: refactor updating current->reclaim_state During reclaim, we keep track of pages reclaimed from other means than LRU-based reclaim through scan_control->reclaim_state->reclaimed_slab, which we stash a pointer to in current task_struct. However, we keep track of more than just reclaimed slab pages through this. We also use it for clean file pages dropped through pruned inodes, and xfs buffer pages freed. Rename reclaimed_slab to reclaimed, and add a helper function that wraps updating it through current, so that future changes to this logic are contained within include/linux/swap.h. Link: https://lkml.kernel.org/r/20230413104034.1086717-4-yosryahmed@google.com Signed-off-by: Yosry Ahmed Acked-by: Michal Hocko Cc: Alexander Viro Cc: Christoph Lameter Cc: Darrick J. Wong Cc: Dave Chinner Cc: David Hildenbrand Cc: David Rientjes Cc: Hyeonggon Yoo <42.hyeyoo@gmail.com> Cc: Johannes Weiner Cc: Joonsoo Kim Cc: Matthew Wilcox Cc: Miaohe Lin Cc: NeilBrown Cc: Peter Xu Cc: Roman Gushchin Cc: Shakeel Butt Cc: Tim Chen Cc: Vlastimil Babka Cc: Yu Zhao Signed-off-by: Andrew Morton --- fs/inode.c | 3 +-- fs/xfs/xfs_buf.c | 3 +-- include/linux/swap.h | 17 ++++++++++++++++- mm/slab.c | 3 +-- mm/slob.c | 6 ++---- mm/slub.c | 5 ++--- 6 files changed, 23 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/fs/inode.c b/fs/inode.c index 4558dc2f1355..e60fcc41faf1 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -864,8 +864,7 @@ static enum lru_status inode_lru_isolate(struct list_head *item, __count_vm_events(KSWAPD_INODESTEAL, reap); else __count_vm_events(PGINODESTEAL, reap); - if (current->reclaim_state) - current->reclaim_state->reclaimed_slab += reap; + mm_account_reclaimed_pages(reap); } iput(inode); spin_lock(lru_lock); diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 54c774af6e1c..15d1e5a7c2d3 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -286,8 +286,7 @@ xfs_buf_free_pages( if (bp->b_pages[i]) __free_page(bp->b_pages[i]); } - if (current->reclaim_state) - current->reclaim_state->reclaimed_slab += bp->b_page_count; + mm_account_reclaimed_pages(bp->b_page_count); if (bp->b_pages != bp->b_page_array) kmem_free(bp->b_pages); diff --git a/include/linux/swap.h b/include/linux/swap.h index bfc3b06b5f8f..7f7d5b9ddf7e 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -153,13 +153,28 @@ union swap_header { * memory reclaim */ struct reclaim_state { - unsigned long reclaimed_slab; + /* pages reclaimed outside of LRU-based reclaim */ + unsigned long reclaimed; #ifdef CONFIG_LRU_GEN /* per-thread mm walk data */ struct lru_gen_mm_walk *mm_walk; #endif }; +/* + * mm_account_reclaimed_pages(): account reclaimed pages outside of LRU-based + * reclaim + * @pages: number of pages reclaimed + * + * If the current process is undergoing a reclaim operation, increment the + * number of reclaimed pages by @pages. + */ +static inline void mm_account_reclaimed_pages(unsigned long pages) +{ + if (current->reclaim_state) + current->reclaim_state->reclaimed += pages; +} + #ifdef __KERNEL__ struct address_space; diff --git a/mm/slab.c b/mm/slab.c index 6b7c172158e5..bb57f7fdbae1 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1392,8 +1392,7 @@ static void kmem_freepages(struct kmem_cache *cachep, struct slab *slab) smp_wmb(); __folio_clear_slab(folio); - if (current->reclaim_state) - current->reclaim_state->reclaimed_slab += 1 << order; + mm_account_reclaimed_pages(1 << order); unaccount_slab(slab, order, cachep); __free_pages(&folio->page, order); } diff --git a/mm/slob.c b/mm/slob.c index fe567fcfa3a3..79cc8680c973 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -61,7 +61,7 @@ #include #include -#include /* struct reclaim_state */ +#include /* mm_account_reclaimed_pages() */ #include #include #include @@ -211,9 +211,7 @@ static void slob_free_pages(void *b, int order) { struct page *sp = virt_to_page(b); - if (current->reclaim_state) - current->reclaim_state->reclaimed_slab += 1 << order; - + mm_account_reclaimed_pages(1 << order); mod_node_page_state(page_pgdat(sp), NR_SLAB_UNRECLAIMABLE_B, -(PAGE_SIZE << order)); __free_pages(sp, order); diff --git a/mm/slub.c b/mm/slub.c index f49d669ff604..2728d5ae4dc0 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -11,7 +11,7 @@ */ #include -#include /* struct reclaim_state */ +#include /* mm_account_reclaimed_pages() */ #include #include #include @@ -2063,8 +2063,7 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) /* Make the mapping reset visible before clearing the flag */ smp_wmb(); __folio_clear_slab(folio); - if (current->reclaim_state) - current->reclaim_state->reclaimed_slab += pages; + mm_account_reclaimed_pages(pages); unaccount_slab(slab, order, s); __free_pages(&folio->page, order); } -- cgit v1.2.3 From 7f63cf2d9b9bbe7b90f808927558a66ff737d399 Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Thu, 13 Apr 2023 14:43:26 -0700 Subject: mm: Multi-gen LRU: remove wait_event_killable() Android 14 and later default to MGLRU [1] and field telemetry showed occasional long tail latency (>100ms) in the reclaim path. Tracing revealed priority inversion in the reclaim path. In try_to_inc_max_seq(), when high priority tasks were blocked on wait_event_killable(), the preemption of the low priority task to call wake_up_all() caused those high priority tasks to wait longer than necessary. In general, this problem is not different from others of its kind, e.g., one caused by mutex_lock(). However, it is specific to MGLRU because it introduced the new wait queue lruvec->mm_state.wait. The purpose of this new wait queue is to avoid the thundering herd problem. If many direct reclaimers rush into try_to_inc_max_seq(), only one can succeed, i.e., the one to wake up the rest, and the rest who failed might cause premature OOM kills if they do not wait. So far there is no evidence supporting this scenario, based on how often the wait has been hit. And this begs the question how useful the wait queue is in practice. Based on Minchan's recommendation, which is in line with his commit 6d4675e60135 ("mm: don't be stuck to rmap lock on reclaim path") and the rest of the MGLRU code which also uses trylock when possible, remove the wait queue. [1] https://android-review.googlesource.com/q/I7ed7fbfd6ef9ce10053347528125dd98c39e50bf Link: https://lkml.kernel.org/r/20230413214326.2147568-1-kaleshsingh@google.com Fixes: bd74fdaea146 ("mm: multi-gen LRU: support page table walks") Signed-off-by: Kalesh Singh Suggested-by: Minchan Kim Reported-by: Wei Wang Acked-by: Yu Zhao Cc: Minchan Kim Cc: Jan Alexander Steffens (heftig) Cc: Oleksandr Natalenko Cc: Suleiman Souhlal Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 8 +--- mm/vmscan.c | 112 ++++++++++++++++++------------------------------- 2 files changed, 42 insertions(+), 78 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 337956748976..a4889c9d4055 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -443,18 +443,14 @@ enum { struct lru_gen_mm_state { /* set to max_seq after each iteration */ unsigned long seq; - /* where the current iteration continues (inclusive) */ + /* where the current iteration continues after */ struct list_head *head; - /* where the last iteration ended (exclusive) */ + /* where the last iteration ended before */ struct list_head *tail; - /* to wait for the last page table walker to finish */ - struct wait_queue_head wait; /* Bloom filters flip after each iteration */ unsigned long *filters[NR_BLOOM_FILTERS]; /* the mm stats for debugging */ unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; - /* the number of concurrent page table walkers */ - int nr_walkers; }; struct lru_gen_mm_walk { diff --git a/mm/vmscan.c b/mm/vmscan.c index 2eb5862e99ec..8c05f4c288ce 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3450,18 +3450,13 @@ void lru_gen_del_mm(struct mm_struct *mm) for_each_node(nid) { struct lruvec *lruvec = get_lruvec(memcg, nid); - /* where the last iteration ended (exclusive) */ + /* where the current iteration continues after */ + if (lruvec->mm_state.head == &mm->lru_gen.list) + lruvec->mm_state.head = lruvec->mm_state.head->prev; + + /* where the last iteration ended before */ if (lruvec->mm_state.tail == &mm->lru_gen.list) lruvec->mm_state.tail = lruvec->mm_state.tail->next; - - /* where the current iteration continues (inclusive) */ - if (lruvec->mm_state.head != &mm->lru_gen.list) - continue; - - lruvec->mm_state.head = lruvec->mm_state.head->next; - /* the deletion ends the current iteration */ - if (lruvec->mm_state.head == &mm_list->fifo) - WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); } list_del_init(&mm->lru_gen.list); @@ -3557,68 +3552,54 @@ static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, struct mm_struct **iter) { bool first = false; - bool last = true; + bool last = false; struct mm_struct *mm = NULL; struct mem_cgroup *memcg = lruvec_memcg(lruvec); struct lru_gen_mm_list *mm_list = get_mm_list(memcg); struct lru_gen_mm_state *mm_state = &lruvec->mm_state; /* - * There are four interesting cases for this page table walker: - * 1. It tries to start a new iteration of mm_list with a stale max_seq; - * there is nothing left to do. - * 2. It's the first of the current generation, and it needs to reset - * the Bloom filter for the next generation. - * 3. It reaches the end of mm_list, and it needs to increment - * mm_state->seq; the iteration is done. - * 4. It's the last of the current generation, and it needs to reset the - * mm stats counters for the next generation. + * mm_state->seq is incremented after each iteration of mm_list. There + * are three interesting cases for this page table walker: + * 1. It tries to start a new iteration with a stale max_seq: there is + * nothing left to do. + * 2. It started the next iteration: it needs to reset the Bloom filter + * so that a fresh set of PTE tables can be recorded. + * 3. It ended the current iteration: it needs to reset the mm stats + * counters and tell its caller to increment max_seq. */ spin_lock(&mm_list->lock); VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); - VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); - VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); - if (walk->max_seq <= mm_state->seq) { - if (!*iter) - last = false; + if (walk->max_seq <= mm_state->seq) goto done; - } - if (!mm_state->nr_walkers) { - VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); + if (!mm_state->head) + mm_state->head = &mm_list->fifo; - mm_state->head = mm_list->fifo.next; + if (mm_state->head == &mm_list->fifo) first = true; - } - - while (!mm && mm_state->head != &mm_list->fifo) { - mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); + do { mm_state->head = mm_state->head->next; + if (mm_state->head == &mm_list->fifo) { + WRITE_ONCE(mm_state->seq, mm_state->seq + 1); + last = true; + break; + } /* force scan for those added after the last iteration */ - if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { - mm_state->tail = mm_state->head; + if (!mm_state->tail || mm_state->tail == mm_state->head) { + mm_state->tail = mm_state->head->next; walk->force_scan = true; } + mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); if (should_skip_mm(mm, walk)) mm = NULL; - } - - if (mm_state->head == &mm_list->fifo) - WRITE_ONCE(mm_state->seq, mm_state->seq + 1); + } while (!mm); done: - if (*iter && !mm) - mm_state->nr_walkers--; - if (!*iter && mm) - mm_state->nr_walkers++; - - if (mm_state->nr_walkers) - last = false; - if (*iter || last) reset_mm_stats(lruvec, walk, last); @@ -3646,9 +3627,9 @@ static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); - if (max_seq > mm_state->seq && !mm_state->nr_walkers) { - VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); - + if (max_seq > mm_state->seq) { + mm_state->head = NULL; + mm_state->tail = NULL; WRITE_ONCE(mm_state->seq, mm_state->seq + 1); reset_mm_stats(lruvec, NULL, true); success = true; @@ -4248,10 +4229,6 @@ restart: walk_pmd_range(&val, addr, next, args); - /* a racy check to curtail the waiting time */ - if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) - return 1; - if (need_resched() || walk->batched >= MAX_LRU_BATCH) { end = (addr | ~PUD_MASK) + 1; goto done; @@ -4284,8 +4261,14 @@ static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_ walk->next_addr = FIRST_USER_ADDRESS; do { + DEFINE_MAX_SEQ(lruvec); + err = -EBUSY; + /* another thread might have called inc_max_seq() */ + if (walk->max_seq != max_seq) + break; + /* folio_update_gen() requires stable folio_memcg() */ if (!mem_cgroup_trylock_pages(memcg)) break; @@ -4518,25 +4501,12 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, success = iterate_mm_list(lruvec, walk, &mm); if (mm) walk_mm(lruvec, mm, walk); - - cond_resched(); } while (mm); done: - if (!success) { - if (sc->priority <= DEF_PRIORITY - 2) - wait_event_killable(lruvec->mm_state.wait, - max_seq < READ_ONCE(lrugen->max_seq)); - return false; - } + if (success) + inc_max_seq(lruvec, can_swap, force_scan); - VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); - - inc_max_seq(lruvec, can_swap, force_scan); - /* either this sees any waiters or they will see updated max_seq */ - if (wq_has_sleeper(&lruvec->mm_state.wait)) - wake_up_all(&lruvec->mm_state.wait); - - return true; + return success; } /****************************************************************************** @@ -6173,7 +6143,6 @@ void lru_gen_init_lruvec(struct lruvec *lruvec) INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]); lruvec->mm_state.seq = MIN_NR_GENS; - init_waitqueue_head(&lruvec->mm_state.wait); } #ifdef CONFIG_MEMCG @@ -6206,7 +6175,6 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg) for_each_node(nid) { struct lruvec *lruvec = get_lruvec(memcg, nid); - VM_WARN_ON_ONCE(lruvec->mm_state.nr_walkers); VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, sizeof(lruvec->lrugen.nr_pages))); -- cgit v1.2.3 From f7b8f70ba44fb63df47b6fc3204d49ac2885de64 Mon Sep 17 00:00:00 2001 From: Luca Vizzarro Date: Fri, 14 Apr 2023 16:24:58 +0100 Subject: memfd: pass argument of memfd_fcntl as int The interface for fcntl expects the argument passed for the command F_ADD_SEALS to be of type int. The current code wrongly treats it as a long. In order to avoid access to undefined bits, we should explicitly cast the argument to int. This commit changes the signature of all the related and helper functions so that they treat the argument as int instead of long. Link: https://lkml.kernel.org/r/20230414152459.816046-5-Luca.Vizzarro@arm.com Signed-off-by: Luca Vizzarro Cc: Alexander Viro Cc: Christian Brauner Cc: Jeff Layton Cc: Chuck Lever Cc: Kevin Brodsky Cc: Vincenzo Frascino Cc: Szabolcs Nagy Cc: "Theodore Ts'o" Cc: David Laight Cc: Mark Rutland Signed-off-by: Andrew Morton --- include/linux/memfd.h | 4 ++-- mm/memfd.c | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memfd.h b/include/linux/memfd.h index 4f1600413f91..e7abf6fa4c52 100644 --- a/include/linux/memfd.h +++ b/include/linux/memfd.h @@ -5,9 +5,9 @@ #include #ifdef CONFIG_MEMFD_CREATE -extern long memfd_fcntl(struct file *file, unsigned int cmd, unsigned long arg); +extern long memfd_fcntl(struct file *file, unsigned int cmd, unsigned int arg); #else -static inline long memfd_fcntl(struct file *f, unsigned int c, unsigned long a) +static inline long memfd_fcntl(struct file *f, unsigned int c, unsigned int a) { return -EINVAL; } diff --git a/mm/memfd.c b/mm/memfd.c index a0a7a37e8177..69b90c31d38c 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -243,16 +243,12 @@ static int memfd_get_seals(struct file *file) return seals ? *seals : -EINVAL; } -long memfd_fcntl(struct file *file, unsigned int cmd, unsigned long arg) +long memfd_fcntl(struct file *file, unsigned int cmd, unsigned int arg) { long error; switch (cmd) { case F_ADD_SEALS: - /* disallow upper 32bit */ - if (arg > UINT_MAX) - return -EINVAL; - error = memfd_add_seals(file, arg); break; case F_GET_SEALS: -- cgit v1.2.3 From b0687c1119b4e8c88a651b6e876b7eae28d076e3 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Tue, 4 Apr 2023 17:13:51 -0500 Subject: lib/rbtree: use '+' instead of '|' for setting color. This has a slight benefit for x86 and has no effect on other targets. The benefit to x86 is it change the codegen for setting a node to block from `mov %r0, %r1; or $RB_BLACK, %r1` to `lea RB_BLACK(%r0), %r1` which saves an instructions. In all other cases it just replace ALU with ALU (or -> and) which perform the same on all machines I am aware of. Total instructions in rbtree.o: Before - 802 After - 782 so it saves about 20 `mov` instructions. Link: https://lkml.kernel.org/r/20230404221350.3806566-1-goldstein.w.n@gmail.com Signed-off-by: Noah Goldstein Cc: Michel Lespinasse Signed-off-by: Andrew Morton --- include/linux/rbtree_augmented.h | 4 ++-- lib/rbtree.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h index d1c53e9d8c75..7ee7ed5de722 100644 --- a/include/linux/rbtree_augmented.h +++ b/include/linux/rbtree_augmented.h @@ -156,13 +156,13 @@ RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p) { - rb->__rb_parent_color = rb_color(rb) | (unsigned long)p; + rb->__rb_parent_color = rb_color(rb) + (unsigned long)p; } static inline void rb_set_parent_color(struct rb_node *rb, struct rb_node *p, int color) { - rb->__rb_parent_color = (unsigned long)p | color; + rb->__rb_parent_color = (unsigned long)p + color; } static inline void diff --git a/lib/rbtree.c b/lib/rbtree.c index c4ac5c2421f2..5114eda6309c 100644 --- a/lib/rbtree.c +++ b/lib/rbtree.c @@ -58,7 +58,7 @@ static inline void rb_set_black(struct rb_node *rb) { - rb->__rb_parent_color |= RB_BLACK; + rb->__rb_parent_color += RB_BLACK; } static inline struct rb_node *rb_red_parent(struct rb_node *red) -- cgit v1.2.3 From a3b2aeac9d154e5e15ddbf19de934c0c606b6acd Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Sat, 8 Apr 2023 17:28:35 +0800 Subject: delayacct: track delays from IRQ/SOFTIRQ Delay accounting does not track the delay of IRQ/SOFTIRQ. While IRQ/SOFTIRQ could have obvious impact on some workloads productivity, such as when workloads are running on system which is busy handling network IRQ/SOFTIRQ. Get the delay of IRQ/SOFTIRQ could help users to reduce such delay. Such as setting interrupt affinity or task affinity, using kernel thread for NAPI etc. This is inspired by "sched/psi: Add PSI_IRQ to track IRQ/SOFTIRQ pressure"[1]. Also fix some code indent problems of older code. And update tools/accounting/getdelays.c: / # ./getdelays -p 156 -di print delayacct stats ON printing IO accounting PID 156 CPU count real total virtual total delay total delay average 15 15836008 16218149 275700790 18.380ms IO count delay total delay average 0 0 0.000ms SWAP count delay total delay average 0 0 0.000ms RECLAIM count delay total delay average 0 0 0.000ms THRASHING count delay total delay average 0 0 0.000ms COMPACT count delay total delay average 0 0 0.000ms WPCOPY count delay total delay average 36 7586118 0.211ms IRQ count delay total delay average 42 929161 0.022ms [1] commit 52b1364ba0b1("sched/psi: Add PSI_IRQ to track IRQ/SOFTIRQ pressure") Link: https://lkml.kernel.org/r/202304081728353557233@zte.com.cn Signed-off-by: Yang Yang Cc: Jiang Xuexin Cc: wangyong Cc: junhua huang Cc: Balbir Singh Cc: Ingo Molnar Cc: Jonathan Corbet Cc: Juri Lelli Cc: Peter Zijlstra Signed-off-by: Andrew Morton --- Documentation/accounting/delay-accounting.rst | 7 +++++-- include/linux/delayacct.h | 15 ++++++++++++++ include/uapi/linux/taskstats.h | 6 +++++- kernel/delayacct.c | 14 +++++++++++++ kernel/sched/core.c | 1 + tools/accounting/getdelays.c | 30 ++++++++++++++++----------- 6 files changed, 58 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/Documentation/accounting/delay-accounting.rst b/Documentation/accounting/delay-accounting.rst index 79f537c9f160..f61c01fc376e 100644 --- a/Documentation/accounting/delay-accounting.rst +++ b/Documentation/accounting/delay-accounting.rst @@ -16,6 +16,7 @@ d) memory reclaim e) thrashing f) direct compact g) write-protect copy +h) IRQ/SOFTIRQ and makes these statistics available to userspace through the taskstats interface. @@ -49,7 +50,7 @@ this structure. See for a description of the fields pertaining to delay accounting. It will generally be in the form of counters returning the cumulative delay seen for cpu, sync block I/O, swapin, memory reclaim, thrash page -cache, direct compact, write-protect copy etc. +cache, direct compact, write-protect copy, IRQ/SOFTIRQ etc. Taking the difference of two successive readings of a given counter (say cpu_delay_total) for a task will give the delay @@ -118,7 +119,9 @@ Get sum of delays, since system boot, for all pids with tgid 5:: 0 0 0.000ms COMPACT count delay total delay average 0 0 0.000ms - WPCOPY count delay total delay average + WPCOPY count delay total delay average + 0 0 0.000ms + IRQ count delay total delay average 0 0 0.000ms Get IO accounting for pid 1, it works only with -p:: diff --git a/include/linux/delayacct.h b/include/linux/delayacct.h index 0da97dba9ef8..6639f48dac36 100644 --- a/include/linux/delayacct.h +++ b/include/linux/delayacct.h @@ -48,10 +48,13 @@ struct task_delay_info { u64 wpcopy_start; u64 wpcopy_delay; /* wait for write-protect copy */ + u64 irq_delay; /* wait for IRQ/SOFTIRQ */ + u32 freepages_count; /* total count of memory reclaim */ u32 thrashing_count; /* total count of thrash waits */ u32 compact_count; /* total count of memory compact */ u32 wpcopy_count; /* total count of write-protect copy */ + u32 irq_count; /* total count of IRQ/SOFTIRQ */ }; #endif @@ -81,6 +84,7 @@ extern void __delayacct_compact_start(void); extern void __delayacct_compact_end(void); extern void __delayacct_wpcopy_start(void); extern void __delayacct_wpcopy_end(void); +extern void __delayacct_irq(struct task_struct *task, u32 delta); static inline void delayacct_tsk_init(struct task_struct *tsk) { @@ -215,6 +219,15 @@ static inline void delayacct_wpcopy_end(void) __delayacct_wpcopy_end(); } +static inline void delayacct_irq(struct task_struct *task, u32 delta) +{ + if (!static_branch_unlikely(&delayacct_key)) + return; + + if (task->delays) + __delayacct_irq(task, delta); +} + #else static inline void delayacct_init(void) {} @@ -253,6 +266,8 @@ static inline void delayacct_wpcopy_start(void) {} static inline void delayacct_wpcopy_end(void) {} +static inline void delayacct_irq(struct task_struct *task, u32 delta) +{} #endif /* CONFIG_TASK_DELAY_ACCT */ diff --git a/include/uapi/linux/taskstats.h b/include/uapi/linux/taskstats.h index a7f5b11a8f1b..b50b2eb257a0 100644 --- a/include/uapi/linux/taskstats.h +++ b/include/uapi/linux/taskstats.h @@ -34,7 +34,7 @@ */ -#define TASKSTATS_VERSION 13 +#define TASKSTATS_VERSION 14 #define TS_COMM_LEN 32 /* should be >= TASK_COMM_LEN * in linux/sched.h */ @@ -198,6 +198,10 @@ struct taskstats { /* v13: Delay waiting for write-protect copy */ __u64 wpcopy_count; __u64 wpcopy_delay_total; + + /* v14: Delay waiting for IRQ/SOFTIRQ */ + __u64 irq_count; + __u64 irq_delay_total; }; diff --git a/kernel/delayacct.c b/kernel/delayacct.c index e39cb696cfbd..6f0c358e73d8 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -179,12 +179,15 @@ int delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk) d->compact_delay_total = (tmp < d->compact_delay_total) ? 0 : tmp; tmp = d->wpcopy_delay_total + tsk->delays->wpcopy_delay; d->wpcopy_delay_total = (tmp < d->wpcopy_delay_total) ? 0 : tmp; + tmp = d->irq_delay_total + tsk->delays->irq_delay; + d->irq_delay_total = (tmp < d->irq_delay_total) ? 0 : tmp; d->blkio_count += tsk->delays->blkio_count; d->swapin_count += tsk->delays->swapin_count; d->freepages_count += tsk->delays->freepages_count; d->thrashing_count += tsk->delays->thrashing_count; d->compact_count += tsk->delays->compact_count; d->wpcopy_count += tsk->delays->wpcopy_count; + d->irq_count += tsk->delays->irq_count; raw_spin_unlock_irqrestore(&tsk->delays->lock, flags); return 0; @@ -274,3 +277,14 @@ void __delayacct_wpcopy_end(void) ¤t->delays->wpcopy_delay, ¤t->delays->wpcopy_count); } + +void __delayacct_irq(struct task_struct *task, u32 delta) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&task->delays->lock, flags); + task->delays->irq_delay += delta; + task->delays->irq_count++; + raw_spin_unlock_irqrestore(&task->delays->lock, flags); +} + diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0d18c3969f90..5473e831daf3 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -704,6 +704,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) rq->prev_irq_time += irq_delta; delta -= irq_delta; psi_account_irqtime(rq->curr, irq_delta); + delayacct_irq(rq->curr, irq_delta); #endif #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING if (static_key_false((¶virt_steal_rq_enabled))) { diff --git a/tools/accounting/getdelays.c b/tools/accounting/getdelays.c index 23a15d8f2bf4..1334214546d7 100644 --- a/tools/accounting/getdelays.c +++ b/tools/accounting/getdelays.c @@ -198,17 +198,19 @@ static void print_delayacct(struct taskstats *t) printf("\n\nCPU %15s%15s%15s%15s%15s\n" " %15llu%15llu%15llu%15llu%15.3fms\n" "IO %15s%15s%15s\n" - " %15llu%15llu%15.3fms\n" + " %15llu%15llu%15.3fms\n" "SWAP %15s%15s%15s\n" - " %15llu%15llu%15.3fms\n" + " %15llu%15llu%15.3fms\n" "RECLAIM %12s%15s%15s\n" - " %15llu%15llu%15.3fms\n" + " %15llu%15llu%15.3fms\n" "THRASHING%12s%15s%15s\n" - " %15llu%15llu%15.3fms\n" + " %15llu%15llu%15.3fms\n" "COMPACT %12s%15s%15s\n" - " %15llu%15llu%15.3fms\n" + " %15llu%15llu%15.3fms\n" "WPCOPY %12s%15s%15s\n" - " %15llu%15llu%15.3fms\n", + " %15llu%15llu%15.3fms\n" + "IRQ %15s%15s%15s\n" + " %15llu%15llu%15.3fms\n", "count", "real total", "virtual total", "delay total", "delay average", (unsigned long long)t->cpu_count, @@ -219,27 +221,31 @@ static void print_delayacct(struct taskstats *t) "count", "delay total", "delay average", (unsigned long long)t->blkio_count, (unsigned long long)t->blkio_delay_total, - average_ms((double)t->blkio_delay_total, t->blkio_count), + average_ms((double)t->blkio_delay_total, t->blkio_count), "count", "delay total", "delay average", (unsigned long long)t->swapin_count, (unsigned long long)t->swapin_delay_total, - average_ms((double)t->swapin_delay_total, t->swapin_count), + average_ms((double)t->swapin_delay_total, t->swapin_count), "count", "delay total", "delay average", (unsigned long long)t->freepages_count, (unsigned long long)t->freepages_delay_total, - average_ms((double)t->freepages_delay_total, t->freepages_count), + average_ms((double)t->freepages_delay_total, t->freepages_count), "count", "delay total", "delay average", (unsigned long long)t->thrashing_count, (unsigned long long)t->thrashing_delay_total, - average_ms((double)t->thrashing_delay_total, t->thrashing_count), + average_ms((double)t->thrashing_delay_total, t->thrashing_count), "count", "delay total", "delay average", (unsigned long long)t->compact_count, (unsigned long long)t->compact_delay_total, - average_ms((double)t->compact_delay_total, t->compact_count), + average_ms((double)t->compact_delay_total, t->compact_count), "count", "delay total", "delay average", (unsigned long long)t->wpcopy_count, (unsigned long long)t->wpcopy_delay_total, - average_ms((double)t->wpcopy_delay_total, t->wpcopy_count)); + average_ms((double)t->wpcopy_delay_total, t->wpcopy_count), + "count", "delay total", "delay average", + (unsigned long long)t->irq_count, + (unsigned long long)t->irq_delay_total, + average_ms((double)t->irq_delay_total, t->irq_count)); } static void task_context_switch_counts(struct taskstats *t) -- cgit v1.2.3 From 4248d0083ec5817eebfb916c54950d100b3468ee Mon Sep 17 00:00:00 2001 From: Longlong Xia Date: Fri, 14 Apr 2023 10:17:41 +0800 Subject: mm: ksm: support hwpoison for ksm page hwpoison_user_mappings() is updated to support ksm pages, and add collect_procs_ksm() to collect processes when the error hit an ksm page. The difference from collect_procs_anon() is that it also needs to traverse the rmap-item list on the stable node of the ksm page. At the same time, add_to_kill_ksm() is added to handle ksm pages. And task_in_to_kill_list() is added to avoid duplicate addition of tsk to the to_kill list. This is because when scanning the list, if the pages that make up the ksm page all come from the same process, they may be added repeatedly. Link: https://lkml.kernel.org/r/20230414021741.2597273-3-xialonglong1@huawei.com Signed-off-by: Longlong Xia Tested-by: Naoya Horiguchi Reviewed-by: Kefeng Wang Cc: Miaohe Lin Cc: Nanyong Sun Signed-off-by: Andrew Morton --- include/linux/ksm.h | 11 +++++++++++ include/linux/mm.h | 7 +++++++ mm/ksm.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ mm/memory-failure.c | 34 +++++++++++++++++++++++++--------- 4 files changed, 88 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 7e232ba59b86..d9e701326a88 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -51,6 +51,10 @@ struct page *ksm_might_need_to_copy(struct page *page, void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc); void folio_migrate_ksm(struct folio *newfolio, struct folio *folio); +#ifdef CONFIG_MEMORY_FAILURE +void collect_procs_ksm(struct page *page, struct list_head *to_kill, + int force_early); +#endif #else /* !CONFIG_KSM */ static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) @@ -62,6 +66,13 @@ static inline void ksm_exit(struct mm_struct *mm) { } +#ifdef CONFIG_MEMORY_FAILURE +static inline void collect_procs_ksm(struct page *page, + struct list_head *to_kill, int force_early) +{ +} +#endif + #ifdef CONFIG_MMU static inline int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, unsigned long *vm_flags) diff --git a/include/linux/mm.h b/include/linux/mm.h index 2772a1800cf3..37554b08bb28 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3604,6 +3604,7 @@ extern int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, bool *migratable_cleared); void num_poisoned_pages_inc(unsigned long pfn); void num_poisoned_pages_sub(unsigned long pfn, long i); +struct task_struct *task_early_kill(struct task_struct *tsk, int force_early); #else static inline void memory_failure_queue(unsigned long pfn, int flags) { @@ -3624,6 +3625,12 @@ static inline void num_poisoned_pages_sub(unsigned long pfn, long i) } #endif +#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_KSM) +void add_to_kill_ksm(struct task_struct *tsk, struct page *p, + struct vm_area_struct *vma, struct list_head *to_kill, + unsigned long ksm_addr); +#endif + #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) extern void memblk_nr_poison_inc(unsigned long pfn); extern void memblk_nr_poison_sub(unsigned long pfn, long i); diff --git a/mm/ksm.c b/mm/ksm.c index 290a3eb6d8de..37c63310bc4e 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -2738,6 +2738,51 @@ again: goto again; } +#ifdef CONFIG_MEMORY_FAILURE +/* + * Collect processes when the error hit an ksm page. + */ +void collect_procs_ksm(struct page *page, struct list_head *to_kill, + int force_early) +{ + struct ksm_stable_node *stable_node; + struct ksm_rmap_item *rmap_item; + struct folio *folio = page_folio(page); + struct vm_area_struct *vma; + struct task_struct *tsk; + + stable_node = folio_stable_node(folio); + if (!stable_node) + return; + hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { + struct anon_vma *av = rmap_item->anon_vma; + + anon_vma_lock_read(av); + read_lock(&tasklist_lock); + for_each_process(tsk) { + struct anon_vma_chain *vmac; + unsigned long addr; + struct task_struct *t = + task_early_kill(tsk, force_early); + if (!t) + continue; + anon_vma_interval_tree_foreach(vmac, &av->rb_root, 0, + ULONG_MAX) + { + vma = vmac->vma; + if (vma->vm_mm == t->mm) { + addr = rmap_item->address & PAGE_MASK; + add_to_kill_ksm(t, page, vma, to_kill, + addr); + } + } + } + read_unlock(&tasklist_lock); + anon_vma_unlock_read(av); + } +} +#endif + #ifdef CONFIG_MIGRATION void folio_migrate_ksm(struct folio *newfolio, struct folio *folio) { diff --git a/mm/memory-failure.c b/mm/memory-failure.c index bb09999108ef..f69c410fb9f9 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -455,6 +455,27 @@ static void add_to_kill_anon_file(struct task_struct *tsk, struct page *p, __add_to_kill(tsk, p, vma, to_kill, 0, FSDAX_INVALID_PGOFF); } +#ifdef CONFIG_KSM +static bool task_in_to_kill_list(struct list_head *to_kill, + struct task_struct *tsk) +{ + struct to_kill *tk, *next; + + list_for_each_entry_safe(tk, next, to_kill, nd) { + if (tk->tsk == tsk) + return true; + } + + return false; +} +void add_to_kill_ksm(struct task_struct *tsk, struct page *p, + struct vm_area_struct *vma, struct list_head *to_kill, + unsigned long ksm_addr) +{ + if (!task_in_to_kill_list(to_kill, tsk)) + __add_to_kill(tsk, p, vma, to_kill, ksm_addr, FSDAX_INVALID_PGOFF); +} +#endif /* * Kill the processes that have been collected earlier. * @@ -534,8 +555,7 @@ static struct task_struct *find_early_kill_thread(struct task_struct *tsk) * processes sharing the same error page,if the process is "early kill", the * task_struct of the dedicated thread will also be returned. */ -static struct task_struct *task_early_kill(struct task_struct *tsk, - int force_early) +struct task_struct *task_early_kill(struct task_struct *tsk, int force_early) { if (!tsk->mm) return NULL; @@ -666,8 +686,9 @@ static void collect_procs(struct page *page, struct list_head *tokill, { if (!page->mapping) return; - - if (PageAnon(page)) + if (unlikely(PageKsm(page))) + collect_procs_ksm(page, tokill, force_early); + else if (PageAnon(page)) collect_procs_anon(page, tokill, force_early); else collect_procs_file(page, tokill, force_early); @@ -1522,11 +1543,6 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, if (!page_mapped(hpage)) return true; - if (PageKsm(p)) { - pr_err("%#lx: can't handle KSM pages.\n", pfn); - return false; - } - if (PageSwapCache(p)) { pr_err("%#lx: keeping poisoned page in swap cache\n", pfn); ttu &= ~TTU_HWPOISON; -- cgit v1.2.3 From 691d0b782066a6eeeecbfceb7910a8f6184e6105 Mon Sep 17 00:00:00 2001 From: Dai Ngo Date: Tue, 18 Apr 2023 13:19:02 -0700 Subject: SUNRPC: remove the maximum number of retries in call_bind_status Currently call_bind_status places a hard limit of 3 to the number of retries on EACCES error. This limit was done to prevent NLM unlock requests from being hang forever when the server keeps returning garbage. However this change causes problem for cases when NLM service takes longer than 9 seconds to register with the port mapper after a restart. This patch removes this hard coded limit and let the RPC handles the retry based on the standard hard/soft task semantics. Fixes: 0b760113a3a1 ("NLM: Don't hang forever on NLM unlock requests") Reported-by: Helen Chao Tested-by: Helen Chao Signed-off-by: Dai Ngo Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- include/linux/sunrpc/sched.h | 3 +-- net/sunrpc/clnt.c | 3 --- net/sunrpc/sched.c | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index b8ca3ecaf8d7..8ada7dc802d3 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -90,8 +90,7 @@ struct rpc_task { #endif unsigned char tk_priority : 2,/* Task priority */ tk_garb_retry : 2, - tk_cred_retry : 2, - tk_rebind_retry : 2; + tk_cred_retry : 2; }; typedef void (*rpc_action)(struct rpc_task *); diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index fd7e1c630493..d2ee56634308 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -2050,9 +2050,6 @@ call_bind_status(struct rpc_task *task) status = -EOPNOTSUPP; break; } - if (task->tk_rebind_retry == 0) - break; - task->tk_rebind_retry--; rpc_delay(task, 3*HZ); goto retry_timeout; case -ENOBUFS: diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index be587a308e05..c8321de341ee 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -817,7 +817,6 @@ rpc_init_task_statistics(struct rpc_task *task) /* Initialize retry counters */ task->tk_garb_retry = 2; task->tk_cred_retry = 2; - task->tk_rebind_retry = 2; /* starting timestamp */ task->tk_start = ktime_get(); -- cgit v1.2.3 From 13890626501ffda22b18213ddaf7930473da5792 Mon Sep 17 00:00:00 2001 From: Alan Stern Date: Mon, 10 Apr 2023 15:37:07 -0400 Subject: USB: core: Add routines for endpoint checks in old drivers Many of the older USB drivers in the Linux USB stack were written based simply on a vendor's device specification. They use the endpoint information in the spec and assume these endpoints will always be present, with the properties listed, in any device matching the given vendor and product IDs. While that may have been true back then, with spoofing and fuzzing it is not true any more. More and more we are finding that those old drivers need to perform at least a minimum of checking before they try to use any endpoint other than ep0. To make this checking as simple as possible, we now add a couple of utility routines to the USB core. usb_check_bulk_endpoints() and usb_check_int_endpoints() take an interface pointer together with a list of endpoint addresses (numbers and directions). They check that the interface's current alternate setting includes endpoints with those addresses and that each of these endpoints has the right type: bulk or interrupt, respectively. Although we already have usb_find_common_endpoints() and related routines meant for a similar purpose, they are not well suited for this kind of checking. Those routines find endpoints of various kinds, but only one (either the first or the last) of each kind, and they don't verify that the endpoints' addresses agree with what the caller expects. In theory the new routines could be more general: They could take a particular altsetting as their argument instead of always using the interface's current altsetting. In practice I think this won't matter too much; multiple altsettings tend to be used for transferring media (audio or visual) over isochronous endpoints, not bulk or interrupt. Drivers for such devices will generally require more sophisticated checking than these simplistic routines provide. Signed-off-by: Alan Stern Link: https://lore.kernel.org/r/dd2c8e8c-2c87-44ea-ba17-c64b97e201c9@rowland.harvard.edu Signed-off-by: Greg Kroah-Hartman --- drivers/usb/core/usb.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/usb.h | 5 ++++ 2 files changed, 81 insertions(+) (limited to 'include/linux') diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c index 34742fbbd84d..901ec732321c 100644 --- a/drivers/usb/core/usb.c +++ b/drivers/usb/core/usb.c @@ -206,6 +206,82 @@ int usb_find_common_endpoints_reverse(struct usb_host_interface *alt, } EXPORT_SYMBOL_GPL(usb_find_common_endpoints_reverse); +/** + * usb_find_endpoint() - Given an endpoint address, search for the endpoint's + * usb_host_endpoint structure in an interface's current altsetting. + * @intf: the interface whose current altsetting should be searched + * @ep_addr: the endpoint address (number and direction) to find + * + * Search the altsetting's list of endpoints for one with the specified address. + * + * Return: Pointer to the usb_host_endpoint if found, %NULL otherwise. + */ +static const struct usb_host_endpoint *usb_find_endpoint( + const struct usb_interface *intf, unsigned int ep_addr) +{ + int n; + const struct usb_host_endpoint *ep; + + n = intf->cur_altsetting->desc.bNumEndpoints; + ep = intf->cur_altsetting->endpoint; + for (; n > 0; (--n, ++ep)) { + if (ep->desc.bEndpointAddress == ep_addr) + return ep; + } + return NULL; +} + +/** + * usb_check_bulk_endpoints - Check whether an interface's current altsetting + * contains a set of bulk endpoints with the given addresses. + * @intf: the interface whose current altsetting should be searched + * @ep_addrs: 0-terminated array of the endpoint addresses (number and + * direction) to look for + * + * Search for endpoints with the specified addresses and check their types. + * + * Return: %true if all the endpoints are found and are bulk, %false otherwise. + */ +bool usb_check_bulk_endpoints( + const struct usb_interface *intf, const u8 *ep_addrs) +{ + const struct usb_host_endpoint *ep; + + for (; *ep_addrs; ++ep_addrs) { + ep = usb_find_endpoint(intf, *ep_addrs); + if (!ep || !usb_endpoint_xfer_bulk(&ep->desc)) + return false; + } + return true; +} +EXPORT_SYMBOL_GPL(usb_check_bulk_endpoints); + +/** + * usb_check_int_endpoints - Check whether an interface's current altsetting + * contains a set of interrupt endpoints with the given addresses. + * @intf: the interface whose current altsetting should be searched + * @ep_addrs: 0-terminated array of the endpoint addresses (number and + * direction) to look for + * + * Search for endpoints with the specified addresses and check their types. + * + * Return: %true if all the endpoints are found and are interrupt, + * %false otherwise. + */ +bool usb_check_int_endpoints( + const struct usb_interface *intf, const u8 *ep_addrs) +{ + const struct usb_host_endpoint *ep; + + for (; *ep_addrs; ++ep_addrs) { + ep = usb_find_endpoint(intf, *ep_addrs); + if (!ep || !usb_endpoint_xfer_int(&ep->desc)) + return false; + } + return true; +} +EXPORT_SYMBOL_GPL(usb_check_int_endpoints); + /** * usb_find_alt_setting() - Given a configuration, find the alternate setting * for the given interface. diff --git a/include/linux/usb.h b/include/linux/usb.h index d510fabcafa2..4ae0466c846c 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -291,6 +291,11 @@ void usb_put_intf(struct usb_interface *intf); #define USB_MAXINTERFACES 32 #define USB_MAXIADS (USB_MAXINTERFACES/2) +bool usb_check_bulk_endpoints( + const struct usb_interface *intf, const u8 *ep_addrs); +bool usb_check_int_endpoints( + const struct usb_interface *intf, const u8 *ep_addrs); + /* * USB Resume Timer: Every Host controller driver should drive the resume * signalling on the bus for the amount of time defined by this macro. -- cgit v1.2.3 From 046b6a171009e1ed9ede02194025e9ccd709beb2 Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Wed, 19 Apr 2023 09:41:27 -0700 Subject: device property: make device_property functions take const device * device_property functions do not modify the device pointer passed to them. The underlying of_device and fwnode_ functions actually already take const * arguments. Mark the parameter constant to simplify conversion from of_property to device_property functions, and to let the calling code use const device pointers where possible. Cc: Chris Packham Reviewed-by: Chris Packham Reviewed-by: Sakari Ailus Signed-off-by: Guenter Roeck Link: https://lore.kernel.org/r/20230419164127.3773278-1-linux@roeck-us.net Signed-off-by: Greg Kroah-Hartman --- drivers/base/property.c | 16 ++++++++-------- include/linux/property.h | 36 ++++++++++++++++++------------------ 2 files changed, 26 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/property.c b/drivers/base/property.c index 6a908e9c30f8..f6117ec9805c 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -40,7 +40,7 @@ EXPORT_SYMBOL_GPL(__dev_fwnode_const); * * Return: true if property @propname is present. Otherwise, returns false. */ -bool device_property_present(struct device *dev, const char *propname) +bool device_property_present(const struct device *dev, const char *propname) { return fwnode_property_present(dev_fwnode(dev), propname); } @@ -90,7 +90,7 @@ EXPORT_SYMBOL_GPL(fwnode_property_present); * %-EOVERFLOW if the size of the property is not as expected. * %-ENXIO if no suitable firmware interface is present. */ -int device_property_read_u8_array(struct device *dev, const char *propname, +int device_property_read_u8_array(const struct device *dev, const char *propname, u8 *val, size_t nval) { return fwnode_property_read_u8_array(dev_fwnode(dev), propname, val, nval); @@ -118,7 +118,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u8_array); * %-EOVERFLOW if the size of the property is not as expected. * %-ENXIO if no suitable firmware interface is present. */ -int device_property_read_u16_array(struct device *dev, const char *propname, +int device_property_read_u16_array(const struct device *dev, const char *propname, u16 *val, size_t nval) { return fwnode_property_read_u16_array(dev_fwnode(dev), propname, val, nval); @@ -146,7 +146,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u16_array); * %-EOVERFLOW if the size of the property is not as expected. * %-ENXIO if no suitable firmware interface is present. */ -int device_property_read_u32_array(struct device *dev, const char *propname, +int device_property_read_u32_array(const struct device *dev, const char *propname, u32 *val, size_t nval) { return fwnode_property_read_u32_array(dev_fwnode(dev), propname, val, nval); @@ -174,7 +174,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u32_array); * %-EOVERFLOW if the size of the property is not as expected. * %-ENXIO if no suitable firmware interface is present. */ -int device_property_read_u64_array(struct device *dev, const char *propname, +int device_property_read_u64_array(const struct device *dev, const char *propname, u64 *val, size_t nval) { return fwnode_property_read_u64_array(dev_fwnode(dev), propname, val, nval); @@ -202,7 +202,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u64_array); * %-EOVERFLOW if the size of the property is not as expected. * %-ENXIO if no suitable firmware interface is present. */ -int device_property_read_string_array(struct device *dev, const char *propname, +int device_property_read_string_array(const struct device *dev, const char *propname, const char **val, size_t nval) { return fwnode_property_read_string_array(dev_fwnode(dev), propname, val, nval); @@ -224,7 +224,7 @@ EXPORT_SYMBOL_GPL(device_property_read_string_array); * %-EPROTO or %-EILSEQ if the property type is not a string. * %-ENXIO if no suitable firmware interface is present. */ -int device_property_read_string(struct device *dev, const char *propname, +int device_property_read_string(const struct device *dev, const char *propname, const char **val) { return fwnode_property_read_string(dev_fwnode(dev), propname, val); @@ -246,7 +246,7 @@ EXPORT_SYMBOL_GPL(device_property_read_string); * %-EPROTO if the property is not an array of strings, * %-ENXIO if no suitable firmware interface is present. */ -int device_property_match_string(struct device *dev, const char *propname, +int device_property_match_string(const struct device *dev, const char *propname, const char *string) { return fwnode_property_match_string(dev_fwnode(dev), propname, string); diff --git a/include/linux/property.h b/include/linux/property.h index 59f452198c64..66df1a15d518 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -39,20 +39,20 @@ struct fwnode_handle *__dev_fwnode(struct device *dev); const struct device *: __dev_fwnode_const, \ struct device *: __dev_fwnode)(dev) -bool device_property_present(struct device *dev, const char *propname); -int device_property_read_u8_array(struct device *dev, const char *propname, +bool device_property_present(const struct device *dev, const char *propname); +int device_property_read_u8_array(const struct device *dev, const char *propname, u8 *val, size_t nval); -int device_property_read_u16_array(struct device *dev, const char *propname, +int device_property_read_u16_array(const struct device *dev, const char *propname, u16 *val, size_t nval); -int device_property_read_u32_array(struct device *dev, const char *propname, +int device_property_read_u32_array(const struct device *dev, const char *propname, u32 *val, size_t nval); -int device_property_read_u64_array(struct device *dev, const char *propname, +int device_property_read_u64_array(const struct device *dev, const char *propname, u64 *val, size_t nval); -int device_property_read_string_array(struct device *dev, const char *propname, +int device_property_read_string_array(const struct device *dev, const char *propname, const char **val, size_t nval); -int device_property_read_string(struct device *dev, const char *propname, +int device_property_read_string(const struct device *dev, const char *propname, const char **val); -int device_property_match_string(struct device *dev, +int device_property_match_string(const struct device *dev, const char *propname, const char *string); bool fwnode_property_present(const struct fwnode_handle *fwnode, @@ -142,57 +142,57 @@ int fwnode_irq_get_byname(const struct fwnode_handle *fwnode, const char *name); unsigned int device_get_child_node_count(const struct device *dev); -static inline bool device_property_read_bool(struct device *dev, +static inline bool device_property_read_bool(const struct device *dev, const char *propname) { return device_property_present(dev, propname); } -static inline int device_property_read_u8(struct device *dev, +static inline int device_property_read_u8(const struct device *dev, const char *propname, u8 *val) { return device_property_read_u8_array(dev, propname, val, 1); } -static inline int device_property_read_u16(struct device *dev, +static inline int device_property_read_u16(const struct device *dev, const char *propname, u16 *val) { return device_property_read_u16_array(dev, propname, val, 1); } -static inline int device_property_read_u32(struct device *dev, +static inline int device_property_read_u32(const struct device *dev, const char *propname, u32 *val) { return device_property_read_u32_array(dev, propname, val, 1); } -static inline int device_property_read_u64(struct device *dev, +static inline int device_property_read_u64(const struct device *dev, const char *propname, u64 *val) { return device_property_read_u64_array(dev, propname, val, 1); } -static inline int device_property_count_u8(struct device *dev, const char *propname) +static inline int device_property_count_u8(const struct device *dev, const char *propname) { return device_property_read_u8_array(dev, propname, NULL, 0); } -static inline int device_property_count_u16(struct device *dev, const char *propname) +static inline int device_property_count_u16(const struct device *dev, const char *propname) { return device_property_read_u16_array(dev, propname, NULL, 0); } -static inline int device_property_count_u32(struct device *dev, const char *propname) +static inline int device_property_count_u32(const struct device *dev, const char *propname) { return device_property_read_u32_array(dev, propname, NULL, 0); } -static inline int device_property_count_u64(struct device *dev, const char *propname) +static inline int device_property_count_u64(const struct device *dev, const char *propname) { return device_property_read_u64_array(dev, propname, NULL, 0); } -static inline int device_property_string_array_count(struct device *dev, +static inline int device_property_string_array_count(const struct device *dev, const char *propname) { return device_property_read_string_array(dev, propname, NULL, 0); -- cgit v1.2.3 From ec274aff21b6a94c7973384ca80a503c1bc3b173 Mon Sep 17 00:00:00 2001 From: Petr Tesarik Date: Thu, 20 Apr 2023 11:58:58 +0200 Subject: swiotlb: Omit total_used and used_hiwater if !CONFIG_DEBUG_FS The tracking of used_hiwater adds an atomic operation to the hot path. This is acceptable only when debugging the kernel. To make sure that the fields can never be used by mistake, do not even include them in struct io_tlb_mem if CONFIG_DEBUG_FS is not set. The build fails after doing that. To fix it, it is necessary to remove all code specific to debugfs and instead provide a stub implementation of swiotlb_create_debugfs_files(). As a bonus, this change allows to remove one __maybe_unused attribute. Signed-off-by: Petr Tesarik Signed-off-by: Christoph Hellwig --- include/linux/swiotlb.h | 2 ++ kernel/dma/swiotlb.c | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 6dc4598d2260..44767844e12b 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -107,8 +107,10 @@ struct io_tlb_mem { unsigned int area_nslabs; struct io_tlb_area *areas; struct io_tlb_slot *slots; +#ifdef CONFIG_DEBUG_FS atomic_long_t total_used; atomic_long_t used_hiwater; +#endif }; extern struct io_tlb_mem io_tlb_default_mem; diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 9bbc2802a444..e67dafc255f2 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -966,6 +966,8 @@ bool is_swiotlb_active(struct device *dev) } EXPORT_SYMBOL_GPL(is_swiotlb_active); +#ifdef CONFIG_DEBUG_FS + static int io_tlb_used_get(void *data, u64 *val) { struct io_tlb_mem *mem = data; @@ -1015,15 +1017,22 @@ static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, &fops_io_tlb_hiwater); } -static int __init __maybe_unused swiotlb_create_default_debugfs(void) +static int __init swiotlb_create_default_debugfs(void) { swiotlb_create_debugfs_files(&io_tlb_default_mem, "swiotlb"); return 0; } -#ifdef CONFIG_DEBUG_FS late_initcall(swiotlb_create_default_debugfs); -#endif + +#else /* !CONFIG_DEBUG_FS */ + +static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, + const char *dirname) +{ +} + +#endif /* CONFIG_DEBUG_FS */ #ifdef CONFIG_DMA_RESTRICTED_POOL -- cgit v1.2.3 From 038585573d0544bc717cdc5b3be4e95206d3ee3d Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 18 Apr 2023 15:49:47 +0200 Subject: efi/pe: Import new BTI/IBT header flags from the spec The latest version of your favorite fork of the PE/COFF spec includes a new type of header flag that is intended to be used in the context of EFI firmware to indicate to the image loader that the executable regions of an image can be mapped with BTI/IBT enforcement enabled. So let's import these definitions so we can use them in subsequent patches. Signed-off-by: Ard Biesheuvel --- include/linux/pe.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pe.h b/include/linux/pe.h index 6ffabf1e6d03..5e1e11540870 100644 --- a/include/linux/pe.h +++ b/include/linux/pe.h @@ -118,6 +118,9 @@ #define IMAGE_DLLCHARACTERISTICS_WDM_DRIVER 0x2000 #define IMAGE_DLLCHARACTERISTICS_TERMINAL_SERVER_AWARE 0x8000 +#define IMAGE_DLLCHARACTERISTICS_EX_CET_COMPAT 0x0001 +#define IMAGE_DLLCHARACTERISTICS_EX_FORWARD_CFI_COMPAT 0x0040 + /* they actually defined 0x00000000 as well, but I think we'll skip that one. */ #define IMAGE_SCN_RESERVED_0 0x00000001 #define IMAGE_SCN_RESERVED_1 0x00000002 @@ -165,6 +168,7 @@ #define IMAGE_SCN_MEM_WRITE 0x80000000 /* writeable */ #define IMAGE_DEBUG_TYPE_CODEVIEW 2 +#define IMAGE_DEBUG_TYPE_EX_DLLCHARACTERISTICS 20 #ifndef __ASSEMBLY__ -- cgit v1.2.3 From 211db0ac9e3dc6c46f2dd53395b34d76af929faf Mon Sep 17 00:00:00 2001 From: Namjae Jeon Date: Thu, 16 Mar 2023 07:34:33 +0900 Subject: ksmbd: remove internal.h include Since vfs_path_lookup is exported, It should not be internal. Move vfs_path_lookup prototype in internal.h to linux/namei.h. Suggested-by: Al Viro Reviewed-by: Christian Brauner Signed-off-by: Namjae Jeon Signed-off-by: Al Viro --- fs/internal.h | 2 -- fs/ksmbd/vfs.c | 2 -- include/linux/namei.h | 2 ++ 3 files changed, 2 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/fs/internal.h b/fs/internal.h index dc4eb91a577a..071a7517f1a7 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -59,8 +59,6 @@ extern int finish_clean_context(struct fs_context *fc); */ extern int filename_lookup(int dfd, struct filename *name, unsigned flags, struct path *path, struct path *root); -extern int vfs_path_lookup(struct dentry *, struct vfsmount *, - const char *, unsigned int, struct path *); int do_rmdir(int dfd, struct filename *name); int do_unlinkat(int dfd, struct filename *name); int may_linkat(struct mnt_idmap *idmap, const struct path *link); diff --git a/fs/ksmbd/vfs.c b/fs/ksmbd/vfs.c index 5ea9229dad2c..cef07d7fb7dc 100644 --- a/fs/ksmbd/vfs.c +++ b/fs/ksmbd/vfs.c @@ -19,8 +19,6 @@ #include #include -#include "../internal.h" /* for vfs_path_lookup */ - #include "glob.h" #include "oplock.h" #include "connection.h" diff --git a/include/linux/namei.h b/include/linux/namei.h index 0d797f3367ca..ba9b32b4d1b0 100644 --- a/include/linux/namei.h +++ b/include/linux/namei.h @@ -63,6 +63,8 @@ extern struct dentry *kern_path_create(int, const char *, struct path *, unsigne extern struct dentry *user_path_create(int, const char __user *, struct path *, unsigned int); extern void done_path_create(struct path *, struct dentry *); extern struct dentry *kern_path_locked(const char *, struct path *); +int vfs_path_lookup(struct dentry *, struct vfsmount *, const char *, + unsigned int, struct path *); extern struct dentry *try_lookup_one_len(const char *, struct dentry *, int); extern struct dentry *lookup_one_len(const char *, struct dentry *, int); -- cgit v1.2.3 From 9bc37e04823b5280dd0f22b6680fc23fe81ca325 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Thu, 16 Mar 2023 07:34:34 +0900 Subject: fs: introduce lock_rename_child() helper Pass the dentry of a source file and the dentry of a destination directory to lock parent inodes for rename. As soon as this function returns, ->d_parent of the source file dentry is stable and inodes are properly locked for calling vfs-rename. This helper is needed for ksmbd server. rename request of SMB protocol has to rename an opened file, no matter which directory it's in. Signed-off-by: Al Viro Signed-off-by: Namjae Jeon Signed-off-by: Al Viro --- fs/namei.c | 68 ++++++++++++++++++++++++++++++++++++++++++--------- include/linux/namei.h | 1 + 2 files changed, 58 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/fs/namei.c b/fs/namei.c index edfedfbccaef..6bc1964e2214 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -2980,20 +2980,10 @@ static inline int may_create(struct mnt_idmap *idmap, return inode_permission(idmap, dir, MAY_WRITE | MAY_EXEC); } -/* - * p1 and p2 should be directories on the same fs. - */ -struct dentry *lock_rename(struct dentry *p1, struct dentry *p2) +static struct dentry *lock_two_directories(struct dentry *p1, struct dentry *p2) { struct dentry *p; - if (p1 == p2) { - inode_lock_nested(p1->d_inode, I_MUTEX_PARENT); - return NULL; - } - - mutex_lock(&p1->d_sb->s_vfs_rename_mutex); - p = d_ancestor(p2, p1); if (p) { inode_lock_nested(p2->d_inode, I_MUTEX_PARENT); @@ -3012,8 +3002,64 @@ struct dentry *lock_rename(struct dentry *p1, struct dentry *p2) inode_lock_nested(p2->d_inode, I_MUTEX_PARENT2); return NULL; } + +/* + * p1 and p2 should be directories on the same fs. + */ +struct dentry *lock_rename(struct dentry *p1, struct dentry *p2) +{ + if (p1 == p2) { + inode_lock_nested(p1->d_inode, I_MUTEX_PARENT); + return NULL; + } + + mutex_lock(&p1->d_sb->s_vfs_rename_mutex); + return lock_two_directories(p1, p2); +} EXPORT_SYMBOL(lock_rename); +/* + * c1 and p2 should be on the same fs. + */ +struct dentry *lock_rename_child(struct dentry *c1, struct dentry *p2) +{ + if (READ_ONCE(c1->d_parent) == p2) { + /* + * hopefully won't need to touch ->s_vfs_rename_mutex at all. + */ + inode_lock_nested(p2->d_inode, I_MUTEX_PARENT); + /* + * now that p2 is locked, nobody can move in or out of it, + * so the test below is safe. + */ + if (likely(c1->d_parent == p2)) + return NULL; + + /* + * c1 got moved out of p2 while we'd been taking locks; + * unlock and fall back to slow case. + */ + inode_unlock(p2->d_inode); + } + + mutex_lock(&c1->d_sb->s_vfs_rename_mutex); + /* + * nobody can move out of any directories on this fs. + */ + if (likely(c1->d_parent != p2)) + return lock_two_directories(c1->d_parent, p2); + + /* + * c1 got moved into p2 while we were taking locks; + * we need p2 locked and ->s_vfs_rename_mutex unlocked, + * for consistency with lock_rename(). + */ + inode_lock_nested(p2->d_inode, I_MUTEX_PARENT); + mutex_unlock(&c1->d_sb->s_vfs_rename_mutex); + return NULL; +} +EXPORT_SYMBOL(lock_rename_child); + void unlock_rename(struct dentry *p1, struct dentry *p2) { inode_unlock(p1->d_inode); diff --git a/include/linux/namei.h b/include/linux/namei.h index ba9b32b4d1b0..5864e4d82e56 100644 --- a/include/linux/namei.h +++ b/include/linux/namei.h @@ -83,6 +83,7 @@ extern int follow_down(struct path *path, unsigned int flags); extern int follow_up(struct path *); extern struct dentry *lock_rename(struct dentry *, struct dentry *); +extern struct dentry *lock_rename_child(struct dentry *, struct dentry *); extern void unlock_rename(struct dentry *, struct dentry *); extern int __must_check nd_jump_link(const struct path *path); -- cgit v1.2.3 From 48cd6bc5b22d68b8bbc8601f3c7ddeed99541a0b Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 18 Feb 2023 09:10:31 +0100 Subject: virtio: Reorder fields in 'struct virtqueue' Group some variables based on their sizes to reduce hole and avoid padding. On x86_64, this shrinks the size of 'struct virtqueue' from 72 to 68 bytes. It saves a few bytes of memory. Signed-off-by: Christophe JAILLET Message-Id: <8f3d2e49270a2158717e15008e7ed7228196ba02.1676707807.git.christophe.jaillet@wanadoo.fr> Signed-off-by: Michael S. Tsirkin Acked-by: Peter Lafreniere Reviewed-by: Stefano Garzarella --- include/linux/virtio.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 2b472514c49b..d6e161aa532e 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -34,8 +34,8 @@ struct virtqueue { unsigned int index; unsigned int num_free; unsigned int num_max; - void *priv; bool reset; + void *priv; }; int virtqueue_add_outbuf(struct virtqueue *vq, -- cgit v1.2.3 From 4b6ec919b84830949313c805c586fb34f141ce0c Mon Sep 17 00:00:00 2001 From: Feng Liu Date: Fri, 10 Mar 2023 07:34:28 +0200 Subject: virtio_ring: Use const to annotate read-only pointer params Add const to make the read-only pointer parameters clear, similar to many existing functions. To implement this change, the commit also introduces the use of `container_of_const` to implement `to_vvq`, which ensures the const-ness of read-only parameters and avoids accidental modification of their members. Signed-off-by: Feng Liu Reviewed-by: Jiri Pirko Reviewed-by: Parav Pandit Reviewed-by: Gavin Li Reviewed-by: Bodong Wang Message-Id: <20230310053428.3376-4-feliu@nvidia.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 36 ++++++++++++++++++------------------ include/linux/virtio.h | 14 +++++++------- 2 files changed, 25 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index a26fab91c59f..4c3bb0ddeb9b 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -231,9 +231,9 @@ static void vring_free(struct virtqueue *_vq); * Helpers. */ -#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) +#define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq) -static bool virtqueue_use_indirect(struct vring_virtqueue *vq, +static bool virtqueue_use_indirect(const struct vring_virtqueue *vq, unsigned int total_sg) { /* @@ -269,7 +269,7 @@ static bool virtqueue_use_indirect(struct vring_virtqueue *vq, * unconditionally on data path. */ -static bool vring_use_dma_api(struct virtio_device *vdev) +static bool vring_use_dma_api(const struct virtio_device *vdev) { if (!virtio_has_dma_quirk(vdev)) return true; @@ -289,7 +289,7 @@ static bool vring_use_dma_api(struct virtio_device *vdev) return false; } -size_t virtio_max_dma_size(struct virtio_device *vdev) +size_t virtio_max_dma_size(const struct virtio_device *vdev) { size_t max_segment_size = SIZE_MAX; @@ -423,7 +423,7 @@ static void virtqueue_init(struct vring_virtqueue *vq, u32 num) */ static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq, - struct vring_desc *desc) + const struct vring_desc *desc) { u16 flags; @@ -1183,7 +1183,7 @@ static u16 packed_last_used(u16 last_used_idx) } static void vring_unmap_extra_packed(const struct vring_virtqueue *vq, - struct vring_desc_extra *extra) + const struct vring_desc_extra *extra) { u16 flags; @@ -1206,7 +1206,7 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq, } static void vring_unmap_desc_packed(const struct vring_virtqueue *vq, - struct vring_packed_desc *desc) + const struct vring_packed_desc *desc) { u16 flags; @@ -2786,10 +2786,10 @@ EXPORT_SYMBOL_GPL(vring_transport_features); * Returns the size of the vring. This is mainly used for boasting to * userspace. Unlike other operations, this need not be serialized. */ -unsigned int virtqueue_get_vring_size(struct virtqueue *_vq) +unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq) { - struct vring_virtqueue *vq = to_vvq(_vq); + const struct vring_virtqueue *vq = to_vvq(_vq); return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; } @@ -2819,9 +2819,9 @@ void __virtqueue_unbreak(struct virtqueue *_vq) } EXPORT_SYMBOL_GPL(__virtqueue_unbreak); -bool virtqueue_is_broken(struct virtqueue *_vq) +bool virtqueue_is_broken(const struct virtqueue *_vq) { - struct vring_virtqueue *vq = to_vvq(_vq); + const struct vring_virtqueue *vq = to_vvq(_vq); return READ_ONCE(vq->broken); } @@ -2868,9 +2868,9 @@ void __virtio_unbreak_device(struct virtio_device *dev) } EXPORT_SYMBOL_GPL(__virtio_unbreak_device); -dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq) +dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq) { - struct vring_virtqueue *vq = to_vvq(_vq); + const struct vring_virtqueue *vq = to_vvq(_vq); BUG_ON(!vq->we_own_ring); @@ -2881,9 +2881,9 @@ dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq) } EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr); -dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq) +dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq) { - struct vring_virtqueue *vq = to_vvq(_vq); + const struct vring_virtqueue *vq = to_vvq(_vq); BUG_ON(!vq->we_own_ring); @@ -2895,9 +2895,9 @@ dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq) } EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr); -dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq) +dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq) { - struct vring_virtqueue *vq = to_vvq(_vq); + const struct vring_virtqueue *vq = to_vvq(_vq); BUG_ON(!vq->we_own_ring); @@ -2910,7 +2910,7 @@ dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq) EXPORT_SYMBOL_GPL(virtqueue_get_used_addr); /* Only available for split ring */ -const struct vring *virtqueue_get_vring(struct virtqueue *vq) +const struct vring *virtqueue_get_vring(const struct virtqueue *vq) { return &to_vvq(vq)->split.vring; } diff --git a/include/linux/virtio.h b/include/linux/virtio.h index d6e161aa532e..b93238db94e3 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -84,14 +84,14 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *vq); void *virtqueue_detach_unused_buf(struct virtqueue *vq); -unsigned int virtqueue_get_vring_size(struct virtqueue *vq); +unsigned int virtqueue_get_vring_size(const struct virtqueue *vq); -bool virtqueue_is_broken(struct virtqueue *vq); +bool virtqueue_is_broken(const struct virtqueue *vq); -const struct vring *virtqueue_get_vring(struct virtqueue *vq); -dma_addr_t virtqueue_get_desc_addr(struct virtqueue *vq); -dma_addr_t virtqueue_get_avail_addr(struct virtqueue *vq); -dma_addr_t virtqueue_get_used_addr(struct virtqueue *vq); +const struct vring *virtqueue_get_vring(const struct virtqueue *vq); +dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *vq); +dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *vq); +dma_addr_t virtqueue_get_used_addr(const struct virtqueue *vq); int virtqueue_resize(struct virtqueue *vq, u32 num, void (*recycle)(struct virtqueue *vq, void *buf)); @@ -147,7 +147,7 @@ int virtio_device_restore(struct virtio_device *dev); #endif void virtio_reset_device(struct virtio_device *dev); -size_t virtio_max_dma_size(struct virtio_device *vdev); +size_t virtio_max_dma_size(const struct virtio_device *vdev); #define virtio_device_for_each_vq(vdev, vq) \ list_for_each_entry(vq, &vdev->vqs, list) -- cgit v1.2.3 From 1d24692732fb299c94b0dcc032b48ac8fa85c854 Mon Sep 17 00:00:00 2001 From: Xie Yongji Date: Thu, 23 Mar 2023 13:30:34 +0800 Subject: vdpa: Add set/get_vq_affinity callbacks in vdpa_config_ops This introduces set/get_vq_affinity callbacks in vdpa_config_ops to support virtqueue affinity management for vdpa device drivers. Signed-off-by: Xie Yongji Acked-by: Jason Wang Message-Id: <20230323053043.35-3-xieyongji@bytedance.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_vdpa.c | 28 ++++++++++++++++++++++++++++ include/linux/vdpa.h | 13 +++++++++++++ 2 files changed, 41 insertions(+) (limited to 'include/linux') diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index d7f5af62ddaa..f72696b4c1c2 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -337,6 +337,32 @@ static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) return dev_name(&vdpa->dev); } +static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq, + const struct cpumask *cpu_mask) +{ + struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); + struct vdpa_device *vdpa = vd_dev->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + unsigned int index = vq->index; + + if (ops->set_vq_affinity) + return ops->set_vq_affinity(vdpa, index, cpu_mask); + + return 0; +} + +static const struct cpumask * +virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index) +{ + struct vdpa_device *vdpa = vd_get_vdpa(vdev); + const struct vdpa_config_ops *ops = vdpa->config; + + if (ops->get_vq_affinity) + return ops->get_vq_affinity(vdpa, index); + + return NULL; +} + static const struct virtio_config_ops virtio_vdpa_config_ops = { .get = virtio_vdpa_get, .set = virtio_vdpa_set, @@ -349,6 +375,8 @@ static const struct virtio_config_ops virtio_vdpa_config_ops = { .get_features = virtio_vdpa_get_features, .finalize_features = virtio_vdpa_finalize_features, .bus_name = virtio_vdpa_bus_name, + .set_vq_affinity = virtio_vdpa_set_vq_affinity, + .get_vq_affinity = virtio_vdpa_get_vq_affinity, }; static void virtio_vdpa_release_dev(struct device *_d) diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 43f59ef10cc9..2313db735773 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -250,6 +250,15 @@ struct vdpa_map_file { * @vdev: vdpa device * Returns the iova range supported by * the device. + * @set_vq_affinity: Set the affinity of virtqueue (optional) + * @vdev: vdpa device + * @idx: virtqueue index + * @cpu_mask: the affinity mask + * Returns integer: success (0) or error (< 0) + * @get_vq_affinity: Get the affinity of virtqueue (optional) + * @vdev: vdpa device + * @idx: virtqueue index + * Returns the affinity mask * @set_group_asid: Set address space identifier for a * virtqueue group (optional) * @vdev: vdpa device @@ -340,6 +349,10 @@ struct vdpa_config_ops { const void *buf, unsigned int len); u32 (*get_generation)(struct vdpa_device *vdev); struct vdpa_iova_range (*get_iova_range)(struct vdpa_device *vdev); + int (*set_vq_affinity)(struct vdpa_device *vdev, u16 idx, + const struct cpumask *cpu_mask); + const struct cpumask *(*get_vq_affinity)(struct vdpa_device *vdev, + u16 idx); /* DMA ops */ int (*set_map)(struct vdpa_device *vdev, unsigned int asid, -- cgit v1.2.3 From 5e68470f4e80a4120e9ecec408f6ab4ad386bd4a Mon Sep 17 00:00:00 2001 From: Xie Yongji Date: Thu, 23 Mar 2023 13:30:40 +0800 Subject: vdpa: Add eventfd for the vdpa callback Add eventfd for the vdpa callback so that user can signal it directly instead of triggering the callback. It will be used for vhost-vdpa case. Signed-off-by: Xie Yongji Message-Id: <20230323053043.35-9-xieyongji@bytedance.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vhost/vdpa.c | 2 ++ drivers/virtio/virtio_vdpa.c | 1 + include/linux/vdpa.h | 6 ++++++ 3 files changed, 9 insertions(+) (limited to 'include/linux') diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 7be9d9d8f01c..9cd878e25cff 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -599,9 +599,11 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, if (vq->call_ctx.ctx) { cb.callback = vhost_vdpa_virtqueue_cb; cb.private = vq; + cb.trigger = vq->call_ctx.ctx; } else { cb.callback = NULL; cb.private = NULL; + cb.trigger = NULL; } ops->set_vq_cb(vdpa, idx, &cb); vhost_vdpa_setup_vq_irq(v, idx); diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index f3826f42b704..2a095f37f26b 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -196,6 +196,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, /* Setup virtqueue callback */ cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL; cb.private = info; + cb.trigger = NULL; ops->set_vq_cb(vdpa, index, &cb); ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq)); diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 2313db735773..f2064fa2d2da 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -13,10 +13,16 @@ * struct vdpa_calllback - vDPA callback definition. * @callback: interrupt callback function * @private: the data passed to the callback function + * @trigger: the eventfd for the callback (Optional). + * When it is set, the vDPA driver must guarantee that + * signaling it is functional equivalent to triggering + * the callback. Then vDPA parent can signal it directly + * instead of triggering the callback. */ struct vdpa_callback { irqreturn_t (*callback)(void *data); void *private; + struct eventfd_ctx *trigger; }; /** -- cgit v1.2.3 From c618c84d4ccc6268c3da7609c7388b6cb305c639 Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Tue, 4 Apr 2023 15:13:18 +0200 Subject: vdpa: add bind_mm/unbind_mm callbacks These new optional callbacks is used to bind/unbind the device to a specific address space so the vDPA framework can use VA when these callbacks are implemented. Suggested-by: Jason Wang Acked-by: Jason Wang Signed-off-by: Stefano Garzarella Message-Id: <20230404131326.44403-2-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin --- include/linux/vdpa.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index f2064fa2d2da..2f1af73b3a1a 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -305,6 +305,14 @@ struct vdpa_map_file { * @vdev: vdpa device * @idx: virtqueue index * Returns pointer to structure device or error (NULL) + * @bind_mm: Bind the device to a specific address space + * so the vDPA framework can use VA when this + * callback is implemented. (optional) + * @vdev: vdpa device + * @mm: address space to bind + * @unbind_mm: Unbind the device from the address space + * bound using the bind_mm callback. (optional) + * @vdev: vdpa device * @free: Free resources that belongs to vDPA (optional) * @vdev: vdpa device */ @@ -370,6 +378,8 @@ struct vdpa_config_ops { int (*set_group_asid)(struct vdpa_device *vdev, unsigned int group, unsigned int asid); struct device *(*get_vq_dma_dev)(struct vdpa_device *vdev, u16 idx); + int (*bind_mm)(struct vdpa_device *vdev, struct mm_struct *mm); + void (*unbind_mm)(struct vdpa_device *vdev); /* Free device resources */ void (*free)(struct vdpa_device *vdev); -- cgit v1.2.3 From 42823a871fd4e17b34034f43b36ae57bd2ed8a67 Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Tue, 4 Apr 2023 15:17:16 +0200 Subject: vringh: support VA with iotlb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vDPA supports the possibility to use user VA in the iotlb messages. So, let's add support for user VA in vringh to use it in the vDPA simulators. Acked-by: Eugenio Pérez Signed-off-by: Stefano Garzarella Message-Id: <20230404131716.45855-1-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vringh.c | 171 ++++++++++++++++++++++++++++++++++++++++--------- include/linux/vringh.h | 9 +++ 2 files changed, 148 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c index 3d5f6bb4ac31..955d938eb663 100644 --- a/drivers/vhost/vringh.c +++ b/drivers/vhost/vringh.c @@ -1094,10 +1094,17 @@ EXPORT_SYMBOL(vringh_need_notify_kern); #if IS_REACHABLE(CONFIG_VHOST_IOTLB) +struct iotlb_vec { + union { + struct iovec *iovec; + struct bio_vec *bvec; + } iov; + size_t count; +}; + static int iotlb_translate(const struct vringh *vrh, u64 addr, u64 len, u64 *translated, - struct bio_vec iov[], - int iov_size, u32 perm) + struct iotlb_vec *ivec, u32 perm) { struct vhost_iotlb_map *map; struct vhost_iotlb *iotlb = vrh->iotlb; @@ -1107,9 +1114,11 @@ static int iotlb_translate(const struct vringh *vrh, spin_lock(vrh->iotlb_lock); while (len > s) { - u64 size, pa, pfn; + uintptr_t io_addr; + size_t io_len; + u64 size; - if (unlikely(ret >= iov_size)) { + if (unlikely(ret >= ivec->count)) { ret = -ENOBUFS; break; } @@ -1124,10 +1133,22 @@ static int iotlb_translate(const struct vringh *vrh, } size = map->size - addr + map->start; - pa = map->addr + addr - map->start; - pfn = pa >> PAGE_SHIFT; - bvec_set_page(&iov[ret], pfn_to_page(pfn), min(len - s, size), - pa & (PAGE_SIZE - 1)); + io_len = min(len - s, size); + io_addr = map->addr - map->start + addr; + + if (vrh->use_va) { + struct iovec *iovec = ivec->iov.iovec; + + iovec[ret].iov_len = io_len; + iovec[ret].iov_base = (void __user *)io_addr; + } else { + u64 pfn = io_addr >> PAGE_SHIFT; + struct bio_vec *bvec = ivec->iov.bvec; + + bvec_set_page(&bvec[ret], pfn_to_page(pfn), io_len, + io_addr & (PAGE_SIZE - 1)); + } + s += size; addr += size; ++ret; @@ -1146,23 +1167,36 @@ static int iotlb_translate(const struct vringh *vrh, static inline int copy_from_iotlb(const struct vringh *vrh, void *dst, void *src, size_t len) { + struct iotlb_vec ivec; + union { + struct iovec iovec[IOTLB_IOV_STRIDE]; + struct bio_vec bvec[IOTLB_IOV_STRIDE]; + } iov; u64 total_translated = 0; + ivec.iov.iovec = iov.iovec; + ivec.count = IOTLB_IOV_STRIDE; + while (total_translated < len) { - struct bio_vec iov[IOTLB_IOV_STRIDE]; struct iov_iter iter; u64 translated; int ret; ret = iotlb_translate(vrh, (u64)(uintptr_t)src, len - total_translated, &translated, - iov, ARRAY_SIZE(iov), VHOST_MAP_RO); + &ivec, VHOST_MAP_RO); if (ret == -ENOBUFS) - ret = ARRAY_SIZE(iov); + ret = IOTLB_IOV_STRIDE; else if (ret < 0) return ret; - iov_iter_bvec(&iter, ITER_SOURCE, iov, ret, translated); + if (vrh->use_va) { + iov_iter_init(&iter, ITER_SOURCE, ivec.iov.iovec, ret, + translated); + } else { + iov_iter_bvec(&iter, ITER_SOURCE, ivec.iov.bvec, ret, + translated); + } ret = copy_from_iter(dst, translated, &iter); if (ret < 0) @@ -1179,23 +1213,36 @@ static inline int copy_from_iotlb(const struct vringh *vrh, void *dst, static inline int copy_to_iotlb(const struct vringh *vrh, void *dst, void *src, size_t len) { + struct iotlb_vec ivec; + union { + struct iovec iovec[IOTLB_IOV_STRIDE]; + struct bio_vec bvec[IOTLB_IOV_STRIDE]; + } iov; u64 total_translated = 0; + ivec.iov.iovec = iov.iovec; + ivec.count = IOTLB_IOV_STRIDE; + while (total_translated < len) { - struct bio_vec iov[IOTLB_IOV_STRIDE]; struct iov_iter iter; u64 translated; int ret; ret = iotlb_translate(vrh, (u64)(uintptr_t)dst, len - total_translated, &translated, - iov, ARRAY_SIZE(iov), VHOST_MAP_WO); + &ivec, VHOST_MAP_WO); if (ret == -ENOBUFS) - ret = ARRAY_SIZE(iov); + ret = IOTLB_IOV_STRIDE; else if (ret < 0) return ret; - iov_iter_bvec(&iter, ITER_DEST, iov, ret, translated); + if (vrh->use_va) { + iov_iter_init(&iter, ITER_DEST, ivec.iov.iovec, ret, + translated); + } else { + iov_iter_bvec(&iter, ITER_DEST, ivec.iov.bvec, ret, + translated); + } ret = copy_to_iter(src, translated, &iter); if (ret < 0) @@ -1212,20 +1259,36 @@ static inline int copy_to_iotlb(const struct vringh *vrh, void *dst, static inline int getu16_iotlb(const struct vringh *vrh, u16 *val, const __virtio16 *p) { - struct bio_vec iov; - void *kaddr, *from; + struct iotlb_vec ivec; + union { + struct iovec iovec[1]; + struct bio_vec bvec[1]; + } iov; + __virtio16 tmp; int ret; + ivec.iov.iovec = iov.iovec; + ivec.count = 1; + /* Atomic read is needed for getu16 */ - ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL, - &iov, 1, VHOST_MAP_RO); + ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), + NULL, &ivec, VHOST_MAP_RO); if (ret < 0) return ret; - kaddr = kmap_local_page(iov.bv_page); - from = kaddr + iov.bv_offset; - *val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from)); - kunmap_local(kaddr); + if (vrh->use_va) { + ret = __get_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base); + if (ret) + return ret; + } else { + void *kaddr = kmap_local_page(ivec.iov.bvec[0].bv_page); + void *from = kaddr + ivec.iov.bvec[0].bv_offset; + + tmp = READ_ONCE(*(__virtio16 *)from); + kunmap_local(kaddr); + } + + *val = vringh16_to_cpu(vrh, tmp); return 0; } @@ -1233,20 +1296,36 @@ static inline int getu16_iotlb(const struct vringh *vrh, static inline int putu16_iotlb(const struct vringh *vrh, __virtio16 *p, u16 val) { - struct bio_vec iov; - void *kaddr, *to; + struct iotlb_vec ivec; + union { + struct iovec iovec; + struct bio_vec bvec; + } iov; + __virtio16 tmp; int ret; + ivec.iov.iovec = &iov.iovec; + ivec.count = 1; + /* Atomic write is needed for putu16 */ - ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL, - &iov, 1, VHOST_MAP_WO); + ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), + NULL, &ivec, VHOST_MAP_RO); if (ret < 0) return ret; - kaddr = kmap_local_page(iov.bv_page); - to = kaddr + iov.bv_offset; - WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val)); - kunmap_local(kaddr); + tmp = cpu_to_vringh16(vrh, val); + + if (vrh->use_va) { + ret = __put_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base); + if (ret) + return ret; + } else { + void *kaddr = kmap_local_page(ivec.iov.bvec[0].bv_page); + void *to = kaddr + ivec.iov.bvec[0].bv_offset; + + WRITE_ONCE(*(__virtio16 *)to, tmp); + kunmap_local(kaddr); + } return 0; } @@ -1320,11 +1399,39 @@ int vringh_init_iotlb(struct vringh *vrh, u64 features, struct vring_avail *avail, struct vring_used *used) { + vrh->use_va = false; + return vringh_init_kern(vrh, features, num, weak_barriers, desc, avail, used); } EXPORT_SYMBOL(vringh_init_iotlb); +/** + * vringh_init_iotlb_va - initialize a vringh for a ring with IOTLB containing + * user VA. + * @vrh: the vringh to initialize. + * @features: the feature bits for this ring. + * @num: the number of elements. + * @weak_barriers: true if we only need memory barriers, not I/O. + * @desc: the userspace descriptor pointer. + * @avail: the userspace avail pointer. + * @used: the userspace used pointer. + * + * Returns an error if num is invalid. + */ +int vringh_init_iotlb_va(struct vringh *vrh, u64 features, + unsigned int num, bool weak_barriers, + struct vring_desc *desc, + struct vring_avail *avail, + struct vring_used *used) +{ + vrh->use_va = true; + + return vringh_init_kern(vrh, features, num, weak_barriers, + desc, avail, used); +} +EXPORT_SYMBOL(vringh_init_iotlb_va); + /** * vringh_set_iotlb - initialize a vringh for a ring with IOTLB. * @vrh: the vring diff --git a/include/linux/vringh.h b/include/linux/vringh.h index 1991a02c6431..b4edfadf5479 100644 --- a/include/linux/vringh.h +++ b/include/linux/vringh.h @@ -32,6 +32,9 @@ struct vringh { /* Can we get away with weak barriers? */ bool weak_barriers; + /* Use user's VA */ + bool use_va; + /* Last available index we saw (ie. where we're up to). */ u16 last_avail_idx; @@ -284,6 +287,12 @@ int vringh_init_iotlb(struct vringh *vrh, u64 features, struct vring_avail *avail, struct vring_used *used); +int vringh_init_iotlb_va(struct vringh *vrh, u64 features, + unsigned int num, bool weak_barriers, + struct vring_desc *desc, + struct vring_avail *avail, + struct vring_used *used); + int vringh_getdesc_iotlb(struct vringh *vrh, struct vringh_kiov *riov, struct vringh_kiov *wiov, -- cgit v1.2.3 From 9be5d2d424a19a0c6d643a1baecba3d58e725012 Mon Sep 17 00:00:00 2001 From: Simon Horman Date: Fri, 31 Mar 2023 10:56:55 +0200 Subject: vdpa: address kdoc warnings This patch addresses the following minor kdoc problems. * Incorrect spelling of 'callback' and 'notification' * Unrecognised kdoc format for 'struct vdpa_map_file' * Missing documentation of 'get_vendor_vq_stats' member of 'struct vdpa_config_ops' * Missing documentation of 'max_supported_vqs' and 'supported_features' members of 'struct vdpa_mgmt_dev' Most of these problems were flagged by: $ ./scripts/kernel-doc -Werror -none include/linux/vdpa.h include/linux/vdpa.h:20: warning: expecting prototype for struct vdpa_calllback. Prototype was for struct vdpa_callback instead include/linux/vdpa.h:117: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst * Corresponding file area for device memory mapping include/linux/vdpa.h:357: warning: Function parameter or member 'get_vendor_vq_stats' not described in 'vdpa_config_ops' include/linux/vdpa.h:518: warning: Function parameter or member 'supported_features' not described in 'vdpa_mgmt_dev' include/linux/vdpa.h:518: warning: Function parameter or member 'max_supported_vqs' not described in 'vdpa_mgmt_dev' The misspelling of 'notification' was flagged by: $ ./scripts/checkpatch.pl --codespell --showfile --strict -f include/linux/vdpa.h include/linux/vdpa.h:171: CHECK: 'notifcation' may be misspelled - perhaps 'notification'? ... Signed-off-by: Simon Horman Message-Id: <20230331-vhost-fixes-v1-1-1f046e735b9e@kernel.org> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- include/linux/vdpa.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 2f1af73b3a1a..c0d3f5433af7 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -10,7 +10,7 @@ #include /** - * struct vdpa_calllback - vDPA callback definition. + * struct vdpa_callback - vDPA callback definition. * @callback: interrupt callback function * @private: the data passed to the callback function * @trigger: the eventfd for the callback (Optional). @@ -120,7 +120,7 @@ struct vdpa_dev_set_config { }; /** - * Corresponding file area for device memory mapping + * struct vdpa_map_file - file area for device memory mapping * @file: vma->vm_file for the mapping * @offset: mapping offset in the vm_file */ @@ -171,10 +171,16 @@ struct vdpa_map_file { * @vdev: vdpa device * @idx: virtqueue index * @state: pointer to returned state (last_avail_idx) + * @get_vendor_vq_stats: Get the vendor statistics of a device. + * @vdev: vdpa device + * @idx: virtqueue index + * @msg: socket buffer holding stats message + * @extack: extack for reporting error messages + * Returns integer: success (0) or error (< 0) * @get_vq_notification: Get the notification area for a virtqueue (optional) * @vdev: vdpa device * @idx: virtqueue index - * Returns the notifcation area + * Returns the notification area * @get_vq_irq: Get the irq number of a virtqueue (optional, * but must implemented if require vq irq offloading) * @vdev: vdpa device @@ -535,6 +541,8 @@ struct vdpa_mgmtdev_ops { * @config_attr_mask: bit mask of attributes of type enum vdpa_attr that * management device support during dev_add callback * @list: list entry + * @supported_features: features supported by device + * @max_supported_vqs: maximum number of virtqueues supported by device */ struct vdpa_mgmt_dev { struct device *device; -- cgit v1.2.3 From b2ffaa672edaf1a681537a9f47bc14ab6c9e8845 Mon Sep 17 00:00:00 2001 From: Simon Horman Date: Fri, 31 Mar 2023 10:56:56 +0200 Subject: vringh: address kdoc warnings Address some minor kdoc warnings in vring.h. * Place kdoc for 'struct vringh_config_ops' immediately before the structure * Add missing documentation of members of 'vringh_iov' and 'vringh_kiov' Warnings flagged by: $ ./scripts/kernel-doc -none include/linux/vringh.h include/linux/vringh.h:68: error: Cannot parse struct or union! include/linux/vringh.h:92: warning: Function parameter or member 'iov' not described in 'vringh_iov' include/linux/vringh.h:92: warning: Function parameter or member 'consumed' not described in 'vringh_iov' include/linux/vringh.h:92: warning: Function parameter or member 'i' not described in 'vringh_iov' include/linux/vringh.h:92: warning: Function parameter or member 'used' not described in 'vringh_iov' include/linux/vringh.h:92: warning: Function parameter or member 'max_num' not described in 'vringh_iov' include/linux/vringh.h:104: warning: Function parameter or member 'iov' not described in 'vringh_kiov' include/linux/vringh.h:104: warning: Function parameter or member 'consumed' not described in 'vringh_kiov' include/linux/vringh.h:104: warning: Function parameter or member 'i' not described in 'vringh_kiov' include/linux/vringh.h:104: warning: Function parameter or member 'used' not described in 'vringh_kiov' include/linux/vringh.h:104: warning: Function parameter or member 'max_num' not described in 'vringh_kiov' Signed-off-by: Simon Horman Message-Id: <20230331-vhost-fixes-v1-2-1f046e735b9e@kernel.org> Signed-off-by: Michael S. Tsirkin --- include/linux/vringh.h | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/vringh.h b/include/linux/vringh.h index b4edfadf5479..c3a8117dabe8 100644 --- a/include/linux/vringh.h +++ b/include/linux/vringh.h @@ -57,6 +57,9 @@ struct vringh { void (*notify)(struct vringh *); }; +struct virtio_device; +typedef void vrh_callback_t(struct virtio_device *, struct vringh *); + /** * struct vringh_config_ops - ops for creating a host vring from a virtio driver * @find_vrhs: find the host vrings and instantiate them @@ -68,8 +71,6 @@ struct vringh { * Returns 0 on success or error status * @del_vrhs: free the host vrings found by find_vrhs(). */ -struct virtio_device; -typedef void vrh_callback_t(struct virtio_device *, struct vringh *); struct vringh_config_ops { int (*find_vrhs)(struct virtio_device *vdev, unsigned nhvrs, struct vringh *vrhs[], vrh_callback_t *callbacks[]); @@ -84,6 +85,12 @@ struct vringh_range { /** * struct vringh_iov - iovec mangler. + * @iov: array of iovecs to operate on + * @consumed: number of bytes consumed within iov[i] + * @i: index of current iovec + * @used: number of iovecs present in @iov + * @max_num: maximum number of iovecs. + * corresponds to allocated memory of @iov * * Mangles iovec in place, and restores it. * Remaining data is iov + i, of used - i elements. @@ -96,6 +103,12 @@ struct vringh_iov { /** * struct vringh_kiov - kvec mangler. + * @iov: array of iovecs to operate on + * @consumed: number of bytes consumed within iov[i] + * @i: index of current iovec + * @used: number of iovecs present in @iov + * @max_num: maximum number of iovecs. + * corresponds to allocated memory of @iov * * Mangles kvec in place, and restores it. * Remaining data is iov + i, of used - i elements. -- cgit v1.2.3 From af8ececda185078c096852edb4e1d7a2349e6856 Mon Sep 17 00:00:00 2001 From: Viktor Prutyanov Date: Thu, 13 Apr 2023 11:18:54 +0300 Subject: virtio: add VIRTIO_F_NOTIFICATION_DATA feature support According to VirtIO spec v1.2, VIRTIO_F_NOTIFICATION_DATA feature indicates that the driver passes extra data along with the queue notifications. In a split queue case, the extra data is 16-bit available index. In a packed queue case, the extra data is 1-bit wrap counter and 15-bit available index. Add support for this feature for MMIO, channel I/O and modern PCI transports. Signed-off-by: Viktor Prutyanov Acked-by: Jason Wang Reviewed-by: Xuan Zhuo Message-Id: <20230413081855.36643-2-alvaro.karsz@solid-run.com> Signed-off-by: Michael S. Tsirkin --- drivers/s390/virtio/virtio_ccw.c | 22 +++++++++++++++++++--- drivers/virtio/virtio_mmio.c | 18 +++++++++++++++++- drivers/virtio/virtio_pci_modern.c | 17 ++++++++++++++++- drivers/virtio/virtio_ring.c | 19 +++++++++++++++++++ include/linux/virtio_ring.h | 2 ++ include/uapi/linux/virtio_config.h | 6 ++++++ 6 files changed, 79 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 954fc31b4bc7..02922768b129 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -391,7 +391,7 @@ static void virtio_ccw_drop_indicator(struct virtio_ccw_device *vcdev, ccw_device_dma_free(vcdev->cdev, thinint_area, sizeof(*thinint_area)); } -static bool virtio_ccw_kvm_notify(struct virtqueue *vq) +static inline bool virtio_ccw_do_kvm_notify(struct virtqueue *vq, u32 data) { struct virtio_ccw_vq_info *info = vq->priv; struct virtio_ccw_device *vcdev; @@ -402,12 +402,22 @@ static bool virtio_ccw_kvm_notify(struct virtqueue *vq) BUILD_BUG_ON(sizeof(struct subchannel_id) != sizeof(unsigned int)); info->cookie = kvm_hypercall3(KVM_S390_VIRTIO_CCW_NOTIFY, *((unsigned int *)&schid), - vq->index, info->cookie); + data, info->cookie); if (info->cookie < 0) return false; return true; } +static bool virtio_ccw_kvm_notify(struct virtqueue *vq) +{ + return virtio_ccw_do_kvm_notify(vq, vq->index); +} + +static bool virtio_ccw_kvm_notify_with_data(struct virtqueue *vq) +{ + return virtio_ccw_do_kvm_notify(vq, vring_notification_data(vq)); +} + static int virtio_ccw_read_vq_conf(struct virtio_ccw_device *vcdev, struct ccw1 *ccw, int index) { @@ -495,6 +505,7 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev, struct ccw1 *ccw) { struct virtio_ccw_device *vcdev = to_vc_device(vdev); + bool (*notify)(struct virtqueue *vq); int err; struct virtqueue *vq = NULL; struct virtio_ccw_vq_info *info; @@ -502,6 +513,11 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev, unsigned long flags; bool may_reduce; + if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) + notify = virtio_ccw_kvm_notify_with_data; + else + notify = virtio_ccw_kvm_notify; + /* Allocate queue. */ info = kzalloc(sizeof(struct virtio_ccw_vq_info), GFP_KERNEL); if (!info) { @@ -524,7 +540,7 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev, may_reduce = vcdev->revision > 0; vq = vring_create_virtqueue(i, info->num, KVM_VIRTIO_CCW_RING_ALIGN, vdev, true, may_reduce, ctx, - virtio_ccw_kvm_notify, callback, name); + notify, callback, name); if (!vq) { /* For now, we fail if we can't get the requested size. */ diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 3ff746e3f24a..dd4424c14233 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -285,6 +285,16 @@ static bool vm_notify(struct virtqueue *vq) return true; } +static bool vm_notify_with_data(struct virtqueue *vq) +{ + struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev); + u32 data = vring_notification_data(vq); + + writel(data, vm_dev->base + VIRTIO_MMIO_QUEUE_NOTIFY); + + return true; +} + /* Notify all virtqueues on an interrupt. */ static irqreturn_t vm_interrupt(int irq, void *opaque) { @@ -363,12 +373,18 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in const char *name, bool ctx) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); + bool (*notify)(struct virtqueue *vq); struct virtio_mmio_vq_info *info; struct virtqueue *vq; unsigned long flags; unsigned int num; int err; + if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) + notify = vm_notify_with_data; + else + notify = vm_notify; + if (!name) return NULL; @@ -397,7 +413,7 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in /* Create the vring */ vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev, - true, true, ctx, vm_notify, callback, name); + true, true, ctx, notify, callback, name); if (!vq) { err = -ENOMEM; goto error_new_virtqueue; diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 6e713904d8e8..d6bb68ba84e5 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -288,6 +288,15 @@ static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) return vp_modern_config_vector(&vp_dev->mdev, vector); } +static bool vp_notify_with_data(struct virtqueue *vq) +{ + u32 data = vring_notification_data(vq); + + iowrite32(data, (void __iomem *)vq->priv); + + return true; +} + static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, struct virtio_pci_vq_info *info, unsigned int index, @@ -298,10 +307,16 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, { struct virtio_pci_modern_device *mdev = &vp_dev->mdev; + bool (*notify)(struct virtqueue *vq); struct virtqueue *vq; u16 num; int err; + if (__virtio_test_bit(&vp_dev->vdev, VIRTIO_F_NOTIFICATION_DATA)) + notify = vp_notify_with_data; + else + notify = vp_notify; + if (index >= vp_modern_get_num_queues(mdev)) return ERR_PTR(-EINVAL); @@ -316,7 +331,7 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, vq = vring_create_virtqueue(index, num, SMP_CACHE_BYTES, &vp_dev->vdev, true, true, ctx, - vp_notify, callback, name); + notify, callback, name); if (!vq) return ERR_PTR(-ENOMEM); diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 34e3849629da..c5310eaf8b46 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2762,6 +2762,23 @@ void vring_del_virtqueue(struct virtqueue *_vq) } EXPORT_SYMBOL_GPL(vring_del_virtqueue); +u32 vring_notification_data(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + u16 next; + + if (vq->packed_ring) + next = (vq->packed.next_avail_idx & + ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) | + vq->packed.avail_wrap_counter << + VRING_PACKED_EVENT_F_WRAP_CTR; + else + next = vq->split.avail_idx_shadow; + + return next << 16 | _vq->index; +} +EXPORT_SYMBOL_GPL(vring_notification_data); + /* Manipulates transport-specific feature bits. */ void vring_transport_features(struct virtio_device *vdev) { @@ -2781,6 +2798,8 @@ void vring_transport_features(struct virtio_device *vdev) break; case VIRTIO_F_ORDER_PLATFORM: break; + case VIRTIO_F_NOTIFICATION_DATA: + break; default: /* We don't understand this bit. */ __virtio_clear_bit(vdev, i); diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h index 8b95b69ef694..2550c9170f4f 100644 --- a/include/linux/virtio_ring.h +++ b/include/linux/virtio_ring.h @@ -117,4 +117,6 @@ void vring_del_virtqueue(struct virtqueue *vq); void vring_transport_features(struct virtio_device *vdev); irqreturn_t vring_interrupt(int irq, void *_vq); + +u32 vring_notification_data(struct virtqueue *_vq); #endif /* _LINUX_VIRTIO_RING_H */ diff --git a/include/uapi/linux/virtio_config.h b/include/uapi/linux/virtio_config.h index 3c05162bc988..2c712c654165 100644 --- a/include/uapi/linux/virtio_config.h +++ b/include/uapi/linux/virtio_config.h @@ -99,6 +99,12 @@ */ #define VIRTIO_F_SR_IOV 37 +/* + * This feature indicates that the driver passes extra data (besides + * identifying the virtqueue) in its device notifications. + */ +#define VIRTIO_F_NOTIFICATION_DATA 38 + /* * This feature indicates that the driver can reset a queue individually. */ -- cgit v1.2.3 From 2c4e4a22a3b090e06191f8544d21e0e1d72ce518 Mon Sep 17 00:00:00 2001 From: Alvaro Karsz Date: Thu, 13 Apr 2023 11:18:55 +0300 Subject: virtio-vdpa: add VIRTIO_F_NOTIFICATION_DATA feature support Add VIRTIO_F_NOTIFICATION_DATA support for vDPA transport. If this feature is negotiated, the driver passes extra data when kicking a virtqueue. A device that offers this feature needs to implement the kick_vq_with_data callback. kick_vq_with_data receives the vDPA device and data. data includes: 16 bits vqn and 16 bits next available index for split virtqueues. 16 bits vqs, 15 least significant bits of next available index and 1 bit next_wrap for packed virtqueues. This patch follows a patch [1] by Viktor Prutyanov which adds support for the MMIO, channel I/O and modern PCI transports. Signed-off-by: Alvaro Karsz Message-Id: <20230413081855.36643-3-alvaro.karsz@solid-run.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/virtio/virtio_vdpa.c | 23 +++++++++++++++++++++-- include/linux/vdpa.h | 9 +++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 2a095f37f26b..eb6aee8c06b2 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -113,6 +113,17 @@ static bool virtio_vdpa_notify(struct virtqueue *vq) return true; } +static bool virtio_vdpa_notify_with_data(struct virtqueue *vq) +{ + struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev); + const struct vdpa_config_ops *ops = vdpa->config; + u32 data = vring_notification_data(vq); + + ops->kick_vq_with_data(vdpa, data); + + return true; +} + static irqreturn_t virtio_vdpa_config_cb(void *private) { struct virtio_vdpa_device *vd_dev = private; @@ -139,6 +150,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, struct device *dma_dev; const struct vdpa_config_ops *ops = vdpa->config; struct virtio_vdpa_vq_info *info; + bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify; struct vdpa_callback cb; struct virtqueue *vq; u64 desc_addr, driver_addr, device_addr; @@ -155,6 +167,14 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, if (index >= vdpa->nvqs) return ERR_PTR(-ENOENT); + /* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */ + if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) { + if (ops->kick_vq_with_data) + notify = virtio_vdpa_notify_with_data; + else + __virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA); + } + /* Queue shouldn't already be set up. */ if (ops->get_vq_ready(vdpa, index)) return ERR_PTR(-ENOENT); @@ -184,8 +204,7 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, dma_dev = vdpa_get_dma_dev(vdpa); vq = vring_create_virtqueue_dma(index, max_num, align, vdev, true, may_reduce_num, ctx, - virtio_vdpa_notify, callback, - name, dma_dev); + notify, callback, name, dma_dev); if (!vq) { err = -ENOMEM; goto error_new_virtqueue; diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index c0d3f5433af7..db1b0eaef4eb 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -149,6 +149,14 @@ struct vdpa_map_file { * @kick_vq: Kick the virtqueue * @vdev: vdpa device * @idx: virtqueue index + * @kick_vq_with_data: Kick the virtqueue and supply extra data + * (only if VIRTIO_F_NOTIFICATION_DATA is negotiated) + * @vdev: vdpa device + * @data for split virtqueue: + * 16 bits vqn and 16 bits next available index. + * @data for packed virtqueue: + * 16 bits vqn, 15 least significant bits of + * next available index and 1 bit next_wrap. * @set_vq_cb: Set the interrupt callback function for * a virtqueue * @vdev: vdpa device @@ -329,6 +337,7 @@ struct vdpa_config_ops { u64 device_area); void (*set_vq_num)(struct vdpa_device *vdev, u16 idx, u32 num); void (*kick_vq)(struct vdpa_device *vdev, u16 idx); + void (*kick_vq_with_data)(struct vdpa_device *vdev, u32 data); void (*set_vq_cb)(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb); void (*set_vq_ready)(struct vdpa_device *vdev, u16 idx, bool ready); -- cgit v1.2.3 From 38fc29ea754711e9a8c4e9ca2678ab41353a4662 Mon Sep 17 00:00:00 2001 From: Shunsuke Mie Date: Mon, 17 Apr 2023 11:20:36 +0900 Subject: virtio_ring: add a struct device forward declaration The virtio_ring header file uses the struct device without a forward declaration. Signed-off-by: Shunsuke Mie Message-Id: <20230417022037.917668-1-mie@igel.co.jp> Signed-off-by: Michael S. Tsirkin --- include/linux/virtio_ring.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h index 2550c9170f4f..9b33df741b63 100644 --- a/include/linux/virtio_ring.h +++ b/include/linux/virtio_ring.h @@ -58,6 +58,7 @@ do { \ struct virtio_device; struct virtqueue; +struct device; /* * Creates a virtqueue and allocates the descriptor ring. If -- cgit v1.2.3 From 223baf9d17f25e2608dbdff7232c095c1e612268 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 20 Apr 2023 10:55:48 -0400 Subject: sched: Fix performance regression introduced by mm_cid Introduce per-mm/cpu current concurrency id (mm_cid) to fix a PostgreSQL sysbench regression reported by Aaron Lu. Keep track of the currently allocated mm_cid for each mm/cpu rather than freeing them immediately on context switch. This eliminates most atomic operations when context switching back and forth between threads belonging to different memory spaces in multi-threaded scenarios (many processes, each with many threads). The per-mm/per-cpu mm_cid values are serialized by their respective runqueue locks. Thread migration is handled by introducing invocation to sched_mm_cid_migrate_to() (with destination runqueue lock held) in activate_task() for migrating tasks. If the destination cpu's mm_cid is unset, and if the source runqueue is not actively using its mm_cid, then the source cpu's mm_cid is moved to the destination cpu on migration. Introduce a task-work executed periodically, similarly to NUMA work, which delays reclaim of cid values when they are unused for a period of time. Keep track of the allocation time for each per-cpu cid, and let the task work clear them when they are observed to be older than SCHED_MM_CID_PERIOD_NS and unused. This task work also clears all mm_cids which are greater or equal to the Hamming weight of the mm cidmask to keep concurrency ids compact. Because we want to ensure the mm_cid converges towards the smaller values as migrations happen, the prior optimization that was done when context switching between threads belonging to the same mm is removed, because it could delay the lazy release of the destination runqueue mm_cid after it has been replaced by a migration. Removing this prior optimization is not an issue performance-wise because the introduced per-mm/per-cpu mm_cid tracking also covers this more specific case. Fixes: af7f588d8f73 ("sched: Introduce per-memory-map concurrency ID") Reported-by: Aaron Lu Signed-off-by: Mathieu Desnoyers Signed-off-by: Peter Zijlstra (Intel) Tested-by: Aaron Lu Link: https://lore.kernel.org/lkml/20230327080502.GA570847@ziqianlu-desk2/ --- include/linux/mm_types.h | 82 +++++++- include/linux/sched.h | 3 + include/linux/sched/mm.h | 5 + kernel/fork.c | 9 +- kernel/sched/core.c | 523 +++++++++++++++++++++++++++++++++++++++++++++-- kernel/sched/sched.h | 239 +++++++++++++++++++--- 6 files changed, 804 insertions(+), 57 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a57e6ae78e65..5eab61156f0e 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -550,6 +550,13 @@ struct vm_area_struct { struct vm_userfaultfd_ctx vm_userfaultfd_ctx; } __randomize_layout; +#ifdef CONFIG_SCHED_MM_CID +struct mm_cid { + u64 time; + int cid; +}; +#endif + struct kioctx_table; struct mm_struct { struct { @@ -600,15 +607,19 @@ struct mm_struct { atomic_t mm_count; #ifdef CONFIG_SCHED_MM_CID /** - * @cid_lock: Protect cid bitmap updates vs lookups. + * @pcpu_cid: Per-cpu current cid. * - * Prevent situations where updates to the cid bitmap happen - * concurrently with lookups. Those can lead to situations - * where a lookup cannot find a free bit simply because it was - * unlucky enough to load, non-atomically, bitmap words as they - * were being concurrently updated by the updaters. + * Keep track of the currently allocated mm_cid for each cpu. + * The per-cpu mm_cid values are serialized by their respective + * runqueue locks. */ - raw_spinlock_t cid_lock; + struct mm_cid __percpu *pcpu_cid; + /* + * @mm_cid_next_scan: Next mm_cid scan (in jiffies). + * + * When the next mm_cid scan is due (in jiffies). + */ + unsigned long mm_cid_next_scan; #endif #ifdef CONFIG_MMU atomic_long_t pgtables_bytes; /* size of all page tables */ @@ -873,6 +884,37 @@ static inline void vma_iter_init(struct vma_iterator *vmi, } #ifdef CONFIG_SCHED_MM_CID + +enum mm_cid_state { + MM_CID_UNSET = -1U, /* Unset state has lazy_put flag set. */ + MM_CID_LAZY_PUT = (1U << 31), +}; + +static inline bool mm_cid_is_unset(int cid) +{ + return cid == MM_CID_UNSET; +} + +static inline bool mm_cid_is_lazy_put(int cid) +{ + return !mm_cid_is_unset(cid) && (cid & MM_CID_LAZY_PUT); +} + +static inline bool mm_cid_is_valid(int cid) +{ + return !(cid & MM_CID_LAZY_PUT); +} + +static inline int mm_cid_set_lazy_put(int cid) +{ + return cid | MM_CID_LAZY_PUT; +} + +static inline int mm_cid_clear_lazy_put(int cid) +{ + return cid & ~MM_CID_LAZY_PUT; +} + /* Accessor for struct mm_struct's cidmask. */ static inline cpumask_t *mm_cidmask(struct mm_struct *mm) { @@ -886,16 +928,40 @@ static inline cpumask_t *mm_cidmask(struct mm_struct *mm) static inline void mm_init_cid(struct mm_struct *mm) { - raw_spin_lock_init(&mm->cid_lock); + int i; + + for_each_possible_cpu(i) { + struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i); + + pcpu_cid->cid = MM_CID_UNSET; + pcpu_cid->time = 0; + } cpumask_clear(mm_cidmask(mm)); } +static inline int mm_alloc_cid(struct mm_struct *mm) +{ + mm->pcpu_cid = alloc_percpu(struct mm_cid); + if (!mm->pcpu_cid) + return -ENOMEM; + mm_init_cid(mm); + return 0; +} + +static inline void mm_destroy_cid(struct mm_struct *mm) +{ + free_percpu(mm->pcpu_cid); + mm->pcpu_cid = NULL; +} + static inline unsigned int mm_cid_size(void) { return cpumask_size(); } #else /* CONFIG_SCHED_MM_CID */ static inline void mm_init_cid(struct mm_struct *mm) { } +static inline int mm_alloc_cid(struct mm_struct *mm) { return 0; } +static inline void mm_destroy_cid(struct mm_struct *mm) { } static inline unsigned int mm_cid_size(void) { return 0; diff --git a/include/linux/sched.h b/include/linux/sched.h index 6d654eb4cabd..675298d6eb36 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1314,7 +1314,10 @@ struct task_struct { #ifdef CONFIG_SCHED_MM_CID int mm_cid; /* Current cid in mm */ + int last_mm_cid; /* Most recent cid in mm */ + int migrate_from_cpu; int mm_cid_active; /* Whether cid bitmap is active */ + struct callback_head cid_work; #endif struct tlbflush_unmap_batch tlb_ubc; diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 2a243616f222..f20fc0600fcc 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -37,6 +37,11 @@ static inline void mmgrab(struct mm_struct *mm) atomic_inc(&mm->mm_count); } +static inline void smp_mb__after_mmgrab(void) +{ + smp_mb__after_atomic(); +} + extern void __mmdrop(struct mm_struct *mm); static inline void mmdrop(struct mm_struct *mm) diff --git a/kernel/fork.c b/kernel/fork.c index 0c92f224c68c..ad2ee22272a3 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -793,6 +793,7 @@ void __mmdrop(struct mm_struct *mm) check_mm(mm); put_user_ns(mm->user_ns); mm_pasid_drop(mm); + mm_destroy_cid(mm); for (i = 0; i < NR_MM_COUNTERS; i++) percpu_counter_destroy(&mm->rss_stat[i]); @@ -1057,7 +1058,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) #ifdef CONFIG_SCHED_MM_CID tsk->mm_cid = -1; + tsk->last_mm_cid = -1; tsk->mm_cid_active = 0; + tsk->migrate_from_cpu = -1; #endif return tsk; @@ -1162,18 +1165,22 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, if (init_new_context(p, mm)) goto fail_nocontext; + if (mm_alloc_cid(mm)) + goto fail_cid; + for (i = 0; i < NR_MM_COUNTERS; i++) if (percpu_counter_init(&mm->rss_stat[i], 0, GFP_KERNEL_ACCOUNT)) goto fail_pcpu; mm->user_ns = get_user_ns(user_ns); lru_gen_init_mm(mm); - mm_init_cid(mm); return mm; fail_pcpu: while (i > 0) percpu_counter_destroy(&mm->rss_stat[--i]); + mm_destroy_cid(mm); +fail_cid: fail_nocontext: mm_free_pgd(mm); fail_nopgd: diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 5a97ceb37c13..898fa3bc2765 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2101,6 +2101,8 @@ void activate_task(struct rq *rq, struct task_struct *p, int flags) { if (task_on_rq_migrating(p)) flags |= ENQUEUE_MIGRATED; + if (flags & ENQUEUE_MIGRATED) + sched_mm_cid_migrate_to(rq, p); enqueue_task(rq, p, flags); @@ -3210,6 +3212,7 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) p->sched_class->migrate_task_rq(p, new_cpu); p->se.nr_migrations++; rseq_migrate(p); + sched_mm_cid_migrate_from(p); perf_event_task_migrate(p); } @@ -4483,6 +4486,7 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->wake_entry.u_flags = CSD_TYPE_TTWU; p->migration_pending = NULL; #endif + init_sched_mm_cid(p); } DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); @@ -5129,7 +5133,6 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev, sched_info_switch(rq, prev, next); perf_event_task_sched_out(prev, next); rseq_preempt(prev); - switch_mm_cid(prev, next); fire_sched_out_preempt_notifiers(prev, next); kmap_local_sched_out(); prepare_task(next); @@ -5285,6 +5288,9 @@ context_switch(struct rq *rq, struct task_struct *prev, * * kernel -> user switch + mmdrop() active * user -> user switch + * + * switch_mm_cid() needs to be updated if the barriers provided + * by context_switch() are modified. */ if (!next->mm) { // to kernel enter_lazy_tlb(prev->active_mm, next); @@ -5314,6 +5320,9 @@ context_switch(struct rq *rq, struct task_struct *prev, } } + /* switch_mm_cid() requires the memory barriers above. */ + switch_mm_cid(rq, prev, next); + rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); prepare_lock_switch(rq, next, rf); @@ -5602,6 +5611,7 @@ void scheduler_tick(void) resched_latency = cpu_resched_latency(rq); calc_global_load_tick(rq); sched_core_tick(rq); + task_tick_mm_cid(rq, curr); rq_unlock(rq, &rf); @@ -11469,45 +11479,524 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count) } #ifdef CONFIG_SCHED_MM_CID -void sched_mm_cid_exit_signals(struct task_struct *t) + +/** + * @cid_lock: Guarantee forward-progress of cid allocation. + * + * Concurrency ID allocation within a bitmap is mostly lock-free. The cid_lock + * is only used when contention is detected by the lock-free allocation so + * forward progress can be guaranteed. + */ +DEFINE_RAW_SPINLOCK(cid_lock); + +/** + * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock. + * + * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is + * detected, it is set to 1 to ensure that all newly coming allocations are + * serialized by @cid_lock until the allocation which detected contention + * completes and sets @use_cid_lock back to 0. This guarantees forward progress + * of a cid allocation. + */ +int use_cid_lock; + +/* + * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid + * concurrently with respect to the execution of the source runqueue context + * switch. + * + * There is one basic properties we want to guarantee here: + * + * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively + * used by a task. That would lead to concurrent allocation of the cid and + * userspace corruption. + * + * Provide this guarantee by introducing a Dekker memory ordering to guarantee + * that a pair of loads observe at least one of a pair of stores, which can be + * shown as: + * + * X = Y = 0 + * + * w[X]=1 w[Y]=1 + * MB MB + * r[Y]=y r[X]=x + * + * Which guarantees that x==0 && y==0 is impossible. But rather than using + * values 0 and 1, this algorithm cares about specific state transitions of the + * runqueue current task (as updated by the scheduler context switch), and the + * per-mm/cpu cid value. + * + * Let's introduce task (Y) which has task->mm == mm and task (N) which has + * task->mm != mm for the rest of the discussion. There are two scheduler state + * transitions on context switch we care about: + * + * (TSA) Store to rq->curr with transition from (N) to (Y) + * + * (TSB) Store to rq->curr with transition from (Y) to (N) + * + * On the remote-clear side, there is one transition we care about: + * + * (TMA) cmpxchg to *pcpu_cid to set the LAZY flag + * + * There is also a transition to UNSET state which can be performed from all + * sides (scheduler, remote-clear). It is always performed with a cmpxchg which + * guarantees that only a single thread will succeed: + * + * (TMB) cmpxchg to *pcpu_cid to mark UNSET + * + * Just to be clear, what we do _not_ want to happen is a transition to UNSET + * when a thread is actively using the cid (property (1)). + * + * Let's looks at the relevant combinations of TSA/TSB, and TMA transitions. + * + * Scenario A) (TSA)+(TMA) (from next task perspective) + * + * CPU0 CPU1 + * + * Context switch CS-1 Remote-clear + * - store to rq->curr: (N)->(Y) (TSA) - cmpxchg to *pcpu_id to LAZY (TMA) + * (implied barrier after cmpxchg) + * - switch_mm_cid() + * - memory barrier (see switch_mm_cid() + * comment explaining how this barrier + * is combined with other scheduler + * barriers) + * - mm_cid_get (next) + * - READ_ONCE(*pcpu_cid) - rcu_dereference(src_rq->curr) + * + * This Dekker ensures that either task (Y) is observed by the + * rcu_dereference() or the LAZY flag is observed by READ_ONCE(), or both are + * observed. + * + * If task (Y) store is observed by rcu_dereference(), it means that there is + * still an active task on the cpu. Remote-clear will therefore not transition + * to UNSET, which fulfills property (1). + * + * If task (Y) is not observed, but the lazy flag is observed by READ_ONCE(), + * it will move its state to UNSET, which clears the percpu cid perhaps + * uselessly (which is not an issue for correctness). Because task (Y) is not + * observed, CPU1 can move ahead to set the state to UNSET. Because moving + * state to UNSET is done with a cmpxchg expecting that the old state has the + * LAZY flag set, only one thread will successfully UNSET. + * + * If both states (LAZY flag and task (Y)) are observed, the thread on CPU0 + * will observe the LAZY flag and transition to UNSET (perhaps uselessly), and + * CPU1 will observe task (Y) and do nothing more, which is fine. + * + * What we are effectively preventing with this Dekker is a scenario where + * neither LAZY flag nor store (Y) are observed, which would fail property (1) + * because this would UNSET a cid which is actively used. + */ + +void sched_mm_cid_migrate_from(struct task_struct *t) +{ + t->migrate_from_cpu = task_cpu(t); +} + +static +int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq, + struct task_struct *t, + struct mm_cid *src_pcpu_cid) { struct mm_struct *mm = t->mm; - unsigned long flags; + struct task_struct *src_task; + int src_cid, last_mm_cid; if (!mm) + return -1; + + last_mm_cid = t->last_mm_cid; + /* + * If the migrated task has no last cid, or if the current + * task on src rq uses the cid, it means the source cid does not need + * to be moved to the destination cpu. + */ + if (last_mm_cid == -1) + return -1; + src_cid = READ_ONCE(src_pcpu_cid->cid); + if (!mm_cid_is_valid(src_cid) || last_mm_cid != src_cid) + return -1; + + /* + * If we observe an active task using the mm on this rq, it means we + * are not the last task to be migrated from this cpu for this mm, so + * there is no need to move src_cid to the destination cpu. + */ + rcu_read_lock(); + src_task = rcu_dereference(src_rq->curr); + if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) { + rcu_read_unlock(); + t->last_mm_cid = -1; + return -1; + } + rcu_read_unlock(); + + return src_cid; +} + +static +int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq, + struct task_struct *t, + struct mm_cid *src_pcpu_cid, + int src_cid) +{ + struct task_struct *src_task; + struct mm_struct *mm = t->mm; + int lazy_cid; + + if (src_cid == -1) + return -1; + + /* + * Attempt to clear the source cpu cid to move it to the destination + * cpu. + */ + lazy_cid = mm_cid_set_lazy_put(src_cid); + if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid)) + return -1; + + /* + * The implicit barrier after cmpxchg per-mm/cpu cid before loading + * rq->curr->mm matches the scheduler barrier in context_switch() + * between store to rq->curr and load of prev and next task's + * per-mm/cpu cid. + * + * The implicit barrier after cmpxchg per-mm/cpu cid before loading + * rq->curr->mm_cid_active matches the barrier in + * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and + * sched_mm_cid_after_execve() between store to t->mm_cid_active and + * load of per-mm/cpu cid. + */ + + /* + * If we observe an active task using the mm on this rq after setting + * the lazy-put flag, this task will be responsible for transitioning + * from lazy-put flag set to MM_CID_UNSET. + */ + rcu_read_lock(); + src_task = rcu_dereference(src_rq->curr); + if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) { + rcu_read_unlock(); + /* + * We observed an active task for this mm, there is therefore + * no point in moving this cid to the destination cpu. + */ + t->last_mm_cid = -1; + return -1; + } + rcu_read_unlock(); + + /* + * The src_cid is unused, so it can be unset. + */ + if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET)) + return -1; + return src_cid; +} + +/* + * Migration to dst cpu. Called with dst_rq lock held. + * Interrupts are disabled, which keeps the window of cid ownership without the + * source rq lock held small. + */ +void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) +{ + struct mm_cid *src_pcpu_cid, *dst_pcpu_cid; + struct mm_struct *mm = t->mm; + int src_cid, dst_cid, src_cpu; + struct rq *src_rq; + + lockdep_assert_rq_held(dst_rq); + + if (!mm) + return; + src_cpu = t->migrate_from_cpu; + if (src_cpu == -1) { + t->last_mm_cid = -1; + return; + } + /* + * Move the src cid if the dst cid is unset. This keeps id + * allocation closest to 0 in cases where few threads migrate around + * many cpus. + * + * If destination cid is already set, we may have to just clear + * the src cid to ensure compactness in frequent migrations + * scenarios. + * + * It is not useful to clear the src cid when the number of threads is + * greater or equal to the number of allowed cpus, because user-space + * can expect that the number of allowed cids can reach the number of + * allowed cpus. + */ + dst_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(dst_rq)); + dst_cid = READ_ONCE(dst_pcpu_cid->cid); + if (!mm_cid_is_unset(dst_cid) && + atomic_read(&mm->mm_users) >= t->nr_cpus_allowed) + return; + src_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, src_cpu); + src_rq = cpu_rq(src_cpu); + src_cid = __sched_mm_cid_migrate_from_fetch_cid(src_rq, t, src_pcpu_cid); + if (src_cid == -1) + return; + src_cid = __sched_mm_cid_migrate_from_try_steal_cid(src_rq, t, src_pcpu_cid, + src_cid); + if (src_cid == -1) + return; + if (!mm_cid_is_unset(dst_cid)) { + __mm_cid_put(mm, src_cid); + return; + } + /* Move src_cid to dst cpu. */ + mm_cid_snapshot_time(dst_rq, mm); + WRITE_ONCE(dst_pcpu_cid->cid, src_cid); +} + +static void sched_mm_cid_remote_clear(struct mm_struct *mm, struct mm_cid *pcpu_cid, + int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct task_struct *t; + unsigned long flags; + int cid, lazy_cid; + + cid = READ_ONCE(pcpu_cid->cid); + if (!mm_cid_is_valid(cid)) return; + + /* + * Clear the cpu cid if it is set to keep cid allocation compact. If + * there happens to be other tasks left on the source cpu using this + * mm, the next task using this mm will reallocate its cid on context + * switch. + */ + lazy_cid = mm_cid_set_lazy_put(cid); + if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid)) + return; + + /* + * The implicit barrier after cmpxchg per-mm/cpu cid before loading + * rq->curr->mm matches the scheduler barrier in context_switch() + * between store to rq->curr and load of prev and next task's + * per-mm/cpu cid. + * + * The implicit barrier after cmpxchg per-mm/cpu cid before loading + * rq->curr->mm_cid_active matches the barrier in + * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and + * sched_mm_cid_after_execve() between store to t->mm_cid_active and + * load of per-mm/cpu cid. + */ + + /* + * If we observe an active task using the mm on this rq after setting + * the lazy-put flag, that task will be responsible for transitioning + * from lazy-put flag set to MM_CID_UNSET. + */ + rcu_read_lock(); + t = rcu_dereference(rq->curr); + if (READ_ONCE(t->mm_cid_active) && t->mm == mm) { + rcu_read_unlock(); + return; + } + rcu_read_unlock(); + + /* + * The cid is unused, so it can be unset. + * Disable interrupts to keep the window of cid ownership without rq + * lock small. + */ local_irq_save(flags); - mm_cid_put(mm, t->mm_cid); - t->mm_cid = -1; - t->mm_cid_active = 0; + if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET)) + __mm_cid_put(mm, cid); local_irq_restore(flags); } +static void sched_mm_cid_remote_clear_old(struct mm_struct *mm, int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct mm_cid *pcpu_cid; + struct task_struct *curr; + u64 rq_clock; + + /* + * rq->clock load is racy on 32-bit but one spurious clear once in a + * while is irrelevant. + */ + rq_clock = READ_ONCE(rq->clock); + pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu); + + /* + * In order to take care of infrequently scheduled tasks, bump the time + * snapshot associated with this cid if an active task using the mm is + * observed on this rq. + */ + rcu_read_lock(); + curr = rcu_dereference(rq->curr); + if (READ_ONCE(curr->mm_cid_active) && curr->mm == mm) { + WRITE_ONCE(pcpu_cid->time, rq_clock); + rcu_read_unlock(); + return; + } + rcu_read_unlock(); + + if (rq_clock < pcpu_cid->time + SCHED_MM_CID_PERIOD_NS) + return; + sched_mm_cid_remote_clear(mm, pcpu_cid, cpu); +} + +static void sched_mm_cid_remote_clear_weight(struct mm_struct *mm, int cpu, + int weight) +{ + struct mm_cid *pcpu_cid; + int cid; + + pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu); + cid = READ_ONCE(pcpu_cid->cid); + if (!mm_cid_is_valid(cid) || cid < weight) + return; + sched_mm_cid_remote_clear(mm, pcpu_cid, cpu); +} + +static void task_mm_cid_work(struct callback_head *work) +{ + unsigned long now = jiffies, old_scan, next_scan; + struct task_struct *t = current; + struct cpumask *cidmask; + struct mm_struct *mm; + int weight, cpu; + + SCHED_WARN_ON(t != container_of(work, struct task_struct, cid_work)); + + work->next = work; /* Prevent double-add */ + if (t->flags & PF_EXITING) + return; + mm = t->mm; + if (!mm) + return; + old_scan = READ_ONCE(mm->mm_cid_next_scan); + next_scan = now + msecs_to_jiffies(MM_CID_SCAN_DELAY); + if (!old_scan) { + unsigned long res; + + res = cmpxchg(&mm->mm_cid_next_scan, old_scan, next_scan); + if (res != old_scan) + old_scan = res; + else + old_scan = next_scan; + } + if (time_before(now, old_scan)) + return; + if (!try_cmpxchg(&mm->mm_cid_next_scan, &old_scan, next_scan)) + return; + cidmask = mm_cidmask(mm); + /* Clear cids that were not recently used. */ + for_each_possible_cpu(cpu) + sched_mm_cid_remote_clear_old(mm, cpu); + weight = cpumask_weight(cidmask); + /* + * Clear cids that are greater or equal to the cidmask weight to + * recompact it. + */ + for_each_possible_cpu(cpu) + sched_mm_cid_remote_clear_weight(mm, cpu, weight); +} + +void init_sched_mm_cid(struct task_struct *t) +{ + struct mm_struct *mm = t->mm; + int mm_users = 0; + + if (mm) { + mm_users = atomic_read(&mm->mm_users); + if (mm_users == 1) + mm->mm_cid_next_scan = jiffies + msecs_to_jiffies(MM_CID_SCAN_DELAY); + } + t->cid_work.next = &t->cid_work; /* Protect against double add */ + init_task_work(&t->cid_work, task_mm_cid_work); +} + +void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) +{ + struct callback_head *work = &curr->cid_work; + unsigned long now = jiffies; + + if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || + work->next != work) + return; + if (time_before(now, READ_ONCE(curr->mm->mm_cid_next_scan))) + return; + task_work_add(curr, work, TWA_RESUME); +} + +void sched_mm_cid_exit_signals(struct task_struct *t) +{ + struct mm_struct *mm = t->mm; + struct rq_flags rf; + struct rq *rq; + + if (!mm) + return; + + preempt_disable(); + rq = this_rq(); + rq_lock_irqsave(rq, &rf); + preempt_enable_no_resched(); /* holding spinlock */ + WRITE_ONCE(t->mm_cid_active, 0); + /* + * Store t->mm_cid_active before loading per-mm/cpu cid. + * Matches barrier in sched_mm_cid_remote_clear_old(). + */ + smp_mb(); + mm_cid_put(mm); + t->last_mm_cid = t->mm_cid = -1; + rq_unlock_irqrestore(rq, &rf); +} + void sched_mm_cid_before_execve(struct task_struct *t) { struct mm_struct *mm = t->mm; - unsigned long flags; + struct rq_flags rf; + struct rq *rq; if (!mm) return; - local_irq_save(flags); - mm_cid_put(mm, t->mm_cid); - t->mm_cid = -1; - t->mm_cid_active = 0; - local_irq_restore(flags); + + preempt_disable(); + rq = this_rq(); + rq_lock_irqsave(rq, &rf); + preempt_enable_no_resched(); /* holding spinlock */ + WRITE_ONCE(t->mm_cid_active, 0); + /* + * Store t->mm_cid_active before loading per-mm/cpu cid. + * Matches barrier in sched_mm_cid_remote_clear_old(). + */ + smp_mb(); + mm_cid_put(mm); + t->last_mm_cid = t->mm_cid = -1; + rq_unlock_irqrestore(rq, &rf); } void sched_mm_cid_after_execve(struct task_struct *t) { struct mm_struct *mm = t->mm; - unsigned long flags; + struct rq_flags rf; + struct rq *rq; if (!mm) return; - local_irq_save(flags); - t->mm_cid = mm_cid_get(mm); - t->mm_cid_active = 1; - local_irq_restore(flags); + + preempt_disable(); + rq = this_rq(); + rq_lock_irqsave(rq, &rf); + preempt_enable_no_resched(); /* holding spinlock */ + WRITE_ONCE(t->mm_cid_active, 1); + /* + * Store t->mm_cid_active before loading per-mm/cpu cid. + * Matches barrier in sched_mm_cid_remote_clear_old(). + */ + smp_mb(); + t->last_mm_cid = t->mm_cid = mm_cid_get(rq, mm); + rq_unlock_irqrestore(rq, &rf); rseq_set_notify_resume(t); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 060616944d7a..ec7b3e0a2b20 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3253,61 +3253,238 @@ static inline void update_current_exec_runtime(struct task_struct *curr, } #ifdef CONFIG_SCHED_MM_CID -static inline int __mm_cid_get(struct mm_struct *mm) + +#define SCHED_MM_CID_PERIOD_NS (100ULL * 1000000) /* 100ms */ +#define MM_CID_SCAN_DELAY 100 /* 100ms */ + +extern raw_spinlock_t cid_lock; +extern int use_cid_lock; + +extern void sched_mm_cid_migrate_from(struct task_struct *t); +extern void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t); +extern void task_tick_mm_cid(struct rq *rq, struct task_struct *curr); +extern void init_sched_mm_cid(struct task_struct *t); + +static inline void __mm_cid_put(struct mm_struct *mm, int cid) +{ + if (cid < 0) + return; + cpumask_clear_cpu(cid, mm_cidmask(mm)); +} + +/* + * The per-mm/cpu cid can have the MM_CID_LAZY_PUT flag set or transition to + * the MM_CID_UNSET state without holding the rq lock, but the rq lock needs to + * be held to transition to other states. + * + * State transitions synchronized with cmpxchg or try_cmpxchg need to be + * consistent across cpus, which prevents use of this_cpu_cmpxchg. + */ +static inline void mm_cid_put_lazy(struct task_struct *t) +{ + struct mm_struct *mm = t->mm; + struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid; + int cid; + + lockdep_assert_irqs_disabled(); + cid = __this_cpu_read(pcpu_cid->cid); + if (!mm_cid_is_lazy_put(cid) || + !try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET)) + return; + __mm_cid_put(mm, mm_cid_clear_lazy_put(cid)); +} + +static inline int mm_cid_pcpu_unset(struct mm_struct *mm) +{ + struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid; + int cid, res; + + lockdep_assert_irqs_disabled(); + cid = __this_cpu_read(pcpu_cid->cid); + for (;;) { + if (mm_cid_is_unset(cid)) + return MM_CID_UNSET; + /* + * Attempt transition from valid or lazy-put to unset. + */ + res = cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, cid, MM_CID_UNSET); + if (res == cid) + break; + cid = res; + } + return cid; +} + +static inline void mm_cid_put(struct mm_struct *mm) +{ + int cid; + + lockdep_assert_irqs_disabled(); + cid = mm_cid_pcpu_unset(mm); + if (cid == MM_CID_UNSET) + return; + __mm_cid_put(mm, mm_cid_clear_lazy_put(cid)); +} + +static inline int __mm_cid_try_get(struct mm_struct *mm) { struct cpumask *cpumask; int cid; cpumask = mm_cidmask(mm); - cid = cpumask_first_zero(cpumask); - if (cid >= nr_cpu_ids) + /* + * Retry finding first zero bit if the mask is temporarily + * filled. This only happens during concurrent remote-clear + * which owns a cid without holding a rq lock. + */ + for (;;) { + cid = cpumask_first_zero(cpumask); + if (cid < nr_cpu_ids) + break; + cpu_relax(); + } + if (cpumask_test_and_set_cpu(cid, cpumask)) return -1; - __cpumask_set_cpu(cid, cpumask); return cid; } -static inline void mm_cid_put(struct mm_struct *mm, int cid) +/* + * Save a snapshot of the current runqueue time of this cpu + * with the per-cpu cid value, allowing to estimate how recently it was used. + */ +static inline void mm_cid_snapshot_time(struct rq *rq, struct mm_struct *mm) { - lockdep_assert_irqs_disabled(); - if (cid < 0) - return; - raw_spin_lock(&mm->cid_lock); - __cpumask_clear_cpu(cid, mm_cidmask(mm)); - raw_spin_unlock(&mm->cid_lock); + struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(rq)); + + lockdep_assert_rq_held(rq); + WRITE_ONCE(pcpu_cid->time, rq->clock); } -static inline int mm_cid_get(struct mm_struct *mm) +static inline int __mm_cid_get(struct rq *rq, struct mm_struct *mm) { - int ret; + int cid; - lockdep_assert_irqs_disabled(); - raw_spin_lock(&mm->cid_lock); - ret = __mm_cid_get(mm); - raw_spin_unlock(&mm->cid_lock); - return ret; + /* + * All allocations (even those using the cid_lock) are lock-free. If + * use_cid_lock is set, hold the cid_lock to perform cid allocation to + * guarantee forward progress. + */ + if (!READ_ONCE(use_cid_lock)) { + cid = __mm_cid_try_get(mm); + if (cid >= 0) + goto end; + raw_spin_lock(&cid_lock); + } else { + raw_spin_lock(&cid_lock); + cid = __mm_cid_try_get(mm); + if (cid >= 0) + goto unlock; + } + + /* + * cid concurrently allocated. Retry while forcing following + * allocations to use the cid_lock to ensure forward progress. + */ + WRITE_ONCE(use_cid_lock, 1); + /* + * Set use_cid_lock before allocation. Only care about program order + * because this is only required for forward progress. + */ + barrier(); + /* + * Retry until it succeeds. It is guaranteed to eventually succeed once + * all newcoming allocations observe the use_cid_lock flag set. + */ + do { + cid = __mm_cid_try_get(mm); + cpu_relax(); + } while (cid < 0); + /* + * Allocate before clearing use_cid_lock. Only care about + * program order because this is for forward progress. + */ + barrier(); + WRITE_ONCE(use_cid_lock, 0); +unlock: + raw_spin_unlock(&cid_lock); +end: + mm_cid_snapshot_time(rq, mm); + return cid; } -static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) +static inline int mm_cid_get(struct rq *rq, struct mm_struct *mm) { + struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid; + struct cpumask *cpumask; + int cid; + + lockdep_assert_rq_held(rq); + cpumask = mm_cidmask(mm); + cid = __this_cpu_read(pcpu_cid->cid); + if (mm_cid_is_valid(cid)) { + mm_cid_snapshot_time(rq, mm); + return cid; + } + if (mm_cid_is_lazy_put(cid)) { + if (try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET)) + __mm_cid_put(mm, mm_cid_clear_lazy_put(cid)); + } + cid = __mm_cid_get(rq, mm); + __this_cpu_write(pcpu_cid->cid, cid); + return cid; +} + +static inline void switch_mm_cid(struct rq *rq, + struct task_struct *prev, + struct task_struct *next) +{ + /* + * Provide a memory barrier between rq->curr store and load of + * {prev,next}->mm->pcpu_cid[cpu] on rq->curr->mm transition. + * + * Should be adapted if context_switch() is modified. + */ + if (!next->mm) { // to kernel + /* + * user -> kernel transition does not guarantee a barrier, but + * we can use the fact that it performs an atomic operation in + * mmgrab(). + */ + if (prev->mm) // from user + smp_mb__after_mmgrab(); + /* + * kernel -> kernel transition does not change rq->curr->mm + * state. It stays NULL. + */ + } else { // to user + /* + * kernel -> user transition does not provide a barrier + * between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu]. + * Provide it here. + */ + if (!prev->mm) // from kernel + smp_mb(); + /* + * user -> user transition guarantees a memory barrier through + * switch_mm() when current->mm changes. If current->mm is + * unchanged, no barrier is needed. + */ + } if (prev->mm_cid_active) { - if (next->mm_cid_active && next->mm == prev->mm) { - /* - * Context switch between threads in same mm, hand over - * the mm_cid from prev to next. - */ - next->mm_cid = prev->mm_cid; - prev->mm_cid = -1; - return; - } - mm_cid_put(prev->mm, prev->mm_cid); + mm_cid_snapshot_time(rq, prev->mm); + mm_cid_put_lazy(prev); prev->mm_cid = -1; } if (next->mm_cid_active) - next->mm_cid = mm_cid_get(next->mm); + next->last_mm_cid = next->mm_cid = mm_cid_get(rq, next->mm); } #else -static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { } +static inline void switch_mm_cid(struct rq *rq, struct task_struct *prev, struct task_struct *next) { } +static inline void sched_mm_cid_migrate_from(struct task_struct *t) { } +static inline void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) { } +static inline void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) { } +static inline void init_sched_mm_cid(struct task_struct *t) { } #endif #endif /* _KERNEL_SCHED_SCHED_H */ -- cgit v1.2.3 From f6c73a11133ef991284311387ca707b48bf53912 Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Fri, 21 Apr 2023 08:52:42 -0700 Subject: fs.h: Add TRACE_IOCB_STRINGS for use in trace points Add TRACE_IOCB_STRINGS macro which can be used in the trace point patch to print different flag values with meaningful string output. Tested-by: Disha Goel Reviewed-by: Christoph Hellwig Signed-off-by: Ritesh Harjani (IBM) Reviewed-by: Darrick J. Wong [djwong: line up strings all prettylike] Signed-off-by: Darrick J. Wong --- include/linux/fs.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index c85916e9f7db..75d24fa082e5 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -340,6 +340,20 @@ enum rw_hint { /* can use bio alloc cache */ #define IOCB_ALLOC_CACHE (1 << 21) +/* for use in trace events */ +#define TRACE_IOCB_STRINGS \ + { IOCB_HIPRI, "HIPRI" }, \ + { IOCB_DSYNC, "DSYNC" }, \ + { IOCB_SYNC, "SYNC" }, \ + { IOCB_NOWAIT, "NOWAIT" }, \ + { IOCB_APPEND, "APPEND" }, \ + { IOCB_EVENTFD, "EVENTFD"}, \ + { IOCB_DIRECT, "DIRECT" }, \ + { IOCB_WRITE, "WRITE" }, \ + { IOCB_WAITQ, "WAITQ" }, \ + { IOCB_NOIO, "NOIO" }, \ + { IOCB_ALLOC_CACHE, "ALLOC_CACHE" } + struct kiocb { struct file *ki_filp; loff_t ki_pos; -- cgit v1.2.3 From d3bff1fc50d4fcaccddbd63917dd94172e80c40e Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Fri, 21 Apr 2023 08:52:43 -0700 Subject: iomap: Remove IOMAP_DIO_NOSYNC unused dio flag IOMAP_DIO_NOSYNC earlier was added for use in btrfs. But it seems for aio dsync writes this is not useful anyway. For aio dsync case, we we queue the request and return -EIOCBQUEUED. Now, since IOMAP_DIO_NOSYNC doesn't let iomap_dio_complete() to call generic_write_sync(), hence we may lose the sync write. Hence kill this flag as it is not in use by any FS now. Tested-by: Disha Goel Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Ritesh Harjani (IBM) Signed-off-by: Darrick J. Wong --- fs/iomap/direct-io.c | 2 +- include/linux/iomap.h | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) (limited to 'include/linux') diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index f771001574d0..36ab1152dbea 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -541,7 +541,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, } /* for data sync or sync, we need sync completion processing */ - if (iocb_is_dsync(iocb) && !(dio_flags & IOMAP_DIO_NOSYNC)) { + if (iocb_is_dsync(iocb)) { dio->flags |= IOMAP_DIO_NEED_SYNC; /* diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 0f8123504e5e..e2b836c2e119 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -377,12 +377,6 @@ struct iomap_dio_ops { */ #define IOMAP_DIO_PARTIAL (1 << 2) -/* - * The caller will sync the write if needed; do not sync it within - * iomap_dio_rw. Overrides IOMAP_DIO_FORCE_WAIT. - */ -#define IOMAP_DIO_NOSYNC (1 << 3) - ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, unsigned int dio_flags, void *private, size_t done_before); -- cgit v1.2.3 From 465e5e6a1698f3325f5d2262d2875779b06b50c7 Mon Sep 17 00:00:00 2001 From: Pankaj Raghav Date: Mon, 17 Apr 2023 14:36:15 +0200 Subject: fs/buffer: add folio_set_bh helper Patch series "convert create_page_buffers to folio_create_buffers". One of the first kernel panic we hit when we try to increase the block size > 4k is inside create_page_buffers()[1]. Even though buffer.c function do not support large folios (folios > PAGE_SIZE) at the moment, these changes are required when we want to remove that constraint. This patch (of 4): The folio version of set_bh_page(). This is required to convert create_page_buffers() to folio_create_buffers() later in the series. Link: https://lkml.kernel.org/r/20230417123618.22094-1-p.raghav@samsung.com Link: https://lkml.kernel.org/r/20230417123618.22094-2-p.raghav@samsung.com Signed-off-by: Pankaj Raghav Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Hannes Reinecke Cc: Alexander Viro Cc: Christian Brauner Cc: Luis Chamberlain Signed-off-by: Andrew Morton --- fs/buffer.c | 15 +++++++++++++++ include/linux/buffer_head.h | 2 ++ 2 files changed, 17 insertions(+) (limited to 'include/linux') diff --git a/fs/buffer.c b/fs/buffer.c index 9e1e2add541e..009d950fbf42 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1485,6 +1485,21 @@ void set_bh_page(struct buffer_head *bh, } EXPORT_SYMBOL(set_bh_page); +void folio_set_bh(struct buffer_head *bh, struct folio *folio, + unsigned long offset) +{ + bh->b_folio = folio; + BUG_ON(offset >= folio_size(folio)); + if (folio_test_highmem(folio)) + /* + * This catches illegal uses and preserves the offset: + */ + bh->b_data = (char *)(0 + offset); + else + bh->b_data = folio_address(folio) + offset; +} +EXPORT_SYMBOL(folio_set_bh); + /* * Called when truncating a buffer on a page completely. */ diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 8f14dca5fed7..7e92d23f4782 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -196,6 +196,8 @@ void mark_buffer_write_io_error(struct buffer_head *bh); void touch_buffer(struct buffer_head *bh); void set_bh_page(struct buffer_head *bh, struct page *page, unsigned long offset); +void folio_set_bh(struct buffer_head *bh, struct folio *folio, + unsigned long offset); bool try_to_free_buffers(struct folio *); struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, bool retry); -- cgit v1.2.3 From c71124a8afa441af203d2001a92c91f114446687 Mon Sep 17 00:00:00 2001 From: Pankaj Raghav Date: Mon, 17 Apr 2023 14:36:16 +0200 Subject: buffer: add folio_alloc_buffers() helper Folio version of alloc_page_buffers() helper. This is required to convert create_page_buffers() to folio_create_buffers() later in the series. alloc_page_buffers() has been modified to call folio_alloc_buffers() which adds one call to compound_head() but folio_alloc_buffers() removes one call to compound_head() compared to the existing alloc_page_buffers() implementation. Link: https://lkml.kernel.org/r/20230417123618.22094-3-p.raghav@samsung.com Signed-off-by: Pankaj Raghav Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Hannes Reinecke Cc: Alexander Viro Cc: Christian Brauner Cc: Luis Chamberlain Signed-off-by: Andrew Morton --- fs/buffer.c | 23 +++++++++++++++-------- include/linux/buffer_head.h | 2 ++ 2 files changed, 17 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/fs/buffer.c b/fs/buffer.c index 009d950fbf42..bbd4aa670262 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -843,7 +843,7 @@ int remove_inode_buffers(struct inode *inode) } /* - * Create the appropriate buffers when given a page for data area and + * Create the appropriate buffers when given a folio for data area and * the size of each buffer.. Use the bh->b_this_page linked list to * follow the buffers created. Return NULL if unable to create more * buffers. @@ -851,8 +851,8 @@ int remove_inode_buffers(struct inode *inode) * The retry flag is used to differentiate async IO (paging, swapping) * which may not fail from ordinary buffer allocations. */ -struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, - bool retry) +struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size, + bool retry) { struct buffer_head *bh, *head; gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT; @@ -862,12 +862,12 @@ struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, if (retry) gfp |= __GFP_NOFAIL; - /* The page lock pins the memcg */ - memcg = page_memcg(page); + /* The folio lock pins the memcg */ + memcg = folio_memcg(folio); old_memcg = set_active_memcg(memcg); head = NULL; - offset = PAGE_SIZE; + offset = folio_size(folio); while ((offset -= size) >= 0) { bh = alloc_buffer_head(gfp); if (!bh) @@ -879,8 +879,8 @@ struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, bh->b_size = size; - /* Link the buffer to its page */ - set_bh_page(bh, page, offset); + /* Link the buffer to its folio */ + folio_set_bh(bh, folio, offset); } out: set_active_memcg(old_memcg); @@ -899,6 +899,13 @@ no_grow: goto out; } +EXPORT_SYMBOL_GPL(folio_alloc_buffers); + +struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, + bool retry) +{ + return folio_alloc_buffers(page_folio(page), size, retry); +} EXPORT_SYMBOL_GPL(alloc_page_buffers); static inline void diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 7e92d23f4782..0b14eab41bd1 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -199,6 +199,8 @@ void set_bh_page(struct buffer_head *bh, void folio_set_bh(struct buffer_head *bh, struct folio *folio, unsigned long offset); bool try_to_free_buffers(struct folio *); +struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size, + bool retry); struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, bool retry); void create_empty_buffers(struct page *, unsigned long, -- cgit v1.2.3 From 8e2e17560bed73c2a73c94cbe378719f6cf850ee Mon Sep 17 00:00:00 2001 From: Pankaj Raghav Date: Mon, 17 Apr 2023 14:36:17 +0200 Subject: fs/buffer: add folio_create_empty_buffers helper Folio version of create_empty_buffers(). This is required to convert create_page_buffers() to folio_create_buffers() later in the series. It removes several calls to compound_head() as it works directly on folio compared to create_empty_buffers(). Hence, create_empty_buffers() has been modified to call folio_create_empty_buffers(). Link: https://lkml.kernel.org/r/20230417123618.22094-4-p.raghav@samsung.com Signed-off-by: Pankaj Raghav Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Hannes Reinecke Cc: Alexander Viro Cc: Christian Brauner Cc: Luis Chamberlain Signed-off-by: Andrew Morton --- fs/buffer.c | 28 +++++++++++++++++----------- include/linux/buffer_head.h | 2 ++ 2 files changed, 19 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/fs/buffer.c b/fs/buffer.c index bbd4aa670262..e6266bf7eeea 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1594,18 +1594,17 @@ out: } EXPORT_SYMBOL(block_invalidate_folio); - /* * We attach and possibly dirty the buffers atomically wrt * block_dirty_folio() via private_lock. try_to_free_buffers - * is already excluded via the page lock. + * is already excluded via the folio lock. */ -void create_empty_buffers(struct page *page, - unsigned long blocksize, unsigned long b_state) +void folio_create_empty_buffers(struct folio *folio, unsigned long blocksize, + unsigned long b_state) { struct buffer_head *bh, *head, *tail; - head = alloc_page_buffers(page, blocksize, true); + head = folio_alloc_buffers(folio, blocksize, true); bh = head; do { bh->b_state |= b_state; @@ -1614,19 +1613,26 @@ void create_empty_buffers(struct page *page, } while (bh); tail->b_this_page = head; - spin_lock(&page->mapping->private_lock); - if (PageUptodate(page) || PageDirty(page)) { + spin_lock(&folio->mapping->private_lock); + if (folio_test_uptodate(folio) || folio_test_dirty(folio)) { bh = head; do { - if (PageDirty(page)) + if (folio_test_dirty(folio)) set_buffer_dirty(bh); - if (PageUptodate(page)) + if (folio_test_uptodate(folio)) set_buffer_uptodate(bh); bh = bh->b_this_page; } while (bh != head); } - attach_page_private(page, head); - spin_unlock(&page->mapping->private_lock); + folio_attach_private(folio, head); + spin_unlock(&folio->mapping->private_lock); +} +EXPORT_SYMBOL(folio_create_empty_buffers); + +void create_empty_buffers(struct page *page, + unsigned long blocksize, unsigned long b_state) +{ + folio_create_empty_buffers(page_folio(page), blocksize, b_state); } EXPORT_SYMBOL(create_empty_buffers); diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 0b14eab41bd1..1520793c72da 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -205,6 +205,8 @@ struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, bool retry); void create_empty_buffers(struct page *, unsigned long, unsigned long b_state); +void folio_create_empty_buffers(struct folio *folio, unsigned long blocksize, + unsigned long b_state); void end_buffer_read_sync(struct buffer_head *bh, int uptodate); void end_buffer_write_sync(struct buffer_head *bh, int uptodate); void end_buffer_async_write(struct buffer_head *bh, int uptodate); -- cgit v1.2.3 From 4bf4f155bfbc77a12ad2c9e12d9f57718adb9a5c Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Mon, 17 Apr 2023 19:48:07 +0800 Subject: mm: correct arg in reclaim_pages()/reclaim_clean_pages_from_list() Both of them change the arg from page_list to folio_list when convert them to use a folio, but not the declaration, let's correct it, also move the reclaim_pages() from swap.h to internal.h as it only used in mm. Link: https://lkml.kernel.org/r/20230417114807.186786-1-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Reviwed-by: Matthew Wilcox (Oracle) Reviewed-by: SeongJae Park Reviewed-by: David Hildenbrand Cc: Hugh Dickins Signed-off-by: Andrew Morton --- include/linux/swap.h | 1 - mm/internal.h | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index 7f7d5b9ddf7e..3c69cb653cb9 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -442,7 +442,6 @@ extern unsigned long shrink_all_memory(unsigned long nr_pages); extern int vm_swappiness; long remove_mapping(struct address_space *mapping, struct folio *folio); -extern unsigned long reclaim_pages(struct list_head *page_list); #ifdef CONFIG_NUMA extern int node_reclaim_mode; extern int sysctl_min_unmapped_ratio; diff --git a/mm/internal.h b/mm/internal.h index 6483db57a31f..68410c6d97ac 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -783,8 +783,9 @@ extern unsigned long __must_check vm_mmap_pgoff(struct file *, unsigned long, unsigned long, unsigned long); extern void set_pageblock_order(void); +unsigned long reclaim_pages(struct list_head *folio_list); unsigned int reclaim_clean_pages_from_list(struct zone *zone, - struct list_head *page_list); + struct list_head *folio_list); /* The ALLOC_WMARK bits are used as an index to zone->watermark */ #define ALLOC_WMARK_MIN WMARK_MIN #define ALLOC_WMARK_LOW WMARK_LOW -- cgit v1.2.3 From d7597f59d1d33e9efbffa7060deb9ee5bd119e62 Mon Sep 17 00:00:00 2001 From: Stefan Roesch Date: Mon, 17 Apr 2023 22:13:40 -0700 Subject: mm: add new api to enable ksm per process Patch series "mm: process/cgroup ksm support", v9. So far KSM can only be enabled by calling madvise for memory regions. To be able to use KSM for more workloads, KSM needs to have the ability to be enabled / disabled at the process / cgroup level. Use case 1: The madvise call is not available in the programming language. An example for this are programs with forked workloads using a garbage collected language without pointers. In such a language madvise cannot be made available. In addition the addresses of objects get moved around as they are garbage collected. KSM sharing needs to be enabled "from the outside" for these type of workloads. Use case 2: The same interpreter can also be used for workloads where KSM brings no benefit or even has overhead. We'd like to be able to enable KSM on a workload by workload basis. Use case 3: With the madvise call sharing opportunities are only enabled for the current process: it is a workload-local decision. A considerable number of sharing opportunities may exist across multiple workloads or jobs (if they are part of the same security domain). Only a higler level entity like a job scheduler or container can know for certain if its running one or more instances of a job. That job scheduler however doesn't have the necessary internal workload knowledge to make targeted madvise calls. Security concerns: In previous discussions security concerns have been brought up. The problem is that an individual workload does not have the knowledge about what else is running on a machine. Therefore it has to be very conservative in what memory areas can be shared or not. However, if the system is dedicated to running multiple jobs within the same security domain, its the job scheduler that has the knowledge that sharing can be safely enabled and is even desirable. Performance: Experiments with using UKSM have shown a capacity increase of around 20%. Here are the metrics from an instagram workload (taken from a machine with 64GB main memory): full_scans: 445 general_profit: 20158298048 max_page_sharing: 256 merge_across_nodes: 1 pages_shared: 129547 pages_sharing: 5119146 pages_to_scan: 4000 pages_unshared: 1760924 pages_volatile: 10761341 run: 1 sleep_millisecs: 20 stable_node_chains: 167 stable_node_chains_prune_millisecs: 2000 stable_node_dups: 2751 use_zero_pages: 0 zero_pages_sharing: 0 After the service is running for 30 minutes to an hour, 4 to 5 million shared pages are common for this workload when using KSM. Detailed changes: 1. New options for prctl system command This patch series adds two new options to the prctl system call. The first one allows to enable KSM at the process level and the second one to query the setting. The setting will be inherited by child processes. With the above setting, KSM can be enabled for the seed process of a cgroup and all processes in the cgroup will inherit the setting. 2. Changes to KSM processing When KSM is enabled at the process level, the KSM code will iterate over all the VMA's and enable KSM for the eligible VMA's. When forking a process that has KSM enabled, the setting will be inherited by the new child process. 3. Add general_profit metric The general_profit metric of KSM is specified in the documentation, but not calculated. This adds the general profit metric to /sys/kernel/debug/mm/ksm. 4. Add more metrics to ksm_stat This adds the process profit metric to /proc//ksm_stat. 5. Add more tests to ksm_tests and ksm_functional_tests This adds an option to specify the merge type to the ksm_tests. This allows to test madvise and prctl KSM. It also adds a two new tests to ksm_functional_tests: one to test the new prctl options and the other one is a fork test to verify that the KSM process setting is inherited by client processes. This patch (of 3): So far KSM can only be enabled by calling madvise for memory regions. To be able to use KSM for more workloads, KSM needs to have the ability to be enabled / disabled at the process / cgroup level. 1. New options for prctl system command This patch series adds two new options to the prctl system call. The first one allows to enable KSM at the process level and the second one to query the setting. The setting will be inherited by child processes. With the above setting, KSM can be enabled for the seed process of a cgroup and all processes in the cgroup will inherit the setting. 2. Changes to KSM processing When KSM is enabled at the process level, the KSM code will iterate over all the VMA's and enable KSM for the eligible VMA's. When forking a process that has KSM enabled, the setting will be inherited by the new child process. 1) Introduce new MMF_VM_MERGE_ANY flag This introduces the new flag MMF_VM_MERGE_ANY flag. When this flag is set, kernel samepage merging (ksm) gets enabled for all vma's of a process. 2) Setting VM_MERGEABLE on VMA creation When a VMA is created, if the MMF_VM_MERGE_ANY flag is set, the VM_MERGEABLE flag will be set for this VMA. 3) support disabling of ksm for a process This adds the ability to disable ksm for a process if ksm has been enabled for the process with prctl. 4) add new prctl option to get and set ksm for a process This adds two new options to the prctl system call - enable ksm for all vmas of a process (if the vmas support it). - query if ksm has been enabled for a process. 3. Disabling MMF_VM_MERGE_ANY for storage keys in s390 In the s390 architecture when storage keys are used, the MMF_VM_MERGE_ANY will be disabled. Link: https://lkml.kernel.org/r/20230418051342.1919757-1-shr@devkernel.io Link: https://lkml.kernel.org/r/20230418051342.1919757-2-shr@devkernel.io Signed-off-by: Stefan Roesch Acked-by: David Hildenbrand Cc: David Hildenbrand Cc: Johannes Weiner Cc: Michal Hocko Cc: Rik van Riel Cc: Bagas Sanjaya Signed-off-by: Andrew Morton --- arch/s390/mm/gmap.c | 7 +++ include/linux/ksm.h | 21 ++++++++- include/linux/sched/coredump.h | 1 + include/uapi/linux/prctl.h | 2 + kernel/sys.c | 27 +++++++++++ mm/ksm.c | 104 ++++++++++++++++++++++++++++++++++------- mm/mmap.c | 3 ++ 7 files changed, 146 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 5a716bdcba05..0949811761e6 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -2591,6 +2591,13 @@ int gmap_mark_unmergeable(void) int ret; VMA_ITERATOR(vmi, mm, 0); + /* + * Make sure to disable KSM (if enabled for the whole process or + * individual VMAs). Note that nothing currently hinders user space + * from re-enabling it. + */ + clear_bit(MMF_VM_MERGE_ANY, &mm->flags); + for_each_vma(vmi, vma) { /* Copy vm_flags to avoid partial modifications in ksm_madvise */ vm_flags = vma->vm_flags; diff --git a/include/linux/ksm.h b/include/linux/ksm.h index d9e701326a88..4647b0c70c12 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -18,13 +18,26 @@ #ifdef CONFIG_KSM int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, unsigned long *vm_flags); + +void ksm_add_vma(struct vm_area_struct *vma); +int ksm_enable_merge_any(struct mm_struct *mm); + int __ksm_enter(struct mm_struct *mm); void __ksm_exit(struct mm_struct *mm); static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) { - if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags)) - return __ksm_enter(mm); + int ret; + + if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags)) { + ret = __ksm_enter(mm); + if (ret) + return ret; + } + + if (test_bit(MMF_VM_MERGE_ANY, &oldmm->flags)) + set_bit(MMF_VM_MERGE_ANY, &mm->flags); + return 0; } @@ -57,6 +70,10 @@ void collect_procs_ksm(struct page *page, struct list_head *to_kill, #endif #else /* !CONFIG_KSM */ +static inline void ksm_add_vma(struct vm_area_struct *vma) +{ +} + static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) { return 0; diff --git a/include/linux/sched/coredump.h b/include/linux/sched/coredump.h index 0e17ae7fbfd3..0ee96ea7a0e9 100644 --- a/include/linux/sched/coredump.h +++ b/include/linux/sched/coredump.h @@ -90,4 +90,5 @@ static inline int get_dumpable(struct mm_struct *mm) #define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\ MMF_DISABLE_THP_MASK | MMF_HAS_MDWE_MASK) +#define MMF_VM_MERGE_ANY 29 #endif /* _LINUX_SCHED_COREDUMP_H */ diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index b99c0be72577..f23d9a16507f 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -292,4 +292,6 @@ struct prctl_mm_map { #define PR_GET_AUXV 0x41555856 +#define PR_SET_MEMORY_MERGE 67 +#define PR_GET_MEMORY_MERGE 68 #endif /* _LINUX_PRCTL_H */ diff --git a/kernel/sys.c b/kernel/sys.c index 26c1399e0654..72cdb16e2636 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -2687,6 +2688,32 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, case PR_SET_VMA: error = prctl_set_vma(arg2, arg3, arg4, arg5); break; +#ifdef CONFIG_KSM + case PR_SET_MEMORY_MERGE: + if (arg3 || arg4 || arg5) + return -EINVAL; + if (mmap_write_lock_killable(me->mm)) + return -EINTR; + + if (arg2) { + error = ksm_enable_merge_any(me->mm); + } else { + /* + * TODO: we might want disable KSM on all VMAs and + * trigger unsharing to completely disable KSM. + */ + clear_bit(MMF_VM_MERGE_ANY, &me->mm->flags); + error = 0; + } + mmap_write_unlock(me->mm); + break; + case PR_GET_MEMORY_MERGE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + + error = !!test_bit(MMF_VM_MERGE_ANY, &me->mm->flags); + break; +#endif default: error = -EINVAL; break; diff --git a/mm/ksm.c b/mm/ksm.c index 37c63310bc4e..35ac6c741572 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -515,6 +515,28 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr) return (ret & VM_FAULT_OOM) ? -ENOMEM : 0; } +static bool vma_ksm_compatible(struct vm_area_struct *vma) +{ + if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE | VM_PFNMAP | + VM_IO | VM_DONTEXPAND | VM_HUGETLB | + VM_MIXEDMAP)) + return false; /* just ignore the advice */ + + if (vma_is_dax(vma)) + return false; + +#ifdef VM_SAO + if (vma->vm_flags & VM_SAO) + return false; +#endif +#ifdef VM_SPARC_ADI + if (vma->vm_flags & VM_SPARC_ADI) + return false; +#endif + + return true; +} + static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm, unsigned long addr) { @@ -1026,6 +1048,7 @@ mm_exiting: mm_slot_free(mm_slot_cache, mm_slot); clear_bit(MMF_VM_MERGEABLE, &mm->flags); + clear_bit(MMF_VM_MERGE_ANY, &mm->flags); mmdrop(mm); } else spin_unlock(&ksm_mmlist_lock); @@ -2408,6 +2431,7 @@ no_vmas: mm_slot_free(mm_slot_cache, mm_slot); clear_bit(MMF_VM_MERGEABLE, &mm->flags); + clear_bit(MMF_VM_MERGE_ANY, &mm->flags); mmap_read_unlock(mm); mmdrop(mm); } else { @@ -2485,6 +2509,66 @@ static int ksm_scan_thread(void *nothing) return 0; } +static void __ksm_add_vma(struct vm_area_struct *vma) +{ + unsigned long vm_flags = vma->vm_flags; + + if (vm_flags & VM_MERGEABLE) + return; + + if (vma_ksm_compatible(vma)) + vm_flags_set(vma, VM_MERGEABLE); +} + +/** + * ksm_add_vma - Mark vma as mergeable if compatible + * + * @vma: Pointer to vma + */ +void ksm_add_vma(struct vm_area_struct *vma) +{ + struct mm_struct *mm = vma->vm_mm; + + if (test_bit(MMF_VM_MERGE_ANY, &mm->flags)) + __ksm_add_vma(vma); +} + +static void ksm_add_vmas(struct mm_struct *mm) +{ + struct vm_area_struct *vma; + + VMA_ITERATOR(vmi, mm, 0); + for_each_vma(vmi, vma) + __ksm_add_vma(vma); +} + +/** + * ksm_enable_merge_any - Add mm to mm ksm list and enable merging on all + * compatible VMA's + * + * @mm: Pointer to mm + * + * Returns 0 on success, otherwise error code + */ +int ksm_enable_merge_any(struct mm_struct *mm) +{ + int err; + + if (test_bit(MMF_VM_MERGE_ANY, &mm->flags)) + return 0; + + if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { + err = __ksm_enter(mm); + if (err) + return err; + } + + set_bit(MMF_VM_MERGE_ANY, &mm->flags); + ksm_add_vmas(mm); + + return 0; +} + int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, unsigned long *vm_flags) { @@ -2493,25 +2577,10 @@ int ksm_madvise(struct vm_area_struct *vma, unsigned long start, switch (advice) { case MADV_MERGEABLE: - /* - * Be somewhat over-protective for now! - */ - if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE | - VM_PFNMAP | VM_IO | VM_DONTEXPAND | - VM_HUGETLB | VM_MIXEDMAP)) - return 0; /* just ignore the advice */ - - if (vma_is_dax(vma)) + if (vma->vm_flags & VM_MERGEABLE) return 0; - -#ifdef VM_SAO - if (*vm_flags & VM_SAO) + if (!vma_ksm_compatible(vma)) return 0; -#endif -#ifdef VM_SPARC_ADI - if (*vm_flags & VM_SPARC_ADI) - return 0; -#endif if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { err = __ksm_enter(mm); @@ -2615,6 +2684,7 @@ void __ksm_exit(struct mm_struct *mm) if (easy_to_free) { mm_slot_free(mm_slot_cache, mm_slot); + clear_bit(MMF_VM_MERGE_ANY, &mm->flags); clear_bit(MMF_VM_MERGEABLE, &mm->flags); mmdrop(mm); } else if (mm_slot) { diff --git a/mm/mmap.c b/mm/mmap.c index 790cc62c0038..51b6976fd525 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -46,6 +46,7 @@ #include #include #include +#include #include #include @@ -2729,6 +2730,7 @@ unmap_writable: if (file && vm_flags & VM_SHARED) mapping_unmap_writable(file->f_mapping); file = vma->vm_file; + ksm_add_vma(vma); expanded: perf_event_mmap(vma); @@ -3001,6 +3003,7 @@ static int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, goto mas_store_fail; mm->map_count++; + ksm_add_vma(vma); out: perf_event_mmap(vma); mm->total_vm += len >> PAGE_SHIFT; -- cgit v1.2.3 From d21077fbc2fc987c2e593c34dc3b4d84e546dc9f Mon Sep 17 00:00:00 2001 From: Stefan Roesch Date: Mon, 17 Apr 2023 22:13:41 -0700 Subject: mm: add new KSM process and sysfs knobs This adds the general_profit KSM sysfs knob and the process profit metric knobs to ksm_stat. 1) expose general_profit metric The documentation mentions a general profit metric, however this metric is not calculated. In addition the formula depends on the size of internal structures, which makes it more difficult for an administrator to make the calculation. Adding the metric for a better user experience. 2) document general_profit sysfs knob 3) calculate ksm process profit metric The ksm documentation mentions the process profit metric and how to calculate it. This adds the calculation of the metric. 4) mm: expose ksm process profit metric in ksm_stat This exposes the ksm process profit metric in /proc//ksm_stat. The documentation mentions the formula for the ksm process profit metric, however it does not calculate it. In addition the formula depends on the size of internal structures. So it makes sense to expose it. 5) document new procfs ksm knobs Link: https://lkml.kernel.org/r/20230418051342.1919757-3-shr@devkernel.io Signed-off-by: Stefan Roesch Reviewed-by: Bagas Sanjaya Acked-by: David Hildenbrand Cc: David Hildenbrand Cc: Johannes Weiner Cc: Michal Hocko Cc: Rik van Riel Signed-off-by: Andrew Morton --- Documentation/ABI/testing/sysfs-kernel-mm-ksm | 8 ++++++++ Documentation/admin-guide/mm/ksm.rst | 5 ++++- fs/proc/base.c | 3 +++ include/linux/ksm.h | 5 +++++ mm/ksm.c | 21 +++++++++++++++++++++ 5 files changed, 41 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-ksm b/Documentation/ABI/testing/sysfs-kernel-mm-ksm index d244674a9480..6041a025b65a 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-ksm +++ b/Documentation/ABI/testing/sysfs-kernel-mm-ksm @@ -51,3 +51,11 @@ Description: Control merging pages across different NUMA nodes. When it is set to 0 only pages from the same node are merged, otherwise pages from all nodes can be merged together (default). + +What: /sys/kernel/mm/ksm/general_profit +Date: April 2023 +KernelVersion: 6.4 +Contact: Linux memory management mailing list +Description: Measure how effective KSM is. + general_profit: how effective is KSM. The formula for the + calculation is in Documentation/admin-guide/mm/ksm.rst. diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst index eed51a910c94..551083a396fb 100644 --- a/Documentation/admin-guide/mm/ksm.rst +++ b/Documentation/admin-guide/mm/ksm.rst @@ -157,6 +157,8 @@ stable_node_chains_prune_millisecs The effectiveness of KSM and MADV_MERGEABLE is shown in ``/sys/kernel/mm/ksm/``: +general_profit + how effective is KSM. The calculation is explained below. pages_shared how many shared pages are being used pages_sharing @@ -207,7 +209,8 @@ several times, which are unprofitable memory consumed. ksm_rmap_items * sizeof(rmap_item). where ksm_merging_pages is shown under the directory ``/proc//``, - and ksm_rmap_items is shown in ``/proc//ksm_stat``. + and ksm_rmap_items is shown in ``/proc//ksm_stat``. The process profit + is also shown in ``/proc//ksm_stat`` as ksm_process_profit. From the perspective of application, a high ratio of ``ksm_rmap_items`` to ``ksm_merging_pages`` means a bad madvise-applied policy, so developers or diff --git a/fs/proc/base.c b/fs/proc/base.c index 5e0e0ccd47aa..96a6a08c8235 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -96,6 +96,7 @@ #include #include #include +#include #include #include "internal.h" #include "fd.h" @@ -3207,6 +3208,8 @@ static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, mm = get_task_mm(task); if (mm) { seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items); + seq_printf(m, "ksm_merging_pages %lu\n", mm->ksm_merging_pages); + seq_printf(m, "ksm_process_profit %ld\n", ksm_process_profit(mm)); mmput(mm); } diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 4647b0c70c12..7a9b76fb6c3f 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -68,6 +68,11 @@ void folio_migrate_ksm(struct folio *newfolio, struct folio *folio); void collect_procs_ksm(struct page *page, struct list_head *to_kill, int force_early); #endif + +#ifdef CONFIG_PROC_FS +long ksm_process_profit(struct mm_struct *); +#endif /* CONFIG_PROC_FS */ + #else /* !CONFIG_KSM */ static inline void ksm_add_vma(struct vm_area_struct *vma) diff --git a/mm/ksm.c b/mm/ksm.c index 35ac6c741572..9e48258985d2 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -3007,6 +3007,14 @@ static void wait_while_offlining(void) } #endif /* CONFIG_MEMORY_HOTREMOVE */ +#ifdef CONFIG_PROC_FS +long ksm_process_profit(struct mm_struct *mm) +{ + return mm->ksm_merging_pages * PAGE_SIZE - + mm->ksm_rmap_items * sizeof(struct ksm_rmap_item); +} +#endif /* CONFIG_PROC_FS */ + #ifdef CONFIG_SYSFS /* * This all compiles without CONFIG_SYSFS, but is a waste of space. @@ -3271,6 +3279,18 @@ static ssize_t pages_volatile_show(struct kobject *kobj, } KSM_ATTR_RO(pages_volatile); +static ssize_t general_profit_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + long general_profit; + + general_profit = ksm_pages_sharing * PAGE_SIZE - + ksm_rmap_items * sizeof(struct ksm_rmap_item); + + return sysfs_emit(buf, "%ld\n", general_profit); +} +KSM_ATTR_RO(general_profit); + static ssize_t stable_node_dups_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -3335,6 +3355,7 @@ static struct attribute *ksm_attrs[] = { &stable_node_dups_attr.attr, &stable_node_chains_prune_millisecs_attr.attr, &use_zero_pages_attr.attr, + &general_profit_attr.attr, NULL, }; -- cgit v1.2.3 From f724392415b3e1ae844d2766669c0d955fe9a17b Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Tue, 18 Apr 2023 22:22:02 -0700 Subject: hugetlb: pte_alloc_huge() to replace huge pte_alloc_map() Some architectures can have their hugetlb pages down at the lowest PTE level: their huge_pte_alloc() using pte_alloc_map(), but without any following pte_unmap(). Since none of these arches uses CONFIG_HIGHPTE, this is not seen as a problem at present; but would become a problem if forthcoming changes were to add an rcu_read_lock() into pte_offset_map(), with the rcu_read_unlock() expected in pte_unmap(). Similarly in their huge_pte_offset(): pte_offset_kernel() is good enough for that, but it's probably less confusing if we define pte_offset_huge() along with pte_alloc_huge(). Only define them without CONFIG_HIGHPTE: so there would be a build error to signal if ever more work is needed. For ease of development, define these now for 6.4-rc1, ahead of any use: then architectures can integrate patches using them, independent from mm. Link: https://lkml.kernel.org/r/ae9e7d98-8a3a-cfd9-4762-bcddffdf96cf@google.com Signed-off-by: Hugh Dickins Reviewed-by: Mike Kravetz Cc: Muchun Song Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 28703fe22386..cbe1e97a15a1 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -191,6 +191,23 @@ extern struct list_head huge_boot_pages; /* arch callbacks */ +#ifndef CONFIG_HIGHPTE +/* + * pte_offset_huge() and pte_alloc_huge() are helpers for those architectures + * which may go down to the lowest PTE level in their huge_pte_offset() and + * huge_pte_alloc(): to avoid reliance on pte_offset_map() without pte_unmap(). + */ +static inline pte_t *pte_offset_huge(pmd_t *pmd, unsigned long address) +{ + return pte_offset_kernel(pmd, address); +} +static inline pte_t *pte_alloc_huge(struct mm_struct *mm, pmd_t *pmd, + unsigned long address) +{ + return pte_alloc(mm, pmd) ? NULL : pte_offset_huge(pmd, address); +} +#endif + pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, unsigned long sz); /* -- cgit v1.2.3 From 74d7970febf7e9005375aeda0df821d2edffc9f7 Mon Sep 17 00:00:00 2001 From: Namjae Jeon Date: Fri, 21 Apr 2023 16:09:01 +0900 Subject: ksmbd: fix racy issue from using ->d_parent and ->d_name Al pointed out that ksmbd has racy issue from using ->d_parent and ->d_name in ksmbd_vfs_unlink and smb2_vfs_rename(). and use new lock_rename_child() to lock stable parent while underlying rename racy. Introduce vfs_path_parent_lookup helper to avoid out of share access and export vfs functions like the following ones to use vfs_path_parent_lookup(). - rename __lookup_hash() to lookup_one_qstr_excl(). - export lookup_one_qstr_excl(). - export getname_kernel() and putname(). vfs_path_parent_lookup() is used for parent lookup of destination file using absolute pathname given from FILE_RENAME_INFORMATION request. Signed-off-by: Namjae Jeon Signed-off-by: Steve French --- fs/ksmbd/smb2pdu.c | 147 ++++------------- fs/ksmbd/vfs.c | 435 ++++++++++++++++++++++---------------------------- fs/ksmbd/vfs.h | 19 +-- fs/ksmbd/vfs_cache.c | 5 +- fs/namei.c | 57 +++++-- include/linux/namei.h | 6 + 6 files changed, 283 insertions(+), 386 deletions(-) (limited to 'include/linux') diff --git a/fs/ksmbd/smb2pdu.c b/fs/ksmbd/smb2pdu.c index 215fea2883f6..bbc9e92935fb 100644 --- a/fs/ksmbd/smb2pdu.c +++ b/fs/ksmbd/smb2pdu.c @@ -2408,7 +2408,7 @@ static int smb2_creat(struct ksmbd_work *work, struct path *path, char *name, return rc; } - rc = ksmbd_vfs_kern_path(work, name, 0, path, 0); + rc = ksmbd_vfs_kern_path_locked(work, name, 0, path, 0); if (rc) { pr_err("cannot get linux path (%s), err = %d\n", name, rc); @@ -2699,8 +2699,10 @@ int smb2_open(struct ksmbd_work *work) goto err_out1; } - rc = ksmbd_vfs_kern_path(work, name, LOOKUP_NO_SYMLINKS, &path, 1); + rc = ksmbd_vfs_kern_path_locked(work, name, LOOKUP_NO_SYMLINKS, &path, 1); if (!rc) { + file_present = true; + if (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) { /* * If file exists with under flags, return access @@ -2709,7 +2711,6 @@ int smb2_open(struct ksmbd_work *work) if (req->CreateDisposition == FILE_OVERWRITE_IF_LE || req->CreateDisposition == FILE_OPEN_IF_LE) { rc = -EACCES; - path_put(&path); goto err_out; } @@ -2717,26 +2718,23 @@ int smb2_open(struct ksmbd_work *work) ksmbd_debug(SMB, "User does not have write permission\n"); rc = -EACCES; - path_put(&path); goto err_out; } } else if (d_is_symlink(path.dentry)) { rc = -EACCES; - path_put(&path); goto err_out; } - } - if (rc) { + file_present = true; + idmap = mnt_idmap(path.mnt); + } else { if (rc != -ENOENT) goto err_out; ksmbd_debug(SMB, "can not get linux path for %s, rc = %d\n", name, rc); rc = 0; - } else { - file_present = true; - idmap = mnt_idmap(path.mnt); } + if (stream_name) { if (req->CreateOptions & FILE_DIRECTORY_FILE_LE) { if (s_type == DATA_STREAM) { @@ -2864,8 +2862,9 @@ int smb2_open(struct ksmbd_work *work) if ((daccess & FILE_DELETE_LE) || (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE)) { - rc = ksmbd_vfs_may_delete(idmap, - path.dentry); + rc = inode_permission(idmap, + d_inode(path.dentry->d_parent), + MAY_EXEC | MAY_WRITE); if (rc) goto err_out; } @@ -3236,10 +3235,13 @@ int smb2_open(struct ksmbd_work *work) } err_out: - if (file_present || created) - path_put(&path); + if (file_present || created) { + inode_unlock(d_inode(path.dentry->d_parent)); + dput(path.dentry); + } ksmbd_revert_fsids(work); err_out1: + if (rc) { if (rc == -EINVAL) rsp->hdr.Status = STATUS_INVALID_PARAMETER; @@ -5390,44 +5392,19 @@ int smb2_echo(struct ksmbd_work *work) static int smb2_rename(struct ksmbd_work *work, struct ksmbd_file *fp, - struct mnt_idmap *idmap, struct smb2_file_rename_info *file_info, struct nls_table *local_nls) { struct ksmbd_share_config *share = fp->tcon->share_conf; - char *new_name = NULL, *abs_oldname = NULL, *old_name = NULL; - char *pathname = NULL; - struct path path; - bool file_present = true; - int rc; + char *new_name = NULL; + int rc, flags = 0; ksmbd_debug(SMB, "setting FILE_RENAME_INFO\n"); - pathname = kmalloc(PATH_MAX, GFP_KERNEL); - if (!pathname) - return -ENOMEM; - - abs_oldname = file_path(fp->filp, pathname, PATH_MAX); - if (IS_ERR(abs_oldname)) { - rc = -EINVAL; - goto out; - } - old_name = strrchr(abs_oldname, '/'); - if (old_name && old_name[1] != '\0') { - old_name++; - } else { - ksmbd_debug(SMB, "can't get last component in path %s\n", - abs_oldname); - rc = -ENOENT; - goto out; - } - new_name = smb2_get_name(file_info->FileName, le32_to_cpu(file_info->FileNameLength), local_nls); - if (IS_ERR(new_name)) { - rc = PTR_ERR(new_name); - goto out; - } + if (IS_ERR(new_name)) + return PTR_ERR(new_name); if (strchr(new_name, ':')) { int s_type; @@ -5453,7 +5430,7 @@ static int smb2_rename(struct ksmbd_work *work, if (rc) goto out; - rc = ksmbd_vfs_setxattr(idmap, + rc = ksmbd_vfs_setxattr(file_mnt_idmap(fp->filp), fp->filp->f_path.dentry, xattr_stream_name, NULL, 0, 0); @@ -5468,47 +5445,18 @@ static int smb2_rename(struct ksmbd_work *work, } ksmbd_debug(SMB, "new name %s\n", new_name); - rc = ksmbd_vfs_kern_path(work, new_name, LOOKUP_NO_SYMLINKS, &path, 1); - if (rc) { - if (rc != -ENOENT) - goto out; - file_present = false; - } else { - path_put(&path); - } - if (ksmbd_share_veto_filename(share, new_name)) { rc = -ENOENT; ksmbd_debug(SMB, "Can't rename vetoed file: %s\n", new_name); goto out; } - if (file_info->ReplaceIfExists) { - if (file_present) { - rc = ksmbd_vfs_remove_file(work, new_name); - if (rc) { - if (rc != -ENOTEMPTY) - rc = -EINVAL; - ksmbd_debug(SMB, "cannot delete %s, rc %d\n", - new_name, rc); - goto out; - } - } - } else { - if (file_present && - strncmp(old_name, path.dentry->d_name.name, strlen(old_name))) { - rc = -EEXIST; - ksmbd_debug(SMB, - "cannot rename already existing file\n"); - goto out; - } - } + if (!file_info->ReplaceIfExists) + flags = RENAME_NOREPLACE; - rc = ksmbd_vfs_fp_rename(work, fp, new_name); + rc = ksmbd_vfs_rename(work, &fp->filp->f_path, new_name, flags); out: - kfree(pathname); - if (!IS_ERR(new_name)) - kfree(new_name); + kfree(new_name); return rc; } @@ -5548,18 +5496,17 @@ static int smb2_create_link(struct ksmbd_work *work, } ksmbd_debug(SMB, "target name is %s\n", target_name); - rc = ksmbd_vfs_kern_path(work, link_name, LOOKUP_NO_SYMLINKS, &path, 0); + rc = ksmbd_vfs_kern_path_locked(work, link_name, LOOKUP_NO_SYMLINKS, + &path, 0); if (rc) { if (rc != -ENOENT) goto out; file_present = false; - } else { - path_put(&path); } if (file_info->ReplaceIfExists) { if (file_present) { - rc = ksmbd_vfs_remove_file(work, link_name); + rc = ksmbd_vfs_remove_file(work, &path); if (rc) { rc = -EINVAL; ksmbd_debug(SMB, "cannot delete %s\n", @@ -5579,6 +5526,10 @@ static int smb2_create_link(struct ksmbd_work *work, if (rc) rc = -EINVAL; out: + if (file_present) { + inode_unlock(d_inode(path.dentry->d_parent)); + path_put(&path); + } if (!IS_ERR(link_name)) kfree(link_name); kfree(pathname); @@ -5756,12 +5707,6 @@ static int set_rename_info(struct ksmbd_work *work, struct ksmbd_file *fp, struct smb2_file_rename_info *rename_info, unsigned int buf_len) { - struct mnt_idmap *idmap; - struct ksmbd_file *parent_fp; - struct dentry *parent; - struct dentry *dentry = fp->filp->f_path.dentry; - int ret; - if (!(fp->daccess & FILE_DELETE_LE)) { pr_err("no right to delete : 0x%x\n", fp->daccess); return -EACCES; @@ -5771,32 +5716,10 @@ static int set_rename_info(struct ksmbd_work *work, struct ksmbd_file *fp, le32_to_cpu(rename_info->FileNameLength)) return -EINVAL; - idmap = file_mnt_idmap(fp->filp); - if (ksmbd_stream_fd(fp)) - goto next; - - parent = dget_parent(dentry); - ret = ksmbd_vfs_lock_parent(idmap, parent, dentry); - if (ret) { - dput(parent); - return ret; - } - - parent_fp = ksmbd_lookup_fd_inode(d_inode(parent)); - inode_unlock(d_inode(parent)); - dput(parent); + if (!le32_to_cpu(rename_info->FileNameLength)) + return -EINVAL; - if (parent_fp) { - if (parent_fp->daccess & FILE_DELETE_LE) { - pr_err("parent dir is opened with delete access\n"); - ksmbd_fd_put(work, parent_fp); - return -ESHARE; - } - ksmbd_fd_put(work, parent_fp); - } -next: - return smb2_rename(work, fp, idmap, rename_info, - work->conn->local_nls); + return smb2_rename(work, fp, rename_info, work->conn->local_nls); } static int set_file_disposition_info(struct ksmbd_file *fp, diff --git a/fs/ksmbd/vfs.c b/fs/ksmbd/vfs.c index cef07d7fb7dc..778c152708e4 100644 --- a/fs/ksmbd/vfs.c +++ b/fs/ksmbd/vfs.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "glob.h" #include "oplock.h" @@ -35,19 +36,6 @@ #include "mgmt/user_session.h" #include "mgmt/user_config.h" -static char *extract_last_component(char *path) -{ - char *p = strrchr(path, '/'); - - if (p && p[1] != '\0') { - *p = '\0'; - p++; - } else { - p = NULL; - } - return p; -} - static void ksmbd_vfs_inherit_owner(struct ksmbd_work *work, struct inode *parent_inode, struct inode *inode) @@ -61,65 +49,77 @@ static void ksmbd_vfs_inherit_owner(struct ksmbd_work *work, /** * ksmbd_vfs_lock_parent() - lock parent dentry if it is stable - * - * the parent dentry got by dget_parent or @parent could be - * unstable, we try to lock a parent inode and lookup the - * child dentry again. - * - * the reference count of @parent isn't incremented. */ -int ksmbd_vfs_lock_parent(struct mnt_idmap *idmap, struct dentry *parent, - struct dentry *child) +int ksmbd_vfs_lock_parent(struct dentry *parent, struct dentry *child) { - struct dentry *dentry; - int ret = 0; - inode_lock_nested(d_inode(parent), I_MUTEX_PARENT); - dentry = lookup_one(idmap, child->d_name.name, parent, - child->d_name.len); - if (IS_ERR(dentry)) { - ret = PTR_ERR(dentry); - goto out_err; - } - - if (dentry != child) { - ret = -ESTALE; - dput(dentry); - goto out_err; + if (child->d_parent != parent) { + inode_unlock(d_inode(parent)); + return -ENOENT; } - dput(dentry); return 0; -out_err: - inode_unlock(d_inode(parent)); - return ret; } -int ksmbd_vfs_may_delete(struct mnt_idmap *idmap, - struct dentry *dentry) +static int ksmbd_vfs_path_lookup_locked(struct ksmbd_share_config *share_conf, + char *pathname, unsigned int flags, + struct path *path) { - struct dentry *parent; - int ret; + struct qstr last; + struct filename *filename; + struct path *root_share_path = &share_conf->vfs_path; + int err, type; + struct path parent_path; + struct dentry *d; + + if (pathname[0] == '\0') { + pathname = share_conf->path; + root_share_path = NULL; + } else { + flags |= LOOKUP_BENEATH; + } - parent = dget_parent(dentry); - ret = ksmbd_vfs_lock_parent(idmap, parent, dentry); - if (ret) { - dput(parent); - return ret; + filename = getname_kernel(pathname); + if (IS_ERR(filename)) + return PTR_ERR(filename); + + err = vfs_path_parent_lookup(filename, flags, + &parent_path, &last, &type, + root_share_path); + putname(filename); + if (err) + return err; + + if (unlikely(type != LAST_NORM)) { + path_put(&parent_path); + return -ENOENT; } - ret = inode_permission(idmap, d_inode(parent), - MAY_EXEC | MAY_WRITE); + inode_lock_nested(parent_path.dentry->d_inode, I_MUTEX_PARENT); + d = lookup_one_qstr_excl(&last, parent_path.dentry, 0); + if (IS_ERR(d)) + goto err_out; - inode_unlock(d_inode(parent)); - dput(parent); - return ret; + if (d_is_negative(d)) { + dput(d); + goto err_out; + } + + path->dentry = d; + path->mnt = share_conf->vfs_path.mnt; + path_put(&parent_path); + + return 0; + +err_out: + inode_unlock(parent_path.dentry->d_inode); + path_put(&parent_path); + return -ENOENT; } int ksmbd_vfs_query_maximal_access(struct mnt_idmap *idmap, struct dentry *dentry, __le32 *daccess) { - struct dentry *parent; int ret = 0; *daccess = cpu_to_le32(FILE_READ_ATTRIBUTES | READ_CONTROL); @@ -136,18 +136,9 @@ int ksmbd_vfs_query_maximal_access(struct mnt_idmap *idmap, if (!inode_permission(idmap, d_inode(dentry), MAY_OPEN | MAY_EXEC)) *daccess |= FILE_EXECUTE_LE; - parent = dget_parent(dentry); - ret = ksmbd_vfs_lock_parent(idmap, parent, dentry); - if (ret) { - dput(parent); - return ret; - } - - if (!inode_permission(idmap, d_inode(parent), MAY_EXEC | MAY_WRITE)) + if (!inode_permission(idmap, d_inode(dentry->d_parent), MAY_EXEC | MAY_WRITE)) *daccess |= FILE_DELETE_LE; - inode_unlock(d_inode(parent)); - dput(parent); return ret; } @@ -580,54 +571,32 @@ int ksmbd_vfs_fsync(struct ksmbd_work *work, u64 fid, u64 p_id) * * Return: 0 on success, otherwise error */ -int ksmbd_vfs_remove_file(struct ksmbd_work *work, char *name) +int ksmbd_vfs_remove_file(struct ksmbd_work *work, const struct path *path) { struct mnt_idmap *idmap; - struct path path; - struct dentry *parent; + struct dentry *parent = path->dentry->d_parent; int err; if (ksmbd_override_fsids(work)) return -ENOMEM; - err = ksmbd_vfs_kern_path(work, name, LOOKUP_NO_SYMLINKS, &path, false); - if (err) { - ksmbd_debug(VFS, "can't get %s, err %d\n", name, err); - ksmbd_revert_fsids(work); - return err; - } - - idmap = mnt_idmap(path.mnt); - parent = dget_parent(path.dentry); - err = ksmbd_vfs_lock_parent(idmap, parent, path.dentry); - if (err) { - dput(parent); - path_put(&path); - ksmbd_revert_fsids(work); - return err; - } - - if (!d_inode(path.dentry)->i_nlink) { + if (!d_inode(path->dentry)->i_nlink) { err = -ENOENT; goto out_err; } - if (S_ISDIR(d_inode(path.dentry)->i_mode)) { - err = vfs_rmdir(idmap, d_inode(parent), path.dentry); + idmap = mnt_idmap(path->mnt); + if (S_ISDIR(d_inode(path->dentry)->i_mode)) { + err = vfs_rmdir(idmap, d_inode(parent), path->dentry); if (err && err != -ENOTEMPTY) - ksmbd_debug(VFS, "%s: rmdir failed, err %d\n", name, - err); + ksmbd_debug(VFS, "rmdir failed, err %d\n", err); } else { - err = vfs_unlink(idmap, d_inode(parent), path.dentry, NULL); + err = vfs_unlink(idmap, d_inode(parent), path->dentry, NULL); if (err) - ksmbd_debug(VFS, "%s: unlink failed, err %d\n", name, - err); + ksmbd_debug(VFS, "unlink failed, err %d\n", err); } out_err: - inode_unlock(d_inode(parent)); - dput(parent); - path_put(&path); ksmbd_revert_fsids(work); return err; } @@ -686,149 +655,114 @@ out1: return err; } -static int ksmbd_validate_entry_in_use(struct dentry *src_dent) +int ksmbd_vfs_rename(struct ksmbd_work *work, const struct path *old_path, + char *newname, int flags) { - struct dentry *dst_dent; - - spin_lock(&src_dent->d_lock); - list_for_each_entry(dst_dent, &src_dent->d_subdirs, d_child) { - struct ksmbd_file *child_fp; + struct dentry *old_parent, *new_dentry, *trap; + struct dentry *old_child = old_path->dentry; + struct path new_path; + struct qstr new_last; + struct renamedata rd; + struct filename *to; + struct ksmbd_share_config *share_conf = work->tcon->share_conf; + struct ksmbd_file *parent_fp; + int new_type; + int err, lookup_flags = LOOKUP_NO_SYMLINKS; - if (d_really_is_negative(dst_dent)) - continue; + if (ksmbd_override_fsids(work)) + return -ENOMEM; - child_fp = ksmbd_lookup_fd_inode(d_inode(dst_dent)); - if (child_fp) { - spin_unlock(&src_dent->d_lock); - ksmbd_debug(VFS, "Forbid rename, sub file/dir is in use\n"); - return -EACCES; - } + to = getname_kernel(newname); + if (IS_ERR(to)) { + err = PTR_ERR(to); + goto revert_fsids; } - spin_unlock(&src_dent->d_lock); - return 0; -} +retry: + err = vfs_path_parent_lookup(to, lookup_flags | LOOKUP_BENEATH, + &new_path, &new_last, &new_type, + &share_conf->vfs_path); + if (err) + goto out1; -static int __ksmbd_vfs_rename(struct ksmbd_work *work, - struct mnt_idmap *src_idmap, - struct dentry *src_dent_parent, - struct dentry *src_dent, - struct mnt_idmap *dst_idmap, - struct dentry *dst_dent_parent, - struct dentry *trap_dent, - char *dst_name) -{ - struct dentry *dst_dent; - int err; + if (old_path->mnt != new_path.mnt) { + err = -EXDEV; + goto out2; + } - if (!work->tcon->posix_extensions) { - err = ksmbd_validate_entry_in_use(src_dent); - if (err) - return err; + trap = lock_rename_child(old_child, new_path.dentry); + + old_parent = dget(old_child->d_parent); + if (d_unhashed(old_child)) { + err = -EINVAL; + goto out3; } - if (d_really_is_negative(src_dent_parent)) - return -ENOENT; - if (d_really_is_negative(dst_dent_parent)) - return -ENOENT; - if (d_really_is_negative(src_dent)) - return -ENOENT; - if (src_dent == trap_dent) - return -EINVAL; + parent_fp = ksmbd_lookup_fd_inode(d_inode(old_child->d_parent)); + if (parent_fp) { + if (parent_fp->daccess & FILE_DELETE_LE) { + pr_err("parent dir is opened with delete access\n"); + err = -ESHARE; + ksmbd_fd_put(work, parent_fp); + goto out3; + } + ksmbd_fd_put(work, parent_fp); + } - if (ksmbd_override_fsids(work)) - return -ENOMEM; + new_dentry = lookup_one_qstr_excl(&new_last, new_path.dentry, + lookup_flags | LOOKUP_RENAME_TARGET); + if (IS_ERR(new_dentry)) { + err = PTR_ERR(new_dentry); + goto out3; + } - dst_dent = lookup_one(dst_idmap, dst_name, - dst_dent_parent, strlen(dst_name)); - err = PTR_ERR(dst_dent); - if (IS_ERR(dst_dent)) { - pr_err("lookup failed %s [%d]\n", dst_name, err); - goto out; + if (d_is_symlink(new_dentry)) { + err = -EACCES; + goto out4; } - err = -ENOTEMPTY; - if (dst_dent != trap_dent && !d_really_is_positive(dst_dent)) { - struct renamedata rd = { - .old_mnt_idmap = src_idmap, - .old_dir = d_inode(src_dent_parent), - .old_dentry = src_dent, - .new_mnt_idmap = dst_idmap, - .new_dir = d_inode(dst_dent_parent), - .new_dentry = dst_dent, - }; - err = vfs_rename(&rd); + if ((flags & RENAME_NOREPLACE) && d_is_positive(new_dentry)) { + err = -EEXIST; + goto out4; } - if (err) - pr_err("vfs_rename failed err %d\n", err); - if (dst_dent) - dput(dst_dent); -out: - ksmbd_revert_fsids(work); - return err; -} -int ksmbd_vfs_fp_rename(struct ksmbd_work *work, struct ksmbd_file *fp, - char *newname) -{ - struct mnt_idmap *idmap; - struct path dst_path; - struct dentry *src_dent_parent, *dst_dent_parent; - struct dentry *src_dent, *trap_dent, *src_child; - char *dst_name; - int err; + if (old_child == trap) { + err = -EINVAL; + goto out4; + } - dst_name = extract_last_component(newname); - if (!dst_name) { - dst_name = newname; - newname = ""; + if (new_dentry == trap) { + err = -ENOTEMPTY; + goto out4; } - src_dent_parent = dget_parent(fp->filp->f_path.dentry); - src_dent = fp->filp->f_path.dentry; + rd.old_mnt_idmap = mnt_idmap(old_path->mnt), + rd.old_dir = d_inode(old_parent), + rd.old_dentry = old_child, + rd.new_mnt_idmap = mnt_idmap(new_path.mnt), + rd.new_dir = new_path.dentry->d_inode, + rd.new_dentry = new_dentry, + rd.flags = flags, + err = vfs_rename(&rd); + if (err) + ksmbd_debug(VFS, "vfs_rename failed err %d\n", err); - err = ksmbd_vfs_kern_path(work, newname, - LOOKUP_NO_SYMLINKS | LOOKUP_DIRECTORY, - &dst_path, false); - if (err) { - ksmbd_debug(VFS, "Cannot get path for %s [%d]\n", newname, err); - goto out; +out4: + dput(new_dentry); +out3: + dput(old_parent); + unlock_rename(old_parent, new_path.dentry); +out2: + path_put(&new_path); + + if (retry_estale(err, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; } - dst_dent_parent = dst_path.dentry; - - trap_dent = lock_rename(src_dent_parent, dst_dent_parent); - dget(src_dent); - dget(dst_dent_parent); - idmap = file_mnt_idmap(fp->filp); - src_child = lookup_one(idmap, src_dent->d_name.name, src_dent_parent, - src_dent->d_name.len); - if (IS_ERR(src_child)) { - err = PTR_ERR(src_child); - goto out_lock; - } - - if (src_child != src_dent) { - err = -ESTALE; - dput(src_child); - goto out_lock; - } - dput(src_child); - - err = __ksmbd_vfs_rename(work, - idmap, - src_dent_parent, - src_dent, - mnt_idmap(dst_path.mnt), - dst_dent_parent, - trap_dent, - dst_name); -out_lock: - dput(src_dent); - dput(dst_dent_parent); - unlock_rename(src_dent_parent, dst_dent_parent); - path_put(&dst_path); -out: - dput(src_dent_parent); +out1: + putname(to); +revert_fsids: + ksmbd_revert_fsids(work); return err; } @@ -1079,14 +1013,16 @@ int ksmbd_vfs_remove_xattr(struct mnt_idmap *idmap, return vfs_removexattr(idmap, dentry, attr_name); } -int ksmbd_vfs_unlink(struct mnt_idmap *idmap, - struct dentry *dir, struct dentry *dentry) +int ksmbd_vfs_unlink(struct file *filp) { int err = 0; + struct dentry *dir, *dentry = filp->f_path.dentry; + struct mnt_idmap *idmap = file_mnt_idmap(filp); - err = ksmbd_vfs_lock_parent(idmap, dir, dentry); + dir = dget_parent(dentry); + err = ksmbd_vfs_lock_parent(dir, dentry); if (err) - return err; + goto out; dget(dentry); if (S_ISDIR(d_inode(dentry)->i_mode)) @@ -1098,6 +1034,8 @@ int ksmbd_vfs_unlink(struct mnt_idmap *idmap, inode_unlock(d_inode(dir)); if (err) ksmbd_debug(VFS, "failed to delete, err %d\n", err); +out: + dput(dir); return err; } @@ -1200,7 +1138,7 @@ static int ksmbd_vfs_lookup_in_dir(const struct path *dir, char *name, } /** - * ksmbd_vfs_kern_path() - lookup a file and get path info + * ksmbd_vfs_kern_path_locked() - lookup a file and get path info * @name: file path that is relative to share * @flags: lookup flags * @path: if lookup succeed, return path info @@ -1208,24 +1146,20 @@ static int ksmbd_vfs_lookup_in_dir(const struct path *dir, char *name, * * Return: 0 on success, otherwise error */ -int ksmbd_vfs_kern_path(struct ksmbd_work *work, char *name, - unsigned int flags, struct path *path, bool caseless) +int ksmbd_vfs_kern_path_locked(struct ksmbd_work *work, char *name, + unsigned int flags, struct path *path, + bool caseless) { struct ksmbd_share_config *share_conf = work->tcon->share_conf; int err; + struct path parent_path; - flags |= LOOKUP_BENEATH; - err = vfs_path_lookup(share_conf->vfs_path.dentry, - share_conf->vfs_path.mnt, - name, - flags, - path); + err = ksmbd_vfs_path_lookup_locked(share_conf, name, flags, path); if (!err) - return 0; + return err; if (caseless) { char *filepath; - struct path parent; size_t path_len, remain_len; filepath = kstrdup(name, GFP_KERNEL); @@ -1235,10 +1169,10 @@ int ksmbd_vfs_kern_path(struct ksmbd_work *work, char *name, path_len = strlen(filepath); remain_len = path_len; - parent = share_conf->vfs_path; - path_get(&parent); + parent_path = share_conf->vfs_path; + path_get(&parent_path); - while (d_can_lookup(parent.dentry)) { + while (d_can_lookup(parent_path.dentry)) { char *filename = filepath + path_len - remain_len; char *next = strchrnul(filename, '/'); size_t filename_len = next - filename; @@ -1247,12 +1181,11 @@ int ksmbd_vfs_kern_path(struct ksmbd_work *work, char *name, if (filename_len == 0) break; - err = ksmbd_vfs_lookup_in_dir(&parent, filename, + err = ksmbd_vfs_lookup_in_dir(&parent_path, filename, filename_len, work->conn->um); - path_put(&parent); if (err) - goto out; + goto out2; next[0] = '\0'; @@ -1260,23 +1193,31 @@ int ksmbd_vfs_kern_path(struct ksmbd_work *work, char *name, share_conf->vfs_path.mnt, filepath, flags, - &parent); + path); if (err) - goto out; - else if (is_last) { - *path = parent; - goto out; - } + goto out2; + else if (is_last) + goto out1; + path_put(&parent_path); + parent_path = *path; next[0] = '/'; remain_len -= filename_len + 1; } - path_put(&parent); err = -EINVAL; -out: +out2: + path_put(&parent_path); +out1: kfree(filepath); } + + if (!err) { + err = ksmbd_vfs_lock_parent(parent_path.dentry, path->dentry); + if (err) + dput(path->dentry); + path_put(&parent_path); + } return err; } diff --git a/fs/ksmbd/vfs.h b/fs/ksmbd/vfs.h index 9d676ab0cd25..a4ae89f3230d 100644 --- a/fs/ksmbd/vfs.h +++ b/fs/ksmbd/vfs.h @@ -71,9 +71,7 @@ struct ksmbd_kstat { __le32 file_attributes; }; -int ksmbd_vfs_lock_parent(struct mnt_idmap *idmap, struct dentry *parent, - struct dentry *child); -int ksmbd_vfs_may_delete(struct mnt_idmap *idmap, struct dentry *dentry); +int ksmbd_vfs_lock_parent(struct dentry *parent, struct dentry *child); int ksmbd_vfs_query_maximal_access(struct mnt_idmap *idmap, struct dentry *dentry, __le32 *daccess); int ksmbd_vfs_create(struct ksmbd_work *work, const char *name, umode_t mode); @@ -84,12 +82,12 @@ int ksmbd_vfs_write(struct ksmbd_work *work, struct ksmbd_file *fp, char *buf, size_t count, loff_t *pos, bool sync, ssize_t *written); int ksmbd_vfs_fsync(struct ksmbd_work *work, u64 fid, u64 p_id); -int ksmbd_vfs_remove_file(struct ksmbd_work *work, char *name); +int ksmbd_vfs_remove_file(struct ksmbd_work *work, const struct path *path); int ksmbd_vfs_link(struct ksmbd_work *work, const char *oldname, const char *newname); int ksmbd_vfs_getattr(const struct path *path, struct kstat *stat); -int ksmbd_vfs_fp_rename(struct ksmbd_work *work, struct ksmbd_file *fp, - char *newname); +int ksmbd_vfs_rename(struct ksmbd_work *work, const struct path *old_path, + char *newname, int flags); int ksmbd_vfs_truncate(struct ksmbd_work *work, struct ksmbd_file *fp, loff_t size); struct srv_copychunk; @@ -116,9 +114,9 @@ int ksmbd_vfs_xattr_stream_name(char *stream_name, char **xattr_stream_name, size_t *xattr_stream_name_size, int s_type); int ksmbd_vfs_remove_xattr(struct mnt_idmap *idmap, struct dentry *dentry, char *attr_name); -int ksmbd_vfs_kern_path(struct ksmbd_work *work, - char *name, unsigned int flags, struct path *path, - bool caseless); +int ksmbd_vfs_kern_path_locked(struct ksmbd_work *work, char *name, + unsigned int flags, struct path *path, + bool caseless); struct dentry *ksmbd_vfs_kern_path_create(struct ksmbd_work *work, const char *name, unsigned int flags, @@ -131,8 +129,7 @@ struct file_allocated_range_buffer; int ksmbd_vfs_fqar_lseek(struct ksmbd_file *fp, loff_t start, loff_t length, struct file_allocated_range_buffer *ranges, unsigned int in_count, unsigned int *out_count); -int ksmbd_vfs_unlink(struct mnt_idmap *idmap, struct dentry *dir, - struct dentry *dentry); +int ksmbd_vfs_unlink(struct file *filp); void *ksmbd_vfs_init_kstat(char **p, struct ksmbd_kstat *ksmbd_kstat); int ksmbd_vfs_fill_dentry_attrs(struct ksmbd_work *work, struct mnt_idmap *idmap, diff --git a/fs/ksmbd/vfs_cache.c b/fs/ksmbd/vfs_cache.c index 054a7d2e0f48..2d0138e72d78 100644 --- a/fs/ksmbd/vfs_cache.c +++ b/fs/ksmbd/vfs_cache.c @@ -244,7 +244,6 @@ void ksmbd_release_inode_hash(void) static void __ksmbd_inode_close(struct ksmbd_file *fp) { - struct dentry *dir, *dentry; struct ksmbd_inode *ci = fp->f_ci; int err; struct file *filp; @@ -263,11 +262,9 @@ static void __ksmbd_inode_close(struct ksmbd_file *fp) if (atomic_dec_and_test(&ci->m_count)) { write_lock(&ci->m_lock); if (ci->m_flags & (S_DEL_ON_CLS | S_DEL_PENDING)) { - dentry = filp->f_path.dentry; - dir = dentry->d_parent; ci->m_flags &= ~(S_DEL_ON_CLS | S_DEL_PENDING); write_unlock(&ci->m_lock); - ksmbd_vfs_unlink(file_mnt_idmap(filp), dir, dentry); + ksmbd_vfs_unlink(filp); write_lock(&ci->m_lock); } write_unlock(&ci->m_lock); diff --git a/fs/namei.c b/fs/namei.c index 6bc1964e2214..c1d59ae5af83 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -254,6 +254,7 @@ getname_kernel(const char * filename) return result; } +EXPORT_SYMBOL(getname_kernel); void putname(struct filename *name) { @@ -271,6 +272,7 @@ void putname(struct filename *name) } else __putname(name); } +EXPORT_SYMBOL(putname); /** * check_acl - perform ACL permission checking @@ -1581,8 +1583,9 @@ static struct dentry *lookup_dcache(const struct qstr *name, * when directory is guaranteed to have no in-lookup children * at all. */ -static struct dentry *__lookup_hash(const struct qstr *name, - struct dentry *base, unsigned int flags) +struct dentry *lookup_one_qstr_excl(const struct qstr *name, + struct dentry *base, + unsigned int flags) { struct dentry *dentry = lookup_dcache(name, base, flags); struct dentry *old; @@ -1606,6 +1609,7 @@ static struct dentry *__lookup_hash(const struct qstr *name, } return dentry; } +EXPORT_SYMBOL(lookup_one_qstr_excl); static struct dentry *lookup_fast(struct nameidata *nd) { @@ -2532,16 +2536,17 @@ static int path_parentat(struct nameidata *nd, unsigned flags, } /* Note: this does not consume "name" */ -static int filename_parentat(int dfd, struct filename *name, - unsigned int flags, struct path *parent, - struct qstr *last, int *type) +static int __filename_parentat(int dfd, struct filename *name, + unsigned int flags, struct path *parent, + struct qstr *last, int *type, + const struct path *root) { int retval; struct nameidata nd; if (IS_ERR(name)) return PTR_ERR(name); - set_nameidata(&nd, dfd, name, NULL); + set_nameidata(&nd, dfd, name, root); retval = path_parentat(&nd, flags | LOOKUP_RCU, parent); if (unlikely(retval == -ECHILD)) retval = path_parentat(&nd, flags, parent); @@ -2556,6 +2561,13 @@ static int filename_parentat(int dfd, struct filename *name, return retval; } +static int filename_parentat(int dfd, struct filename *name, + unsigned int flags, struct path *parent, + struct qstr *last, int *type) +{ + return __filename_parentat(dfd, name, flags, parent, last, type, NULL); +} + /* does lookup, returns the object with parent locked */ static struct dentry *__kern_path_locked(struct filename *name, struct path *path) { @@ -2571,7 +2583,7 @@ static struct dentry *__kern_path_locked(struct filename *name, struct path *pat return ERR_PTR(-EINVAL); } inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT); - d = __lookup_hash(&last, path->dentry, 0); + d = lookup_one_qstr_excl(&last, path->dentry, 0); if (IS_ERR(d)) { inode_unlock(path->dentry->d_inode); path_put(path); @@ -2599,6 +2611,24 @@ int kern_path(const char *name, unsigned int flags, struct path *path) } EXPORT_SYMBOL(kern_path); +/** + * vfs_path_parent_lookup - lookup a parent path relative to a dentry-vfsmount pair + * @filename: filename structure + * @flags: lookup flags + * @parent: pointer to struct path to fill + * @last: last component + * @type: type of the last component + * @root: pointer to struct path of the base directory + */ +int vfs_path_parent_lookup(struct filename *filename, unsigned int flags, + struct path *parent, struct qstr *last, int *type, + const struct path *root) +{ + return __filename_parentat(AT_FDCWD, filename, flags, parent, last, + type, root); +} +EXPORT_SYMBOL(vfs_path_parent_lookup); + /** * vfs_path_lookup - lookup a file path relative to a dentry-vfsmount pair * @dentry: pointer to dentry of the base directory @@ -3852,7 +3882,8 @@ static struct dentry *filename_create(int dfd, struct filename *name, if (last.name[last.len] && !want_dir) create_flags = 0; inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT); - dentry = __lookup_hash(&last, path->dentry, reval_flag | create_flags); + dentry = lookup_one_qstr_excl(&last, path->dentry, + reval_flag | create_flags); if (IS_ERR(dentry)) goto unlock; @@ -4212,7 +4243,7 @@ retry: goto exit2; inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT); - dentry = __lookup_hash(&last, path.dentry, lookup_flags); + dentry = lookup_one_qstr_excl(&last, path.dentry, lookup_flags); error = PTR_ERR(dentry); if (IS_ERR(dentry)) goto exit3; @@ -4345,7 +4376,7 @@ retry: goto exit2; retry_deleg: inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT); - dentry = __lookup_hash(&last, path.dentry, lookup_flags); + dentry = lookup_one_qstr_excl(&last, path.dentry, lookup_flags); error = PTR_ERR(dentry); if (!IS_ERR(dentry)) { @@ -4909,7 +4940,8 @@ retry: retry_deleg: trap = lock_rename(new_path.dentry, old_path.dentry); - old_dentry = __lookup_hash(&old_last, old_path.dentry, lookup_flags); + old_dentry = lookup_one_qstr_excl(&old_last, old_path.dentry, + lookup_flags); error = PTR_ERR(old_dentry); if (IS_ERR(old_dentry)) goto exit3; @@ -4917,7 +4949,8 @@ retry_deleg: error = -ENOENT; if (d_is_negative(old_dentry)) goto exit4; - new_dentry = __lookup_hash(&new_last, new_path.dentry, lookup_flags | target_flags); + new_dentry = lookup_one_qstr_excl(&new_last, new_path.dentry, + lookup_flags | target_flags); error = PTR_ERR(new_dentry); if (IS_ERR(new_dentry)) goto exit4; diff --git a/include/linux/namei.h b/include/linux/namei.h index 5864e4d82e56..1463cbda4888 100644 --- a/include/linux/namei.h +++ b/include/linux/namei.h @@ -57,12 +57,18 @@ static inline int user_path_at(int dfd, const char __user *name, unsigned flags, return user_path_at_empty(dfd, name, flags, path, NULL); } +struct dentry *lookup_one_qstr_excl(const struct qstr *name, + struct dentry *base, + unsigned int flags); extern int kern_path(const char *, unsigned, struct path *); extern struct dentry *kern_path_create(int, const char *, struct path *, unsigned int); extern struct dentry *user_path_create(int, const char __user *, struct path *, unsigned int); extern void done_path_create(struct path *, struct dentry *); extern struct dentry *kern_path_locked(const char *, struct path *); +int vfs_path_parent_lookup(struct filename *filename, unsigned int flags, + struct path *parent, struct qstr *last, int *type, + const struct path *root); int vfs_path_lookup(struct dentry *, struct vfsmount *, const char *, unsigned int, struct path *); -- cgit v1.2.3 From 96928d9032a7c34f12a88df879665562bcebf59a Mon Sep 17 00:00:00 2001 From: Sergey Senozhatsky Date: Sat, 15 Apr 2023 19:01:10 +0900 Subject: seq_buf: Add seq_buf_do_printk() helper Sometimes we use seq_buf to format a string buffer, which we then pass to printk(). However, in certain situations the seq_buf string buffer can get too big, exceeding the PRINTKRB_RECORD_MAX bytes limit, and causing printk() to truncate the string. Add a new seq_buf helper. This helper prints the seq_buf string buffer line by line, using \n as a delimiter, rather than passing the whole string buffer to printk() at once. Link: https://lkml.kernel.org/r/20230415100110.1419872-1-senozhatsky@chromium.org Cc: Andrew Morton Signed-off-by: Sergey Senozhatsky Reviewed-by: Petr Mladek Tested-by: Yosry Ahmed Signed-off-by: Steven Rostedt (Google) --- include/linux/seq_buf.h | 2 ++ lib/seq_buf.c | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) (limited to 'include/linux') diff --git a/include/linux/seq_buf.h b/include/linux/seq_buf.h index 5b31c5147969..515d7fcb9634 100644 --- a/include/linux/seq_buf.h +++ b/include/linux/seq_buf.h @@ -159,4 +159,6 @@ extern int seq_buf_bprintf(struct seq_buf *s, const char *fmt, const u32 *binary); #endif +void seq_buf_do_printk(struct seq_buf *s, const char *lvl); + #endif /* _LINUX_SEQ_BUF_H */ diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 0a68f7aa85d6..45c450f423fa 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -93,6 +93,38 @@ int seq_buf_printf(struct seq_buf *s, const char *fmt, ...) } EXPORT_SYMBOL_GPL(seq_buf_printf); +/** + * seq_buf_do_printk - printk seq_buf line by line + * @s: seq_buf descriptor + * @lvl: printk level + * + * printk()-s a multi-line sequential buffer line by line. The function + * makes sure that the buffer in @s is nul terminated and safe to read + * as a string. + */ +void seq_buf_do_printk(struct seq_buf *s, const char *lvl) +{ + const char *start, *lf; + + if (s->size == 0 || s->len == 0) + return; + + seq_buf_terminate(s); + + start = s->buffer; + while ((lf = strchr(start, '\n'))) { + int len = lf - start + 1; + + printk("%s%.*s", lvl, len, start); + start = ++lf; + } + + /* No trailing LF */ + if (start < s->buffer + s->len) + printk("%s%s\n", lvl, start); +} +EXPORT_SYMBOL_GPL(seq_buf_do_printk); + #ifdef CONFIG_BINARY_PRINTF /** * seq_buf_bprintf - Write the printf string from binary arguments -- cgit v1.2.3 From f736d2c0caa8348b0bcd897972e470f7a596caad Mon Sep 17 00:00:00 2001 From: David Virag Date: Tue, 31 Jan 2023 19:30:06 +0100 Subject: mfd: sec: Remove PMICs without compatibles The S5M8751 and S5M8763 PMIC chips have no corresponding compatible values, so since board file support was removed for this driver, there is no way to specify these PMICs as present in boards anymore. Remove leftovers of these chips since it's dead code. Signed-off-by: David Virag Reviewed-by: Krzysztof Kozlowski Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20230131183008.4451-2-virag.david003@gmail.com --- drivers/mfd/sec-core.c | 46 ------------------- drivers/mfd/sec-irq.c | 89 ------------------------------------ include/linux/mfd/samsung/core.h | 1 - include/linux/mfd/samsung/s5m8763.h | 90 ------------------------------------- 4 files changed, 226 deletions(-) delete mode 100644 include/linux/mfd/samsung/s5m8763.h (limited to 'include/linux') diff --git a/drivers/mfd/sec-core.c b/drivers/mfd/sec-core.c index b03edda56009..c2d0ed496959 100644 --- a/drivers/mfd/sec-core.c +++ b/drivers/mfd/sec-core.c @@ -24,22 +24,9 @@ #include #include #include -#include #include #include -static const struct mfd_cell s5m8751_devs[] = { - { .name = "s5m8751-pmic", }, - { .name = "s5m-charger", }, - { .name = "s5m8751-codec", }, -}; - -static const struct mfd_cell s5m8763_devs[] = { - { .name = "s5m8763-pmic", }, - { .name = "s5m-rtc", }, - { .name = "s5m-charger", }, -}; - static const struct mfd_cell s5m8767_devs[] = { { .name = "s5m8767-pmic", }, { .name = "s5m-rtc", }, @@ -158,19 +145,6 @@ static bool s2mpu02_volatile(struct device *dev, unsigned int reg) } } -static bool s5m8763_volatile(struct device *dev, unsigned int reg) -{ - switch (reg) { - case S5M8763_REG_IRQM1: - case S5M8763_REG_IRQM2: - case S5M8763_REG_IRQM3: - case S5M8763_REG_IRQM4: - return false; - default: - return true; - } -} - static const struct regmap_config sec_regmap_config = { .reg_bits = 8, .val_bits = 8, @@ -230,15 +204,6 @@ static const struct regmap_config s2mpu02_regmap_config = { .cache_type = REGCACHE_FLAT, }; -static const struct regmap_config s5m8763_regmap_config = { - .reg_bits = 8, - .val_bits = 8, - - .max_register = S5M8763_REG_LBCNFG2, - .volatile_reg = s5m8763_volatile, - .cache_type = REGCACHE_FLAT, -}; - static const struct regmap_config s5m8767_regmap_config = { .reg_bits = 8, .val_bits = 8, @@ -348,9 +313,6 @@ static int sec_pmic_probe(struct i2c_client *i2c) case S2MPS15X: regmap = &s2mps15_regmap_config; break; - case S5M8763X: - regmap = &s5m8763_regmap_config; - break; case S5M8767X: regmap = &s5m8767_regmap_config; break; @@ -375,14 +337,6 @@ static int sec_pmic_probe(struct i2c_client *i2c) pm_runtime_set_active(sec_pmic->dev); switch (sec_pmic->device_type) { - case S5M8751X: - sec_devs = s5m8751_devs; - num_sec_devs = ARRAY_SIZE(s5m8751_devs); - break; - case S5M8763X: - sec_devs = s5m8763_devs; - num_sec_devs = ARRAY_SIZE(s5m8763_devs); - break; case S5M8767X: sec_devs = s5m8767_devs; num_sec_devs = ARRAY_SIZE(s5m8767_devs); diff --git a/drivers/mfd/sec-irq.c b/drivers/mfd/sec-irq.c index f5f59fdc72fe..e191aeb0c07c 100644 --- a/drivers/mfd/sec-irq.c +++ b/drivers/mfd/sec-irq.c @@ -14,7 +14,6 @@ #include #include #include -#include #include static const struct regmap_irq s2mps11_irqs[] = { @@ -297,81 +296,6 @@ static const struct regmap_irq s5m8767_irqs[] = { }, }; -static const struct regmap_irq s5m8763_irqs[] = { - [S5M8763_IRQ_DCINF] = { - .reg_offset = 0, - .mask = S5M8763_IRQ_DCINF_MASK, - }, - [S5M8763_IRQ_DCINR] = { - .reg_offset = 0, - .mask = S5M8763_IRQ_DCINR_MASK, - }, - [S5M8763_IRQ_JIGF] = { - .reg_offset = 0, - .mask = S5M8763_IRQ_JIGF_MASK, - }, - [S5M8763_IRQ_JIGR] = { - .reg_offset = 0, - .mask = S5M8763_IRQ_JIGR_MASK, - }, - [S5M8763_IRQ_PWRONF] = { - .reg_offset = 0, - .mask = S5M8763_IRQ_PWRONF_MASK, - }, - [S5M8763_IRQ_PWRONR] = { - .reg_offset = 0, - .mask = S5M8763_IRQ_PWRONR_MASK, - }, - [S5M8763_IRQ_WTSREVNT] = { - .reg_offset = 1, - .mask = S5M8763_IRQ_WTSREVNT_MASK, - }, - [S5M8763_IRQ_SMPLEVNT] = { - .reg_offset = 1, - .mask = S5M8763_IRQ_SMPLEVNT_MASK, - }, - [S5M8763_IRQ_ALARM1] = { - .reg_offset = 1, - .mask = S5M8763_IRQ_ALARM1_MASK, - }, - [S5M8763_IRQ_ALARM0] = { - .reg_offset = 1, - .mask = S5M8763_IRQ_ALARM0_MASK, - }, - [S5M8763_IRQ_ONKEY1S] = { - .reg_offset = 2, - .mask = S5M8763_IRQ_ONKEY1S_MASK, - }, - [S5M8763_IRQ_TOPOFFR] = { - .reg_offset = 2, - .mask = S5M8763_IRQ_TOPOFFR_MASK, - }, - [S5M8763_IRQ_DCINOVPR] = { - .reg_offset = 2, - .mask = S5M8763_IRQ_DCINOVPR_MASK, - }, - [S5M8763_IRQ_CHGRSTF] = { - .reg_offset = 2, - .mask = S5M8763_IRQ_CHGRSTF_MASK, - }, - [S5M8763_IRQ_DONER] = { - .reg_offset = 2, - .mask = S5M8763_IRQ_DONER_MASK, - }, - [S5M8763_IRQ_CHGFAULT] = { - .reg_offset = 2, - .mask = S5M8763_IRQ_CHGFAULT_MASK, - }, - [S5M8763_IRQ_LOBAT1] = { - .reg_offset = 3, - .mask = S5M8763_IRQ_LOBAT1_MASK, - }, - [S5M8763_IRQ_LOBAT2] = { - .reg_offset = 3, - .mask = S5M8763_IRQ_LOBAT2_MASK, - }, -}; - static const struct regmap_irq_chip s2mps11_irq_chip = { .name = "s2mps11", .irqs = s2mps11_irqs, @@ -425,16 +349,6 @@ static const struct regmap_irq_chip s5m8767_irq_chip = { .ack_base = S5M8767_REG_INT1, }; -static const struct regmap_irq_chip s5m8763_irq_chip = { - .name = "s5m8763", - .irqs = s5m8763_irqs, - .num_irqs = ARRAY_SIZE(s5m8763_irqs), - .num_regs = 4, - .status_base = S5M8763_REG_IRQ1, - .mask_base = S5M8763_REG_IRQM1, - .ack_base = S5M8763_REG_IRQ1, -}; - int sec_irq_init(struct sec_pmic_dev *sec_pmic) { int ret = 0; @@ -448,9 +362,6 @@ int sec_irq_init(struct sec_pmic_dev *sec_pmic) } switch (type) { - case S5M8763X: - sec_irq_chip = &s5m8763_irq_chip; - break; case S5M8767X: sec_irq_chip = &s5m8767_irq_chip; break; diff --git a/include/linux/mfd/samsung/core.h b/include/linux/mfd/samsung/core.h index f92fe090473d..07aae649a86f 100644 --- a/include/linux/mfd/samsung/core.h +++ b/include/linux/mfd/samsung/core.h @@ -36,7 +36,6 @@ struct gpio_desc; enum sec_device_type { - S5M8751X, S5M8763X, S5M8767X, S2MPA01, diff --git a/include/linux/mfd/samsung/s5m8763.h b/include/linux/mfd/samsung/s5m8763.h deleted file mode 100644 index c534f086ca16..000000000000 --- a/include/linux/mfd/samsung/s5m8763.h +++ /dev/null @@ -1,90 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * Copyright (c) 2011 Samsung Electronics Co., Ltd - * http://www.samsung.com - */ - -#ifndef __LINUX_MFD_S5M8763_H -#define __LINUX_MFD_S5M8763_H - -/* S5M8763 registers */ -enum s5m8763_reg { - S5M8763_REG_IRQ1, - S5M8763_REG_IRQ2, - S5M8763_REG_IRQ3, - S5M8763_REG_IRQ4, - S5M8763_REG_IRQM1, - S5M8763_REG_IRQM2, - S5M8763_REG_IRQM3, - S5M8763_REG_IRQM4, - S5M8763_REG_STATUS1, - S5M8763_REG_STATUS2, - S5M8763_REG_STATUSM1, - S5M8763_REG_STATUSM2, - S5M8763_REG_CHGR1, - S5M8763_REG_CHGR2, - S5M8763_REG_LDO_ACTIVE_DISCHARGE1, - S5M8763_REG_LDO_ACTIVE_DISCHARGE2, - S5M8763_REG_BUCK_ACTIVE_DISCHARGE3, - S5M8763_REG_ONOFF1, - S5M8763_REG_ONOFF2, - S5M8763_REG_ONOFF3, - S5M8763_REG_ONOFF4, - S5M8763_REG_BUCK1_VOLTAGE1, - S5M8763_REG_BUCK1_VOLTAGE2, - S5M8763_REG_BUCK1_VOLTAGE3, - S5M8763_REG_BUCK1_VOLTAGE4, - S5M8763_REG_BUCK2_VOLTAGE1, - S5M8763_REG_BUCK2_VOLTAGE2, - S5M8763_REG_BUCK3, - S5M8763_REG_BUCK4, - S5M8763_REG_LDO1_LDO2, - S5M8763_REG_LDO3, - S5M8763_REG_LDO4, - S5M8763_REG_LDO5, - S5M8763_REG_LDO6, - S5M8763_REG_LDO7, - S5M8763_REG_LDO7_LDO8, - S5M8763_REG_LDO9_LDO10, - S5M8763_REG_LDO11, - S5M8763_REG_LDO12, - S5M8763_REG_LDO13, - S5M8763_REG_LDO14, - S5M8763_REG_LDO15, - S5M8763_REG_LDO16, - S5M8763_REG_BKCHR, - S5M8763_REG_LBCNFG1, - S5M8763_REG_LBCNFG2, -}; - -/* S5M8763 regulator ids */ -enum s5m8763_regulators { - S5M8763_LDO1, - S5M8763_LDO2, - S5M8763_LDO3, - S5M8763_LDO4, - S5M8763_LDO5, - S5M8763_LDO6, - S5M8763_LDO7, - S5M8763_LDO8, - S5M8763_LDO9, - S5M8763_LDO10, - S5M8763_LDO11, - S5M8763_LDO12, - S5M8763_LDO13, - S5M8763_LDO14, - S5M8763_LDO15, - S5M8763_LDO16, - S5M8763_BUCK1, - S5M8763_BUCK2, - S5M8763_BUCK3, - S5M8763_BUCK4, - S5M8763_AP_EN32KHZ, - S5M8763_CP_EN32KHZ, - S5M8763_ENCHGVI, - S5M8763_ESAFEUSB1, - S5M8763_ESAFEUSB2, -}; - -#define S5M8763_ENRAMP (1 << 4) -#endif /* __LINUX_MFD_S5M8763_H */ -- cgit v1.2.3 From a3165abaa9bda3e729b434a3e684fcfa205f252a Mon Sep 17 00:00:00 2001 From: David Virag Date: Tue, 31 Jan 2023 19:30:07 +0100 Subject: rtc: s5m: Drop S5M8763 support The S5M8763 MFD has no device tree compatible, and since board file support for it was removed, there's no way to use this MFD. After removing the remaining code for it from the MFD driver, also remove support for it in the s5m RTC driver, and all remaining references to it. Signed-off-by: David Virag Acked-by: Alexandre Belloni Reviewed-by: Krzysztof Kozlowski Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20230131183008.4451-3-virag.david003@gmail.com --- drivers/rtc/rtc-s5m.c | 82 ++-------------------------------------- include/linux/mfd/samsung/core.h | 1 - include/linux/mfd/samsung/irq.h | 50 ------------------------ 3 files changed, 3 insertions(+), 130 deletions(-) (limited to 'include/linux') diff --git a/drivers/rtc/rtc-s5m.c b/drivers/rtc/rtc-s5m.c index 4243fe6d3842..dad294a0ce2a 100644 --- a/drivers/rtc/rtc-s5m.c +++ b/drivers/rtc/rtc-s5m.c @@ -85,7 +85,7 @@ struct s5m_rtc_reg_config { unsigned int write_alarm_udr_mask; }; -/* Register map for S5M8763 and S5M8767 */ +/* Register map for S5M8767 */ static const struct s5m_rtc_reg_config s5m_rtc_regs = { .regs_count = 8, .time = S5M_RTC_SEC, @@ -236,7 +236,6 @@ static int s5m_check_peding_alarm_interrupt(struct s5m_rtc_info *info, switch (info->device_type) { case S5M8767X: - case S5M8763X: ret = regmap_read(info->regmap, S5M_RTC_STATUS, &val); val &= S5M_ALARM0_STATUS; break; @@ -299,7 +298,6 @@ static int s5m8767_rtc_set_alarm_reg(struct s5m_rtc_info *info) data |= info->regs->write_alarm_udr_mask; switch (info->device_type) { - case S5M8763X: case S5M8767X: data &= ~S5M_RTC_TIME_EN_MASK; break; @@ -329,38 +327,6 @@ static int s5m8767_rtc_set_alarm_reg(struct s5m_rtc_info *info) return ret; } -static void s5m8763_data_to_tm(u8 *data, struct rtc_time *tm) -{ - tm->tm_sec = bcd2bin(data[RTC_SEC]); - tm->tm_min = bcd2bin(data[RTC_MIN]); - - if (data[RTC_HOUR] & HOUR_12) { - tm->tm_hour = bcd2bin(data[RTC_HOUR] & 0x1f); - if (data[RTC_HOUR] & HOUR_PM) - tm->tm_hour += 12; - } else { - tm->tm_hour = bcd2bin(data[RTC_HOUR] & 0x3f); - } - - tm->tm_wday = data[RTC_WEEKDAY] & 0x07; - tm->tm_mday = bcd2bin(data[RTC_DATE]); - tm->tm_mon = bcd2bin(data[RTC_MONTH]); - tm->tm_year = bcd2bin(data[RTC_YEAR1]) + bcd2bin(data[RTC_YEAR2]) * 100; - tm->tm_year -= 1900; -} - -static void s5m8763_tm_to_data(struct rtc_time *tm, u8 *data) -{ - data[RTC_SEC] = bin2bcd(tm->tm_sec); - data[RTC_MIN] = bin2bcd(tm->tm_min); - data[RTC_HOUR] = bin2bcd(tm->tm_hour); - data[RTC_WEEKDAY] = tm->tm_wday; - data[RTC_DATE] = bin2bcd(tm->tm_mday); - data[RTC_MONTH] = bin2bcd(tm->tm_mon); - data[RTC_YEAR1] = bin2bcd(tm->tm_year % 100); - data[RTC_YEAR2] = bin2bcd((tm->tm_year + 1900) / 100); -} - static int s5m_rtc_read_time(struct device *dev, struct rtc_time *tm) { struct s5m_rtc_info *info = dev_get_drvdata(dev); @@ -385,10 +351,6 @@ static int s5m_rtc_read_time(struct device *dev, struct rtc_time *tm) return ret; switch (info->device_type) { - case S5M8763X: - s5m8763_data_to_tm(data, tm); - break; - case S5M8767X: case S2MPS15X: case S2MPS14X: @@ -412,9 +374,6 @@ static int s5m_rtc_set_time(struct device *dev, struct rtc_time *tm) int ret = 0; switch (info->device_type) { - case S5M8763X: - s5m8763_tm_to_data(tm, data); - break; case S5M8767X: case S2MPS15X: case S2MPS14X: @@ -444,7 +403,6 @@ static int s5m_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) { struct s5m_rtc_info *info = dev_get_drvdata(dev); u8 data[RTC_MAX_NUM_TIME_REGS]; - unsigned int val; int ret, i; ret = regmap_bulk_read(info->regmap, info->regs->alarm0, data, @@ -453,15 +411,6 @@ static int s5m_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) return ret; switch (info->device_type) { - case S5M8763X: - s5m8763_data_to_tm(data, &alrm->time); - ret = regmap_read(info->regmap, S5M_ALARM0_CONF, &val); - if (ret < 0) - return ret; - - alrm->enabled = !!val; - break; - case S5M8767X: case S2MPS15X: case S2MPS14X: @@ -500,10 +449,6 @@ static int s5m_rtc_stop_alarm(struct s5m_rtc_info *info) dev_dbg(info->dev, "%s: %ptR(%d)\n", __func__, &tm, tm.tm_wday); switch (info->device_type) { - case S5M8763X: - ret = regmap_write(info->regmap, S5M_ALARM0_CONF, 0); - break; - case S5M8767X: case S2MPS15X: case S2MPS14X: @@ -531,7 +476,6 @@ static int s5m_rtc_start_alarm(struct s5m_rtc_info *info) { int ret; u8 data[RTC_MAX_NUM_TIME_REGS]; - u8 alarm0_conf; struct rtc_time tm; ret = regmap_bulk_read(info->regmap, info->regs->alarm0, data, @@ -543,11 +487,6 @@ static int s5m_rtc_start_alarm(struct s5m_rtc_info *info) dev_dbg(info->dev, "%s: %ptR(%d)\n", __func__, &tm, tm.tm_wday); switch (info->device_type) { - case S5M8763X: - alarm0_conf = 0x77; - ret = regmap_write(info->regmap, S5M_ALARM0_CONF, alarm0_conf); - break; - case S5M8767X: case S2MPS15X: case S2MPS14X: @@ -585,10 +524,6 @@ static int s5m_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) int ret; switch (info->device_type) { - case S5M8763X: - s5m8763_tm_to_data(&alrm->time, data); - break; - case S5M8767X: case S2MPS15X: case S2MPS14X: @@ -655,7 +590,6 @@ static int s5m8767_rtc_init_reg(struct s5m_rtc_info *info) int ret; switch (info->device_type) { - case S5M8763X: case S5M8767X: /* UDR update time. Default of 7.32 ms is too long. */ ret = regmap_update_bits(info->regmap, S5M_RTC_UDR_CON, @@ -729,11 +663,6 @@ static int s5m_rtc_probe(struct platform_device *pdev) info->regs = &s2mps13_rtc_regs; alarm_irq = S2MPS14_IRQ_RTCA0; break; - case S5M8763X: - regmap_cfg = &s5m_rtc_regmap_config; - info->regs = &s5m_rtc_regs; - alarm_irq = S5M8763_IRQ_ALARM0; - break; case S5M8767X: regmap_cfg = &s5m_rtc_regmap_config; info->regs = &s5m_rtc_regs; @@ -786,13 +715,8 @@ static int s5m_rtc_probe(struct platform_device *pdev) info->rtc_dev->ops = &s5m_rtc_ops; - if (info->device_type == S5M8763X) { - info->rtc_dev->range_min = RTC_TIMESTAMP_BEGIN_0000; - info->rtc_dev->range_max = RTC_TIMESTAMP_END_9999; - } else { - info->rtc_dev->range_min = RTC_TIMESTAMP_BEGIN_2000; - info->rtc_dev->range_max = RTC_TIMESTAMP_END_2099; - } + info->rtc_dev->range_min = RTC_TIMESTAMP_BEGIN_2000; + info->rtc_dev->range_max = RTC_TIMESTAMP_END_2099; if (!info->irq) { clear_bit(RTC_FEATURE_ALARM, info->rtc_dev->features); diff --git a/include/linux/mfd/samsung/core.h b/include/linux/mfd/samsung/core.h index 07aae649a86f..a212b9f72bc9 100644 --- a/include/linux/mfd/samsung/core.h +++ b/include/linux/mfd/samsung/core.h @@ -36,7 +36,6 @@ struct gpio_desc; enum sec_device_type { - S5M8763X, S5M8767X, S2MPA01, S2MPS11X, diff --git a/include/linux/mfd/samsung/irq.h b/include/linux/mfd/samsung/irq.h index 6cfe4201a106..3fd2775eb9bb 100644 --- a/include/linux/mfd/samsung/irq.h +++ b/include/linux/mfd/samsung/irq.h @@ -194,54 +194,4 @@ enum s5m8767_irq { #define S5M8767_IRQ_RTC1S_MASK (1 << 4) #define S5M8767_IRQ_WTSR_MASK (1 << 5) -enum s5m8763_irq { - S5M8763_IRQ_DCINF, - S5M8763_IRQ_DCINR, - S5M8763_IRQ_JIGF, - S5M8763_IRQ_JIGR, - S5M8763_IRQ_PWRONF, - S5M8763_IRQ_PWRONR, - - S5M8763_IRQ_WTSREVNT, - S5M8763_IRQ_SMPLEVNT, - S5M8763_IRQ_ALARM1, - S5M8763_IRQ_ALARM0, - - S5M8763_IRQ_ONKEY1S, - S5M8763_IRQ_TOPOFFR, - S5M8763_IRQ_DCINOVPR, - S5M8763_IRQ_CHGRSTF, - S5M8763_IRQ_DONER, - S5M8763_IRQ_CHGFAULT, - - S5M8763_IRQ_LOBAT1, - S5M8763_IRQ_LOBAT2, - - S5M8763_IRQ_NR, -}; - -#define S5M8763_IRQ_DCINF_MASK (1 << 2) -#define S5M8763_IRQ_DCINR_MASK (1 << 3) -#define S5M8763_IRQ_JIGF_MASK (1 << 4) -#define S5M8763_IRQ_JIGR_MASK (1 << 5) -#define S5M8763_IRQ_PWRONF_MASK (1 << 6) -#define S5M8763_IRQ_PWRONR_MASK (1 << 7) - -#define S5M8763_IRQ_WTSREVNT_MASK (1 << 0) -#define S5M8763_IRQ_SMPLEVNT_MASK (1 << 1) -#define S5M8763_IRQ_ALARM1_MASK (1 << 2) -#define S5M8763_IRQ_ALARM0_MASK (1 << 3) - -#define S5M8763_IRQ_ONKEY1S_MASK (1 << 0) -#define S5M8763_IRQ_TOPOFFR_MASK (1 << 2) -#define S5M8763_IRQ_DCINOVPR_MASK (1 << 3) -#define S5M8763_IRQ_CHGRSTF_MASK (1 << 4) -#define S5M8763_IRQ_DONER_MASK (1 << 5) -#define S5M8763_IRQ_CHGFAULT_MASK (1 << 7) - -#define S5M8763_IRQ_LOBAT1_MASK (1 << 0) -#define S5M8763_IRQ_LOBAT2_MASK (1 << 1) - -#define S5M8763_ENRAMP (1 << 4) - #endif /* __LINUX_MFD_SEC_IRQ_H */ -- cgit v1.2.3 From 86c6bb0edffa9fc02b4e3801b48c8e82114f1352 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Tue, 14 Feb 2023 09:58:59 +0100 Subject: mfd: core: Reorder fields in 'struct mfd_cell' to save some memory Group some variables based on their sizes to reduce hole and avoid padding. On x86_64, this shrinks the size from 144 to 128 bytes. As an example: $ size drivers/mfd/as3722.o (Before) text data bss dec hex filename 9441 680 16 10137 2799 drivers/mfd/as3722.o $ size drivers/mfd/as3722.o (After) text data bss dec hex filename 9345 680 16 10041 2739 drivers/mfd/as3722.o Signed-off-by: Christophe JAILLET Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/bb631974888dfe1af593b6280cf30fb913d2d1a4.1676365116.git.christophe.jaillet@wanadoo.fr --- include/linux/mfd/core.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mfd/core.h b/include/linux/mfd/core.h index 14ca7b471576..fc4a0e9fb3bb 100644 --- a/include/linux/mfd/core.h +++ b/include/linux/mfd/core.h @@ -78,6 +78,9 @@ struct mfd_cell { void *platform_data; size_t pdata_size; + /* Matches ACPI */ + const struct mfd_cell_acpi_match *acpi_match; + /* Software node for the device. */ const struct software_node *swnode; @@ -97,9 +100,6 @@ struct mfd_cell { /* Set to 'true' to use 'of_reg' (above) - allows for of_reg=0 */ bool use_of_reg; - /* Matches ACPI */ - const struct mfd_cell_acpi_match *acpi_match; - /* * These resources can be specified relative to the parent device. * For accessing hardware you should use resources from the platform dev @@ -119,8 +119,8 @@ struct mfd_cell { /* A list of regulator supplies that should be mapped to the MFD * device rather than the child device when requested */ - const char * const *parent_supplies; int num_parent_supplies; + const char * const *parent_supplies; }; /* -- cgit v1.2.3 From b8fd17d9505e18001d10ceaa350c118a82b467bb Mon Sep 17 00:00:00 2001 From: Jakob Hauser Date: Tue, 28 Feb 2023 23:32:20 +0100 Subject: mfd: rt5033: Fix comments and style in includes Fix comments and remove some empty lines in rt5033-private.h. Align struct rt5033_charger in rt5033.h. Signed-off-by: Jakob Hauser Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/606950da6f4b36f5a124ff13756c78644fc89804.1677620677.git.jahau@rocketmail.com --- include/linux/mfd/rt5033-private.h | 17 +++++++---------- include/linux/mfd/rt5033.h | 7 +++---- 2 files changed, 10 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mfd/rt5033-private.h b/include/linux/mfd/rt5033-private.h index 2d1895c3efbf..6bb432f6a96c 100644 --- a/include/linux/mfd/rt5033-private.h +++ b/include/linux/mfd/rt5033-private.h @@ -107,14 +107,13 @@ enum rt5033_reg { #define RT5033_LDO_CTRL_MASK 0x1f /* RT5033 charger property - model, manufacturer */ - #define RT5033_CHARGER_MODEL "RT5033WSC Charger" #define RT5033_MANUFACTURER "Richtek Technology Corporation" /* - * RT5033 charger fast-charge current lmits (as in CHGCTRL1 register), - * AICR mode limits the input current for example, - * the AIRC 100 mode limits the input current to 100 mA. + * While RT5033 charger can limit the fast-charge current (as in CHGCTRL1 + * register), AICR mode limits the input current. For example, the AIRC 100 + * mode limits the input current to 100 mA. */ #define RT5033_AICR_100_MODE 0x20 #define RT5033_AICR_500_MODE 0x40 @@ -139,10 +138,9 @@ enum rt5033_reg { #define RT5033_TE_ENABLE_MASK 0x08 /* - * RT5033 charger opa mode. RT50300 have two opa mode charger mode - * and boost mode for OTG + * RT5033 charger opa mode. RT5033 has two opa modes for OTG: charger mode + * and boost mode. */ - #define RT5033_CHARGER_MODE 0x00 #define RT5033_BOOST_MODE 0x01 @@ -181,18 +179,17 @@ enum rt5033_reg { * RT5033 charger pre-charge threshold volt limits * (as in CHGCTRL5 register), uV */ - #define RT5033_CHARGER_PRE_THRESHOLD_LIMIT_MIN 2300000U #define RT5033_CHARGER_PRE_THRESHOLD_STEP_NUM 100000U #define RT5033_CHARGER_PRE_THRESHOLD_LIMIT_MAX 3800000U /* - * RT5033 charger enable UUG, If UUG enable MOS auto control by H/W charger + * RT5033 charger UUG. It enables MOS auto control by H/W charger * circuit. */ #define RT5033_CHARGER_UUG_ENABLE 0x02 -/* RT5033 charger High impedance mode */ +/* RT5033 charger high impedance mode */ #define RT5033_CHARGER_HZ_DISABLE 0x00 #define RT5033_CHARGER_HZ_ENABLE 0x01 diff --git a/include/linux/mfd/rt5033.h b/include/linux/mfd/rt5033.h index 3c23b6220c04..8f306ac15a27 100644 --- a/include/linux/mfd/rt5033.h +++ b/include/linux/mfd/rt5033.h @@ -49,10 +49,9 @@ struct rt5033_charger_data { }; struct rt5033_charger { - struct device *dev; - struct rt5033_dev *rt5033; - struct power_supply psy; - + struct device *dev; + struct rt5033_dev *rt5033; + struct power_supply psy; struct rt5033_charger_data *chg; }; -- cgit v1.2.3 From 0742c2a6335281608ac9e9aee67493e9e30f6195 Mon Sep 17 00:00:00 2001 From: Patrick Rudolph Date: Tue, 7 Mar 2023 13:12:45 +0100 Subject: mfd: max597x: Add support for MAX5970 and MAX5978 Implement a regulator driver with IRQ support for fault management. Written against documentation [1] and [2] and tested on real hardware. Every channel has it's own regulator supply nammed 'vss1-supply' and 'vss2-supply'. The regulator supply is used to determine the output voltage, as the smart switch provides no output regulation. The driver requires the 'shunt-resistor-micro-ohms' to be present in the devicetree to properly calculate current related values. You must specify compatible devictree layout: regulator@3a { reg = <0x3a>; vss1-supply = <&p3v3>; compatible = "maxim,max5978"; ... regulators { sw0_ref: SW0 { regulator-compatible = "SW0"; shunt-resistor-micro-ohms = <12000>; ... } } } 1: https://datasheets.maximintegrated.com/en/ds/MAX5970.pdf 2: https://datasheets.maximintegrated.com/en/ds/MAX5978.pdf ... Changes in V12: - Use simple_mfd_i2c driver and remove previous implementation. - Remove newline - Use _MFD_MAX597X_H in header file - Successfull build need following patch from regulator: https://lore.kernel.org/r/20230216075302.68935-1-Naresh.Solanki@9elements.com https://lore.kernel.org/r/20230210163225.1208035-1-Naresh.Solanki@9elements.com Signed-off-by: Patrick Rudolph Signed-off-by: Marcello Sylvester Bauer Signed-off-by: Naresh Solanki Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20230307121246.127425-2-Naresh.Solanki@9elements.com --- drivers/mfd/Kconfig | 10 +++++ drivers/mfd/simple-mfd-i2c.c | 13 ++++++ include/linux/mfd/max597x.h | 96 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 include/linux/mfd/max597x.h (limited to 'include/linux') diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index fcc141e067b9..d381d0e63455 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -266,6 +266,16 @@ config MFD_MADERA_SPI Support for the Cirrus Logic Madera platform audio SoC core functionality controlled via SPI. +config MFD_MAX597X + tristate "Maxim 597x power switch and monitor" + depends on (I2C && OF) + select MFD_SIMPLE_MFD_I2C + help + This driver controls a Maxim 5970/5978 switch via I2C bus. + The MAX5970/5978 is a smart switch with no output regulation, but + fault protection and voltage and current monitoring capabilities. + Also it supports upto 4 indication leds. + config MFD_CS47L15 bool "Cirrus Logic CS47L15" select PINCTRL_CS47L15 diff --git a/drivers/mfd/simple-mfd-i2c.c b/drivers/mfd/simple-mfd-i2c.c index e31f13fd6a79..20782b4dd172 100644 --- a/drivers/mfd/simple-mfd-i2c.c +++ b/drivers/mfd/simple-mfd-i2c.c @@ -72,9 +72,22 @@ static const struct simple_mfd_data silergy_sy7636a = { .mfd_cell_size = ARRAY_SIZE(sy7636a_cells), }; +static const struct mfd_cell max597x_cells[] = { + { .name = "max597x-regulator", }, + { .name = "max597x-iio", }, + { .name = "max597x-led", }, +}; + +static const struct simple_mfd_data maxim_max597x = { + .mfd_cell = max597x_cells, + .mfd_cell_size = ARRAY_SIZE(max597x_cells), +}; + static const struct of_device_id simple_mfd_i2c_of_match[] = { { .compatible = "kontron,sl28cpld" }, { .compatible = "silergy,sy7636a", .data = &silergy_sy7636a}, + { .compatible = "maxim,max5970", .data = &maxim_max597x}, + { .compatible = "maxim,max5978", .data = &maxim_max597x}, {} }; MODULE_DEVICE_TABLE(of, simple_mfd_i2c_of_match); diff --git a/include/linux/mfd/max597x.h b/include/linux/mfd/max597x.h new file mode 100644 index 000000000000..a850b2e02e6a --- /dev/null +++ b/include/linux/mfd/max597x.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Device driver for regulators in MAX5970 and MAX5978 IC + * + * Copyright (c) 2022 9elements GmbH + * + * Author: Patrick Rudolph + */ + +#ifndef _MFD_MAX597X_H +#define _MFD_MAX597X_H + +#include + +#define MAX5970_NUM_SWITCHES 2 +#define MAX5978_NUM_SWITCHES 1 +#define MAX597X_NUM_LEDS 4 + +struct max597x_data { + int num_switches; + u32 irng[MAX5970_NUM_SWITCHES]; + u32 mon_rng[MAX5970_NUM_SWITCHES]; + u32 shunt_micro_ohms[MAX5970_NUM_SWITCHES]; +}; + +enum max597x_chip_type { + MAX597x_TYPE_MAX5978 = 1, + MAX597x_TYPE_MAX5970, +}; + +#define MAX5970_REG_CURRENT_L(ch) (0x01 + (ch) * 4) +#define MAX5970_REG_CURRENT_H(ch) (0x00 + (ch) * 4) +#define MAX5970_REG_VOLTAGE_L(ch) (0x03 + (ch) * 4) +#define MAX5970_REG_VOLTAGE_H(ch) (0x02 + (ch) * 4) +#define MAX5970_REG_MON_RANGE 0x18 +#define MAX5970_MON_MASK 0x3 +#define MAX5970_MON(reg, ch) (((reg) >> ((ch) * 2)) & MAX5970_MON_MASK) +#define MAX5970_MON_MAX_RANGE_UV 16000000 + +#define MAX5970_REG_CH_UV_WARN_H(ch) (0x1A + (ch) * 10) +#define MAX5970_REG_CH_UV_WARN_L(ch) (0x1B + (ch) * 10) +#define MAX5970_REG_CH_UV_CRIT_H(ch) (0x1C + (ch) * 10) +#define MAX5970_REG_CH_UV_CRIT_L(ch) (0x1D + (ch) * 10) +#define MAX5970_REG_CH_OV_WARN_H(ch) (0x1E + (ch) * 10) +#define MAX5970_REG_CH_OV_WARN_L(ch) (0x1F + (ch) * 10) +#define MAX5970_REG_CH_OV_CRIT_H(ch) (0x20 + (ch) * 10) +#define MAX5970_REG_CH_OV_CRIT_L(ch) (0x21 + (ch) * 10) + +#define MAX5970_VAL2REG_H(x) (((x) >> 2) & 0xFF) +#define MAX5970_VAL2REG_L(x) ((x) & 0x3) + +#define MAX5970_REG_DAC_FAST(ch) (0x2E + (ch)) + +#define MAX5970_FAST2SLOW_RATIO 200 + +#define MAX5970_REG_STATUS0 0x31 +#define MAX5970_CB_IFAULTF(ch) (1 << (ch)) +#define MAX5970_CB_IFAULTS(ch) (1 << ((ch) + 4)) + +#define MAX5970_REG_STATUS1 0x32 +#define STATUS1_PROT_MASK 0x3 +#define STATUS1_PROT(reg) \ + (((reg) >> 6) & STATUS1_PROT_MASK) +#define STATUS1_PROT_SHUTDOWN 0 +#define STATUS1_PROT_CLEAR_PG 1 +#define STATUS1_PROT_ALERT_ONLY 2 + +#define MAX5970_REG_STATUS2 0x33 +#define MAX5970_IRNG_MASK 0x3 +#define MAX5970_IRNG(reg, ch) \ + (((reg) >> ((ch) * 2)) & MAX5970_IRNG_MASK) + +#define MAX5970_REG_STATUS3 0x34 +#define MAX5970_STATUS3_ALERT BIT(4) +#define MAX5970_STATUS3_PG(ch) BIT(ch) + +#define MAX5970_REG_FAULT0 0x35 +#define UV_STATUS_WARN(ch) (1 << (ch)) +#define UV_STATUS_CRIT(ch) (1 << ((ch) + 4)) + +#define MAX5970_REG_FAULT1 0x36 +#define OV_STATUS_WARN(ch) (1 << (ch)) +#define OV_STATUS_CRIT(ch) (1 << ((ch) + 4)) + +#define MAX5970_REG_FAULT2 0x37 +#define OC_STATUS_WARN(ch) (1 << (ch)) + +#define MAX5970_REG_CHXEN 0x3b +#define CHXEN(ch) (3 << ((ch) * 2)) + +#define MAX5970_REG_LED_FLASH 0x43 + +#define MAX_REGISTERS 0x49 +#define ADC_MASK 0x3FF + +#endif /* _MFD_MAX597X_H */ -- cgit v1.2.3 From a0b9becad8a7d62a81530035a69f744c0e389737 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Wed, 8 Mar 2023 10:12:57 +0100 Subject: mfd: core: Remove .enable() and .disable() callbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With commit dd77f5fa97d3 ("mfd: Remove toshiba tmio drivers") the last mfd driver that implements these callbacks is gone and since commit 652719b1003a ("w1: remove ds1wm driver") the last user is gone. The corresponding functions mfd_cell_enable() and mfd_cell_disable() are also unused (since commit 0ca222c81977 ("leds: Remove asic3 driver")). Signed-off-by: Uwe Kleine-König Acked-by: Arnd Bergmann Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20230308091257.2404932-1-u.kleine-koenig@pengutronix.de --- drivers/mfd/mfd-core.c | 26 -------------------------- include/linux/mfd/core.h | 12 ------------ 2 files changed, 38 deletions(-) (limited to 'include/linux') diff --git a/drivers/mfd/mfd-core.c b/drivers/mfd/mfd-core.c index 16d1861e9682..695d50b3bac6 100644 --- a/drivers/mfd/mfd-core.c +++ b/drivers/mfd/mfd-core.c @@ -33,32 +33,6 @@ static struct device_type mfd_dev_type = { .name = "mfd_device", }; -int mfd_cell_enable(struct platform_device *pdev) -{ - const struct mfd_cell *cell = mfd_get_cell(pdev); - - if (!cell->enable) { - dev_dbg(&pdev->dev, "No .enable() call-back registered\n"); - return 0; - } - - return cell->enable(pdev); -} -EXPORT_SYMBOL(mfd_cell_enable); - -int mfd_cell_disable(struct platform_device *pdev) -{ - const struct mfd_cell *cell = mfd_get_cell(pdev); - - if (!cell->disable) { - dev_dbg(&pdev->dev, "No .disable() call-back registered\n"); - return 0; - } - - return cell->disable(pdev); -} -EXPORT_SYMBOL(mfd_cell_disable); - #if IS_ENABLED(CONFIG_ACPI) struct match_ids_walk_data { struct acpi_device_id *ids; diff --git a/include/linux/mfd/core.h b/include/linux/mfd/core.h index fc4a0e9fb3bb..47e7a3a61ce6 100644 --- a/include/linux/mfd/core.h +++ b/include/linux/mfd/core.h @@ -68,9 +68,6 @@ struct mfd_cell { int id; int level; - int (*enable)(struct platform_device *dev); - int (*disable)(struct platform_device *dev); - int (*suspend)(struct platform_device *dev); int (*resume)(struct platform_device *dev); @@ -123,15 +120,6 @@ struct mfd_cell { const char * const *parent_supplies; }; -/* - * Convenience functions for clients using shared cells. Refcounting - * happens automatically, with the cell's enable/disable callbacks - * being called only when a device is first being enabled or no other - * clients are making use of it. - */ -extern int mfd_cell_enable(struct platform_device *pdev); -extern int mfd_cell_disable(struct platform_device *pdev); - /* * Given a platform device that's been created by mfd_add_devices(), fetch * the mfd_cell that created it. -- cgit v1.2.3 From 67d6c76fc815cddc77de9529221f9ff8dd1fb10e Mon Sep 17 00:00:00 2001 From: Min Li Date: Mon, 27 Mar 2023 14:39:53 -0400 Subject: mfd: rsmu: Support 32-bit address space We used to assume 0x2010xxxx address. Now that we need to access 0x2011xxxx address, we need to support read/write the whole 32-bit address space. Also defined RSMU_MAX_WRITE_COUNT and RSMU_MAX_READ_COUNT for readability Signed-off-by: Min Li Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/MW5PR03MB693295AF31ABCAF6AE52EE74A08B9@MW5PR03MB6932.namprd03.prod.outlook.com --- drivers/mfd/rsmu.h | 2 + drivers/mfd/rsmu_i2c.c | 171 +++++++++++++++++++++++++++++++++++++---------- drivers/mfd/rsmu_spi.c | 52 ++++++++------ include/linux/mfd/rsmu.h | 5 +- 4 files changed, 174 insertions(+), 56 deletions(-) (limited to 'include/linux') diff --git a/drivers/mfd/rsmu.h b/drivers/mfd/rsmu.h index bb88597d189f..1bb04cafa45d 100644 --- a/drivers/mfd/rsmu.h +++ b/drivers/mfd/rsmu.h @@ -10,6 +10,8 @@ #include +#define RSMU_CM_SCSR_BASE 0x20100000 + int rsmu_core_init(struct rsmu_ddata *rsmu); void rsmu_core_exit(struct rsmu_ddata *rsmu); diff --git a/drivers/mfd/rsmu_i2c.c b/drivers/mfd/rsmu_i2c.c index 15d25b081434..221023faaadf 100644 --- a/drivers/mfd/rsmu_i2c.c +++ b/drivers/mfd/rsmu_i2c.c @@ -18,11 +18,12 @@ #include "rsmu.h" /* - * 16-bit register address: the lower 8 bits of the register address come - * from the offset addr byte and the upper 8 bits come from the page register. + * 32-bit register address: the lower 8 bits of the register address come + * from the offset addr byte and the upper 24 bits come from the page register. */ -#define RSMU_CM_PAGE_ADDR 0xFD -#define RSMU_CM_PAGE_WINDOW 256 +#define RSMU_CM_PAGE_ADDR 0xFC +#define RSMU_CM_PAGE_MASK 0xFFFFFF00 +#define RSMU_CM_ADDRESS_MASK 0x000000FF /* * 15-bit register address: the lower 7 bits of the register address come @@ -31,18 +32,6 @@ #define RSMU_SABRE_PAGE_ADDR 0x7F #define RSMU_SABRE_PAGE_WINDOW 128 -static const struct regmap_range_cfg rsmu_cm_range_cfg[] = { - { - .range_min = 0, - .range_max = 0xD000, - .selector_reg = RSMU_CM_PAGE_ADDR, - .selector_mask = 0xFF, - .selector_shift = 0, - .window_start = 0, - .window_len = RSMU_CM_PAGE_WINDOW, - } -}; - static const struct regmap_range_cfg rsmu_sabre_range_cfg[] = { { .range_min = 0, @@ -55,35 +44,141 @@ static const struct regmap_range_cfg rsmu_sabre_range_cfg[] = { } }; -static bool rsmu_cm_volatile_reg(struct device *dev, unsigned int reg) +static bool rsmu_sabre_volatile_reg(struct device *dev, unsigned int reg) { switch (reg) { - case RSMU_CM_PAGE_ADDR: + case RSMU_SABRE_PAGE_ADDR: return false; default: return true; } } -static bool rsmu_sabre_volatile_reg(struct device *dev, unsigned int reg) +static int rsmu_read_device(struct rsmu_ddata *rsmu, u8 reg, u8 *buf, u16 bytes) { - switch (reg) { - case RSMU_SABRE_PAGE_ADDR: - return false; - default: - return true; + struct i2c_client *client = to_i2c_client(rsmu->dev); + struct i2c_msg msg[2]; + int cnt; + + msg[0].addr = client->addr; + msg[0].flags = 0; + msg[0].len = 1; + msg[0].buf = ® + + msg[1].addr = client->addr; + msg[1].flags = I2C_M_RD; + msg[1].len = bytes; + msg[1].buf = buf; + + cnt = i2c_transfer(client->adapter, msg, 2); + + if (cnt < 0) { + dev_err(rsmu->dev, "i2c_transfer failed at addr: %04x!", reg); + return cnt; + } else if (cnt != 2) { + dev_err(rsmu->dev, + "i2c_transfer sent only %d of 2 messages", cnt); + return -EIO; + } + + return 0; +} + +static int rsmu_write_device(struct rsmu_ddata *rsmu, u8 reg, u8 *buf, u16 bytes) +{ + struct i2c_client *client = to_i2c_client(rsmu->dev); + u8 msg[RSMU_MAX_WRITE_COUNT + 1]; /* 1 Byte added for the device register */ + int cnt; + + if (bytes > RSMU_MAX_WRITE_COUNT) + return -EINVAL; + + msg[0] = reg; + memcpy(&msg[1], buf, bytes); + + cnt = i2c_master_send(client, msg, bytes + 1); + + if (cnt < 0) { + dev_err(&client->dev, + "i2c_master_send failed at addr: %04x!", reg); + return cnt; } + + return 0; +} + +static int rsmu_write_page_register(struct rsmu_ddata *rsmu, u32 reg) +{ + u32 page = reg & RSMU_CM_PAGE_MASK; + u8 buf[4]; + int err; + + /* Do not modify offset register for none-scsr registers */ + if (reg < RSMU_CM_SCSR_BASE) + return 0; + + /* Simply return if we are on the same page */ + if (rsmu->page == page) + return 0; + + buf[0] = 0x0; + buf[1] = (u8)((page >> 8) & 0xFF); + buf[2] = (u8)((page >> 16) & 0xFF); + buf[3] = (u8)((page >> 24) & 0xFF); + + err = rsmu_write_device(rsmu, RSMU_CM_PAGE_ADDR, buf, sizeof(buf)); + if (err) + dev_err(rsmu->dev, "Failed to set page offset 0x%x\n", page); + else + /* Remember the last page */ + rsmu->page = page; + + return err; +} + +static int rsmu_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + struct rsmu_ddata *rsmu = i2c_get_clientdata((struct i2c_client *)context); + u8 addr = (u8)(reg & RSMU_CM_ADDRESS_MASK); + int err; + + err = rsmu_write_page_register(rsmu, reg); + if (err) + return err; + + err = rsmu_read_device(rsmu, addr, (u8 *)val, 1); + if (err) + dev_err(rsmu->dev, "Failed to read offset address 0x%x\n", addr); + + return err; +} + +static int rsmu_reg_write(void *context, unsigned int reg, unsigned int val) +{ + struct rsmu_ddata *rsmu = i2c_get_clientdata((struct i2c_client *)context); + u8 addr = (u8)(reg & RSMU_CM_ADDRESS_MASK); + u8 data = (u8)val; + int err; + + err = rsmu_write_page_register(rsmu, reg); + if (err) + return err; + + err = rsmu_write_device(rsmu, addr, &data, 1); + if (err) + dev_err(rsmu->dev, + "Failed to write offset address 0x%x\n", addr); + + return err; } static const struct regmap_config rsmu_cm_regmap_config = { - .reg_bits = 8, + .reg_bits = 32, .val_bits = 8, - .max_register = 0xD000, - .ranges = rsmu_cm_range_cfg, - .num_ranges = ARRAY_SIZE(rsmu_cm_range_cfg), - .volatile_reg = rsmu_cm_volatile_reg, - .cache_type = REGCACHE_RBTREE, - .can_multi_write = true, + .max_register = 0x20120000, + .reg_read = rsmu_reg_read, + .reg_write = rsmu_reg_write, + .cache_type = REGCACHE_NONE, }; static const struct regmap_config rsmu_sabre_regmap_config = { @@ -101,14 +196,14 @@ static const struct regmap_config rsmu_sl_regmap_config = { .reg_bits = 16, .val_bits = 8, .reg_format_endian = REGMAP_ENDIAN_BIG, - .max_register = 0x339, + .max_register = 0x340, .cache_type = REGCACHE_NONE, .can_multi_write = true, }; -static int rsmu_i2c_probe(struct i2c_client *client) +static int rsmu_i2c_probe(struct i2c_client *client, + const struct i2c_device_id *id) { - const struct i2c_device_id *id = i2c_client_get_device_id(client); const struct regmap_config *cfg; struct rsmu_ddata *rsmu; int ret; @@ -136,7 +231,11 @@ static int rsmu_i2c_probe(struct i2c_client *client) dev_err(rsmu->dev, "Unsupported RSMU device type: %d\n", rsmu->type); return -ENODEV; } - rsmu->regmap = devm_regmap_init_i2c(client, cfg); + + if (rsmu->type == RSMU_CM) + rsmu->regmap = devm_regmap_init(&client->dev, NULL, client, cfg); + else + rsmu->regmap = devm_regmap_init_i2c(client, cfg); if (IS_ERR(rsmu->regmap)) { ret = PTR_ERR(rsmu->regmap); dev_err(rsmu->dev, "Failed to allocate register map: %d\n", ret); @@ -180,7 +279,7 @@ static struct i2c_driver rsmu_i2c_driver = { .name = "rsmu-i2c", .of_match_table = of_match_ptr(rsmu_i2c_of_match), }, - .probe_new = rsmu_i2c_probe, + .probe = rsmu_i2c_probe, .remove = rsmu_i2c_remove, .id_table = rsmu_i2c_id, }; diff --git a/drivers/mfd/rsmu_spi.c b/drivers/mfd/rsmu_spi.c index 2428aaa9aaed..a4a595bb8d0d 100644 --- a/drivers/mfd/rsmu_spi.c +++ b/drivers/mfd/rsmu_spi.c @@ -19,19 +19,21 @@ #define RSMU_CM_PAGE_ADDR 0x7C #define RSMU_SABRE_PAGE_ADDR 0x7F -#define RSMU_HIGHER_ADDR_MASK 0xFF80 -#define RSMU_HIGHER_ADDR_SHIFT 7 -#define RSMU_LOWER_ADDR_MASK 0x7F +#define RSMU_PAGE_MASK 0xFFFFFF80 +#define RSMU_ADDR_MASK 0x7F static int rsmu_read_device(struct rsmu_ddata *rsmu, u8 reg, u8 *buf, u16 bytes) { struct spi_device *client = to_spi_device(rsmu->dev); struct spi_transfer xfer = {0}; struct spi_message msg; - u8 cmd[256] = {0}; - u8 rsp[256] = {0}; + u8 cmd[RSMU_MAX_READ_COUNT + 1] = {0}; + u8 rsp[RSMU_MAX_READ_COUNT + 1] = {0}; int ret; + if (bytes > RSMU_MAX_READ_COUNT) + return -EINVAL; + cmd[0] = reg | 0x80; xfer.rx_buf = rsp; xfer.len = bytes + 1; @@ -66,7 +68,10 @@ static int rsmu_write_device(struct rsmu_ddata *rsmu, u8 reg, u8 *buf, u16 bytes struct spi_device *client = to_spi_device(rsmu->dev); struct spi_transfer xfer = {0}; struct spi_message msg; - u8 cmd[256] = {0}; + u8 cmd[RSMU_MAX_WRITE_COUNT + 1] = {0}; + + if (bytes > RSMU_MAX_WRITE_COUNT) + return -EINVAL; cmd[0] = reg; memcpy(&cmd[1], buf, bytes); @@ -86,26 +91,35 @@ static int rsmu_write_device(struct rsmu_ddata *rsmu, u8 reg, u8 *buf, u16 bytes * 16-bit register address: the lower 7 bits of the register address come * from the offset addr byte and the upper 9 bits come from the page register. */ -static int rsmu_write_page_register(struct rsmu_ddata *rsmu, u16 reg) +static int rsmu_write_page_register(struct rsmu_ddata *rsmu, u32 reg) { u8 page_reg; - u8 buf[2]; + u8 buf[4]; u16 bytes; - u16 page; + u32 page; int err; switch (rsmu->type) { case RSMU_CM: + /* Do not modify page register for none-scsr registers */ + if (reg < RSMU_CM_SCSR_BASE) + return 0; page_reg = RSMU_CM_PAGE_ADDR; - page = reg & RSMU_HIGHER_ADDR_MASK; + page = reg & RSMU_PAGE_MASK; buf[0] = (u8)(page & 0xff); buf[1] = (u8)((page >> 8) & 0xff); - bytes = 2; + buf[2] = (u8)((page >> 16) & 0xff); + buf[3] = (u8)((page >> 24) & 0xff); + bytes = 4; break; case RSMU_SABRE: + /* Do not modify page register if reg is page register itself */ + if ((reg & RSMU_ADDR_MASK) == RSMU_ADDR_MASK) + return 0; page_reg = RSMU_SABRE_PAGE_ADDR; - page = reg >> RSMU_HIGHER_ADDR_SHIFT; - buf[0] = (u8)(page & 0xff); + page = reg & RSMU_PAGE_MASK; + /* The three page bits are located in the single Page Register */ + buf[0] = (u8)((page >> 7) & 0x7); bytes = 1; break; default: @@ -129,8 +143,8 @@ static int rsmu_write_page_register(struct rsmu_ddata *rsmu, u16 reg) static int rsmu_reg_read(void *context, unsigned int reg, unsigned int *val) { - struct rsmu_ddata *rsmu = spi_get_drvdata(context); - u8 addr = (u8)(reg & RSMU_LOWER_ADDR_MASK); + struct rsmu_ddata *rsmu = spi_get_drvdata((struct spi_device *)context); + u8 addr = (u8)(reg & RSMU_ADDR_MASK); int err; err = rsmu_write_page_register(rsmu, reg); @@ -146,8 +160,8 @@ static int rsmu_reg_read(void *context, unsigned int reg, unsigned int *val) static int rsmu_reg_write(void *context, unsigned int reg, unsigned int val) { - struct rsmu_ddata *rsmu = spi_get_drvdata(context); - u8 addr = (u8)(reg & RSMU_LOWER_ADDR_MASK); + struct rsmu_ddata *rsmu = spi_get_drvdata((struct spi_device *)context); + u8 addr = (u8)(reg & RSMU_ADDR_MASK); u8 data = (u8)val; int err; @@ -164,9 +178,9 @@ static int rsmu_reg_write(void *context, unsigned int reg, unsigned int val) } static const struct regmap_config rsmu_cm_regmap_config = { - .reg_bits = 16, + .reg_bits = 32, .val_bits = 8, - .max_register = 0xD000, + .max_register = 0x20120000, .reg_read = rsmu_reg_read, .reg_write = rsmu_reg_write, .cache_type = REGCACHE_NONE, diff --git a/include/linux/mfd/rsmu.h b/include/linux/mfd/rsmu.h index 6870de608233..0379aa207428 100644 --- a/include/linux/mfd/rsmu.h +++ b/include/linux/mfd/rsmu.h @@ -8,6 +8,9 @@ #ifndef __LINUX_MFD_RSMU_H #define __LINUX_MFD_RSMU_H +#define RSMU_MAX_WRITE_COUNT (255) +#define RSMU_MAX_READ_COUNT (255) + /* The supported devices are ClockMatrix, Sabre and SnowLotus */ enum rsmu_type { RSMU_CM = 0x34000, @@ -31,6 +34,6 @@ struct rsmu_ddata { struct regmap *regmap; struct mutex lock; enum rsmu_type type; - u16 page; + u32 page; }; #endif /* __LINUX_MFD_RSMU_H */ -- cgit v1.2.3 From 654c293e1687b31819f9bf1ac71b5a85a8053210 Mon Sep 17 00:00:00 2001 From: Biju Das Date: Thu, 30 Mar 2023 12:16:28 +0100 Subject: mfd: Add Renesas RZ/G2L MTU3a core driver The RZ/G2L multi-function timer pulse unit 3 (MTU3a) is embedded in the Renesas RZ/G2L family SoCs. It consists of eight 16-bit timer channels and one 32-bit timer channel. It supports the following functions - Counter - Timer - PWM The 8/16/32 bit registers are mixed in each channel. Add MTU3a core driver for RZ/G2L SoC. The core driver shares the clk and channel register access for the other child devices like Counter, PWM and Clock event. Signed-off-by: Biju Das Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/20230330111632.169434-3-biju.das.jz@bp.renesas.com --- drivers/mfd/Kconfig | 10 ++ drivers/mfd/Makefile | 1 + drivers/mfd/rz-mtu3.c | 391 ++++++++++++++++++++++++++++++++++++++++++++ drivers/mfd/rz-mtu3.h | 147 +++++++++++++++++ include/linux/mfd/rz-mtu3.h | 257 +++++++++++++++++++++++++++++ 5 files changed, 806 insertions(+) create mode 100644 drivers/mfd/rz-mtu3.c create mode 100644 drivers/mfd/rz-mtu3.h create mode 100644 include/linux/mfd/rz-mtu3.h (limited to 'include/linux') diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 51d54a1b8673..e90463c4441c 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -1315,6 +1315,16 @@ config MFD_SC27XX_PMIC This driver provides common support for accessing the SC27xx PMICs, and it also adds the irq_chip parts for handling the PMIC chip events. +config RZ_MTU3 + bool "Renesas RZ/G2L MTU3a core driver" + depends on (ARCH_RZG2L && OF) || COMPILE_TEST + help + Select this option to enable Renesas RZ/G2L MTU3a core driver for + the Multi-Function Timer Pulse Unit 3 (MTU3a) hardware available + on SoCs from Renesas. The core driver shares the clk and channel + register access for the other child devices like Counter, PWM, + Clock Source, and Clock event. + config ABX500_CORE bool "ST-Ericsson ABX500 Mixed Signal Circuit register functions" depends on ARCH_U8500 || COMPILE_TEST diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile index 2f6c89d1e277..1d2392f06f78 100644 --- a/drivers/mfd/Makefile +++ b/drivers/mfd/Makefile @@ -174,6 +174,7 @@ pcf50633-objs := pcf50633-core.o pcf50633-irq.o obj-$(CONFIG_MFD_PCF50633) += pcf50633.o obj-$(CONFIG_PCF50633_ADC) += pcf50633-adc.o obj-$(CONFIG_PCF50633_GPIO) += pcf50633-gpio.o +obj-$(CONFIG_RZ_MTU3) += rz-mtu3.o obj-$(CONFIG_ABX500_CORE) += abx500-core.o obj-$(CONFIG_MFD_DB8500_PRCMU) += db8500-prcmu.o # ab8500-core need to come after db8500-prcmu (which provides the channel) diff --git a/drivers/mfd/rz-mtu3.c b/drivers/mfd/rz-mtu3.c new file mode 100644 index 000000000000..04006f4aa702 --- /dev/null +++ b/drivers/mfd/rz-mtu3.c @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Renesas RZ/G2L Multi-Function Timer Pulse Unit 3(MTU3a) Core driver + * + * Copyright (C) 2023 Renesas Electronics Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rz-mtu3.h" + +struct rz_mtu3_priv { + void __iomem *mmio; + struct reset_control *rstc; + raw_spinlock_t lock; +}; + +/******* MTU3 registers (original offset is +0x1200) *******/ +static const unsigned long rz_mtu3_8bit_ch_reg_offs[][13] = { + [RZ_MTU3_CHAN_0] = MTU_8BIT_CH_0(0x104, 0x090, 0x100, 0x128, 0x101, 0x102, 0x103, 0x126), + [RZ_MTU3_CHAN_1] = MTU_8BIT_CH_1_2(0x184, 0x091, 0x185, 0x180, 0x194, 0x181, 0x182), + [RZ_MTU3_CHAN_2] = MTU_8BIT_CH_1_2(0x204, 0x092, 0x205, 0x200, 0x20c, 0x201, 0x202), + [RZ_MTU3_CHAN_3] = MTU_8BIT_CH_3_4_6_7(0x008, 0x093, 0x02c, 0x000, 0x04c, 0x002, 0x004, 0x005, 0x038), + [RZ_MTU3_CHAN_4] = MTU_8BIT_CH_3_4_6_7(0x009, 0x094, 0x02d, 0x001, 0x04d, 0x003, 0x006, 0x007, 0x039), + [RZ_MTU3_CHAN_5] = MTU_8BIT_CH_5(0xab2, 0x1eb, 0xab4, 0xab6, 0xa84, 0xa85, 0xa86, 0xa94, 0xa95, 0xa96, 0xaa4, 0xaa5, 0xaa6), + [RZ_MTU3_CHAN_6] = MTU_8BIT_CH_3_4_6_7(0x808, 0x893, 0x82c, 0x800, 0x84c, 0x802, 0x804, 0x805, 0x838), + [RZ_MTU3_CHAN_7] = MTU_8BIT_CH_3_4_6_7(0x809, 0x894, 0x82d, 0x801, 0x84d, 0x803, 0x806, 0x807, 0x839), + [RZ_MTU3_CHAN_8] = MTU_8BIT_CH_8(0x404, 0x098, 0x400, 0x406, 0x401, 0x402, 0x403) +}; + +static const unsigned long rz_mtu3_16bit_ch_reg_offs[][12] = { + [RZ_MTU3_CHAN_0] = MTU_16BIT_CH_0(0x106, 0x108, 0x10a, 0x10c, 0x10e, 0x120, 0x122), + [RZ_MTU3_CHAN_1] = MTU_16BIT_CH_1_2(0x186, 0x188, 0x18a), + [RZ_MTU3_CHAN_2] = MTU_16BIT_CH_1_2(0x206, 0x208, 0x20a), + [RZ_MTU3_CHAN_3] = MTU_16BIT_CH_3_6(0x010, 0x018, 0x01a, 0x024, 0x026, 0x072), + [RZ_MTU3_CHAN_4] = MTU_16BIT_CH_4_7(0x012, 0x01c, 0x01e, 0x028, 0x2a, 0x074, 0x076, 0x040, 0x044, 0x046, 0x048, 0x04a), + [RZ_MTU3_CHAN_5] = MTU_16BIT_CH_5(0xa80, 0xa82, 0xa90, 0xa92, 0xaa0, 0xaa2), + [RZ_MTU3_CHAN_6] = MTU_16BIT_CH_3_6(0x810, 0x818, 0x81a, 0x824, 0x826, 0x872), + [RZ_MTU3_CHAN_7] = MTU_16BIT_CH_4_7(0x812, 0x81c, 0x81e, 0x828, 0x82a, 0x874, 0x876, 0x840, 0x844, 0x846, 0x848, 0x84a) +}; + +static const unsigned long rz_mtu3_32bit_ch_reg_offs[][5] = { + [RZ_MTU3_CHAN_1] = MTU_32BIT_CH_1(0x1a0, 0x1a4, 0x1a8), + [RZ_MTU3_CHAN_8] = MTU_32BIT_CH_8(0x408, 0x40c, 0x410, 0x414, 0x418) +}; + +static bool rz_mtu3_is_16bit_shared_reg(u16 offset) +{ + return (offset == RZ_MTU3_TDDRA || offset == RZ_MTU3_TDDRB || + offset == RZ_MTU3_TCDRA || offset == RZ_MTU3_TCDRB || + offset == RZ_MTU3_TCBRA || offset == RZ_MTU3_TCBRB || + offset == RZ_MTU3_TCNTSA || offset == RZ_MTU3_TCNTSB); +} + +u16 rz_mtu3_shared_reg_read(struct rz_mtu3_channel *ch, u16 offset) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(ch->dev->parent); + struct rz_mtu3_priv *priv = mtu->priv_data; + + if (rz_mtu3_is_16bit_shared_reg(offset)) + return readw(priv->mmio + offset); + else + return readb(priv->mmio + offset); +} +EXPORT_SYMBOL_GPL(rz_mtu3_shared_reg_read); + +u8 rz_mtu3_8bit_ch_read(struct rz_mtu3_channel *ch, u16 offset) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(ch->dev->parent); + struct rz_mtu3_priv *priv = mtu->priv_data; + u16 ch_offs; + + ch_offs = rz_mtu3_8bit_ch_reg_offs[ch->channel_number][offset]; + + return readb(priv->mmio + ch_offs); +} +EXPORT_SYMBOL_GPL(rz_mtu3_8bit_ch_read); + +u16 rz_mtu3_16bit_ch_read(struct rz_mtu3_channel *ch, u16 offset) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(ch->dev->parent); + struct rz_mtu3_priv *priv = mtu->priv_data; + u16 ch_offs; + + /* MTU8 doesn't have 16-bit registers */ + if (ch->channel_number == RZ_MTU3_CHAN_8) + return 0; + + ch_offs = rz_mtu3_16bit_ch_reg_offs[ch->channel_number][offset]; + + return readw(priv->mmio + ch_offs); +} +EXPORT_SYMBOL_GPL(rz_mtu3_16bit_ch_read); + +u32 rz_mtu3_32bit_ch_read(struct rz_mtu3_channel *ch, u16 offset) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(ch->dev->parent); + struct rz_mtu3_priv *priv = mtu->priv_data; + u16 ch_offs; + + if (ch->channel_number != RZ_MTU3_CHAN_1 && ch->channel_number != RZ_MTU3_CHAN_8) + return 0; + + ch_offs = rz_mtu3_32bit_ch_reg_offs[ch->channel_number][offset]; + + return readl(priv->mmio + ch_offs); +} +EXPORT_SYMBOL_GPL(rz_mtu3_32bit_ch_read); + +void rz_mtu3_8bit_ch_write(struct rz_mtu3_channel *ch, u16 offset, u8 val) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(ch->dev->parent); + struct rz_mtu3_priv *priv = mtu->priv_data; + u16 ch_offs; + + ch_offs = rz_mtu3_8bit_ch_reg_offs[ch->channel_number][offset]; + writeb(val, priv->mmio + ch_offs); +} +EXPORT_SYMBOL_GPL(rz_mtu3_8bit_ch_write); + +void rz_mtu3_16bit_ch_write(struct rz_mtu3_channel *ch, u16 offset, u16 val) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(ch->dev->parent); + struct rz_mtu3_priv *priv = mtu->priv_data; + u16 ch_offs; + + /* MTU8 doesn't have 16-bit registers */ + if (ch->channel_number == RZ_MTU3_CHAN_8) + return; + + ch_offs = rz_mtu3_16bit_ch_reg_offs[ch->channel_number][offset]; + writew(val, priv->mmio + ch_offs); +} +EXPORT_SYMBOL_GPL(rz_mtu3_16bit_ch_write); + +void rz_mtu3_32bit_ch_write(struct rz_mtu3_channel *ch, u16 offset, u32 val) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(ch->dev->parent); + struct rz_mtu3_priv *priv = mtu->priv_data; + u16 ch_offs; + + if (ch->channel_number != RZ_MTU3_CHAN_1 && ch->channel_number != RZ_MTU3_CHAN_8) + return; + + ch_offs = rz_mtu3_32bit_ch_reg_offs[ch->channel_number][offset]; + writel(val, priv->mmio + ch_offs); +} +EXPORT_SYMBOL_GPL(rz_mtu3_32bit_ch_write); + +void rz_mtu3_shared_reg_write(struct rz_mtu3_channel *ch, u16 offset, u16 value) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(ch->dev->parent); + struct rz_mtu3_priv *priv = mtu->priv_data; + + if (rz_mtu3_is_16bit_shared_reg(offset)) + writew(value, priv->mmio + offset); + else + writeb((u8)value, priv->mmio + offset); +} +EXPORT_SYMBOL_GPL(rz_mtu3_shared_reg_write); + +void rz_mtu3_shared_reg_update_bit(struct rz_mtu3_channel *ch, u16 offset, + u16 pos, u8 val) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(ch->dev->parent); + struct rz_mtu3_priv *priv = mtu->priv_data; + unsigned long tmdr, flags; + + raw_spin_lock_irqsave(&priv->lock, flags); + tmdr = rz_mtu3_shared_reg_read(ch, offset); + __assign_bit(pos, &tmdr, !!val); + rz_mtu3_shared_reg_write(ch, offset, tmdr); + raw_spin_unlock_irqrestore(&priv->lock, flags); +} +EXPORT_SYMBOL_GPL(rz_mtu3_shared_reg_update_bit); + +static u16 rz_mtu3_get_tstr_offset(struct rz_mtu3_channel *ch) +{ + u16 offset; + + switch (ch->channel_number) { + case RZ_MTU3_CHAN_0: + case RZ_MTU3_CHAN_1: + case RZ_MTU3_CHAN_2: + case RZ_MTU3_CHAN_3: + case RZ_MTU3_CHAN_4: + case RZ_MTU3_CHAN_8: + offset = RZ_MTU3_TSTRA; + break; + case RZ_MTU3_CHAN_5: + offset = RZ_MTU3_TSTR; + break; + case RZ_MTU3_CHAN_6: + case RZ_MTU3_CHAN_7: + offset = RZ_MTU3_TSTRB; + break; + default: + offset = 0; + break; + } + + return offset; +} + +static u8 rz_mtu3_get_tstr_bit_pos(struct rz_mtu3_channel *ch) +{ + u8 bitpos; + + switch (ch->channel_number) { + case RZ_MTU3_CHAN_0: + case RZ_MTU3_CHAN_1: + case RZ_MTU3_CHAN_2: + case RZ_MTU3_CHAN_6: + case RZ_MTU3_CHAN_7: + bitpos = ch->channel_number; + break; + case RZ_MTU3_CHAN_3: + bitpos = 6; + break; + case RZ_MTU3_CHAN_4: + bitpos = 7; + break; + case RZ_MTU3_CHAN_5: + bitpos = 2; + break; + case RZ_MTU3_CHAN_8: + bitpos = 3; + break; + default: + bitpos = 0; + break; + } + + return bitpos; +} + +static void rz_mtu3_start_stop_ch(struct rz_mtu3_channel *ch, bool start) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(ch->dev->parent); + struct rz_mtu3_priv *priv = mtu->priv_data; + unsigned long flags, tstr; + u16 offset; + u8 bitpos; + + /* start stop register shared by multiple timer channels */ + raw_spin_lock_irqsave(&priv->lock, flags); + + offset = rz_mtu3_get_tstr_offset(ch); + bitpos = rz_mtu3_get_tstr_bit_pos(ch); + tstr = rz_mtu3_shared_reg_read(ch, offset); + __assign_bit(bitpos, &tstr, start); + rz_mtu3_shared_reg_write(ch, offset, tstr); + + raw_spin_unlock_irqrestore(&priv->lock, flags); +} + +bool rz_mtu3_is_enabled(struct rz_mtu3_channel *ch) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(ch->dev->parent); + struct rz_mtu3_priv *priv = mtu->priv_data; + unsigned long flags, tstr; + bool ret = false; + u16 offset; + u8 bitpos; + + /* start stop register shared by multiple timer channels */ + raw_spin_lock_irqsave(&priv->lock, flags); + + offset = rz_mtu3_get_tstr_offset(ch); + bitpos = rz_mtu3_get_tstr_bit_pos(ch); + tstr = rz_mtu3_shared_reg_read(ch, offset); + ret = tstr & BIT(bitpos); + + raw_spin_unlock_irqrestore(&priv->lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(rz_mtu3_is_enabled); + +int rz_mtu3_enable(struct rz_mtu3_channel *ch) +{ + /* enable channel */ + rz_mtu3_start_stop_ch(ch, true); + + return 0; +} +EXPORT_SYMBOL_GPL(rz_mtu3_enable); + +void rz_mtu3_disable(struct rz_mtu3_channel *ch) +{ + /* disable channel */ + rz_mtu3_start_stop_ch(ch, false); +} +EXPORT_SYMBOL_GPL(rz_mtu3_disable); + +static void rz_mtu3_reset_assert(void *data) +{ + struct rz_mtu3 *mtu = dev_get_drvdata(data); + struct rz_mtu3_priv *priv = mtu->priv_data; + + mfd_remove_devices(data); + reset_control_assert(priv->rstc); +} + +static const struct mfd_cell rz_mtu3_devs[] = { + { + .name = "rz-mtu3-counter", + }, + { + .name = "pwm-rz-mtu3", + }, +}; + +static int rz_mtu3_probe(struct platform_device *pdev) +{ + struct rz_mtu3_priv *priv; + struct rz_mtu3 *ddata; + unsigned int i; + int ret; + + ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL); + if (!ddata) + return -ENOMEM; + + ddata->priv_data = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!ddata->priv_data) + return -ENOMEM; + + priv = ddata->priv_data; + + priv->mmio = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->mmio)) + return PTR_ERR(priv->mmio); + + priv->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); + if (IS_ERR(priv->rstc)) + return PTR_ERR(priv->rstc); + + ddata->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(ddata->clk)) + return PTR_ERR(ddata->clk); + + reset_control_deassert(priv->rstc); + raw_spin_lock_init(&priv->lock); + platform_set_drvdata(pdev, ddata); + + for (i = 0; i < RZ_MTU_NUM_CHANNELS; i++) { + ddata->channels[i].channel_number = i; + ddata->channels[i].is_busy = false; + mutex_init(&ddata->channels[i].lock); + } + + ret = mfd_add_devices(&pdev->dev, 0, rz_mtu3_devs, + ARRAY_SIZE(rz_mtu3_devs), NULL, 0, NULL); + if (ret < 0) + goto err_assert; + + return devm_add_action_or_reset(&pdev->dev, rz_mtu3_reset_assert, + &pdev->dev); + +err_assert: + reset_control_assert(priv->rstc); + return ret; +} + +static const struct of_device_id rz_mtu3_of_match[] = { + { .compatible = "renesas,rz-mtu3", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, rz_mtu3_of_match); + +static struct platform_driver rz_mtu3_driver = { + .probe = rz_mtu3_probe, + .driver = { + .name = "rz-mtu3", + .of_match_table = rz_mtu3_of_match, + }, +}; +module_platform_driver(rz_mtu3_driver); + +MODULE_AUTHOR("Biju Das "); +MODULE_DESCRIPTION("Renesas RZ/G2L MTU3a Core Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/mfd/rz-mtu3.h b/drivers/mfd/rz-mtu3.h new file mode 100644 index 000000000000..51a1298b0613 --- /dev/null +++ b/drivers/mfd/rz-mtu3.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * MFD internals for Renesas RZ/G2L MTU3 Core driver + * + * Copyright (C) 2023 Renesas Electronics Corporation + */ + +#ifndef RZ_MTU3_MFD_H +#define RZ_MTU3_MFD_H + +#define MTU_8BIT_CH_0(_tier, _nfcr, _tcr, _tcr2, _tmdr1, _tiorh, _tiorl, _tbtm) \ + { \ + [RZ_MTU3_TIER] = _tier, \ + [RZ_MTU3_NFCR] = _nfcr, \ + [RZ_MTU3_TCR] = _tcr, \ + [RZ_MTU3_TCR2] = _tcr2, \ + [RZ_MTU3_TMDR1] = _tmdr1, \ + [RZ_MTU3_TIORH] = _tiorh, \ + [RZ_MTU3_TIORL] = _tiorl, \ + [RZ_MTU3_TBTM] = _tbtm \ + } + +#define MTU_8BIT_CH_1_2(_tier, _nfcr, _tsr, _tcr, _tcr2, _tmdr1, _tior) \ + { \ + [RZ_MTU3_TIER] = _tier, \ + [RZ_MTU3_NFCR] = _nfcr, \ + [RZ_MTU3_TSR] = _tsr, \ + [RZ_MTU3_TCR] = _tcr, \ + [RZ_MTU3_TCR2] = _tcr2, \ + [RZ_MTU3_TMDR1] = _tmdr1, \ + [RZ_MTU3_TIOR] = _tior \ + } \ + +#define MTU_8BIT_CH_3_4_6_7(_tier, _nfcr, _tsr, _tcr, _tcr2, _tmdr1, _tiorh, _tiorl, _tbtm) \ + { \ + [RZ_MTU3_TIER] = _tier, \ + [RZ_MTU3_NFCR] = _nfcr, \ + [RZ_MTU3_TSR] = _tsr, \ + [RZ_MTU3_TCR] = _tcr, \ + [RZ_MTU3_TCR2] = _tcr2, \ + [RZ_MTU3_TMDR1] = _tmdr1, \ + [RZ_MTU3_TIORH] = _tiorh, \ + [RZ_MTU3_TIORL] = _tiorl, \ + [RZ_MTU3_TBTM] = _tbtm \ + } \ + +#define MTU_8BIT_CH_5(_tier, _nfcr, _tstr, _tcntcmpclr, _tcru, _tcr2u, _tioru, \ + _tcrv, _tcr2v, _tiorv, _tcrw, _tcr2w, _tiorw) \ + { \ + [RZ_MTU3_TIER] = _tier, \ + [RZ_MTU3_NFCR] = _nfcr, \ + [RZ_MTU3_TSTR] = _tstr, \ + [RZ_MTU3_TCNTCMPCLR] = _tcntcmpclr, \ + [RZ_MTU3_TCRU] = _tcru, \ + [RZ_MTU3_TCR2U] = _tcr2u, \ + [RZ_MTU3_TIORU] = _tioru, \ + [RZ_MTU3_TCRV] = _tcrv, \ + [RZ_MTU3_TCR2V] = _tcr2v, \ + [RZ_MTU3_TIORV] = _tiorv, \ + [RZ_MTU3_TCRW] = _tcrw, \ + [RZ_MTU3_TCR2W] = _tcr2w, \ + [RZ_MTU3_TIORW] = _tiorw \ + } \ + +#define MTU_8BIT_CH_8(_tier, _nfcr, _tcr, _tcr2, _tmdr1, _tiorh, _tiorl) \ + { \ + [RZ_MTU3_TIER] = _tier, \ + [RZ_MTU3_NFCR] = _nfcr, \ + [RZ_MTU3_TCR] = _tcr, \ + [RZ_MTU3_TCR2] = _tcr2, \ + [RZ_MTU3_TMDR1] = _tmdr1, \ + [RZ_MTU3_TIORH] = _tiorh, \ + [RZ_MTU3_TIORL] = _tiorl \ + } \ + +#define MTU_16BIT_CH_0(_tcnt, _tgra, _tgrb, _tgrc, _tgrd, _tgre, _tgrf) \ + { \ + [RZ_MTU3_TCNT] = _tcnt, \ + [RZ_MTU3_TGRA] = _tgra, \ + [RZ_MTU3_TGRB] = _tgrb, \ + [RZ_MTU3_TGRC] = _tgrc, \ + [RZ_MTU3_TGRD] = _tgrd, \ + [RZ_MTU3_TGRE] = _tgre, \ + [RZ_MTU3_TGRF] = _tgrf \ + } + +#define MTU_16BIT_CH_1_2(_tcnt, _tgra, _tgrb) \ + { \ + [RZ_MTU3_TCNT] = _tcnt, \ + [RZ_MTU3_TGRA] = _tgra, \ + [RZ_MTU3_TGRB] = _tgrb \ + } + +#define MTU_16BIT_CH_3_6(_tcnt, _tgra, _tgrb, _tgrc, _tgrd, _tgre) \ + { \ + [RZ_MTU3_TCNT] = _tcnt, \ + [RZ_MTU3_TGRA] = _tgra, \ + [RZ_MTU3_TGRB] = _tgrb, \ + [RZ_MTU3_TGRC] = _tgrc, \ + [RZ_MTU3_TGRD] = _tgrd, \ + [RZ_MTU3_TGRE] = _tgre \ + } + +#define MTU_16BIT_CH_4_7(_tcnt, _tgra, _tgrb, _tgrc, _tgrd, _tgre, _tgrf, \ + _tadcr, _tadcora, _tadcorb, _tadcobra, _tadcobrb) \ + { \ + [RZ_MTU3_TCNT] = _tcnt, \ + [RZ_MTU3_TGRA] = _tgra, \ + [RZ_MTU3_TGRB] = _tgrb, \ + [RZ_MTU3_TGRC] = _tgrc, \ + [RZ_MTU3_TGRD] = _tgrd, \ + [RZ_MTU3_TGRE] = _tgre, \ + [RZ_MTU3_TGRF] = _tgrf, \ + [RZ_MTU3_TADCR] = _tadcr, \ + [RZ_MTU3_TADCORA] = _tadcora, \ + [RZ_MTU3_TADCORB] = _tadcorb, \ + [RZ_MTU3_TADCOBRA] = _tadcobra, \ + [RZ_MTU3_TADCOBRB] = _tadcobrb \ + } + +#define MTU_16BIT_CH_5(_tcntu, _tgru, _tcntv, _tgrv, _tcntw, _tgrw) \ + { \ + [RZ_MTU3_TCNTU] = _tcntu, \ + [RZ_MTU3_TGRU] = _tgru, \ + [RZ_MTU3_TCNTV] = _tcntv, \ + [RZ_MTU3_TGRV] = _tgrv, \ + [RZ_MTU3_TCNTW] = _tcntw, \ + [RZ_MTU3_TGRW] = _tgrw \ + } + +#define MTU_32BIT_CH_1(_tcntlw, _tgralw, _tgrblw) \ + { \ + [RZ_MTU3_TCNTLW] = _tcntlw, \ + [RZ_MTU3_TGRALW] = _tgralw, \ + [RZ_MTU3_TGRBLW] = _tgrblw \ + } + +#define MTU_32BIT_CH_8(_tcnt, _tgra, _tgrb, _tgrc, _tgrd) \ + { \ + [RZ_MTU3_TCNT] = _tcnt, \ + [RZ_MTU3_TGRA] = _tgra, \ + [RZ_MTU3_TGRB] = _tgrb, \ + [RZ_MTU3_TGRC] = _tgrc, \ + [RZ_MTU3_TGRD] = _tgrd \ + } + +#endif diff --git a/include/linux/mfd/rz-mtu3.h b/include/linux/mfd/rz-mtu3.h new file mode 100644 index 000000000000..c5173bc06270 --- /dev/null +++ b/include/linux/mfd/rz-mtu3.h @@ -0,0 +1,257 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022 Renesas Electronics Corporation + */ +#ifndef __MFD_RZ_MTU3_H__ +#define __MFD_RZ_MTU3_H__ + +#include +#include +#include + +/* 8-bit shared register offsets macros */ +#define RZ_MTU3_TSTRA 0x080 /* Timer start register A */ +#define RZ_MTU3_TSTRB 0x880 /* Timer start register B */ + +/* 16-bit shared register offset macros */ +#define RZ_MTU3_TDDRA 0x016 /* Timer dead time data register A */ +#define RZ_MTU3_TDDRB 0x816 /* Timer dead time data register B */ +#define RZ_MTU3_TCDRA 0x014 /* Timer cycle data register A */ +#define RZ_MTU3_TCDRB 0x814 /* Timer cycle data register B */ +#define RZ_MTU3_TCBRA 0x022 /* Timer cycle buffer register A */ +#define RZ_MTU3_TCBRB 0x822 /* Timer cycle buffer register B */ +#define RZ_MTU3_TCNTSA 0x020 /* Timer subcounter A */ +#define RZ_MTU3_TCNTSB 0x820 /* Timer subcounter B */ + +/* + * MTU5 contains 3 timer counter registers and is totaly different + * from other channels, so we must separate its offset + */ + +/* 8-bit register offset macros of MTU3 channels except MTU5 */ +#define RZ_MTU3_TIER 0 /* Timer interrupt register */ +#define RZ_MTU3_NFCR 1 /* Noise filter control register */ +#define RZ_MTU3_TSR 2 /* Timer status register */ +#define RZ_MTU3_TCR 3 /* Timer control register */ +#define RZ_MTU3_TCR2 4 /* Timer control register 2 */ + +/* Timer mode register 1 */ +#define RZ_MTU3_TMDR1 5 +#define RZ_MTU3_TMDR1_MD GENMASK(3, 0) +#define RZ_MTU3_TMDR1_MD_NORMAL FIELD_PREP(RZ_MTU3_TMDR1_MD, 0) +#define RZ_MTU3_TMDR1_MD_PWMMODE1 FIELD_PREP(RZ_MTU3_TMDR1_MD, 2) + +#define RZ_MTU3_TIOR 6 /* Timer I/O control register */ +#define RZ_MTU3_TIORH 6 /* Timer I/O control register H */ +#define RZ_MTU3_TIORL 7 /* Timer I/O control register L */ +/* Only MTU3/4/6/7 have TBTM registers */ +#define RZ_MTU3_TBTM 8 /* Timer buffer operation transfer mode register */ + +/* 8-bit MTU5 register offset macros */ +#define RZ_MTU3_TSTR 2 /* MTU5 Timer start register */ +#define RZ_MTU3_TCNTCMPCLR 3 /* MTU5 Timer compare match clear register */ +#define RZ_MTU3_TCRU 4 /* Timer control register U */ +#define RZ_MTU3_TCR2U 5 /* Timer control register 2U */ +#define RZ_MTU3_TIORU 6 /* Timer I/O control register U */ +#define RZ_MTU3_TCRV 7 /* Timer control register V */ +#define RZ_MTU3_TCR2V 8 /* Timer control register 2V */ +#define RZ_MTU3_TIORV 9 /* Timer I/O control register V */ +#define RZ_MTU3_TCRW 10 /* Timer control register W */ +#define RZ_MTU3_TCR2W 11 /* Timer control register 2W */ +#define RZ_MTU3_TIORW 12 /* Timer I/O control register W */ + +/* 16-bit register offset macros of MTU3 channels except MTU5 */ +#define RZ_MTU3_TCNT 0 /* Timer counter */ +#define RZ_MTU3_TGRA 1 /* Timer general register A */ +#define RZ_MTU3_TGRB 2 /* Timer general register B */ +#define RZ_MTU3_TGRC 3 /* Timer general register C */ +#define RZ_MTU3_TGRD 4 /* Timer general register D */ +#define RZ_MTU3_TGRE 5 /* Timer general register E */ +#define RZ_MTU3_TGRF 6 /* Timer general register F */ +/* Timer A/D converter start request registers */ +#define RZ_MTU3_TADCR 7 /* control register */ +#define RZ_MTU3_TADCORA 8 /* cycle set register A */ +#define RZ_MTU3_TADCORB 9 /* cycle set register B */ +#define RZ_MTU3_TADCOBRA 10 /* cycle set buffer register A */ +#define RZ_MTU3_TADCOBRB 11 /* cycle set buffer register B */ + +/* 16-bit MTU5 register offset macros */ +#define RZ_MTU3_TCNTU 0 /* MTU5 Timer counter U */ +#define RZ_MTU3_TGRU 1 /* MTU5 Timer general register U */ +#define RZ_MTU3_TCNTV 2 /* MTU5 Timer counter V */ +#define RZ_MTU3_TGRV 3 /* MTU5 Timer general register V */ +#define RZ_MTU3_TCNTW 4 /* MTU5 Timer counter W */ +#define RZ_MTU3_TGRW 5 /* MTU5 Timer general register W */ + +/* 32-bit register offset */ +#define RZ_MTU3_TCNTLW 0 /* Timer longword counter */ +#define RZ_MTU3_TGRALW 1 /* Timer longword general register A */ +#define RZ_MTU3_TGRBLW 2 /* Timer longowrd general register B */ + +#define RZ_MTU3_TMDR3 0x191 /* MTU1 Timer Mode Register 3 */ + +/* Macros for setting registers */ +#define RZ_MTU3_TCR_CCLR GENMASK(7, 5) +#define RZ_MTU3_TCR_CKEG GENMASK(4, 3) +#define RZ_MTU3_TCR_TPCS GENMASK(2, 0) +#define RZ_MTU3_TCR_CCLR_TGRA BIT(5) +#define RZ_MTU3_TCR_CCLR_TGRC FIELD_PREP(RZ_MTU3_TCR_CCLR, 5) +#define RZ_MTU3_TCR_CKEG_RISING FIELD_PREP(RZ_MTU3_TCR_CKEG, 0) + +#define RZ_MTU3_TIOR_IOB GENMASK(7, 4) +#define RZ_MTU3_TIOR_IOA GENMASK(3, 0) +#define RZ_MTU3_TIOR_OC_RETAIN 0 +#define RZ_MTU3_TIOR_OC_INIT_OUT_LO_HI_OUT 2 +#define RZ_MTU3_TIOR_OC_INIT_OUT_HI_TOGGLE_OUT 7 + +#define RZ_MTU3_TIOR_OC_IOA_H_COMP_MATCH \ + FIELD_PREP(RZ_MTU3_TIOR_IOA, RZ_MTU3_TIOR_OC_INIT_OUT_LO_HI_OUT) +#define RZ_MTU3_TIOR_OC_IOB_TOGGLE \ + FIELD_PREP(RZ_MTU3_TIOR_IOB, RZ_MTU3_TIOR_OC_INIT_OUT_HI_TOGGLE_OUT) + +enum rz_mtu3_channels { + RZ_MTU3_CHAN_0, + RZ_MTU3_CHAN_1, + RZ_MTU3_CHAN_2, + RZ_MTU3_CHAN_3, + RZ_MTU3_CHAN_4, + RZ_MTU3_CHAN_5, + RZ_MTU3_CHAN_6, + RZ_MTU3_CHAN_7, + RZ_MTU3_CHAN_8, + RZ_MTU_NUM_CHANNELS +}; + +/** + * struct rz_mtu3_channel - MTU3 channel private data + * + * @dev: device handle + * @channel_number: channel number + * @lock: Lock to protect channel state + * @is_busy: channel state + */ +struct rz_mtu3_channel { + struct device *dev; + unsigned int channel_number; + struct mutex lock; + bool is_busy; +}; + +/** + * struct rz_mtu3 - MTU3 core private data + * + * @clk: MTU3 module clock + * @rz_mtu3_channel: HW channels + * @priv_data: MTU3 core driver private data + */ +struct rz_mtu3 { + struct clk *clk; + struct rz_mtu3_channel channels[RZ_MTU_NUM_CHANNELS]; + + void *priv_data; +}; + +#if IS_ENABLED(CONFIG_RZ_MTU3) +static inline bool rz_mtu3_request_channel(struct rz_mtu3_channel *ch) +{ + mutex_lock(&ch->lock); + if (ch->is_busy) { + mutex_unlock(&ch->lock); + return false; + } + + ch->is_busy = true; + mutex_unlock(&ch->lock); + + return true; +} + +static inline void rz_mtu3_release_channel(struct rz_mtu3_channel *ch) +{ + mutex_lock(&ch->lock); + ch->is_busy = false; + mutex_unlock(&ch->lock); +} + +bool rz_mtu3_is_enabled(struct rz_mtu3_channel *ch); +void rz_mtu3_disable(struct rz_mtu3_channel *ch); +int rz_mtu3_enable(struct rz_mtu3_channel *ch); + +u8 rz_mtu3_8bit_ch_read(struct rz_mtu3_channel *ch, u16 off); +u16 rz_mtu3_16bit_ch_read(struct rz_mtu3_channel *ch, u16 off); +u32 rz_mtu3_32bit_ch_read(struct rz_mtu3_channel *ch, u16 off); +u16 rz_mtu3_shared_reg_read(struct rz_mtu3_channel *ch, u16 off); + +void rz_mtu3_8bit_ch_write(struct rz_mtu3_channel *ch, u16 off, u8 val); +void rz_mtu3_16bit_ch_write(struct rz_mtu3_channel *ch, u16 off, u16 val); +void rz_mtu3_32bit_ch_write(struct rz_mtu3_channel *ch, u16 off, u32 val); +void rz_mtu3_shared_reg_write(struct rz_mtu3_channel *ch, u16 off, u16 val); +void rz_mtu3_shared_reg_update_bit(struct rz_mtu3_channel *ch, u16 off, + u16 pos, u8 val); +#else +static inline bool rz_mtu3_request_channel(struct rz_mtu3_channel *ch) +{ + return false; +} + +static inline void rz_mtu3_release_channel(struct rz_mtu3_channel *ch) +{ +} + +static inline bool rz_mtu3_is_enabled(struct rz_mtu3_channel *ch) +{ + return false; +} + +static inline void rz_mtu3_disable(struct rz_mtu3_channel *ch) +{ +} + +static inline int rz_mtu3_enable(struct rz_mtu3_channel *ch) +{ + return 0; +} + +static inline u8 rz_mtu3_8bit_ch_read(struct rz_mtu3_channel *ch, u16 off) +{ + return 0; +} + +static inline u16 rz_mtu3_16bit_ch_read(struct rz_mtu3_channel *ch, u16 off) +{ + return 0; +} + +static inline u32 rz_mtu3_32bit_ch_read(struct rz_mtu3_channel *ch, u16 off) +{ + return 0; +} + +static inline u16 rz_mtu3_shared_reg_read(struct rz_mtu3_channel *ch, u16 off) +{ + return 0; +} + +static inline void rz_mtu3_8bit_ch_write(struct rz_mtu3_channel *ch, u16 off, u8 val) +{ +} + +static inline void rz_mtu3_16bit_ch_write(struct rz_mtu3_channel *ch, u16 off, u16 val) +{ +} + +static inline void rz_mtu3_32bit_ch_write(struct rz_mtu3_channel *ch, u16 off, u32 val) +{ +} + +static inline void rz_mtu3_shared_reg_write(struct rz_mtu3_channel *ch, u16 off, u16 val) +{ +} + +static inline void rz_mtu3_shared_reg_update_bit(struct rz_mtu3_channel *ch, + u16 off, u16 pos, u8 val) +{ +} +#endif + +#endif /* __MFD_RZ_MTU3_H__ */ -- cgit v1.2.3 From dcb779fcd4ed5984ad15991d574943d12a8693d1 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Wed, 15 Feb 2023 06:53:54 -0500 Subject: nfsd: allow reaping files still under writeback On most filesystems, there is no reason to delay reaping an nfsd_file just because its underlying inode is still under writeback. nfsd just relies on client activity or the local flusher threads to do writeback. The main exception is NFS, which flushes all of its dirty data on last close. Add a new EXPORT_OP_FLUSH_ON_CLOSE flag to allow filesystems to signal that they do this, and only skip closing files under writeback on such filesystems. Also, remove a redundant NULL file pointer check in nfsd_file_check_writeback, and clean up nfs's export op flag definitions. Signed-off-by: Jeff Layton Acked-by: Anna Schumaker Signed-off-by: Chuck Lever --- fs/nfs/export.c | 9 ++++++--- fs/nfsd/filecache.c | 12 +++++++++++- include/linux/exportfs.h | 1 + 3 files changed, 18 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/export.c b/fs/nfs/export.c index d6a6d1ebb8fd..be686b8e0c54 100644 --- a/fs/nfs/export.c +++ b/fs/nfs/export.c @@ -149,7 +149,10 @@ const struct export_operations nfs_export_ops = { .encode_fh = nfs_encode_fh, .fh_to_dentry = nfs_fh_to_dentry, .get_parent = nfs_get_parent, - .flags = EXPORT_OP_NOWCC|EXPORT_OP_NOSUBTREECHK| - EXPORT_OP_CLOSE_BEFORE_UNLINK|EXPORT_OP_REMOTE_FS| - EXPORT_OP_NOATOMIC_ATTR, + .flags = EXPORT_OP_NOWCC | + EXPORT_OP_NOSUBTREECHK | + EXPORT_OP_CLOSE_BEFORE_UNLINK | + EXPORT_OP_REMOTE_FS | + EXPORT_OP_NOATOMIC_ATTR | + EXPORT_OP_FLUSH_ON_CLOSE, }; diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 9b7082fdd211..a6fa6e980277 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -402,13 +402,23 @@ nfsd_file_check_writeback(struct nfsd_file *nf) struct file *file = nf->nf_file; struct address_space *mapping; - if (!file || !(file->f_mode & FMODE_WRITE)) + /* File not open for write? */ + if (!(file->f_mode & FMODE_WRITE)) return false; + + /* + * Some filesystems (e.g. NFS) flush all dirty data on close. + * On others, there is no need to wait for writeback. + */ + if (!(file_inode(file)->i_sb->s_export_op->flags & EXPORT_OP_FLUSH_ON_CLOSE)) + return false; + mapping = file->f_mapping; return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) || mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK); } + static bool nfsd_file_lru_add(struct nfsd_file *nf) { set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags); diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h index 601700fedc91..9edb29101ec8 100644 --- a/include/linux/exportfs.h +++ b/include/linux/exportfs.h @@ -220,6 +220,7 @@ struct export_operations { #define EXPORT_OP_NOATOMIC_ATTR (0x10) /* Filesystem cannot supply atomic attribute updates */ +#define EXPORT_OP_FLUSH_ON_CLOSE (0x20) /* fs flushes file data on close */ unsigned long flags; }; -- cgit v1.2.3 From c88c680c6de5077292667fae4a147fd5a6523479 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Fri, 3 Mar 2023 07:15:58 -0500 Subject: lockd: remove 2 unused helper functions Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/lockd/lockd.h | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h index 0168ac9fdda8..26c2aed31a0c 100644 --- a/include/linux/lockd/lockd.h +++ b/include/linux/lockd/lockd.h @@ -99,21 +99,11 @@ struct nsm_handle { /* * Rigorous type checking on sockaddr type conversions */ -static inline struct sockaddr_in *nlm_addr_in(const struct nlm_host *host) -{ - return (struct sockaddr_in *)&host->h_addr; -} - static inline struct sockaddr *nlm_addr(const struct nlm_host *host) { return (struct sockaddr *)&host->h_addr; } -static inline struct sockaddr_in *nlm_srcaddr_in(const struct nlm_host *host) -{ - return (struct sockaddr_in *)&host->h_srcaddr; -} - static inline struct sockaddr *nlm_srcaddr(const struct nlm_host *host) { return (struct sockaddr *)&host->h_srcaddr; -- cgit v1.2.3 From f0aa4852e63f9c1cfd4322c770e69d7e6817e906 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Fri, 3 Mar 2023 07:15:59 -0500 Subject: lockd: move struct nlm_wait to lockd.h The next patch needs struct nlm_wait in fs/lockd/clntproc.c, so move the definition to a shared header file. As an added clean-up, drop the unused b_reclaim field. Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/lockd/clntlock.c | 12 ------------ include/linux/lockd/lockd.h | 11 ++++++++++- 2 files changed, 10 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/fs/lockd/clntlock.c b/fs/lockd/clntlock.c index 82b19a30e0f0..464cb15c1a06 100644 --- a/fs/lockd/clntlock.c +++ b/fs/lockd/clntlock.c @@ -29,18 +29,6 @@ static int reclaimer(void *ptr); * client perspective. */ -/* - * This is the representation of a blocked client lock. - */ -struct nlm_wait { - struct list_head b_list; /* linked list */ - wait_queue_head_t b_wait; /* where to wait on */ - struct nlm_host * b_host; - struct file_lock * b_lock; /* local file lock */ - unsigned short b_reclaim; /* got to reclaim lock */ - __be32 b_status; /* grant callback status */ -}; - static LIST_HEAD(nlm_blocked); static DEFINE_SPINLOCK(nlm_blocked_lock); diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h index 26c2aed31a0c..de5ad2c2179a 100644 --- a/include/linux/lockd/lockd.h +++ b/include/linux/lockd/lockd.h @@ -121,7 +121,16 @@ struct nlm_lockowner { uint32_t pid; }; -struct nlm_wait; +/* + * This is the representation of a blocked client lock. + */ +struct nlm_wait { + struct list_head b_list; /* linked list */ + wait_queue_head_t b_wait; /* where to wait on */ + struct nlm_host *b_host; + struct file_lock *b_lock; /* local file lock */ + __be32 b_status; /* grant callback status */ +}; /* * Memory chunk for NLM client RPC request. -- cgit v1.2.3 From 2005f5b9c35bd736c81e9f24f5c5051967c022ee Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Fri, 3 Mar 2023 07:16:00 -0500 Subject: lockd: fix races in client GRANTED_MSG wait logic After the wait for a grant is done (for whatever reason), nlmclnt_block updates the status of the nlm_rqst with the status of the block. At the point it does this, however, the block is still queued its status could change at any time. This is particularly a problem when the waiting task is signaled during the wait. We can end up giving up on the lock just before the GRANTED_MSG callback comes in, and accept it even though the lock request gets back an error, leaving a dangling lock on the server. Since the nlm_wait never lives beyond the end of nlmclnt_lock, put it on the stack and add functions to allow us to enqueue and dequeue the block. Enqueue it just before the lock/wait loop, and dequeue it just after we exit the loop instead of waiting until the end of the function. Also, scrape the status at the time that we dequeue it to ensure that it's final. Reported-by: Yongcheng Yang Link: https://bugzilla.redhat.com/show_bug.cgi?id=2063818 Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/lockd/clntlock.c | 42 +++++++++++++++++++++--------------------- fs/lockd/clntproc.c | 28 ++++++++++++++++++---------- include/linux/lockd/lockd.h | 8 +++++--- 3 files changed, 44 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/fs/lockd/clntlock.c b/fs/lockd/clntlock.c index 464cb15c1a06..c374ee072db3 100644 --- a/fs/lockd/clntlock.c +++ b/fs/lockd/clntlock.c @@ -82,41 +82,42 @@ void nlmclnt_done(struct nlm_host *host) } EXPORT_SYMBOL_GPL(nlmclnt_done); +void nlmclnt_prepare_block(struct nlm_wait *block, struct nlm_host *host, struct file_lock *fl) +{ + block->b_host = host; + block->b_lock = fl; + init_waitqueue_head(&block->b_wait); + block->b_status = nlm_lck_blocked; +} + /* * Queue up a lock for blocking so that the GRANTED request can see it */ -struct nlm_wait *nlmclnt_prepare_block(struct nlm_host *host, struct file_lock *fl) +void nlmclnt_queue_block(struct nlm_wait *block) { - struct nlm_wait *block; - - block = kmalloc(sizeof(*block), GFP_KERNEL); - if (block != NULL) { - block->b_host = host; - block->b_lock = fl; - init_waitqueue_head(&block->b_wait); - block->b_status = nlm_lck_blocked; - - spin_lock(&nlm_blocked_lock); - list_add(&block->b_list, &nlm_blocked); - spin_unlock(&nlm_blocked_lock); - } - return block; + spin_lock(&nlm_blocked_lock); + list_add(&block->b_list, &nlm_blocked); + spin_unlock(&nlm_blocked_lock); } -void nlmclnt_finish_block(struct nlm_wait *block) +/* + * Dequeue the block and return its final status + */ +__be32 nlmclnt_dequeue_block(struct nlm_wait *block) { - if (block == NULL) - return; + __be32 status; + spin_lock(&nlm_blocked_lock); list_del(&block->b_list); + status = block->b_status; spin_unlock(&nlm_blocked_lock); - kfree(block); + return status; } /* * Block on a lock */ -int nlmclnt_block(struct nlm_wait *block, struct nlm_rqst *req, long timeout) +int nlmclnt_wait(struct nlm_wait *block, struct nlm_rqst *req, long timeout) { long ret; @@ -142,7 +143,6 @@ int nlmclnt_block(struct nlm_wait *block, struct nlm_rqst *req, long timeout) /* Reset the lock status after a server reboot so we resend */ if (block->b_status == nlm_lck_denied_grace_period) block->b_status = nlm_lck_blocked; - req->a_res.status = block->b_status; return 0; } diff --git a/fs/lockd/clntproc.c b/fs/lockd/clntproc.c index 16b4de868cd2..a14c9110719c 100644 --- a/fs/lockd/clntproc.c +++ b/fs/lockd/clntproc.c @@ -516,9 +516,10 @@ nlmclnt_lock(struct nlm_rqst *req, struct file_lock *fl) const struct cred *cred = nfs_file_cred(fl->fl_file); struct nlm_host *host = req->a_host; struct nlm_res *resp = &req->a_res; - struct nlm_wait *block = NULL; + struct nlm_wait block; unsigned char fl_flags = fl->fl_flags; unsigned char fl_type; + __be32 b_status; int status = -ENOLCK; if (nsm_monitor(host) < 0) @@ -531,31 +532,41 @@ nlmclnt_lock(struct nlm_rqst *req, struct file_lock *fl) if (status < 0) goto out; - block = nlmclnt_prepare_block(host, fl); + nlmclnt_prepare_block(&block, host, fl); again: /* * Initialise resp->status to a valid non-zero value, * since 0 == nlm_lck_granted */ resp->status = nlm_lck_blocked; - for(;;) { + + /* + * A GRANTED callback can come at any time -- even before the reply + * to the LOCK request arrives, so we queue the wait before + * requesting the lock. + */ + nlmclnt_queue_block(&block); + for (;;) { /* Reboot protection */ fl->fl_u.nfs_fl.state = host->h_state; status = nlmclnt_call(cred, req, NLMPROC_LOCK); if (status < 0) break; /* Did a reclaimer thread notify us of a server reboot? */ - if (resp->status == nlm_lck_denied_grace_period) + if (resp->status == nlm_lck_denied_grace_period) continue; if (resp->status != nlm_lck_blocked) break; /* Wait on an NLM blocking lock */ - status = nlmclnt_block(block, req, NLMCLNT_POLL_TIMEOUT); + status = nlmclnt_wait(&block, req, NLMCLNT_POLL_TIMEOUT); if (status < 0) break; - if (resp->status != nlm_lck_blocked) + if (block.b_status != nlm_lck_blocked) break; } + b_status = nlmclnt_dequeue_block(&block); + if (resp->status == nlm_lck_blocked) + resp->status = b_status; /* if we were interrupted while blocking, then cancel the lock request * and exit @@ -564,7 +575,7 @@ again: if (!req->a_args.block) goto out_unlock; if (nlmclnt_cancel(host, req->a_args.block, fl) == 0) - goto out_unblock; + goto out; } if (resp->status == nlm_granted) { @@ -593,8 +604,6 @@ again: status = -ENOLCK; else status = nlm_stat_to_errno(resp->status); -out_unblock: - nlmclnt_finish_block(block); out: nlmclnt_release_call(req); return status; @@ -602,7 +611,6 @@ out_unlock: /* Fatal error: ensure that we remove the lock altogether */ dprintk("lockd: lock attempt ended in fatal error.\n" " Attempting to unlock.\n"); - nlmclnt_finish_block(block); fl_type = fl->fl_type; fl->fl_type = F_UNLCK; down_read(&host->h_rwsem); diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h index de5ad2c2179a..f42594a9efe0 100644 --- a/include/linux/lockd/lockd.h +++ b/include/linux/lockd/lockd.h @@ -211,9 +211,11 @@ struct nlm_rqst * nlm_alloc_call(struct nlm_host *host); int nlm_async_call(struct nlm_rqst *, u32, const struct rpc_call_ops *); int nlm_async_reply(struct nlm_rqst *, u32, const struct rpc_call_ops *); void nlmclnt_release_call(struct nlm_rqst *); -struct nlm_wait * nlmclnt_prepare_block(struct nlm_host *host, struct file_lock *fl); -void nlmclnt_finish_block(struct nlm_wait *block); -int nlmclnt_block(struct nlm_wait *block, struct nlm_rqst *req, long timeout); +void nlmclnt_prepare_block(struct nlm_wait *block, struct nlm_host *host, + struct file_lock *fl); +void nlmclnt_queue_block(struct nlm_wait *block); +__be32 nlmclnt_dequeue_block(struct nlm_wait *block); +int nlmclnt_wait(struct nlm_wait *block, struct nlm_rqst *req, long timeout); __be32 nlmclnt_grant(const struct sockaddr *addr, const struct nlm_lock *lock); void nlmclnt_recovery(struct nlm_host *); -- cgit v1.2.3 From e59fb6749ed833deee5b3cfd7e89925296d41f49 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Fri, 3 Mar 2023 07:16:02 -0500 Subject: nfs: move nfs_fhandle_hash to common include file lockd needs to be able to hash filehandles for tracepoints. Move the nfs_fhandle_hash() helper to a common nfs include file. Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfs/internal.h | 15 --------------- include/linux/nfs.h | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 2a65fe2a63ab..10fb5e7573eb 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -846,27 +846,12 @@ u64 nfs_timespec_to_change_attr(const struct timespec64 *ts) } #ifdef CONFIG_CRC32 -/** - * nfs_fhandle_hash - calculate the crc32 hash for the filehandle - * @fh - pointer to filehandle - * - * returns a crc32 hash for the filehandle that is compatible with - * the one displayed by "wireshark". - */ -static inline u32 nfs_fhandle_hash(const struct nfs_fh *fh) -{ - return ~crc32_le(0xFFFFFFFF, &fh->data[0], fh->size); -} static inline u32 nfs_stateid_hash(const nfs4_stateid *stateid) { return ~crc32_le(0xFFFFFFFF, &stateid->other[0], NFS4_STATEID_OTHER_SIZE); } #else -static inline u32 nfs_fhandle_hash(const struct nfs_fh *fh) -{ - return 0; -} static inline u32 nfs_stateid_hash(nfs4_stateid *stateid) { return 0; diff --git a/include/linux/nfs.h b/include/linux/nfs.h index b06375e88e58..ceb70a926b95 100644 --- a/include/linux/nfs.h +++ b/include/linux/nfs.h @@ -10,6 +10,7 @@ #include #include +#include #include /* @@ -44,4 +45,23 @@ enum nfs3_stable_how { /* used by direct.c to mark verf as invalid */ NFS_INVALID_STABLE_HOW = -1 }; + +#ifdef CONFIG_CRC32 +/** + * nfs_fhandle_hash - calculate the crc32 hash for the filehandle + * @fh - pointer to filehandle + * + * returns a crc32 hash for the filehandle that is compatible with + * the one displayed by "wireshark". + */ +static inline u32 nfs_fhandle_hash(const struct nfs_fh *fh) +{ + return ~crc32_le(0xFFFFFFFF, &fh->data[0], fh->size); +} +#else /* CONFIG_CRC32 */ +static inline u32 nfs_fhandle_hash(const struct nfs_fh *fh) +{ + return 0; +} +#endif /* CONFIG_CRC32 */ #endif /* _LINUX_NFS_H */ -- cgit v1.2.3 From cf64b9bce95095b80f4589e4f54572cc5d8c1538 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Wed, 8 Mar 2023 17:51:00 +1100 Subject: SUNRPC: return proper error from get_expiry() The get_expiry() function currently returns a timestamp, and uses the special return value of 0 to indicate an error. Unfortunately this causes a problem when 0 is the correct return value. On a system with no RTC it is possible that the boot time will be seen to be "3". When exportfs probes to see if a particular filesystem supports NFS export it tries to cache information with an expiry time of "3". The intention is for this to be "long in the past". Even with no RTC it will not be far in the future (at most a second or two) so this is harmless. But if the boot time happens to have been calculated to be "3", then get_expiry will fail incorrectly as it converts the number to "seconds since bootime" - 0. To avoid this problem we change get_expiry() to report the error quite separately from the expiry time. The error is now the return value. The expiry time is reported through a by-reference parameter. Reported-by: Jerry Zhang Tested-by: Jerry Zhang Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/export.c | 13 ++++++------- fs/nfsd/nfs4idmap.c | 8 ++++---- include/linux/sunrpc/cache.h | 15 ++++++++------- net/sunrpc/auth_gss/svcauth_gss.c | 12 ++++++------ net/sunrpc/svcauth_unix.c | 12 ++++++------ 5 files changed, 30 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 668c7527b17e..6da74aebe1fb 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -123,11 +123,11 @@ static int expkey_parse(struct cache_detail *cd, char *mesg, int mlen) /* OK, we seem to have a valid key */ key.h.flags = 0; - key.h.expiry_time = get_expiry(&mesg); - if (key.h.expiry_time == 0) + err = get_expiry(&mesg, &key.h.expiry_time); + if (err) goto out; - key.ek_client = dom; + key.ek_client = dom; key.ek_fsidtype = fsidtype; memcpy(key.ek_fsid, buf, len); @@ -610,9 +610,8 @@ static int svc_export_parse(struct cache_detail *cd, char *mesg, int mlen) exp.ex_devid_map = NULL; /* expiry */ - err = -EINVAL; - exp.h.expiry_time = get_expiry(&mesg); - if (exp.h.expiry_time == 0) + err = get_expiry(&mesg, &exp.h.expiry_time); + if (err) goto out3; /* flags */ @@ -624,7 +623,7 @@ static int svc_export_parse(struct cache_detail *cd, char *mesg, int mlen) if (err || an_int < 0) goto out3; exp.ex_flags= an_int; - + /* anon uid */ err = get_int(&mesg, &an_int); if (err) diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c index 5e9809aff37e..7a806ac13e31 100644 --- a/fs/nfsd/nfs4idmap.c +++ b/fs/nfsd/nfs4idmap.c @@ -240,8 +240,8 @@ idtoname_parse(struct cache_detail *cd, char *buf, int buflen) goto out; /* expiry */ - ent.h.expiry_time = get_expiry(&buf); - if (ent.h.expiry_time == 0) + error = get_expiry(&buf, &ent.h.expiry_time); + if (error) goto out; error = -ENOMEM; @@ -408,8 +408,8 @@ nametoid_parse(struct cache_detail *cd, char *buf, int buflen) memcpy(ent.name, buf1, sizeof(ent.name)); /* expiry */ - ent.h.expiry_time = get_expiry(&buf); - if (ent.h.expiry_time == 0) + error = get_expiry(&buf, &ent.h.expiry_time); + if (error) goto out; /* ID */ diff --git a/include/linux/sunrpc/cache.h b/include/linux/sunrpc/cache.h index ec5a555df96f..518bd28f5ab8 100644 --- a/include/linux/sunrpc/cache.h +++ b/include/linux/sunrpc/cache.h @@ -300,17 +300,18 @@ static inline int get_time(char **bpp, time64_t *time) return 0; } -static inline time64_t get_expiry(char **bpp) +static inline int get_expiry(char **bpp, time64_t *rvp) { - time64_t rv; + int error; struct timespec64 boot; - if (get_time(bpp, &rv)) - return 0; - if (rv < 0) - return 0; + error = get_time(bpp, rvp); + if (error) + return error; + getboottime64(&boot); - return rv - boot.tv_sec; + (*rvp) -= boot.tv_sec; + return 0; } #endif /* _LINUX_SUNRPC_CACHE_H_ */ diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index 9c843974bb48..c4a566737085 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -257,11 +257,11 @@ static int rsi_parse(struct cache_detail *cd, rsii.h.flags = 0; /* expiry */ - expiry = get_expiry(&mesg); - status = -EINVAL; - if (expiry == 0) + status = get_expiry(&mesg, &expiry); + if (status) goto out; + status = -EINVAL; /* major/minor */ len = qword_get(&mesg, buf, mlen); if (len <= 0) @@ -483,11 +483,11 @@ static int rsc_parse(struct cache_detail *cd, rsci.h.flags = 0; /* expiry */ - expiry = get_expiry(&mesg); - status = -EINVAL; - if (expiry == 0) + status = get_expiry(&mesg, &expiry); + if (status) goto out; + status = -EINVAL; rscp = rsc_lookup(cd, &rsci); if (!rscp) goto out; diff --git a/net/sunrpc/svcauth_unix.c b/net/sunrpc/svcauth_unix.c index 4246363cb095..4485088ce27b 100644 --- a/net/sunrpc/svcauth_unix.c +++ b/net/sunrpc/svcauth_unix.c @@ -225,9 +225,9 @@ static int ip_map_parse(struct cache_detail *cd, return -EINVAL; } - expiry = get_expiry(&mesg); - if (expiry ==0) - return -EINVAL; + err = get_expiry(&mesg, &expiry); + if (err) + return err; /* domainname, or empty for NEGATIVE */ len = qword_get(&mesg, buf, mlen); @@ -506,9 +506,9 @@ static int unix_gid_parse(struct cache_detail *cd, uid = make_kuid(current_user_ns(), id); ug.uid = uid; - expiry = get_expiry(&mesg); - if (expiry == 0) - return -EINVAL; + err = get_expiry(&mesg, &expiry); + if (err) + return err; rv = get_int(&mesg, &gids); if (rv || gids < 0 || gids > 8192) -- cgit v1.2.3 From 0f5162480bd25bd97b91c9153db7afbd89698804 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 17 Mar 2023 17:09:20 -0400 Subject: NFSD: Watch for rq_pages bounds checking errors in nfsd_splice_actor() There have been several bugs over the years where the NFSD splice actor has attempted to write outside the rq_pages array. This is a "should never happen" condition, but if for some reason the pipe splice actor should attempt to walk past the end of rq_pages, it needs to terminate the READ operation to prevent corruption of the pointer addresses in the fields just beyond the array. A server crash is thus prevented. Since the code is not behaving, the READ operation returns -EIO to the client. None of the READ payload data can be trusted if the splice actor isn't operating as expected. Suggested-by: Jeff Layton Signed-off-by: Chuck Lever Reviewed-by: Jeff Layton --- fs/nfsd/vfs.c | 6 +++++- include/linux/sunrpc/svc.h | 2 +- include/trace/events/sunrpc.h | 25 +++++++++++++++++++++++++ net/sunrpc/svc.c | 15 ++++++++++++++- 4 files changed, 45 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 5783209f17fc..10aa68ca82ef 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -930,6 +930,9 @@ nfsd_open_verified(struct svc_rqst *rqstp, struct svc_fh *fhp, int may_flags, * Grab and keep cached pages associated with a file in the svc_rqst * so that they can be passed to the network sendmsg/sendpage routines * directly. They will be released after the sending has completed. + * + * Return values: Number of bytes consumed, or -EIO if there are no + * remaining pages in rqstp->rq_pages. */ static int nfsd_splice_actor(struct pipe_inode_info *pipe, struct pipe_buffer *buf, @@ -948,7 +951,8 @@ nfsd_splice_actor(struct pipe_inode_info *pipe, struct pipe_buffer *buf, */ if (page == *(rqstp->rq_next_page - 1)) continue; - svc_rqst_replace_page(rqstp, page); + if (unlikely(!svc_rqst_replace_page(rqstp, page))) + return -EIO; } if (rqstp->rq_res.page_len == 0) // first call rqstp->rq_res.page_base = offset % PAGE_SIZE; diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 877891536c2f..f5af055280ff 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -422,7 +422,7 @@ struct svc_serv *svc_create(struct svc_program *, unsigned int, int (*threadfn)(void *data)); struct svc_rqst *svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node); -void svc_rqst_replace_page(struct svc_rqst *rqstp, +bool svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page); void svc_rqst_free(struct svc_rqst *); void svc_exit_thread(struct svc_rqst *); diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h index 3ca54536f8f7..5a3bb42e1f50 100644 --- a/include/trace/events/sunrpc.h +++ b/include/trace/events/sunrpc.h @@ -1790,6 +1790,31 @@ DEFINE_EVENT(svc_rqst_status, svc_send, TP_PROTO(const struct svc_rqst *rqst, int status), TP_ARGS(rqst, status)); +TRACE_EVENT(svc_replace_page_err, + TP_PROTO(const struct svc_rqst *rqst), + + TP_ARGS(rqst), + TP_STRUCT__entry( + SVC_RQST_ENDPOINT_FIELDS(rqst) + + __field(const void *, begin) + __field(const void *, respages) + __field(const void *, nextpage) + ), + + TP_fast_assign( + SVC_RQST_ENDPOINT_ASSIGNMENTS(rqst); + + __entry->begin = rqst->rq_pages; + __entry->respages = rqst->rq_respages; + __entry->nextpage = rqst->rq_next_page; + ), + + TP_printk(SVC_RQST_ENDPOINT_FORMAT " begin=%p respages=%p nextpage=%p", + SVC_RQST_ENDPOINT_VARARGS, + __entry->begin, __entry->respages, __entry->nextpage) +); + TRACE_EVENT(svc_stats_latency, TP_PROTO( const struct svc_rqst *rqst diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index fea7ce8fba14..633aa1eb476b 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -842,9 +842,21 @@ EXPORT_SYMBOL_GPL(svc_set_num_threads); * * When replacing a page in rq_pages, batch the release of the * replaced pages to avoid hammering the page allocator. + * + * Return values: + * %true: page replaced + * %false: array bounds checking failed */ -void svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page) +bool svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page) { + struct page **begin = rqstp->rq_pages; + struct page **end = &rqstp->rq_pages[RPCSVC_MAXPAGES]; + + if (unlikely(rqstp->rq_next_page < begin || rqstp->rq_next_page > end)) { + trace_svc_replace_page_err(rqstp); + return false; + } + if (*rqstp->rq_next_page) { if (!pagevec_space(&rqstp->rq_pvec)) __pagevec_release(&rqstp->rq_pvec); @@ -853,6 +865,7 @@ void svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page) get_page(page); *(rqstp->rq_next_page++) = page; + return true; } EXPORT_SYMBOL_GPL(svc_rqst_replace_page); -- cgit v1.2.3 From 55fcc7d9159de886296626e47db2c81f8578c7e1 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 3 Apr 2023 13:53:07 -0400 Subject: SUNRPC: Ignore return value of ->xpo_sendto Clean up: All callers of svc_process() ignore its return value, so svc_process() can safely be converted to return void. Ditto for svc_send(). The return value of ->xpo_sendto() is now used only as part of a trace event. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc.h | 2 +- include/linux/sunrpc/svcsock.h | 2 +- net/sunrpc/svc.c | 13 +++++++------ net/sunrpc/svc_xprt.c | 21 +++++++++------------ 4 files changed, 18 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index f5af055280ff..2d31121fc2e6 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -430,7 +430,7 @@ struct svc_serv * svc_create_pooled(struct svc_program *, unsigned int, int (*threadfn)(void *data)); int svc_set_num_threads(struct svc_serv *, struct svc_pool *, int); int svc_pool_stats_open(struct svc_serv *serv, struct file *file); -int svc_process(struct svc_rqst *); +void svc_process(struct svc_rqst *rqstp); int bc_svc_process(struct svc_serv *, struct rpc_rqst *, struct svc_rqst *); int svc_register(const struct svc_serv *, struct net *, const int, diff --git a/include/linux/sunrpc/svcsock.h b/include/linux/sunrpc/svcsock.h index bcc555c7ae9c..dd73fa174af5 100644 --- a/include/linux/sunrpc/svcsock.h +++ b/include/linux/sunrpc/svcsock.h @@ -56,7 +56,7 @@ static inline u32 svc_sock_final_rec(struct svc_sock *svsk) */ void svc_close_net(struct svc_serv *, struct net *); int svc_recv(struct svc_rqst *, long); -int svc_send(struct svc_rqst *); +void svc_send(struct svc_rqst *rqstp); void svc_drop(struct svc_rqst *); void svc_sock_update_bufs(struct svc_serv *serv); bool svc_alien_sock(struct net *net, int fd); diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 633aa1eb476b..0aa8892fad63 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -1444,11 +1444,12 @@ err_system_err: goto sendit; } -/* - * Process the RPC request. +/** + * svc_process - Execute one RPC transaction + * @rqstp: RPC transaction context + * */ -int -svc_process(struct svc_rqst *rqstp) +void svc_process(struct svc_rqst *rqstp) { struct kvec *resv = &rqstp->rq_res.head[0]; __be32 *p; @@ -1484,7 +1485,8 @@ svc_process(struct svc_rqst *rqstp) if (!svc_process_common(rqstp)) goto out_drop; - return svc_send(rqstp); + svc_send(rqstp); + return; out_baddir: svc_printk(rqstp, "bad direction 0x%08x, dropping request\n", @@ -1492,7 +1494,6 @@ out_baddir: rqstp->rq_server->sv_stats->rpcbadfmt++; out_drop: svc_drop(rqstp); - return 0; } EXPORT_SYMBOL_GPL(svc_process); diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index ba629297da4e..36c79b718323 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -909,18 +909,20 @@ void svc_drop(struct svc_rqst *rqstp) } EXPORT_SYMBOL_GPL(svc_drop); -/* - * Return reply to client. +/** + * svc_send - Return reply to client + * @rqstp: RPC transaction context + * */ -int svc_send(struct svc_rqst *rqstp) +void svc_send(struct svc_rqst *rqstp) { struct svc_xprt *xprt; - int len = -EFAULT; struct xdr_buf *xb; + int status; xprt = rqstp->rq_xprt; if (!xprt) - goto out; + return; /* calculate over-all length */ xb = &rqstp->rq_res; @@ -930,15 +932,10 @@ int svc_send(struct svc_rqst *rqstp) trace_svc_xdr_sendto(rqstp->rq_xid, xb); trace_svc_stats_latency(rqstp); - len = xprt->xpt_ops->xpo_sendto(rqstp); + status = xprt->xpt_ops->xpo_sendto(rqstp); - trace_svc_send(rqstp, len); + trace_svc_send(rqstp, status); svc_xprt_release(rqstp); - - if (len == -ECONNREFUSED || len == -ENOTCONN || len == -EAGAIN) - len = 0; -out: - return len; } /* -- cgit v1.2.3 From b20cb39def085723868972182fb58fa906839a4f Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 14 Apr 2023 20:17:56 -0400 Subject: SUNRPC: Relocate svc_free_res_pages() Clean-up: There doesn't seem to be a reason why this function is stuck in a header. One thing it prevents is the convenient addition of tracing. Moving it to a source file also makes the rq_respages clean-up logic easier to find. Reviewed-by: Calum Mackay Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc.h | 12 +----------- net/sunrpc/svc.c | 19 +++++++++++++++++++ net/sunrpc/svc_xprt.c | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 2d31121fc2e6..762d7231e574 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -309,17 +309,6 @@ static inline struct sockaddr *svc_daddr(const struct svc_rqst *rqst) return (struct sockaddr *) &rqst->rq_daddr; } -static inline void svc_free_res_pages(struct svc_rqst *rqstp) -{ - while (rqstp->rq_next_page != rqstp->rq_respages) { - struct page **pp = --rqstp->rq_next_page; - if (*pp) { - put_page(*pp); - *pp = NULL; - } - } -} - struct svc_deferred_req { u32 prot; /* protocol (UDP or TCP) */ struct svc_xprt *xprt; @@ -424,6 +413,7 @@ struct svc_rqst *svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node); bool svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page); +void svc_rqst_release_pages(struct svc_rqst *rqstp); void svc_rqst_free(struct svc_rqst *); void svc_exit_thread(struct svc_rqst *); struct svc_serv * svc_create_pooled(struct svc_program *, unsigned int, diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 0aa8892fad63..0fc70cc405b2 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -869,6 +869,25 @@ bool svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page) } EXPORT_SYMBOL_GPL(svc_rqst_replace_page); +/** + * svc_rqst_release_pages - Release Reply buffer pages + * @rqstp: RPC transaction context + * + * Release response pages that might still be in flight after + * svc_send, and any spliced filesystem-owned pages. + */ +void svc_rqst_release_pages(struct svc_rqst *rqstp) +{ + while (rqstp->rq_next_page != rqstp->rq_respages) { + struct page **pp = --rqstp->rq_next_page; + + if (*pp) { + put_page(*pp); + *pp = NULL; + } + } +} + /* * Called from a server thread as it's exiting. Caller must hold the "service * mutex" for the service. diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 36c79b718323..533e08c4f319 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -542,7 +542,7 @@ static void svc_xprt_release(struct svc_rqst *rqstp) rqstp->rq_deferred = NULL; pagevec_release(&rqstp->rq_pvec); - svc_free_res_pages(rqstp); + svc_rqst_release_pages(rqstp); rqstp->rq_res.page_len = 0; rqstp->rq_res.page_base = 0; -- cgit v1.2.3 From e0f8ad2a705367518b5c56bf9d6da89681467c02 Mon Sep 17 00:00:00 2001 From: Shengyu Qu Date: Fri, 21 Apr 2023 23:08:15 +0800 Subject: mfd: axp20x: Add support for AXP15060 PMIC The AXP15060 is a PMIC chip produced by X-Powers, and could be connected via an I2C bus. Describe the regmap and the MFD bits, along with the registers exposed via I2C. Eventually advertise the device using a new compatible string and add support for power off the system. The driver would disable PEK function if IRQ is not configured in device tree, since some boards (For example, Starfive Visionfive 2) didn't connect IRQ line of PMIC to SOC. GPIO function isn't enabled in this commit, since its configuration operation is different from any existing AXP PMICs and needs logic modification on existing driver. GPIO support might come in later patches. Signed-off-by: Shengyu Qu Reviewed-by: Krzysztof Kozlowski Signed-off-by: Lee Jones Link: https://lore.kernel.org/r/TY3P286MB261162D57695AC8164ED50E298609@TY3P286MB2611.JPNP286.PROD.OUTLOOK.COM --- drivers/mfd/axp20x-i2c.c | 2 + drivers/mfd/axp20x.c | 107 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/mfd/axp20x.h | 85 +++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+) (limited to 'include/linux') diff --git a/drivers/mfd/axp20x-i2c.c b/drivers/mfd/axp20x-i2c.c index f49fbd307958..b4f5cb457117 100644 --- a/drivers/mfd/axp20x-i2c.c +++ b/drivers/mfd/axp20x-i2c.c @@ -65,6 +65,7 @@ static const struct of_device_id axp20x_i2c_of_match[] = { { .compatible = "x-powers,axp223", .data = (void *)AXP223_ID }, { .compatible = "x-powers,axp803", .data = (void *)AXP803_ID }, { .compatible = "x-powers,axp806", .data = (void *)AXP806_ID }, + { .compatible = "x-powers,axp15060", .data = (void *)AXP15060_ID }, { }, }; MODULE_DEVICE_TABLE(of, axp20x_i2c_of_match); @@ -78,6 +79,7 @@ static const struct i2c_device_id axp20x_i2c_id[] = { { "axp223", 0 }, { "axp803", 0 }, { "axp806", 0 }, + { "axp15060", 0 }, { }, }; MODULE_DEVICE_TABLE(i2c, axp20x_i2c_id); diff --git a/drivers/mfd/axp20x.c b/drivers/mfd/axp20x.c index 7720ac15c7d4..72b87aae60cc 100644 --- a/drivers/mfd/axp20x.c +++ b/drivers/mfd/axp20x.c @@ -43,6 +43,7 @@ static const char * const axp20x_model_names[] = { "AXP806", "AXP809", "AXP813", + "AXP15060", }; static const struct regmap_range axp152_writeable_ranges[] = { @@ -169,6 +170,31 @@ static const struct regmap_access_table axp806_volatile_table = { .n_yes_ranges = ARRAY_SIZE(axp806_volatile_ranges), }; +static const struct regmap_range axp15060_writeable_ranges[] = { + regmap_reg_range(AXP15060_PWR_OUT_CTRL1, AXP15060_DCDC_MODE_CTRL2), + regmap_reg_range(AXP15060_OUTPUT_MONITOR_DISCHARGE, AXP15060_CPUSLDO_V_CTRL), + regmap_reg_range(AXP15060_PWR_WAKEUP_CTRL, AXP15060_PWR_DISABLE_DOWN_SEQ), + regmap_reg_range(AXP15060_PEK_KEY, AXP15060_PEK_KEY), + regmap_reg_range(AXP15060_IRQ1_EN, AXP15060_IRQ2_EN), + regmap_reg_range(AXP15060_IRQ1_STATE, AXP15060_IRQ2_STATE), +}; + +static const struct regmap_range axp15060_volatile_ranges[] = { + regmap_reg_range(AXP15060_STARTUP_SRC, AXP15060_STARTUP_SRC), + regmap_reg_range(AXP15060_PWR_WAKEUP_CTRL, AXP15060_PWR_DISABLE_DOWN_SEQ), + regmap_reg_range(AXP15060_IRQ1_STATE, AXP15060_IRQ2_STATE), +}; + +static const struct regmap_access_table axp15060_writeable_table = { + .yes_ranges = axp15060_writeable_ranges, + .n_yes_ranges = ARRAY_SIZE(axp15060_writeable_ranges), +}; + +static const struct regmap_access_table axp15060_volatile_table = { + .yes_ranges = axp15060_volatile_ranges, + .n_yes_ranges = ARRAY_SIZE(axp15060_volatile_ranges), +}; + static const struct resource axp152_pek_resources[] = { DEFINE_RES_IRQ_NAMED(AXP152_IRQ_PEK_RIS_EDGE, "PEK_DBR"), DEFINE_RES_IRQ_NAMED(AXP152_IRQ_PEK_FAL_EDGE, "PEK_DBF"), @@ -237,6 +263,11 @@ static const struct resource axp809_pek_resources[] = { DEFINE_RES_IRQ_NAMED(AXP809_IRQ_PEK_FAL_EDGE, "PEK_DBF"), }; +static const struct resource axp15060_pek_resources[] = { + DEFINE_RES_IRQ_NAMED(AXP15060_IRQ_PEK_RIS_EDGE, "PEK_DBR"), + DEFINE_RES_IRQ_NAMED(AXP15060_IRQ_PEK_FAL_EDGE, "PEK_DBF"), +}; + static const struct regmap_config axp152_regmap_config = { .reg_bits = 8, .val_bits = 8, @@ -282,6 +313,15 @@ static const struct regmap_config axp806_regmap_config = { .cache_type = REGCACHE_RBTREE, }; +static const struct regmap_config axp15060_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .wr_table = &axp15060_writeable_table, + .volatile_table = &axp15060_volatile_table, + .max_register = AXP15060_IRQ2_STATE, + .cache_type = REGCACHE_RBTREE, +}; + #define INIT_REGMAP_IRQ(_variant, _irq, _off, _mask) \ [_variant##_IRQ_##_irq] = { .reg_offset = (_off), .mask = BIT(_mask) } @@ -503,6 +543,23 @@ static const struct regmap_irq axp809_regmap_irqs[] = { INIT_REGMAP_IRQ(AXP809, GPIO0_INPUT, 4, 0), }; +static const struct regmap_irq axp15060_regmap_irqs[] = { + INIT_REGMAP_IRQ(AXP15060, DIE_TEMP_HIGH_LV1, 0, 0), + INIT_REGMAP_IRQ(AXP15060, DIE_TEMP_HIGH_LV2, 0, 1), + INIT_REGMAP_IRQ(AXP15060, DCDC1_V_LOW, 0, 2), + INIT_REGMAP_IRQ(AXP15060, DCDC2_V_LOW, 0, 3), + INIT_REGMAP_IRQ(AXP15060, DCDC3_V_LOW, 0, 4), + INIT_REGMAP_IRQ(AXP15060, DCDC4_V_LOW, 0, 5), + INIT_REGMAP_IRQ(AXP15060, DCDC5_V_LOW, 0, 6), + INIT_REGMAP_IRQ(AXP15060, DCDC6_V_LOW, 0, 7), + INIT_REGMAP_IRQ(AXP15060, PEK_LONG, 1, 0), + INIT_REGMAP_IRQ(AXP15060, PEK_SHORT, 1, 1), + INIT_REGMAP_IRQ(AXP15060, GPIO1_INPUT, 1, 2), + INIT_REGMAP_IRQ(AXP15060, PEK_FAL_EDGE, 1, 3), + INIT_REGMAP_IRQ(AXP15060, PEK_RIS_EDGE, 1, 4), + INIT_REGMAP_IRQ(AXP15060, GPIO2_INPUT, 1, 5), +}; + static const struct regmap_irq_chip axp152_regmap_irq_chip = { .name = "axp152_irq_chip", .status_base = AXP152_IRQ1_STATE, @@ -582,6 +639,17 @@ static const struct regmap_irq_chip axp809_regmap_irq_chip = { .num_regs = 5, }; +static const struct regmap_irq_chip axp15060_regmap_irq_chip = { + .name = "axp15060", + .status_base = AXP15060_IRQ1_STATE, + .ack_base = AXP15060_IRQ1_STATE, + .unmask_base = AXP15060_IRQ1_EN, + .init_ack_masked = true, + .irqs = axp15060_regmap_irqs, + .num_irqs = ARRAY_SIZE(axp15060_regmap_irqs), + .num_regs = 2, +}; + static const struct mfd_cell axp20x_cells[] = { { .name = "axp20x-gpio", @@ -826,6 +894,23 @@ static const struct mfd_cell axp813_cells[] = { }, }; +static const struct mfd_cell axp15060_cells[] = { + { + .name = "axp221-pek", + .num_resources = ARRAY_SIZE(axp15060_pek_resources), + .resources = axp15060_pek_resources, + }, { + .name = "axp20x-regulator", + }, +}; + +/* For boards that don't have IRQ line connected to SOC. */ +static const struct mfd_cell axp_regulator_only_cells[] = { + { + .name = "axp20x-regulator", + }, +}; + static int axp20x_power_off(struct sys_off_data *data) { struct axp20x_dev *axp20x = data->cb_data; @@ -935,6 +1020,28 @@ int axp20x_match_device(struct axp20x_dev *axp20x) */ axp20x->regmap_irq_chip = &axp803_regmap_irq_chip; break; + case AXP15060_ID: + /* + * Don't register the power key part if there is no interrupt + * line. + * + * Since most use cases of AXP PMICs are Allwinner SOCs, board + * designers follow Allwinner's reference design and connects + * IRQ line to SOC, there's no need for those variants to deal + * with cases that IRQ isn't connected. However, AXP15660 is + * used by some other vendors' SOCs that didn't connect IRQ + * line, we need to deal with this case. + */ + if (axp20x->irq > 0) { + axp20x->nr_cells = ARRAY_SIZE(axp15060_cells); + axp20x->cells = axp15060_cells; + } else { + axp20x->nr_cells = ARRAY_SIZE(axp_regulator_only_cells); + axp20x->cells = axp_regulator_only_cells; + } + axp20x->regmap_cfg = &axp15060_regmap_config; + axp20x->regmap_irq_chip = &axp15060_regmap_irq_chip; + break; default: dev_err(dev, "unsupported AXP20X ID %lu\n", axp20x->variant); return -EINVAL; diff --git a/include/linux/mfd/axp20x.h b/include/linux/mfd/axp20x.h index 2058194807bd..beb3f44f85c5 100644 --- a/include/linux/mfd/axp20x.h +++ b/include/linux/mfd/axp20x.h @@ -21,6 +21,7 @@ enum axp20x_variants { AXP806_ID, AXP809_ID, AXP813_ID, + AXP15060_ID, NR_AXP20X_VARIANTS, }; @@ -131,6 +132,39 @@ enum axp20x_variants { /* Other DCDC regulator control registers are the same as AXP803 */ #define AXP813_DCDC7_V_OUT 0x26 +#define AXP15060_STARTUP_SRC 0x00 +#define AXP15060_PWR_OUT_CTRL1 0x10 +#define AXP15060_PWR_OUT_CTRL2 0x11 +#define AXP15060_PWR_OUT_CTRL3 0x12 +#define AXP15060_DCDC1_V_CTRL 0x13 +#define AXP15060_DCDC2_V_CTRL 0x14 +#define AXP15060_DCDC3_V_CTRL 0x15 +#define AXP15060_DCDC4_V_CTRL 0x16 +#define AXP15060_DCDC5_V_CTRL 0x17 +#define AXP15060_DCDC6_V_CTRL 0x18 +#define AXP15060_ALDO1_V_CTRL 0x19 +#define AXP15060_DCDC_MODE_CTRL1 0x1a +#define AXP15060_DCDC_MODE_CTRL2 0x1b +#define AXP15060_OUTPUT_MONITOR_DISCHARGE 0x1e +#define AXP15060_IRQ_PWROK_VOFF 0x1f +#define AXP15060_ALDO2_V_CTRL 0x20 +#define AXP15060_ALDO3_V_CTRL 0x21 +#define AXP15060_ALDO4_V_CTRL 0x22 +#define AXP15060_ALDO5_V_CTRL 0x23 +#define AXP15060_BLDO1_V_CTRL 0x24 +#define AXP15060_BLDO2_V_CTRL 0x25 +#define AXP15060_BLDO3_V_CTRL 0x26 +#define AXP15060_BLDO4_V_CTRL 0x27 +#define AXP15060_BLDO5_V_CTRL 0x28 +#define AXP15060_CLDO1_V_CTRL 0x29 +#define AXP15060_CLDO2_V_CTRL 0x2a +#define AXP15060_CLDO3_V_CTRL 0x2b +#define AXP15060_CLDO4_V_CTRL 0x2d +#define AXP15060_CPUSLDO_V_CTRL 0x2e +#define AXP15060_PWR_WAKEUP_CTRL 0x31 +#define AXP15060_PWR_DISABLE_DOWN_SEQ 0x32 +#define AXP15060_PEK_KEY 0x36 + /* Interrupt */ #define AXP152_IRQ1_EN 0x40 #define AXP152_IRQ2_EN 0x41 @@ -152,6 +186,11 @@ enum axp20x_variants { #define AXP20X_IRQ5_STATE 0x4c #define AXP20X_IRQ6_STATE 0x4d +#define AXP15060_IRQ1_EN 0x40 +#define AXP15060_IRQ2_EN 0x41 +#define AXP15060_IRQ1_STATE 0x48 +#define AXP15060_IRQ2_STATE 0x49 + /* ADC */ #define AXP20X_ACIN_V_ADC_H 0x56 #define AXP20X_ACIN_V_ADC_L 0x57 @@ -222,6 +261,8 @@ enum axp20x_variants { #define AXP22X_GPIO_STATE 0x94 #define AXP22X_GPIO_PULL_DOWN 0x95 +#define AXP15060_CLDO4_GPIO2_MODESET 0x2c + /* Battery */ #define AXP20X_CHRG_CC_31_24 0xb0 #define AXP20X_CHRG_CC_23_16 0xb1 @@ -419,6 +460,33 @@ enum { AXP813_REG_ID_MAX, }; +enum { + AXP15060_DCDC1 = 0, + AXP15060_DCDC2, + AXP15060_DCDC3, + AXP15060_DCDC4, + AXP15060_DCDC5, + AXP15060_DCDC6, + AXP15060_ALDO1, + AXP15060_ALDO2, + AXP15060_ALDO3, + AXP15060_ALDO4, + AXP15060_ALDO5, + AXP15060_BLDO1, + AXP15060_BLDO2, + AXP15060_BLDO3, + AXP15060_BLDO4, + AXP15060_BLDO5, + AXP15060_CLDO1, + AXP15060_CLDO2, + AXP15060_CLDO3, + AXP15060_CLDO4, + AXP15060_CPUSLDO, + AXP15060_SW, + AXP15060_RTC_LDO, + AXP15060_REG_ID_MAX, +}; + /* IRQs */ enum { AXP152_IRQ_LDO0IN_CONNECT = 1, @@ -637,6 +705,23 @@ enum axp809_irqs { AXP809_IRQ_GPIO0_INPUT, }; +enum axp15060_irqs { + AXP15060_IRQ_DIE_TEMP_HIGH_LV1 = 1, + AXP15060_IRQ_DIE_TEMP_HIGH_LV2, + AXP15060_IRQ_DCDC1_V_LOW, + AXP15060_IRQ_DCDC2_V_LOW, + AXP15060_IRQ_DCDC3_V_LOW, + AXP15060_IRQ_DCDC4_V_LOW, + AXP15060_IRQ_DCDC5_V_LOW, + AXP15060_IRQ_DCDC6_V_LOW, + AXP15060_IRQ_PEK_LONG, + AXP15060_IRQ_PEK_SHORT, + AXP15060_IRQ_GPIO1_INPUT, + AXP15060_IRQ_PEK_FAL_EDGE, + AXP15060_IRQ_PEK_RIS_EDGE, + AXP15060_IRQ_GPIO2_INPUT, +}; + struct axp20x_dev { struct device *dev; int irq; -- cgit v1.2.3 From ff53cd52d9bdbf4074d2bbe9b591729997780bd3 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Sat, 18 Mar 2023 17:36:25 +0000 Subject: blk-integrity: register sysfs attributes on struct device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "integrity" kobject only acted as a holder for static sysfs entries. It also was embedded into struct gendisk without managing it, violating assumptions of the driver core. Instead register the sysfs entries directly onto the struct device. Also drop the now unused member integrity_kobj from struct gendisk. Suggested-by: Christoph Hellwig Signed-off-by: Thomas Weißschuh Reviewed-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Link: https://lore.kernel.org/r/20230309-kobj_release-gendisk_integrity-v3-3-ceccb4493c46@weissschuh.net Signed-off-by: Jens Axboe --- block/blk-integrity.c | 55 +++----------------------------------------------- block/blk.h | 10 +-------- block/genhd.c | 12 ++++------- include/linux/blkdev.h | 3 --- 4 files changed, 8 insertions(+), 72 deletions(-) (limited to 'include/linux') diff --git a/block/blk-integrity.c b/block/blk-integrity.c index 1cbfdea88c72..d4e9b4556d14 100644 --- a/block/blk-integrity.c +++ b/block/blk-integrity.c @@ -212,31 +212,6 @@ bool blk_integrity_merge_bio(struct request_queue *q, struct request *req, return true; } -static ssize_t integrity_attr_show(struct kobject *kobj, struct attribute *attr, - char *page) -{ - struct gendisk *disk = container_of(kobj, struct gendisk, integrity_kobj); - struct device *dev = disk_to_dev(disk); - struct device_attribute *dev_attr = - container_of(attr, struct device_attribute, attr); - - return dev_attr->show(dev, dev_attr, page); -} - -static ssize_t integrity_attr_store(struct kobject *kobj, - struct attribute *attr, const char *page, - size_t count) -{ - struct gendisk *disk = container_of(kobj, struct gendisk, integrity_kobj); - struct device *dev = disk_to_dev(disk); - struct device_attribute *dev_attr = - container_of(attr, struct device_attribute, attr); - - if (!dev_attr->store) - return 0; - return dev_attr->store(dev, dev_attr, page, count); -} - static inline struct blk_integrity *dev_to_bi(struct device *dev) { return &dev_to_disk(dev)->queue->integrity; @@ -345,16 +320,10 @@ static struct attribute *integrity_attrs[] = { &dev_attr_device_is_integrity_capable.attr, NULL }; -ATTRIBUTE_GROUPS(integrity); -static const struct sysfs_ops integrity_ops = { - .show = &integrity_attr_show, - .store = &integrity_attr_store, -}; - -static const struct kobj_type integrity_ktype = { - .default_groups = integrity_groups, - .sysfs_ops = &integrity_ops, +const struct attribute_group blk_integrity_attr_group = { + .name = "integrity", + .attrs = integrity_attrs, }; static blk_status_t blk_integrity_nop_fn(struct blk_integrity_iter *iter) @@ -433,21 +402,3 @@ void blk_integrity_unregister(struct gendisk *disk) memset(bi, 0, sizeof(*bi)); } EXPORT_SYMBOL(blk_integrity_unregister); - -int blk_integrity_add(struct gendisk *disk) -{ - int ret; - - ret = kobject_init_and_add(&disk->integrity_kobj, &integrity_ktype, - &disk_to_dev(disk)->kobj, "%s", "integrity"); - if (!ret) - kobject_uevent(&disk->integrity_kobj, KOBJ_ADD); - return ret; -} - -void blk_integrity_del(struct gendisk *disk) -{ - kobject_uevent(&disk->integrity_kobj, KOBJ_REMOVE); - kobject_del(&disk->integrity_kobj); - kobject_put(&disk->integrity_kobj); -} diff --git a/block/blk.h b/block/blk.h index 564119a76bc5..45547bcf1119 100644 --- a/block/blk.h +++ b/block/blk.h @@ -214,8 +214,7 @@ static inline bool integrity_req_gap_front_merge(struct request *req, bip_next->bip_vec[0].bv_offset); } -int blk_integrity_add(struct gendisk *disk); -void blk_integrity_del(struct gendisk *); +extern const struct attribute_group blk_integrity_attr_group; #else /* CONFIG_BLK_DEV_INTEGRITY */ static inline bool blk_integrity_merge_rq(struct request_queue *rq, struct request *r1, struct request *r2) @@ -248,13 +247,6 @@ static inline bool bio_integrity_endio(struct bio *bio) static inline void bio_integrity_free(struct bio *bio) { } -static inline int blk_integrity_add(struct gendisk *disk) -{ - return 0; -} -static inline void blk_integrity_del(struct gendisk *disk) -{ -} #endif /* CONFIG_BLK_DEV_INTEGRITY */ unsigned long blk_rq_timeout(unsigned long timeout); diff --git a/block/genhd.c b/block/genhd.c index 8d56fb5f08b2..9fa4a7cd978c 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -478,15 +478,11 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk, */ pm_runtime_set_memalloc_noio(ddev, true); - ret = blk_integrity_add(disk); - if (ret) - goto out_del_block_link; - disk->part0->bd_holder_dir = kobject_create_and_add("holders", &ddev->kobj); if (!disk->part0->bd_holder_dir) { ret = -ENOMEM; - goto out_del_integrity; + goto out_del_block_link; } disk->slave_dir = kobject_create_and_add("slaves", &ddev->kobj); if (!disk->slave_dir) { @@ -549,8 +545,6 @@ out_put_slave_dir: disk->slave_dir = NULL; out_put_holder_dir: kobject_put(disk->part0->bd_holder_dir); -out_del_integrity: - blk_integrity_del(disk); out_del_block_link: if (!sysfs_deprecated) sysfs_remove_link(block_depr, dev_name(ddev)); @@ -613,7 +607,6 @@ void del_gendisk(struct gendisk *disk) if (WARN_ON_ONCE(!disk_live(disk) && !(disk->flags & GENHD_FL_HIDDEN))) return; - blk_integrity_del(disk); disk_del_events(disk); mutex_lock(&disk->open_mutex); @@ -1150,6 +1143,9 @@ static const struct attribute_group *disk_attr_groups[] = { &disk_attr_group, #ifdef CONFIG_BLK_DEV_IO_TRACE &blk_trace_attr_group, +#endif +#ifdef CONFIG_BLK_DEV_INTEGRITY + &blk_integrity_attr_group, #endif NULL }; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 6ede578dfbc6..aac5c8d7a9ff 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -157,9 +157,6 @@ struct gendisk { struct timer_rand_state *random; atomic_t sync_io; /* RAID */ struct disk_events *ev; -#ifdef CONFIG_BLK_DEV_INTEGRITY - struct kobject integrity_kobj; -#endif /* CONFIG_BLK_DEV_INTEGRITY */ #ifdef CONFIG_BLK_DEV_ZONED /* -- cgit v1.2.3 From 7cefbaf081eb7c114299f7478ed4fa0d90ec90bb Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Wed, 19 Apr 2023 10:33:38 +0200 Subject: thermal: core: Encapsulate tz->device field There are still some drivers needing to play with the thermal zone device internals. That is not the best but until we can figure out if the information is really needed, let's encapsulate the field used in the thermal zone device structure, so we can move forward relocating the thermal zone device structure definition in the thermal framework private headers. Some drivers are accessing tz->device, that implies they need to have the knowledge of the thermal_zone_device structure but we want to self-encapsulate this structure and reduce the scope of the structure to the thermal core only. By adding this wrapper, these drivers won't need the thermal zone device structure definition and are no longer an obstacle to its relocation to the private thermal core headers. Signed-off-by: Daniel Lezcano Signed-off-by: Rafael J. Wysocki --- drivers/thermal/thermal_core.c | 6 ++++++ include/linux/thermal.h | 1 + 2 files changed, 7 insertions(+) (limited to 'include/linux') diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index c5025aca22ee..842f678c1c3e 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -1398,6 +1398,12 @@ int thermal_zone_device_id(struct thermal_zone_device *tzd) } EXPORT_SYMBOL_GPL(thermal_zone_device_id); +struct device *thermal_zone_device(struct thermal_zone_device *tzd) +{ + return &tzd->device; +} +EXPORT_SYMBOL_GPL(thermal_zone_device); + /** * thermal_zone_device_unregister - removes the registered thermal zone device * @tz: the thermal zone device to remove diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 82ddb32f9876..87837094d549 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -313,6 +313,7 @@ thermal_zone_device_register_with_trips(const char *, struct thermal_trip *, int void *thermal_zone_device_priv(struct thermal_zone_device *tzd); const char *thermal_zone_device_type(struct thermal_zone_device *tzd); int thermal_zone_device_id(struct thermal_zone_device *tzd); +struct device *thermal_zone_device(struct thermal_zone_device *tzd); int thermal_zone_bind_cooling_device(struct thermal_zone_device *, int, struct thermal_cooling_device *, -- cgit v1.2.3 From 4f20b7471c57032860065591a17efd3325216bde Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 21 Apr 2023 16:54:24 +0200 Subject: libgcc: add forward declarations for generic library routines With W=1 on platforms that use the generic gcc library routines (csky/loongarch/mips/riscv/sh/xtensa): lib/ashldi3.c:9:19: warning: no previous prototype for '__ashldi3' [-Wmissing-prototypes] 9 | long long notrace __ashldi3(long long u, word_type b) | ^~~~~~~~~ CC lib/ashrdi3.o lib/ashrdi3.c:9:19: warning: no previous prototype for '__ashrdi3' [-Wmissing-prototypes] 9 | long long notrace __ashrdi3(long long u, word_type b) | ^~~~~~~~~ CC lib/cmpdi2.o lib/cmpdi2.c:9:19: warning: no previous prototype for '__cmpdi2' [-Wmissing-prototypes] 9 | word_type notrace __cmpdi2(long long a, long long b) | ^~~~~~~~ CC lib/lshrdi3.o lib/lshrdi3.c:9:19: warning: no previous prototype for '__lshrdi3' [-Wmissing-prototypes] 9 | long long notrace __lshrdi3(long long u, word_type b) | ^~~~~~~~~ CC lib/muldi3.o lib/muldi3.c:49:19: warning: no previous prototype for '__muldi3' [-Wmissing-prototypes] 49 | long long notrace __muldi3(long long u, long long v) | ^~~~~~~~ CC lib/ucmpdi2.o lib/ucmpdi2.c:8:19: warning: no previous prototype for '__ucmpdi2' [-Wmissing-prototypes] 8 | word_type notrace __ucmpdi2(unsigned long long a, unsigned long long b) | ^~~~~~~~~ Fix this by adding forward declarations to the common libgcc header file. Link: https://lkml.kernel.org/r/5cdbe08296693dd53849f199c3933e16e97b33c1.1682088593.git.geert+renesas@glider.be Signed-off-by: Geert Uytterhoeven Reported-by: kernel test robot Link: https://lore.kernel.org/oe-kbuild-all/202303272214.RxzpA6bP-lkp@intel.com/ Acked-by: Arnd Bergmann Cc: Chris Zankel Cc: Max Filippov Signed-off-by: Andrew Morton --- include/linux/libgcc.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/libgcc.h b/include/linux/libgcc.h index b8dc75f0c830..fc388da6a027 100644 --- a/include/linux/libgcc.h +++ b/include/linux/libgcc.h @@ -27,4 +27,11 @@ typedef union { long long ll; } DWunion; +long long notrace __ashldi3(long long u, word_type b); +long long notrace __ashrdi3(long long u, word_type b); +word_type notrace __cmpdi2(long long a, long long b); +long long notrace __lshrdi3(long long u, word_type b); +long long notrace __muldi3(long long u, long long v); +word_type notrace __ucmpdi2(unsigned long long a, unsigned long long b); + #endif /* __ASM_LIBGCC_H */ -- cgit v1.2.3 From b3cbf98e2fdf3cb147a95161560cd25987284330 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 20 Apr 2023 13:56:24 -0400 Subject: SUNRPC: Support TLS handshake in the server-side TCP socket code This patch adds opportunitistic RPC-with-TLS to the Linux in-kernel NFS server. If the client requests RPC-with-TLS and the user space handshake agent is running, the server will set up a TLS session. There are no policy settings yet. For example, the server cannot yet require the use of RPC-with-TLS to access its data. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_xprt.h | 5 +- include/linux/sunrpc/svcsock.h | 2 + include/trace/events/sunrpc.h | 16 ++++++- net/sunrpc/svc_xprt.c | 5 +- net/sunrpc/svcauth_unix.c | 11 ++++- net/sunrpc/svcsock.c | 101 ++++++++++++++++++++++++++++++++++++++-- 6 files changed, 132 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_xprt.h b/include/linux/sunrpc/svc_xprt.h index 775368802762..867479204840 100644 --- a/include/linux/sunrpc/svc_xprt.h +++ b/include/linux/sunrpc/svc_xprt.h @@ -27,7 +27,7 @@ struct svc_xprt_ops { void (*xpo_detach)(struct svc_xprt *); void (*xpo_free)(struct svc_xprt *); void (*xpo_kill_temp_xprt)(struct svc_xprt *); - void (*xpo_start_tls)(struct svc_xprt *); + void (*xpo_handshake)(struct svc_xprt *xprt); }; struct svc_xprt_class { @@ -70,6 +70,9 @@ struct svc_xprt { #define XPT_LOCAL 12 /* connection from loopback interface */ #define XPT_KILL_TEMP 13 /* call xpo_kill_temp_xprt before closing */ #define XPT_CONG_CTRL 14 /* has congestion control */ +#define XPT_HANDSHAKE 15 /* xprt requests a handshake */ +#define XPT_TLS_SESSION 16 /* transport-layer security established */ +#define XPT_PEER_AUTH 17 /* peer has been authenticated */ struct svc_serv *xpt_server; /* service for transport */ atomic_t xpt_reserved; /* space on outq that is rsvd */ diff --git a/include/linux/sunrpc/svcsock.h b/include/linux/sunrpc/svcsock.h index dd73fa174af5..d16ae621782c 100644 --- a/include/linux/sunrpc/svcsock.h +++ b/include/linux/sunrpc/svcsock.h @@ -38,6 +38,8 @@ struct svc_sock { /* Number of queued send requests */ atomic_t sk_sendqlen; + struct completion sk_handshake_done; + struct page * sk_pages[RPCSVC_MAXPAGES]; /* received data */ }; diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h index 5a3bb42e1f50..31bc7025cb44 100644 --- a/include/trace/events/sunrpc.h +++ b/include/trace/events/sunrpc.h @@ -1857,7 +1857,10 @@ TRACE_EVENT(svc_stats_latency, { BIT(XPT_CACHE_AUTH), "CACHE_AUTH" }, \ { BIT(XPT_LOCAL), "LOCAL" }, \ { BIT(XPT_KILL_TEMP), "KILL_TEMP" }, \ - { BIT(XPT_CONG_CTRL), "CONG_CTRL" }) + { BIT(XPT_CONG_CTRL), "CONG_CTRL" }, \ + { BIT(XPT_HANDSHAKE), "HANDSHAKE" }, \ + { BIT(XPT_TLS_SESSION), "TLS_SESSION" }, \ + { BIT(XPT_PEER_AUTH), "PEER_AUTH" }) TRACE_EVENT(svc_xprt_create_err, TP_PROTO( @@ -1990,6 +1993,17 @@ DEFINE_SVC_XPRT_EVENT(close); DEFINE_SVC_XPRT_EVENT(detach); DEFINE_SVC_XPRT_EVENT(free); +#define DEFINE_SVC_TLS_EVENT(name) \ + DEFINE_EVENT(svc_xprt_event, svc_tls_##name, \ + TP_PROTO(const struct svc_xprt *xprt), \ + TP_ARGS(xprt)) + +DEFINE_SVC_TLS_EVENT(start); +DEFINE_SVC_TLS_EVENT(upcall); +DEFINE_SVC_TLS_EVENT(unavailable); +DEFINE_SVC_TLS_EVENT(not_started); +DEFINE_SVC_TLS_EVENT(timed_out); + TRACE_EVENT(svc_xprt_accept, TP_PROTO( const struct svc_xprt *xprt, diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 3b9708b39e35..84e5d7d31481 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -427,7 +427,7 @@ static bool svc_xprt_ready(struct svc_xprt *xprt) if (xpt_flags & BIT(XPT_BUSY)) return false; - if (xpt_flags & (BIT(XPT_CONN) | BIT(XPT_CLOSE))) + if (xpt_flags & (BIT(XPT_CONN) | BIT(XPT_CLOSE) | BIT(XPT_HANDSHAKE))) return true; if (xpt_flags & (BIT(XPT_DATA) | BIT(XPT_DEFERRED))) { if (xprt->xpt_ops->xpo_has_wspace(xprt) && @@ -828,6 +828,9 @@ static int svc_handle_xprt(struct svc_rqst *rqstp, struct svc_xprt *xprt) module_put(xprt->xpt_class->xcl_owner); } svc_xprt_received(xprt); + } else if (test_bit(XPT_HANDSHAKE, &xprt->xpt_flags)) { + xprt->xpt_ops->xpo_handshake(xprt); + svc_xprt_received(xprt); } else if (svc_xprt_reserve_slot(rqstp, xprt)) { /* XPT_DATA|XPT_DEFERRED case: */ dprintk("svc: server %p, pool %u, transport %p, inuse=%d\n", diff --git a/net/sunrpc/svcauth_unix.c b/net/sunrpc/svcauth_unix.c index 4485088ce27b..174783f804fa 100644 --- a/net/sunrpc/svcauth_unix.c +++ b/net/sunrpc/svcauth_unix.c @@ -17,8 +17,9 @@ #include #include #include -#define RPCDBG_FACILITY RPCDBG_AUTH +#include +#define RPCDBG_FACILITY RPCDBG_AUTH #include "netns.h" @@ -832,6 +833,7 @@ svcauth_tls_accept(struct svc_rqst *rqstp) { struct xdr_stream *xdr = &rqstp->rq_arg_stream; struct svc_cred *cred = &rqstp->rq_cred; + struct svc_xprt *xprt = rqstp->rq_xprt; u32 flavor, len; void *body; __be32 *p; @@ -865,14 +867,19 @@ svcauth_tls_accept(struct svc_rqst *rqstp) if (cred->cr_group_info == NULL) return SVC_CLOSE; - if (rqstp->rq_xprt->xpt_ops->xpo_start_tls) { + if (xprt->xpt_ops->xpo_handshake) { p = xdr_reserve_space(&rqstp->rq_res_stream, XDR_UNIT * 2 + 8); if (!p) return SVC_CLOSE; + trace_svc_tls_start(xprt); *p++ = rpc_auth_null; *p++ = cpu_to_be32(8); memcpy(p, "STARTTLS", 8); + + set_bit(XPT_HANDSHAKE, &xprt->xpt_flags); + svc_xprt_enqueue(xprt); } else { + trace_svc_tls_unavailable(xprt); if (xdr_stream_encode_opaque_auth(&rqstp->rq_res_stream, RPC_AUTH_NULL, NULL, 0) < 0) return SVC_CLOSE; diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index c5b74f523fc4..a51c9b989d58 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -44,9 +44,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -64,6 +66,12 @@ #define RPCDBG_FACILITY RPCDBG_SVCXPRT +/* To-do: to avoid tying up an nfsd thread while waiting for a + * handshake request, the request could instead be deferred. + */ +enum { + SVC_HANDSHAKE_TO = 5U * HZ +}; static struct svc_sock *svc_setup_socket(struct svc_serv *, struct socket *, int flags); @@ -359,6 +367,8 @@ static void svc_data_ready(struct sock *sk) rmb(); svsk->sk_odata(sk); trace_svcsock_data_ready(&svsk->sk_xprt, 0); + if (test_bit(XPT_HANDSHAKE, &svsk->sk_xprt.xpt_flags)) + return; if (!test_and_set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags)) svc_xprt_enqueue(&svsk->sk_xprt); } @@ -396,6 +406,88 @@ static void svc_tcp_kill_temp_xprt(struct svc_xprt *xprt) sock_no_linger(svsk->sk_sock->sk); } +/** + * svc_tcp_handshake_done - Handshake completion handler + * @data: address of xprt to wake + * @status: status of handshake + * @peerid: serial number of key containing the remote peer's identity + * + * If a security policy is specified as an export option, we don't + * have a specific export here to check. So we set a "TLS session + * is present" flag on the xprt and let an upper layer enforce local + * security policy. + */ +static void svc_tcp_handshake_done(void *data, int status, key_serial_t peerid) +{ + struct svc_xprt *xprt = data; + struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); + + if (!status) { + if (peerid != TLS_NO_PEERID) + set_bit(XPT_PEER_AUTH, &xprt->xpt_flags); + set_bit(XPT_TLS_SESSION, &xprt->xpt_flags); + } + clear_bit(XPT_HANDSHAKE, &xprt->xpt_flags); + complete_all(&svsk->sk_handshake_done); +} + +/** + * svc_tcp_handshake - Perform a transport-layer security handshake + * @xprt: connected transport endpoint + * + */ +static void svc_tcp_handshake(struct svc_xprt *xprt) +{ + struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); + struct sock *sk = svsk->sk_sock->sk; + struct tls_handshake_args args = { + .ta_sock = svsk->sk_sock, + .ta_done = svc_tcp_handshake_done, + .ta_data = xprt, + }; + int ret; + + trace_svc_tls_upcall(xprt); + + clear_bit(XPT_TLS_SESSION, &xprt->xpt_flags); + init_completion(&svsk->sk_handshake_done); + + ret = tls_server_hello_x509(&args, GFP_KERNEL); + if (ret) { + trace_svc_tls_not_started(xprt); + goto out_failed; + } + + ret = wait_for_completion_interruptible_timeout(&svsk->sk_handshake_done, + SVC_HANDSHAKE_TO); + if (ret <= 0) { + if (tls_handshake_cancel(sk)) { + trace_svc_tls_timed_out(xprt); + goto out_close; + } + } + + if (!test_bit(XPT_TLS_SESSION, &xprt->xpt_flags)) { + trace_svc_tls_unavailable(xprt); + goto out_close; + } + + /* Mark the transport ready in case the remote sent RPC + * traffic before the kernel received the handshake + * completion downcall. + */ + set_bit(XPT_DATA, &xprt->xpt_flags); + svc_xprt_enqueue(xprt); + return; + +out_close: + set_bit(XPT_CLOSE, &xprt->xpt_flags); +out_failed: + clear_bit(XPT_HANDSHAKE, &xprt->xpt_flags); + set_bit(XPT_DATA, &xprt->xpt_flags); + svc_xprt_enqueue(xprt); +} + /* * See net/ipv6/ip_sockglue.c : ip_cmsg_recv_pktinfo */ @@ -1257,6 +1349,7 @@ static const struct svc_xprt_ops svc_tcp_ops = { .xpo_has_wspace = svc_tcp_has_wspace, .xpo_accept = svc_tcp_accept, .xpo_kill_temp_xprt = svc_tcp_kill_temp_xprt, + .xpo_handshake = svc_tcp_handshake, }; static struct svc_xprt_class svc_tcp_class = { @@ -1580,10 +1673,12 @@ static void svc_tcp_sock_detach(struct svc_xprt *xprt) static void svc_sock_free(struct svc_xprt *xprt) { struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); + struct socket *sock = svsk->sk_sock; - if (svsk->sk_sock->file) - sockfd_put(svsk->sk_sock); + tls_handshake_cancel(sock->sk); + if (sock->file) + sockfd_put(sock); else - sock_release(svsk->sk_sock); + sock_release(sock); kfree(svsk); } -- cgit v1.2.3 From 0c8862de05c1a087795ee0a87bf61a6394306cc0 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Wed, 26 Apr 2023 21:49:37 +0300 Subject: tpm: Re-enable TPM chip boostrapping non-tpm_tis TPM drivers TPM chip bootstrapping was removed from tpm_chip_register(), and it was relocated to tpm_tis_core. This breaks all drivers which are not based on tpm_tis because the chip will not get properly initialized. Take the corrective steps: 1. Rename tpm_chip_startup() as tpm_chip_bootstrap() and make it one-shot. 2. Call tpm_chip_bootstrap() in tpm_chip_register(), which reverts the things as tehy used to be. Cc: Lino Sanfilippo Fixes: 548eb516ec0f ("tpm, tpm_tis: startup chip before testing for interrupts") Reported-by: Pengfei Xu Link: https://lore.kernel.org/all/ZEjqhwHWBnxcaRV5@xpf.sh.intel.com/ Tested-by: Pengfei Xu Signed-off-by: Jarkko Sakkinen --- drivers/char/tpm/tpm-chip.c | 22 +++++++++++++++++++--- drivers/char/tpm/tpm.h | 2 +- drivers/char/tpm/tpm_tis_core.c | 2 +- include/linux/tpm.h | 13 +++++++------ 4 files changed, 28 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c index 6fdfa65a00c3..9c0a6aad8114 100644 --- a/drivers/char/tpm/tpm-chip.c +++ b/drivers/char/tpm/tpm-chip.c @@ -606,13 +606,19 @@ static int tpm_get_pcr_allocation(struct tpm_chip *chip) } /* - * tpm_chip_startup() - performs auto startup and allocates the PCRs + * tpm_chip_bootstrap() - Boostrap TPM chip after power on * @chip: TPM chip to use. + * + * Initialize TPM chip after power on. This a one-shot function: subsequent + * calls will have no effect. */ -int tpm_chip_startup(struct tpm_chip *chip) +int tpm_chip_bootstrap(struct tpm_chip *chip) { int rc; + if (chip->flags & TPM_CHIP_FLAG_BOOTSTRAPPED) + return 0; + rc = tpm_chip_start(chip); if (rc) return rc; @@ -625,9 +631,15 @@ int tpm_chip_startup(struct tpm_chip *chip) stop: tpm_chip_stop(chip); + /* + * Unconditionally set, as driver initialization should cease, when the + * boostrapping process fails. + */ + chip->flags |= TPM_CHIP_FLAG_BOOTSTRAPPED; + return rc; } -EXPORT_SYMBOL_GPL(tpm_chip_startup); +EXPORT_SYMBOL_GPL(tpm_chip_bootstrap); /* * tpm_chip_register() - create a character device for the TPM chip @@ -644,6 +656,10 @@ int tpm_chip_register(struct tpm_chip *chip) { int rc; + rc = tpm_chip_bootstrap(chip); + if (rc) + return rc; + tpm_sysfs_add_device(chip); tpm_bios_log_setup(chip); diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h index 88d3bd76e076..f6c99b3f0045 100644 --- a/drivers/char/tpm/tpm.h +++ b/drivers/char/tpm/tpm.h @@ -263,7 +263,7 @@ static inline void tpm_msleep(unsigned int delay_msec) delay_msec * 1000); }; -int tpm_chip_startup(struct tpm_chip *chip); +int tpm_chip_bootstrap(struct tpm_chip *chip); int tpm_chip_start(struct tpm_chip *chip); void tpm_chip_stop(struct tpm_chip *chip); struct tpm_chip *tpm_find_get_ops(struct tpm_chip *chip); diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index c2421162cf34..02945d53fcef 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -1139,7 +1139,7 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, init_waitqueue_head(&priv->read_queue); init_waitqueue_head(&priv->int_queue); - rc = tpm_chip_startup(chip); + rc = tpm_chip_bootstrap(chip); if (rc) goto out_err; diff --git a/include/linux/tpm.h b/include/linux/tpm.h index 4dc97b9f65fb..77693389c3f9 100644 --- a/include/linux/tpm.h +++ b/include/linux/tpm.h @@ -274,13 +274,14 @@ enum tpm2_cc_attrs { #define TPM_VID_ATML 0x1114 enum tpm_chip_flags { - TPM_CHIP_FLAG_TPM2 = BIT(1), - TPM_CHIP_FLAG_IRQ = BIT(2), - TPM_CHIP_FLAG_VIRTUAL = BIT(3), - TPM_CHIP_FLAG_HAVE_TIMEOUTS = BIT(4), - TPM_CHIP_FLAG_ALWAYS_POWERED = BIT(5), + TPM_CHIP_FLAG_BOOTSTRAPPED = BIT(0), + TPM_CHIP_FLAG_TPM2 = BIT(1), + TPM_CHIP_FLAG_IRQ = BIT(2), + TPM_CHIP_FLAG_VIRTUAL = BIT(3), + TPM_CHIP_FLAG_HAVE_TIMEOUTS = BIT(4), + TPM_CHIP_FLAG_ALWAYS_POWERED = BIT(5), TPM_CHIP_FLAG_FIRMWARE_POWER_MANAGED = BIT(6), - TPM_CHIP_FLAG_FIRMWARE_UPGRADE = BIT(7), + TPM_CHIP_FLAG_FIRMWARE_UPGRADE = BIT(7), }; #define to_tpm_chip(d) container_of(d, struct tpm_chip, dev) -- cgit v1.2.3 From fbd2a05f29a95d5b42b294bf47e55a711424965b Mon Sep 17 00:00:00 2001 From: Anna Schumaker Date: Thu, 6 Apr 2023 15:16:52 -0400 Subject: NFSv4.2: Rework scratch handling for READ_PLUS Instead of using a tiny, static scratch buffer, we should use a kmalloc()-ed buffer that is allocated when checking for read plus usage. This lets us use the buffer before decoding any part of the READ_PLUS operation instead of setting it right before segment decoding, meaning it should be a little more robust. Signed-off-by: Anna Schumaker --- fs/nfs/nfs42xdr.c | 4 ++-- fs/nfs/nfs4proc.c | 17 ++++++++++++----- include/linux/nfs_xdr.h | 1 + 3 files changed, 15 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/fs/nfs/nfs42xdr.c b/fs/nfs/nfs42xdr.c index d80ee88ca996..a6df815a140c 100644 --- a/fs/nfs/nfs42xdr.c +++ b/fs/nfs/nfs42xdr.c @@ -1122,7 +1122,6 @@ static int decode_read_plus(struct xdr_stream *xdr, struct nfs_pgio_res *res) uint32_t segments; struct read_plus_segment *segs; int status, i; - char scratch_buf[16]; __be32 *p; status = decode_op_hdr(xdr, OP_READ_PLUS); @@ -1143,7 +1142,6 @@ static int decode_read_plus(struct xdr_stream *xdr, struct nfs_pgio_res *res) if (!segs) return -ENOMEM; - xdr_set_scratch_buffer(xdr, &scratch_buf, sizeof(scratch_buf)); status = -EIO; for (i = 0; i < segments; i++) { status = decode_read_plus_segment(xdr, &segs[i]); @@ -1348,6 +1346,8 @@ static int nfs4_xdr_dec_read_plus(struct rpc_rqst *rqstp, struct compound_hdr hdr; int status; + xdr_set_scratch_buffer(xdr, res->scratch, sizeof(res->scratch)); + status = decode_compound_hdr(xdr, &hdr); if (status) goto out; diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 5607b1e2b821..18f25ff4bff7 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -5439,6 +5439,8 @@ static bool nfs4_read_plus_not_supported(struct rpc_task *task, static int nfs4_read_done(struct rpc_task *task, struct nfs_pgio_header *hdr) { + if (hdr->res.scratch) + kfree(hdr->res.scratch); if (!nfs4_sequence_done(task, &hdr->res.seq_res)) return -EAGAIN; if (nfs4_read_stateid_changed(task, &hdr->args)) @@ -5452,17 +5454,22 @@ static int nfs4_read_done(struct rpc_task *task, struct nfs_pgio_header *hdr) } #if defined CONFIG_NFS_V4_2 && defined CONFIG_NFS_V4_2_READ_PLUS -static void nfs42_read_plus_support(struct nfs_pgio_header *hdr, +static bool nfs42_read_plus_support(struct nfs_pgio_header *hdr, struct rpc_message *msg) { /* Note: We don't use READ_PLUS with pNFS yet */ - if (nfs_server_capable(hdr->inode, NFS_CAP_READ_PLUS) && !hdr->ds_clp) + if (nfs_server_capable(hdr->inode, NFS_CAP_READ_PLUS) && !hdr->ds_clp) { msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ_PLUS]; + hdr->res.scratch = kmalloc(32, GFP_KERNEL); + return hdr->res.scratch != NULL; + } + return false; } #else -static void nfs42_read_plus_support(struct nfs_pgio_header *hdr, +static bool nfs42_read_plus_support(struct nfs_pgio_header *hdr, struct rpc_message *msg) { + return false; } #endif /* CONFIG_NFS_V4_2 */ @@ -5472,8 +5479,8 @@ static void nfs4_proc_read_setup(struct nfs_pgio_header *hdr, hdr->timestamp = jiffies; if (!hdr->pgio_done_cb) hdr->pgio_done_cb = nfs4_read_done_cb; - msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ]; - nfs42_read_plus_support(hdr, msg); + if (!nfs42_read_plus_support(hdr, msg)) + msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ]; nfs4_init_sequence(&hdr->args.seq_args, &hdr->res.seq_res, 0, 0); } diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index e196ef595908..29a1b39794bf 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -670,6 +670,7 @@ struct nfs_pgio_res { struct { unsigned int replen; /* used by read */ int eof; /* used by read */ + void * scratch; /* used by read */ }; struct { struct nfs_writeverf * verf; /* used by write */ -- cgit v1.2.3 From e6ce9d741163af0b846637ce6550ae8a671b1588 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Wed, 5 Apr 2023 16:17:06 +0200 Subject: locking/atomic: Add generic try_cmpxchg{,64}_local() support Add generic support for try_cmpxchg{,64}_local() and their falbacks. These provides the generic try_cmpxchg_local family of functions from the arch_ prefixed version, also adding explicit instrumentation. Signed-off-by: Uros Bizjak Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Acked-by: Mark Rutland Link: https://lore.kernel.org/r/20230405141710.3551-2-ubizjak@gmail.com Cc: Linus Torvalds --- include/linux/atomic/atomic-arch-fallback.h | 24 +++++++++++++++++++++++- include/linux/atomic/atomic-instrumented.h | 20 +++++++++++++++++++- scripts/atomic/gen-atomic-fallback.sh | 4 ++++ scripts/atomic/gen-atomic-instrumented.sh | 2 +- 4 files changed, 47 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/atomic/atomic-arch-fallback.h index 4226379a232d..a6e4437c5f36 100644 --- a/include/linux/atomic/atomic-arch-fallback.h +++ b/include/linux/atomic/atomic-arch-fallback.h @@ -217,6 +217,28 @@ #endif /* arch_try_cmpxchg64_relaxed */ +#ifndef arch_try_cmpxchg_local +#define arch_try_cmpxchg_local(_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg_local((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg_local */ + +#ifndef arch_try_cmpxchg64_local +#define arch_try_cmpxchg64_local(_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg64_local((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg64_local */ + #ifndef arch_atomic_read_acquire static __always_inline int arch_atomic_read_acquire(const atomic_t *v) @@ -2646,4 +2668,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) #endif #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// 00071fffa021cec66f6290d706d69c91df87bade +// ad2e2b4d168dbc60a73922616047a9bfa446af36 diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h index 0496816738ca..245ba661c493 100644 --- a/include/linux/atomic/atomic-instrumented.h +++ b/include/linux/atomic/atomic-instrumented.h @@ -2132,6 +2132,24 @@ atomic_long_dec_if_positive(atomic_long_t *v) arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ }) +#define try_cmpxchg_local(ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + +#define try_cmpxchg64_local(ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + #define cmpxchg_double(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ @@ -2149,4 +2167,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) }) #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 1b485de9cbaa4900de59e14ee2084357eaeb1c3a +// 97fe4d79aa058d2164df824632cbc4f716d2a407 diff --git a/scripts/atomic/gen-atomic-fallback.sh b/scripts/atomic/gen-atomic-fallback.sh index 3a07695e3c89..6e853f0dad8d 100755 --- a/scripts/atomic/gen-atomic-fallback.sh +++ b/scripts/atomic/gen-atomic-fallback.sh @@ -225,6 +225,10 @@ for cmpxchg in "cmpxchg" "cmpxchg64"; do gen_try_cmpxchg_fallbacks "${cmpxchg}" done +for cmpxchg in "cmpxchg_local" "cmpxchg64_local"; do + gen_try_cmpxchg_fallback "${cmpxchg}" "" +done + grep '^[a-z]' "$1" | while read name meta args; do gen_proto "${meta}" "${name}" "atomic" "int" ${args} done diff --git a/scripts/atomic/gen-atomic-instrumented.sh b/scripts/atomic/gen-atomic-instrumented.sh index 77c06526a574..c8165e9431bf 100755 --- a/scripts/atomic/gen-atomic-instrumented.sh +++ b/scripts/atomic/gen-atomic-instrumented.sh @@ -173,7 +173,7 @@ for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do done done -for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg"; do +for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" ; do gen_xchg "${xchg}" "" "" printf "\n" done -- cgit v1.2.3 From ec570320b09f76d52819e60abdccf372658216b6 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 13 Apr 2023 17:06:44 +0100 Subject: locking/atomic: Correct (cmp)xchg() instrumentation All xchg() and cmpxchg() ops are atomic RMWs, but currently we instrument these with instrument_atomic_write() rather than instrument_atomic_read_write(), missing the read aspect. Similarly, all try_cmpxchg() ops are non-atomic RMWs on *oldp, but we instrument these accesses with instrument_atomic_write() rather than instrument_read_write(), missing the read aspect and erroneously marking these as atomic. Fix the instrumentation for both points. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lkml.kernel.org/r/20230413160644.490976-1-mark.rutland@arm.com Cc: Linus Torvalds --- include/linux/atomic/atomic-instrumented.h | 76 +++++++++++++++--------------- scripts/atomic/gen-atomic-instrumented.sh | 6 +-- 2 files changed, 41 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h index 245ba661c493..03a232a1fa57 100644 --- a/include/linux/atomic/atomic-instrumented.h +++ b/include/linux/atomic/atomic-instrumented.h @@ -1948,14 +1948,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ kcsan_mb(); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg(__ai_ptr, __VA_ARGS__); \ }) #define xchg_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ }) @@ -1963,14 +1963,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ kcsan_release(); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_release(__ai_ptr, __VA_ARGS__); \ }) #define xchg_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ }) @@ -1978,14 +1978,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ kcsan_mb(); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ }) @@ -1993,14 +1993,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ kcsan_release(); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ }) @@ -2008,14 +2008,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ kcsan_mb(); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg64_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ }) @@ -2023,14 +2023,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ kcsan_release(); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg64_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ }) @@ -2039,8 +2039,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ kcsan_mb(); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) @@ -2048,8 +2048,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) @@ -2058,8 +2058,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ kcsan_release(); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) @@ -2067,8 +2067,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) @@ -2077,8 +2077,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ kcsan_mb(); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) @@ -2086,8 +2086,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) @@ -2096,8 +2096,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ kcsan_release(); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) @@ -2105,22 +2105,22 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define cmpxchg_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg64_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ }) @@ -2128,7 +2128,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ kcsan_mb(); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ }) @@ -2136,8 +2136,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) @@ -2145,8 +2145,8 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ typeof(oldp) __ai_oldp = (oldp); \ - instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ - instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) @@ -2154,7 +2154,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) ({ \ typeof(ptr) __ai_ptr = (ptr); \ kcsan_mb(); \ - instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ }) @@ -2162,9 +2162,9 @@ atomic_long_dec_if_positive(atomic_long_t *v) #define cmpxchg_double_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ + instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ }) #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 97fe4d79aa058d2164df824632cbc4f716d2a407 +// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e diff --git a/scripts/atomic/gen-atomic-instrumented.sh b/scripts/atomic/gen-atomic-instrumented.sh index c8165e9431bf..d9ffd74f73ca 100755 --- a/scripts/atomic/gen-atomic-instrumented.sh +++ b/scripts/atomic/gen-atomic-instrumented.sh @@ -104,8 +104,8 @@ cat < Date: Tue, 25 Apr 2023 17:03:13 +0200 Subject: debugobjects,locking: Annotate debug_object_fill_pool() wait type violation There is an explicit wait-type violation in debug_object_fill_pool() for PREEMPT_RT=n kernels which allows them to more easily fill the object pool and reduce the chance of allocation failures. Lockdep's wait-type checks are designed to check the PREEMPT_RT locking rules even for PREEMPT_RT=n kernels and object to this, so create a lockdep annotation to allow this to stand. Specifically, create a 'lock' type that overrides the inner wait-type while it is held -- allowing one to temporarily raise it, such that the violation is hidden. Reported-by: Vlastimil Babka Reported-by: Qi Zheng Signed-off-by: Peter Zijlstra (Intel) Tested-by: Qi Zheng Link: https://lkml.kernel.org/r/20230429100614.GA1489784@hirez.programming.kicks-ass.net --- include/linux/lockdep.h | 14 ++++++++++++++ include/linux/lockdep_types.h | 1 + kernel/locking/lockdep.c | 28 +++++++++++++++++++++------- lib/debugobjects.c | 15 +++++++++++++-- 4 files changed, 49 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 1023f349af71..a3329fb49b33 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -339,6 +339,16 @@ extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie); #define lockdep_repin_lock(l,c) lock_repin_lock(&(l)->dep_map, (c)) #define lockdep_unpin_lock(l,c) lock_unpin_lock(&(l)->dep_map, (c)) +/* + * Must use lock_map_aquire_try() with override maps to avoid + * lockdep thinking they participate in the block chain. + */ +#define DEFINE_WAIT_OVERRIDE_MAP(_name, _wait_type) \ + struct lockdep_map _name = { \ + .name = #_name "-wait-type-override", \ + .wait_type_inner = _wait_type, \ + .lock_type = LD_LOCK_WAIT_OVERRIDE, } + #else /* !CONFIG_LOCKDEP */ static inline void lockdep_init_task(struct task_struct *task) @@ -427,6 +437,9 @@ extern int lockdep_is_held(const void *); #define lockdep_repin_lock(l, c) do { (void)(l); (void)(c); } while (0) #define lockdep_unpin_lock(l, c) do { (void)(l); (void)(c); } while (0) +#define DEFINE_WAIT_OVERRIDE_MAP(_name, _wait_type) \ + struct lockdep_map __maybe_unused _name = {} + #endif /* !LOCKDEP */ enum xhlock_context_t { @@ -551,6 +564,7 @@ do { \ #define rwsem_release(l, i) lock_release(l, i) #define lock_map_acquire(l) lock_acquire_exclusive(l, 0, 0, NULL, _THIS_IP_) +#define lock_map_acquire_try(l) lock_acquire_exclusive(l, 0, 1, NULL, _THIS_IP_) #define lock_map_acquire_read(l) lock_acquire_shared_recursive(l, 0, 0, NULL, _THIS_IP_) #define lock_map_acquire_tryread(l) lock_acquire_shared_recursive(l, 0, 1, NULL, _THIS_IP_) #define lock_map_release(l) lock_release(l, _THIS_IP_) diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h index d22430840b53..59f4fb1626ea 100644 --- a/include/linux/lockdep_types.h +++ b/include/linux/lockdep_types.h @@ -33,6 +33,7 @@ enum lockdep_wait_type { enum lockdep_lock_type { LD_LOCK_NORMAL = 0, /* normal, catch all */ LD_LOCK_PERCPU, /* percpu */ + LD_LOCK_WAIT_OVERRIDE, /* annotation */ LD_LOCK_MAX, }; diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 50d4863974e7..62ef295e07e6 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -2253,6 +2253,9 @@ static inline bool usage_match(struct lock_list *entry, void *mask) static inline bool usage_skip(struct lock_list *entry, void *mask) { + if (entry->class->lock_type == LD_LOCK_NORMAL) + return false; + /* * Skip local_lock() for irq inversion detection. * @@ -2279,14 +2282,16 @@ static inline bool usage_skip(struct lock_list *entry, void *mask) * As a result, we will skip local_lock(), when we search for irq * inversion bugs. */ - if (entry->class->lock_type == LD_LOCK_PERCPU) { - if (DEBUG_LOCKS_WARN_ON(entry->class->wait_type_inner < LD_WAIT_CONFIG)) - return false; + if (entry->class->lock_type == LD_LOCK_PERCPU && + DEBUG_LOCKS_WARN_ON(entry->class->wait_type_inner < LD_WAIT_CONFIG)) + return false; - return true; - } + /* + * Skip WAIT_OVERRIDE for irq inversion detection -- it's not actually + * a lock and only used to override the wait_type. + */ - return false; + return true; } /* @@ -4752,7 +4757,8 @@ static int check_wait_context(struct task_struct *curr, struct held_lock *next) for (; depth < curr->lockdep_depth; depth++) { struct held_lock *prev = curr->held_locks + depth; - u8 prev_inner = hlock_class(prev)->wait_type_inner; + struct lock_class *class = hlock_class(prev); + u8 prev_inner = class->wait_type_inner; if (prev_inner) { /* @@ -4762,6 +4768,14 @@ static int check_wait_context(struct task_struct *curr, struct held_lock *next) * Also due to trylocks. */ curr_inner = min(curr_inner, prev_inner); + + /* + * Allow override for annotations -- this is typically + * only valid/needed for code that only exists when + * CONFIG_PREEMPT_RT=n. + */ + if (unlikely(class->lock_type == LD_LOCK_WAIT_OVERRIDE)) + curr_inner = prev_inner; } } diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 003edc5ebd67..826c617b10a7 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -591,10 +591,21 @@ static void debug_objects_fill_pool(void) { /* * On RT enabled kernels the pool refill must happen in preemptible - * context: + * context -- for !RT kernels we rely on the fact that spinlock_t and + * raw_spinlock_t are basically the same type and this lock-type + * inversion works just fine. */ - if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible()) + if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible()) { + /* + * Annotate away the spinlock_t inside raw_spinlock_t warning + * by temporarily raising the wait-type to WAIT_SLEEP, matching + * the preemptible() condition above. + */ + static DEFINE_WAIT_OVERRIDE_MAP(fill_pool_map, LD_WAIT_SLEEP); + lock_map_acquire_try(&fill_pool_map); fill_pool(); + lock_map_release(&fill_pool_map); + } } static void -- cgit v1.2.3 From 24139c07f413ef4b555482c758343d71392a19bc Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Sat, 22 Apr 2023 22:54:18 +0200 Subject: mm/ksm: unmerge and clear VM_MERGEABLE when setting PR_SET_MEMORY_MERGE=0 Patch series "mm/ksm: improve PR_SET_MEMORY_MERGE=0 handling and cleanup disabling KSM", v2. (1) Make PR_SET_MEMORY_MERGE=0 unmerge pages like setting MADV_UNMERGEABLE does, (2) add a selftest for it and (3) factor out disabling of KSM from s390/gmap code. This patch (of 3): Let's unmerge any KSM pages when setting PR_SET_MEMORY_MERGE=0, and clear the VM_MERGEABLE flag from all VMAs -- just like KSM would. Of course, only do that if we previously set PR_SET_MEMORY_MERGE=1. Link: https://lkml.kernel.org/r/20230422205420.30372-1-david@redhat.com Link: https://lkml.kernel.org/r/20230422205420.30372-2-david@redhat.com Signed-off-by: David Hildenbrand Acked-by: Stefan Roesch Cc: Christian Borntraeger Cc: Claudio Imbrenda Cc: Heiko Carstens Cc: Janosch Frank Cc: Johannes Weiner Cc: Michal Hocko Cc: Rik van Riel Cc: Shuah Khan Cc: Sven Schnelle Cc: Vasily Gorbik Signed-off-by: Andrew Morton --- include/linux/ksm.h | 1 + kernel/sys.c | 12 +++-------- mm/ksm.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 7a9b76fb6c3f..429efa6ff4ae 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -21,6 +21,7 @@ int ksm_madvise(struct vm_area_struct *vma, unsigned long start, void ksm_add_vma(struct vm_area_struct *vma); int ksm_enable_merge_any(struct mm_struct *mm); +int ksm_disable_merge_any(struct mm_struct *mm); int __ksm_enter(struct mm_struct *mm); void __ksm_exit(struct mm_struct *mm); diff --git a/kernel/sys.c b/kernel/sys.c index 72cdb16e2636..339fee3eff6a 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2695,16 +2695,10 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, if (mmap_write_lock_killable(me->mm)) return -EINTR; - if (arg2) { + if (arg2) error = ksm_enable_merge_any(me->mm); - } else { - /* - * TODO: we might want disable KSM on all VMAs and - * trigger unsharing to completely disable KSM. - */ - clear_bit(MMF_VM_MERGE_ANY, &me->mm->flags); - error = 0; - } + else + error = ksm_disable_merge_any(me->mm); mmap_write_unlock(me->mm); break; case PR_GET_MEMORY_MERGE: diff --git a/mm/ksm.c b/mm/ksm.c index 9e48258985d2..823bb3475a68 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -2520,6 +2520,22 @@ static void __ksm_add_vma(struct vm_area_struct *vma) vm_flags_set(vma, VM_MERGEABLE); } +static int __ksm_del_vma(struct vm_area_struct *vma) +{ + int err; + + if (!(vma->vm_flags & VM_MERGEABLE)) + return 0; + + if (vma->anon_vma) { + err = unmerge_ksm_pages(vma, vma->vm_start, vma->vm_end); + if (err) + return err; + } + + vm_flags_clear(vma, VM_MERGEABLE); + return 0; +} /** * ksm_add_vma - Mark vma as mergeable if compatible * @@ -2542,6 +2558,20 @@ static void ksm_add_vmas(struct mm_struct *mm) __ksm_add_vma(vma); } +static int ksm_del_vmas(struct mm_struct *mm) +{ + struct vm_area_struct *vma; + int err; + + VMA_ITERATOR(vmi, mm, 0); + for_each_vma(vmi, vma) { + err = __ksm_del_vma(vma); + if (err) + return err; + } + return 0; +} + /** * ksm_enable_merge_any - Add mm to mm ksm list and enable merging on all * compatible VMA's @@ -2569,6 +2599,35 @@ int ksm_enable_merge_any(struct mm_struct *mm) return 0; } +/** + * ksm_disable_merge_any - Disable merging on all compatible VMA's of the mm, + * previously enabled via ksm_enable_merge_any(). + * + * Disabling merging implies unmerging any merged pages, like setting + * MADV_UNMERGEABLE would. If unmerging fails, the whole operation fails and + * merging on all compatible VMA's remains enabled. + * + * @mm: Pointer to mm + * + * Returns 0 on success, otherwise error code + */ +int ksm_disable_merge_any(struct mm_struct *mm) +{ + int err; + + if (!test_bit(MMF_VM_MERGE_ANY, &mm->flags)) + return 0; + + err = ksm_del_vmas(mm); + if (err) { + ksm_add_vmas(mm); + return err; + } + + clear_bit(MMF_VM_MERGE_ANY, &mm->flags); + return 0; +} + int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, unsigned long *vm_flags) { -- cgit v1.2.3 From 2c281f54f556e1f3266c8cb104adf9eea7a7b742 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Sat, 22 Apr 2023 23:01:56 +0200 Subject: mm/ksm: move disabling KSM from s390/gmap code to KSM code Let's factor out actual disabling of KSM. The existing "mm->def_flags &= ~VM_MERGEABLE;" was essentially a NOP and can be dropped, because def_flags should never include VM_MERGEABLE. Note that we don't currently prevent re-enabling KSM. This should now be faster in case KSM was never enabled, because we only conditionally iterate all VMAs. Further, it certainly looks cleaner. Link: https://lkml.kernel.org/r/20230422210156.33630-1-david@redhat.com Signed-off-by: David Hildenbrand Acked-by: Janosch Frank Acked-by: Stefan Roesch Cc: Christian Borntraeger Cc: Claudio Imbrenda Cc: Heiko Carstens Cc: Johannes Weiner Cc: Michal Hocko Cc: Rik van Riel Cc: Shuah Khan Cc: Sven Schnelle Cc: Vasily Gorbik Signed-off-by: Andrew Morton --- arch/s390/mm/gmap.c | 20 +------------------- include/linux/ksm.h | 6 ++++++ mm/ksm.c | 11 +++++++++++ 3 files changed, 18 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 0949811761e6..dfe905c7bd8e 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -2585,30 +2585,12 @@ EXPORT_SYMBOL_GPL(s390_enable_sie); int gmap_mark_unmergeable(void) { - struct mm_struct *mm = current->mm; - struct vm_area_struct *vma; - unsigned long vm_flags; - int ret; - VMA_ITERATOR(vmi, mm, 0); - /* * Make sure to disable KSM (if enabled for the whole process or * individual VMAs). Note that nothing currently hinders user space * from re-enabling it. */ - clear_bit(MMF_VM_MERGE_ANY, &mm->flags); - - for_each_vma(vmi, vma) { - /* Copy vm_flags to avoid partial modifications in ksm_madvise */ - vm_flags = vma->vm_flags; - ret = ksm_madvise(vma, vma->vm_start, vma->vm_end, - MADV_UNMERGEABLE, &vm_flags); - if (ret) - return ret; - vm_flags_reset(vma, vm_flags); - } - mm->def_flags &= ~VM_MERGEABLE; - return 0; + return ksm_disable(current->mm); } EXPORT_SYMBOL_GPL(gmap_mark_unmergeable); diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 429efa6ff4ae..899a314bc487 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -22,6 +22,7 @@ int ksm_madvise(struct vm_area_struct *vma, unsigned long start, void ksm_add_vma(struct vm_area_struct *vma); int ksm_enable_merge_any(struct mm_struct *mm); int ksm_disable_merge_any(struct mm_struct *mm); +int ksm_disable(struct mm_struct *mm); int __ksm_enter(struct mm_struct *mm); void __ksm_exit(struct mm_struct *mm); @@ -80,6 +81,11 @@ static inline void ksm_add_vma(struct vm_area_struct *vma) { } +static inline int ksm_disable(struct mm_struct *mm) +{ + return 0; +} + static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) { return 0; diff --git a/mm/ksm.c b/mm/ksm.c index 823bb3475a68..0156bded3a66 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -2628,6 +2628,17 @@ int ksm_disable_merge_any(struct mm_struct *mm) return 0; } +int ksm_disable(struct mm_struct *mm) +{ + mmap_assert_write_locked(mm); + + if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) + return 0; + if (test_bit(MMF_VM_MERGE_ANY, &mm->flags)) + return ksm_disable_merge_any(mm); + return ksm_del_vmas(mm); +} + int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, unsigned long *vm_flags) { -- cgit v1.2.3 From 245f0922689364b21163af4937a05ea0ba576fae Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Mon, 17 Apr 2023 12:53:23 +0800 Subject: mm: hwpoison: coredump: support recovery from dump_user_range() dump_user_range() is used to copy the user page to a coredump file, but if a hardware memory error occurred during copy, which called from __kernel_write_iter() in dump_user_range(), it crashes, CPU: 112 PID: 7014 Comm: mca-recover Not tainted 6.3.0-rc2 #425 pc : __memcpy+0x110/0x260 lr : _copy_from_iter+0x3bc/0x4c8 ... Call trace: __memcpy+0x110/0x260 copy_page_from_iter+0xcc/0x130 pipe_write+0x164/0x6d8 __kernel_write_iter+0x9c/0x210 dump_user_range+0xc8/0x1d8 elf_core_dump+0x308/0x368 do_coredump+0x2e8/0xa40 get_signal+0x59c/0x788 do_signal+0x118/0x1f8 do_notify_resume+0xf0/0x280 el0_da+0x130/0x138 el0t_64_sync_handler+0x68/0xc0 el0t_64_sync+0x188/0x190 Generally, the '->write_iter' of file ops will use copy_page_from_iter() and copy_page_from_iter_atomic(), change memcpy() to copy_mc_to_kernel() in both of them to handle #MC during source read, which stop coredump processing and kill the task instead of kernel panic, but the source address may not always a user address, so introduce a new copy_mc flag in struct iov_iter{} to indicate that the iter could do a safe memory copy, also introduce the helpers to set/cleck the flag, for now, it's only used in coredump's dump_user_range(), but it could expand to any other scenarios to fix the similar issue. Link: https://lkml.kernel.org/r/20230417045323.11054-1-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: Alexander Viro Cc: Christian Brauner Cc: Miaohe Lin Cc: Naoya Horiguchi Cc: Tong Tiangen Cc: Jens Axboe Signed-off-by: Andrew Morton --- fs/coredump.c | 1 + include/linux/uio.h | 16 ++++++++++++++++ lib/iov_iter.c | 17 +++++++++++++++-- 3 files changed, 32 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/fs/coredump.c b/fs/coredump.c index 5df1e6e1eb2b..ece7badf701b 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -882,6 +882,7 @@ static int dump_emit_page(struct coredump_params *cprm, struct page *page) pos = file->f_pos; bvec_set_page(&bvec, page, PAGE_SIZE, 0); iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, PAGE_SIZE); + iov_iter_set_copy_mc(&iter); n = __kernel_write_iter(cprm->file, &iter, &pos); if (n != PAGE_SIZE) return 0; diff --git a/include/linux/uio.h b/include/linux/uio.h index 3d386849a758..044c1d8c230c 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -42,6 +42,7 @@ struct iov_iter_state { struct iov_iter { u8 iter_type; + bool copy_mc; bool nofault; bool data_source; bool user_backed; @@ -256,8 +257,22 @@ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i); #ifdef CONFIG_ARCH_HAS_COPY_MC size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i); +static inline void iov_iter_set_copy_mc(struct iov_iter *i) +{ + i->copy_mc = true; +} + +static inline bool iov_iter_is_copy_mc(const struct iov_iter *i) +{ + return i->copy_mc; +} #else #define _copy_mc_to_iter _copy_to_iter +static inline void iov_iter_set_copy_mc(struct iov_iter *i) { } +static inline bool iov_iter_is_copy_mc(const struct iov_iter *i) +{ + return false; +} #endif size_t iov_iter_zero(size_t bytes, struct iov_iter *); @@ -380,6 +395,7 @@ static inline void iov_iter_ubuf(struct iov_iter *i, unsigned int direction, WARN_ON(direction & ~(READ | WRITE)); *i = (struct iov_iter) { .iter_type = ITER_UBUF, + .copy_mc = false, .user_backed = true, .data_source = direction, .ubuf = buf, diff --git a/lib/iov_iter.c b/lib/iov_iter.c index c3dbe994112c..960223ed9199 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -434,6 +434,7 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction, WARN_ON(direction & ~(READ | WRITE)); *i = (struct iov_iter) { .iter_type = ITER_IOVEC, + .copy_mc = false, .nofault = false, .user_backed = true, .data_source = direction, @@ -630,6 +631,14 @@ size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) EXPORT_SYMBOL_GPL(_copy_mc_to_iter); #endif /* CONFIG_ARCH_HAS_COPY_MC */ +static void *memcpy_from_iter(struct iov_iter *i, void *to, const void *from, + size_t size) +{ + if (iov_iter_is_copy_mc(i)) + return (void *)copy_mc_to_kernel(to, from, size); + return memcpy(to, from, size); +} + size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) { if (WARN_ON_ONCE(!i->data_source)) @@ -639,7 +648,7 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) might_fault(); iterate_and_advance(i, bytes, base, len, off, copyin(addr + off, base, len), - memcpy(addr + off, base, len) + memcpy_from_iter(i, addr + off, base, len) ) return bytes; @@ -862,7 +871,7 @@ size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t byt } iterate_and_advance(i, bytes, base, len, off, copyin(p + off, base, len), - memcpy(p + off, base, len) + memcpy_from_iter(i, p + off, base, len) ) kunmap_atomic(kaddr); return bytes; @@ -1043,6 +1052,7 @@ void iov_iter_kvec(struct iov_iter *i, unsigned int direction, WARN_ON(direction & ~(READ | WRITE)); *i = (struct iov_iter){ .iter_type = ITER_KVEC, + .copy_mc = false, .data_source = direction, .kvec = kvec, .nr_segs = nr_segs, @@ -1059,6 +1069,7 @@ void iov_iter_bvec(struct iov_iter *i, unsigned int direction, WARN_ON(direction & ~(READ | WRITE)); *i = (struct iov_iter){ .iter_type = ITER_BVEC, + .copy_mc = false, .data_source = direction, .bvec = bvec, .nr_segs = nr_segs, @@ -1105,6 +1116,7 @@ void iov_iter_xarray(struct iov_iter *i, unsigned int direction, BUG_ON(direction & ~1); *i = (struct iov_iter) { .iter_type = ITER_XARRAY, + .copy_mc = false, .data_source = direction, .xarray = xarray, .xarray_start = start, @@ -1128,6 +1140,7 @@ void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) BUG_ON(direction != READ); *i = (struct iov_iter){ .iter_type = ITER_DISCARD, + .copy_mc = false, .data_source = false, .count = count, .iov_offset = 0 -- cgit v1.2.3 From 0199849acd07d07e2a8e42757653ca8b14a122f5 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Tue, 2 May 2023 18:30:04 -0700 Subject: sysctl: remove register_sysctl_paths() The deprecation for register_sysctl_paths() is over. We can rejoice as we nuke register_sysctl_paths(). The routine register_sysctl_table() was the only user left of register_sysctl_paths(), so we can now just open code and move the implementation over to what used to be to __register_sysctl_paths(). The old dynamic struct ctl_table_set *set is now the point to sysctl_table_root.default_set. The old dynamic const struct ctl_path *path was being used in the routine register_sysctl_paths() with a static: static const struct ctl_path null_path[] = { {} }; Since this is a null path we can now just simplfy the old routine and remove its use as its always empty. This saves us a total of 230 bytes. $ ./scripts/bloat-o-meter vmlinux.old vmlinux add/remove: 2/7 grow/shrink: 1/1 up/down: 1015/-1245 (-230) Function old new delta register_leaf_sysctl_tables.constprop - 524 +524 register_sysctl_table 22 497 +475 __pfx_register_leaf_sysctl_tables.constprop - 16 +16 null_path 8 - -8 __pfx_register_sysctl_paths 16 - -16 __pfx_register_leaf_sysctl_tables 16 - -16 __pfx___register_sysctl_paths 16 - -16 __register_sysctl_base 29 12 -17 register_sysctl_paths 18 - -18 register_leaf_sysctl_tables 534 - -534 __register_sysctl_paths 620 - -620 Total: Before=21259666, After=21259436, chg -0.00% Signed-off-by: Luis Chamberlain --- fs/proc/proc_sysctl.c | 55 ++++------------------------------------------- include/linux/sysctl.h | 12 ----------- scripts/check-sysctl-docs | 16 -------------- 3 files changed, 4 insertions(+), 79 deletions(-) (limited to 'include/linux') diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 81dbb175017e..8038833ff5b0 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -1575,25 +1575,18 @@ out: } /** - * __register_sysctl_paths - register a sysctl table hierarchy - * @set: Sysctl tree to register on - * @path: The path to the directory the sysctl table is in. + * register_sysctl_table - register a sysctl table hierarchy * @table: the top-level table structure * * Register a sysctl table hierarchy. @table should be a filled in ctl_table * array. A completely 0 filled entry terminates the table. * We are slowly deprecating this call so avoid its use. - * - * See __register_sysctl_table for more details. */ -struct ctl_table_header *__register_sysctl_paths( - struct ctl_table_set *set, - const struct ctl_path *path, struct ctl_table *table) +struct ctl_table_header *register_sysctl_table(struct ctl_table *table) { struct ctl_table *ctl_table_arg = table; int nr_subheaders = count_subheaders(table); struct ctl_table_header *header = NULL, **subheaders, **subheader; - const struct ctl_path *component; char *new_path, *pos; pos = new_path = kmalloc(PATH_MAX, GFP_KERNEL); @@ -1601,11 +1594,6 @@ struct ctl_table_header *__register_sysctl_paths( return NULL; pos[0] = '\0'; - for (component = path; component->procname; component++) { - pos = append_path(new_path, pos, component->procname); - if (!pos) - goto out; - } while (table->procname && table->child && !table[1].procname) { pos = append_path(new_path, pos, table->procname); if (!pos) @@ -1613,7 +1601,7 @@ struct ctl_table_header *__register_sysctl_paths( table = table->child; } if (nr_subheaders == 1) { - header = __register_sysctl_table(set, new_path, table); + header = __register_sysctl_table(&sysctl_table_root.default_set, new_path, table); if (header) header->ctl_table_arg = ctl_table_arg; } else { @@ -1627,7 +1615,7 @@ struct ctl_table_header *__register_sysctl_paths( header->ctl_table_arg = ctl_table_arg; if (register_leaf_sysctl_tables(new_path, pos, &subheader, - set, table)) + &sysctl_table_root.default_set, table)) goto err_register_leaves; } @@ -1646,41 +1634,6 @@ err_register_leaves: header = NULL; goto out; } - -/** - * register_sysctl_paths - register a sysctl table hierarchy - * @path: The path to the directory the sysctl table is in. - * @table: the top-level table structure - * - * Register a sysctl table hierarchy. @table should be a filled in ctl_table - * array. A completely 0 filled entry terminates the table. - * We are slowly deprecating this caller so avoid future uses of it. - * - * See __register_sysctl_paths for more details. - */ -struct ctl_table_header *register_sysctl_paths(const struct ctl_path *path, - struct ctl_table *table) -{ - return __register_sysctl_paths(&sysctl_table_root.default_set, - path, table); -} -EXPORT_SYMBOL(register_sysctl_paths); - -/** - * register_sysctl_table - register a sysctl table hierarchy - * @table: the top-level table structure - * - * Register a sysctl table hierarchy. @table should be a filled in ctl_table - * array. A completely 0 filled entry terminates the table. - * - * See register_sysctl_paths for more details. - */ -struct ctl_table_header *register_sysctl_table(struct ctl_table *table) -{ - static const struct ctl_path null_path[] = { {} }; - - return register_sysctl_paths(null_path, table); -} EXPORT_SYMBOL(register_sysctl_table); int __register_sysctl_base(struct ctl_table *base_table) diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 780690dc08cd..3d08277959af 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -221,14 +221,8 @@ extern void retire_sysctl_set(struct ctl_table_set *set); struct ctl_table_header *__register_sysctl_table( struct ctl_table_set *set, const char *path, struct ctl_table *table); -struct ctl_table_header *__register_sysctl_paths( - struct ctl_table_set *set, - const struct ctl_path *path, struct ctl_table *table); struct ctl_table_header *register_sysctl(const char *path, struct ctl_table *table); struct ctl_table_header *register_sysctl_table(struct ctl_table * table); -struct ctl_table_header *register_sysctl_paths(const struct ctl_path *path, - struct ctl_table *table); - void unregister_sysctl_table(struct ctl_table_header * table); extern int sysctl_init_bases(void); @@ -277,12 +271,6 @@ static inline struct ctl_table_header *register_sysctl_mount_point(const char *p return NULL; } -static inline struct ctl_table_header *register_sysctl_paths( - const struct ctl_path *path, struct ctl_table *table) -{ - return NULL; -} - static inline struct ctl_table_header *register_sysctl(const char *path, struct ctl_table *table) { return NULL; diff --git a/scripts/check-sysctl-docs b/scripts/check-sysctl-docs index 8bcb9e26c7bc..edc9a629d79e 100755 --- a/scripts/check-sysctl-docs +++ b/scripts/check-sysctl-docs @@ -156,22 +156,6 @@ curtable && /\.procname[\t ]*=[\t ]*".+"/ { } } -/register_sysctl_paths\(.*\)/ { - match($0, /register_sysctl_paths\(([^)]+), ([^)]+)\)/, tables) - if (debug) print "Attaching table " tables[2] " to path " tables[1] - if (paths[tables[1]] == table) { - for (entry in entries[tables[2]]) { - printentry(entry) - } - } - split(paths[tables[1]], components, "/") - if (length(components) > 1 && components[1] == table) { - # Count the first subdirectory as seen - seen[components[2]]++ - } -} - - END { for (entry in documented) { if (!seen[entry]) { -- cgit v1.2.3 From 5d388143fa6c351d985ffd23ea50c91c8839141b Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 3 Apr 2023 09:49:13 +0200 Subject: i2c: gxp: fix build failure without CONFIG_I2C_SLAVE The gxp_i2c_slave_irq_handler() is hidden in an #ifdef, but the caller uses an IS_ENABLED() check: drivers/i2c/busses/i2c-gxp.c: In function 'gxp_i2c_irq_handler': drivers/i2c/busses/i2c-gxp.c:467:29: error: implicit declaration of function 'gxp_i2c_slave_irq_handler'; did you mean 'gxp_i2c_irq_handler'? [-Werror=implicit-function-declaration] It has to consistently use one method or the other to avoid warnings, so move to IS_ENABLED() here for readability and build coverage, and move the #ifdef in linux/i2c.h to allow building it as dead code. Fixes: 4a55ed6f89f5 ("i2c: Add GXP SoC I2C Controller") Signed-off-by: Arnd Bergmann Reviewed-by: Nick Hawkins Signed-off-by: Wolfram Sang --- drivers/i2c/busses/i2c-gxp.c | 2 -- include/linux/i2c.h | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/i2c/busses/i2c-gxp.c b/drivers/i2c/busses/i2c-gxp.c index d4b55d989a26..8ea3fb5e4c7f 100644 --- a/drivers/i2c/busses/i2c-gxp.c +++ b/drivers/i2c/busses/i2c-gxp.c @@ -353,7 +353,6 @@ static void gxp_i2c_chk_data_ack(struct gxp_i2c_drvdata *drvdata) writew(value, drvdata->base + GXP_I2CMCMD); } -#if IS_ENABLED(CONFIG_I2C_SLAVE) static bool gxp_i2c_slave_irq_handler(struct gxp_i2c_drvdata *drvdata) { u8 value; @@ -437,7 +436,6 @@ static bool gxp_i2c_slave_irq_handler(struct gxp_i2c_drvdata *drvdata) return true; } -#endif static irqreturn_t gxp_i2c_irq_handler(int irq, void *_drvdata) { diff --git a/include/linux/i2c.h b/include/linux/i2c.h index 5ba89663ea86..13a1ce38cb0c 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -385,7 +385,6 @@ static inline void i2c_set_clientdata(struct i2c_client *client, void *data) /* I2C slave support */ -#if IS_ENABLED(CONFIG_I2C_SLAVE) enum i2c_slave_event { I2C_SLAVE_READ_REQUESTED, I2C_SLAVE_WRITE_REQUESTED, @@ -396,9 +395,10 @@ enum i2c_slave_event { int i2c_slave_register(struct i2c_client *client, i2c_slave_cb_t slave_cb); int i2c_slave_unregister(struct i2c_client *client); -bool i2c_detect_slave_mode(struct device *dev); int i2c_slave_event(struct i2c_client *client, enum i2c_slave_event event, u8 *val); +#if IS_ENABLED(CONFIG_I2C_SLAVE) +bool i2c_detect_slave_mode(struct device *dev); #else static inline bool i2c_detect_slave_mode(struct device *dev) { return false; } #endif -- cgit v1.2.3 From fd9b8547bc5c34186dc42ea05fb4380d21695374 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 4 May 2023 05:18:55 -0700 Subject: io_uring: Pass whole sqe to commands Currently uring CMD operation relies on having large SQEs, but future operations might want to use normal SQE. The io_uring_cmd currently only saves the payload (cmd) part of the SQE, but, for commands that use normal SQE size, it might be necessary to access the initial SQE fields outside of the payload/cmd block. So, saves the whole SQE other than just the pdu. This changes slightly how the io_uring_cmd works, since the cmd structures and callbacks are not opaque to io_uring anymore. I.e, the callbacks can look at the SQE entries, not only, in the cmd structure. The main advantage is that we don't need to create custom structures for simple commands. Creates io_uring_sqe_cmd() that returns the cmd private data as a null pointer and avoids casting in the callee side. Also, make most of ublk_drv's sqe->cmd priv structure into const, and use io_uring_sqe_cmd() to get the private structure, removing the unwanted cast. (There is one case where the cast is still needed since the header->{len,addr} is updated in the private structure) Suggested-by: Pavel Begunkov Signed-off-by: Breno Leitao Reviewed-by: Keith Busch Reviewed-by: Christoph Hellwig Reviewed-by: Pavel Begunkov Link: https://lore.kernel.org/r/20230504121856.904491-3-leitao@debian.org Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 26 +++++++++++++------------- drivers/nvme/host/ioctl.c | 2 +- include/linux/io_uring.h | 7 ++++++- io_uring/opdef.c | 2 +- io_uring/uring_cmd.c | 9 +++------ 5 files changed, 24 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index c73cc57ec547..42f4d7ca962e 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -1019,7 +1019,7 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) } static void ublk_commit_completion(struct ublk_device *ub, - struct ublksrv_io_cmd *ub_cmd) + const struct ublksrv_io_cmd *ub_cmd) { u32 qid = ub_cmd->q_id, tag = ub_cmd->tag; struct ublk_queue *ubq = ublk_get_queue(ub, qid); @@ -1263,7 +1263,7 @@ static void ublk_handle_need_get_data(struct ublk_device *ub, int q_id, static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) { - struct ublksrv_io_cmd *ub_cmd = (struct ublksrv_io_cmd *)cmd->cmd; + const struct ublksrv_io_cmd *ub_cmd = io_uring_sqe_cmd(cmd->sqe); struct ublk_device *ub = cmd->file->private_data; struct ublk_queue *ubq; struct ublk_io *io; @@ -1567,7 +1567,7 @@ static struct ublk_device *ublk_get_device_from_id(int idx) static int ublk_ctrl_start_dev(struct ublk_device *ub, struct io_uring_cmd *cmd) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); int ublksrv_pid = (int)header->data[0]; struct gendisk *disk; int ret = -EINVAL; @@ -1630,7 +1630,7 @@ out_unlock: static int ublk_ctrl_get_queue_affinity(struct ublk_device *ub, struct io_uring_cmd *cmd) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); void __user *argp = (void __user *)(unsigned long)header->addr; cpumask_var_t cpumask; unsigned long queue; @@ -1681,7 +1681,7 @@ static inline void ublk_dump_dev_info(struct ublksrv_ctrl_dev_info *info) static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); void __user *argp = (void __user *)(unsigned long)header->addr; struct ublksrv_ctrl_dev_info info; struct ublk_device *ub; @@ -1844,7 +1844,7 @@ static int ublk_ctrl_del_dev(struct ublk_device **p_ub) static inline void ublk_ctrl_cmd_dump(struct io_uring_cmd *cmd) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n", __func__, cmd->cmd_op, header->dev_id, header->queue_id, @@ -1863,7 +1863,7 @@ static int ublk_ctrl_stop_dev(struct ublk_device *ub) static int ublk_ctrl_get_dev_info(struct ublk_device *ub, struct io_uring_cmd *cmd) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); void __user *argp = (void __user *)(unsigned long)header->addr; if (header->len < sizeof(struct ublksrv_ctrl_dev_info) || !header->addr) @@ -1894,7 +1894,7 @@ static void ublk_ctrl_fill_params_devt(struct ublk_device *ub) static int ublk_ctrl_get_params(struct ublk_device *ub, struct io_uring_cmd *cmd) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); void __user *argp = (void __user *)(unsigned long)header->addr; struct ublk_params_header ph; int ret; @@ -1925,7 +1925,7 @@ static int ublk_ctrl_get_params(struct ublk_device *ub, static int ublk_ctrl_set_params(struct ublk_device *ub, struct io_uring_cmd *cmd) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); void __user *argp = (void __user *)(unsigned long)header->addr; struct ublk_params_header ph; int ret = -EFAULT; @@ -1983,7 +1983,7 @@ static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq) static int ublk_ctrl_start_recovery(struct ublk_device *ub, struct io_uring_cmd *cmd) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); int ret = -EINVAL; int i; @@ -2025,7 +2025,7 @@ static int ublk_ctrl_start_recovery(struct ublk_device *ub, static int ublk_ctrl_end_recovery(struct ublk_device *ub, struct io_uring_cmd *cmd) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); int ublksrv_pid = (int)header->data[0]; int ret = -EINVAL; @@ -2092,7 +2092,7 @@ exit: static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, struct io_uring_cmd *cmd) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)io_uring_sqe_cmd(cmd->sqe); bool unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV; void __user *argp = (void __user *)(unsigned long)header->addr; char *dev_path = NULL; @@ -2171,7 +2171,7 @@ exit: static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) { - struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe); struct ublk_device *ub = NULL; int ret = -EINVAL; diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index d24ea2e05156..81c5c9e38477 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -552,7 +552,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, struct io_uring_cmd *ioucmd, unsigned int issue_flags, bool vec) { struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); - const struct nvme_uring_cmd *cmd = ioucmd->cmd; + const struct nvme_uring_cmd *cmd = io_uring_sqe_cmd(ioucmd->sqe); struct request_queue *q = ns ? ns->queue : ctrl->admin_q; struct nvme_uring_data d; struct nvme_command c; diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 35b9328ca335..3399d979ee1c 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -24,7 +24,7 @@ enum io_uring_cmd_flags { struct io_uring_cmd { struct file *file; - const void *cmd; + const struct io_uring_sqe *sqe; union { /* callback to defer completions to task context */ void (*task_work_cb)(struct io_uring_cmd *cmd, unsigned); @@ -66,6 +66,11 @@ static inline void io_uring_free(struct task_struct *tsk) if (tsk->io_uring) __io_uring_free(tsk); } + +static inline const void *io_uring_sqe_cmd(const struct io_uring_sqe *sqe) +{ + return sqe->cmd; +} #else static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, struct iov_iter *iter, void *ioucmd) diff --git a/io_uring/opdef.c b/io_uring/opdef.c index cca7c5b55208..3b9c6489b8b6 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -627,7 +627,7 @@ const struct io_cold_def io_cold_defs[] = { }, [IORING_OP_URING_CMD] = { .name = "URING_CMD", - .async_size = uring_cmd_pdu_size(1), + .async_size = 2 * sizeof(struct io_uring_sqe), .prep_async = io_uring_cmd_prep_async, }, [IORING_OP_SEND_ZC] = { diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 5113c9a48583..ed536d7499db 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -69,15 +69,12 @@ EXPORT_SYMBOL_GPL(io_uring_cmd_done); int io_uring_cmd_prep_async(struct io_kiocb *req) { struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); - size_t cmd_size; BUILD_BUG_ON(uring_cmd_pdu_size(0) != 16); BUILD_BUG_ON(uring_cmd_pdu_size(1) != 80); - cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128); - - memcpy(req->async_data, ioucmd->cmd, cmd_size); - ioucmd->cmd = req->async_data; + memcpy(req->async_data, ioucmd->sqe, uring_sqe_size(req->ctx)); + ioucmd->sqe = req->async_data; return 0; } @@ -103,7 +100,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) req->imu = ctx->user_bufs[index]; io_req_set_rsrc_node(req, ctx, 0); } - ioucmd->cmd = sqe->cmd; + ioucmd->sqe = sqe; ioucmd->cmd_op = READ_ONCE(sqe->cmd_op); return 0; } -- cgit v1.2.3 From 6ce2c04fcbcaa5eb086e5142ab359be306cbc3e1 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 2 May 2023 21:32:33 -0400 Subject: ftrace: Add MODIFIED flag to show if IPMODIFY or direct was attached If a function had ever had IPMODIFY or DIRECT attached to it, where this is how live kernel patching and BPF overrides work, mark them and display an "M" in the enabled_functions and touched_functions files. This can be used for debugging. If a function had been modified and later there's a bug in the code related to that function, this can be used to know if the cause is possibly from a live kernel patch or a BPF program that changed the behavior of the code. Also update the documentation on the enabled_functions and touched_functions output, as it was missing direct callers and CALL_OPS. And include this new modify attribute. Link: https://lore.kernel.org/linux-trace-kernel/20230502213233.004e3ae4@gandalf.local.home Cc: Mark Rutland Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- Documentation/trace/ftrace.rst | 25 +++++++++++++++++++++++++ include/linux/ftrace.h | 4 +++- kernel/trace/ftrace.c | 12 +++++++++--- 3 files changed, 37 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst index aaebb821912e..d5766229c71a 100644 --- a/Documentation/trace/ftrace.rst +++ b/Documentation/trace/ftrace.rst @@ -350,6 +350,19 @@ of ftrace. Here is a list of some of the key files: an 'I' will be displayed on the same line as the function that can be overridden. + If a non ftrace trampoline is attached (BPF) a 'D' will be displayed. + Note, normal ftrace trampolines can also be attached, but only one + "direct" trampoline can be attached to a given function at a time. + + Some architectures can not call direct trampolines, but instead have + the ftrace ops function located above the function entry point. In + such cases an 'O' will be displayed. + + If a function had either the "ip modify" or a "direct" call attached to + it in the past, a 'M' will be shown. This flag is never cleared. It is + used to know if a function was every modified by the ftrace infrastructure, + and can be used for debugging. + If the architecture supports it, it will also show what callback is being directly called by the function. If the count is greater than 1 it most likely will be ftrace_ops_list_func(). @@ -359,6 +372,18 @@ of ftrace. Here is a list of some of the key files: its address will be printed as well as the function that the trampoline calls. + touched_functions: + + This file contains all the functions that ever had a function callback + to it via the ftrace infrastructure. It has the same format as + enabled_functions but shows all functions that have every been + traced. + + To see any function that has every been modified by "ip modify" or a + direct trampoline, one can perform the following command: + + grep ' M ' /sys/kernel/tracing/touched_functions + function_profile_enabled: When set it will enable all functions with either the function diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 327046f1278d..7dffd740e784 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -549,6 +549,7 @@ bool is_ftrace_trampoline(unsigned long addr); * CALL_OPS - the record can use callsite-specific ops * CALL_OPS_EN - the function is set up to use callsite-specific ops * TOUCHED - A callback was added since boot up + * MODIFIED - The function had IPMODIFY or DIRECT attached to it * * When a new ftrace_ops is registered and wants a function to save * pt_regs, the rec->flags REGS is set. When the function has been @@ -569,9 +570,10 @@ enum { FTRACE_FL_CALL_OPS = (1UL << 22), FTRACE_FL_CALL_OPS_EN = (1UL << 21), FTRACE_FL_TOUCHED = (1UL << 20), + FTRACE_FL_MODIFIED = (1UL << 19), }; -#define FTRACE_REF_MAX_SHIFT 20 +#define FTRACE_REF_MAX_SHIFT 19 #define FTRACE_REF_MAX ((1UL << FTRACE_REF_MAX_SHIFT) - 1) #define ftrace_rec_count(rec) ((rec)->flags & FTRACE_REF_MAX) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index db8532a4d5c8..885845fc851d 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -46,7 +46,8 @@ #include "trace_stat.h" /* Flags that do not get reset */ -#define FTRACE_NOCLEAR_FLAGS (FTRACE_FL_DISABLED | FTRACE_FL_TOUCHED) +#define FTRACE_NOCLEAR_FLAGS (FTRACE_FL_DISABLED | FTRACE_FL_TOUCHED | \ + FTRACE_FL_MODIFIED) #define FTRACE_INVALID_FUNCTION "__ftrace_invalid_address__" @@ -2273,6 +2274,10 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) rec->flags &= ~FTRACE_FL_TRAMP_EN; } + /* Keep track of anything that modifies the function */ + if (rec->flags & (FTRACE_FL_DIRECT | FTRACE_FL_IPMODIFY)) + rec->flags |= FTRACE_FL_MODIFIED; + if (flag & FTRACE_FL_DIRECT) { /* * If there's only one user (direct_ops helper) @@ -3866,12 +3871,13 @@ static int t_show(struct seq_file *m, void *v) if (iter->flags & (FTRACE_ITER_ENABLED | FTRACE_ITER_TOUCHED)) { struct ftrace_ops *ops; - seq_printf(m, " (%ld)%s%s%s%s", + seq_printf(m, " (%ld)%s%s%s%s%s", ftrace_rec_count(rec), rec->flags & FTRACE_FL_REGS ? " R" : " ", rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ", rec->flags & FTRACE_FL_DIRECT ? " D" : " ", - rec->flags & FTRACE_FL_CALL_OPS ? " O" : " "); + rec->flags & FTRACE_FL_CALL_OPS ? " O" : " ", + rec->flags & FTRACE_FL_MODIFIED ? " M " : " "); if (rec->flags & FTRACE_FL_TRAMP_EN) { ops = ftrace_find_tramp_ops_any(rec); if (ops) { -- cgit v1.2.3 From 23a5b8bb022c1e071ca91b1a9c10f0ad6a0966e9 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 27 Apr 2023 00:33:36 -0500 Subject: x86/amd_nb: Add PCI ID for family 19h model 78h Commit 310e782a99c7 ("platform/x86/amd: pmc: Utilize SMN index 0 for driver probe") switched to using amd_smn_read() which relies upon the misc PCI ID used by DF function 3 being included in a table. The ID for model 78h is missing in that table, so amd_smn_read() doesn't work. Add the missing ID into amd_nb, restoring s2idle on this system. [ bp: Simplify commit message. ] Fixes: 310e782a99c7 ("platform/x86/amd: pmc: Utilize SMN index 0 for driver probe") Signed-off-by: Mario Limonciello Signed-off-by: Borislav Petkov (AMD) Acked-by: Bjorn Helgaas # pci_ids.h Acked-by: Guenter Roeck Link: https://lore.kernel.org/r/20230427053338.16653-2-mario.limonciello@amd.com --- arch/x86/kernel/amd_nb.c | 2 ++ include/linux/pci_ids.h | 1 + 2 files changed, 3 insertions(+) (limited to 'include/linux') diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index 4266b64631a4..7e331e8f3692 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -36,6 +36,7 @@ #define PCI_DEVICE_ID_AMD_19H_M50H_DF_F4 0x166e #define PCI_DEVICE_ID_AMD_19H_M60H_DF_F4 0x14e4 #define PCI_DEVICE_ID_AMD_19H_M70H_DF_F4 0x14f4 +#define PCI_DEVICE_ID_AMD_19H_M78H_DF_F4 0x12fc /* Protect the PCI config register pairs used for SMN. */ static DEFINE_MUTEX(smn_mutex); @@ -79,6 +80,7 @@ static const struct pci_device_id amd_nb_misc_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M50H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M60H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M70H_DF_F3) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_M78H_DF_F3) }, {} }; diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 45c3d62e616d..95f33dadb2be 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -567,6 +567,7 @@ #define PCI_DEVICE_ID_AMD_19H_M50H_DF_F3 0x166d #define PCI_DEVICE_ID_AMD_19H_M60H_DF_F3 0x14e3 #define PCI_DEVICE_ID_AMD_19H_M70H_DF_F3 0x14f3 +#define PCI_DEVICE_ID_AMD_19H_M78H_DF_F3 0x12fb #define PCI_DEVICE_ID_AMD_CNB17H_F3 0x1703 #define PCI_DEVICE_ID_AMD_LANCE 0x2000 #define PCI_DEVICE_ID_AMD_LANCE_HOME 0x2001 -- cgit v1.2.3 From c00bc80462afc7963f449d7f21d896d2f629cacc Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 15 Apr 2023 20:23:34 +0200 Subject: power: supply: bq27xxx: Fix poll_interval handling and races on remove Before this patch bq27xxx_battery_teardown() was setting poll_interval = 0 to avoid bq27xxx_battery_update() requeuing the delayed_work item. There are 2 problems with this: 1. If the driver is unbound through sysfs, rather then the module being rmmod-ed, this changes poll_interval unexpectedly 2. This is racy, after it being set poll_interval could be changed before bq27xxx_battery_update() checks it through /sys/module/bq27xxx_battery/parameters/poll_interval Fix this by added a removed attribute to struct bq27xxx_device_info and using that instead of setting poll_interval to 0. There also is another poll_interval related race on remove(), writing /sys/module/bq27xxx_battery/parameters/poll_interval will requeue the delayed_work item for all devices on the bq27xxx_battery_devices list and the device being removed was only removed from that list after cancelling the delayed_work item. Fix this by moving the removal from the bq27xxx_battery_devices list to before cancelling the delayed_work item. Fixes: 8cfaaa811894 ("bq27x00_battery: Fix OOPS caused by unregistring bq27x00 driver") Signed-off-by: Hans de Goede Signed-off-by: Sebastian Reichel --- drivers/power/supply/bq27xxx_battery.c | 22 +++++++++------------- include/linux/power/bq27xxx_battery.h | 1 + 2 files changed, 10 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 79aedf83cc8d..7411f1cf741b 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1801,7 +1801,7 @@ static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di) di->last_update = jiffies; - if (poll_interval > 0) + if (!di->removed && poll_interval > 0) mod_delayed_work(system_wq, &di->work, poll_interval * HZ); } @@ -2132,22 +2132,18 @@ EXPORT_SYMBOL_GPL(bq27xxx_battery_setup); void bq27xxx_battery_teardown(struct bq27xxx_device_info *di) { - /* - * power_supply_unregister call bq27xxx_battery_get_property which - * call bq27xxx_battery_poll. - * Make sure that bq27xxx_battery_poll will not call - * schedule_delayed_work again after unregister (which cause OOPS). - */ - poll_interval = 0; - - cancel_delayed_work_sync(&di->work); - - power_supply_unregister(di->bat); - mutex_lock(&bq27xxx_list_lock); list_del(&di->list); mutex_unlock(&bq27xxx_list_lock); + /* Set removed to avoid bq27xxx_battery_update() re-queuing the work */ + mutex_lock(&di->lock); + di->removed = true; + mutex_unlock(&di->lock); + + cancel_delayed_work_sync(&di->work); + + power_supply_unregister(di->bat); mutex_destroy(&di->lock); } EXPORT_SYMBOL_GPL(bq27xxx_battery_teardown); diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index a1aa68141d0b..e3322dad9c85 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -68,6 +68,7 @@ struct bq27xxx_device_info { struct bq27xxx_access_methods bus; struct bq27xxx_reg_cache cache; int charge_design_full; + bool removed; unsigned long last_update; struct delayed_work work; struct power_supply *bat; -- cgit v1.2.3 From 939a116142012926e25de0ea6b7e2f8d86a5f1b6 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 15 Apr 2023 20:23:37 +0200 Subject: power: supply: bq27xxx: Ensure power_supply_changed() is called on current sign changes On gauges where the current register is signed, there is no charging flag in the flags register. So only checking flags will not result in power_supply_changed() getting called when e.g. a charger is plugged in and the current sign changes from negative (discharging) to positive (charging). This causes userspace's notion of the status to lag until userspace does a poll. And when a power_supply_leds.c LED trigger is used to indicate charging status with a LED, this LED will lag until the capacity percentage changes, which may take many minutes (because the LED trigger only is updated on power_supply_changed() calls). Fix this by calling bq27xxx_battery_current_and_status() on gauges with a signed current register and checking if the status has changed. Fixes: 297a533b3e62 ("bq27x00: Cache battery registers") Signed-off-by: Hans de Goede Signed-off-by: Sebastian Reichel --- drivers/power/supply/bq27xxx_battery.c | 13 ++++++++++++- include/linux/power/bq27xxx_battery.h | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 8f2995e9850a..f98b51ce19b5 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1836,6 +1836,7 @@ static int bq27xxx_battery_current_and_status( static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di) { + union power_supply_propval status = di->last_status; struct bq27xxx_reg_cache cache = {0, }; bool has_singe_flag = di->opts & BQ27XXX_O_ZERO; @@ -1860,14 +1861,24 @@ static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di) if (di->regs[BQ27XXX_REG_CYCT] != INVALID_REG_ADDR) cache.cycle_count = bq27xxx_battery_read_cyct(di); + /* + * On gauges with signed current reporting the current must be + * checked to detect charging <-> discharging status changes. + */ + if (!(di->opts & BQ27XXX_O_ZERO)) + bq27xxx_battery_current_and_status(di, NULL, &status, &cache); + /* We only have to read charge design full once */ if (di->charge_design_full <= 0) di->charge_design_full = bq27xxx_battery_read_dcap(di); } if ((di->cache.capacity != cache.capacity) || - (di->cache.flags != cache.flags)) + (di->cache.flags != cache.flags) || + (di->last_status.intval != status.intval)) { + di->last_status.intval = status.intval; power_supply_changed(di->bat); + } if (memcmp(&di->cache, &cache, sizeof(cache)) != 0) di->cache = cache; diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index e3322dad9c85..7c8d65414a70 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -2,6 +2,8 @@ #ifndef __LINUX_BQ27X00_BATTERY_H__ #define __LINUX_BQ27X00_BATTERY_H__ +#include + enum bq27xxx_chip { BQ27000 = 1, /* bq27000, bq27200 */ BQ27010, /* bq27010, bq27210 */ @@ -70,6 +72,7 @@ struct bq27xxx_device_info { int charge_design_full; bool removed; unsigned long last_update; + union power_supply_propval last_status; struct delayed_work work; struct power_supply *bat; struct list_head list; -- cgit v1.2.3 From 19b8766459c41c6f318f8a548cc1c66dffd18363 Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Thu, 20 Apr 2023 16:06:03 +0100 Subject: firmware: arm_ffa: Fix FFA device names for logical partitions Each physical partition can provide multiple services each with UUID. Each such service can be presented as logical partition with a unique combination of VM ID and UUID. The number of distinct UUID in a system will be less than or equal to the number of logical partitions. However, currently it fails to register more than one logical partition or service within a physical partition as the device name contains only VM ID while both VM ID and UUID are maintained in the partition information. The kernel complains with the below message: | sysfs: cannot create duplicate filename '/devices/arm-ffa-8001' | CPU: 1 PID: 1 Comm: swapper/0 Not tainted 6.3.0-rc7 #8 | Hardware name: FVP Base RevC (DT) | Call trace: | dump_backtrace+0xf8/0x118 | show_stack+0x18/0x24 | dump_stack_lvl+0x50/0x68 | dump_stack+0x18/0x24 | sysfs_create_dir_ns+0xe0/0x13c | kobject_add_internal+0x220/0x3d4 | kobject_add+0x94/0x100 | device_add+0x144/0x5d8 | device_register+0x20/0x30 | ffa_device_register+0x88/0xd8 | ffa_setup_partitions+0x108/0x1b8 | ffa_init+0x2ec/0x3a4 | do_one_initcall+0xcc/0x240 | do_initcall_level+0x8c/0xac | do_initcalls+0x54/0x94 | do_basic_setup+0x1c/0x28 | kernel_init_freeable+0x100/0x16c | kernel_init+0x20/0x1a0 | ret_from_fork+0x10/0x20 | kobject_add_internal failed for arm-ffa-8001 with -EEXIST, don't try to | register things with the same name in the same directory. | arm_ffa arm-ffa: unable to register device arm-ffa-8001 err=-17 | ARM FF-A: ffa_setup_partitions: failed to register partition ID 0x8001 By virtue of being random enough to avoid collisions when generated in a distributed system, there is no way to compress UUID keys to the number of bits required to identify each. We can eliminate '-' in the name but it is not worth eliminating 4 bytes and add unnecessary logic for doing that. Also v1.0 doesn't provide the UUID of the partitions which makes it hard to use the same for the device name. So to keep it simple, let us alloc an ID using ida_alloc() and append the same to "arm-ffa" to make up a unique device name. Also stash the id value in ffa_dev to help freeing the ID later when the device is destroyed. Fixes: e781858488b9 ("firmware: arm_ffa: Add initial FFA bus support for device enumeration") Reported-by: Lucian Paul-Trifu Link: https://lore.kernel.org/r/20230419-ffa_fixes_6-4-v2-3-d9108e43a176@arm.com Signed-off-by: Sudeep Holla --- drivers/firmware/arm_ffa/bus.c | 16 +++++++++++++--- include/linux/arm_ffa.h | 1 + 2 files changed, 14 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/arm_ffa/bus.c b/drivers/firmware/arm_ffa/bus.c index 36bd5423c2f0..2b8bfcd010f5 100644 --- a/drivers/firmware/arm_ffa/bus.c +++ b/drivers/firmware/arm_ffa/bus.c @@ -15,6 +15,8 @@ #include "common.h" +static DEFINE_IDA(ffa_bus_id); + static int ffa_device_match(struct device *dev, struct device_driver *drv) { const struct ffa_device_id *id_table; @@ -131,6 +133,7 @@ static void ffa_release_device(struct device *dev) { struct ffa_device *ffa_dev = to_ffa_dev(dev); + ida_free(&ffa_bus_id, ffa_dev->id); kfree(ffa_dev); } @@ -171,18 +174,24 @@ bool ffa_device_is_valid(struct ffa_device *ffa_dev) struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id, const struct ffa_ops *ops) { - int ret; + int id, ret; struct device *dev; struct ffa_device *ffa_dev; + id = ida_alloc_min(&ffa_bus_id, 1, GFP_KERNEL); + if (id < 0) + return NULL; + ffa_dev = kzalloc(sizeof(*ffa_dev), GFP_KERNEL); - if (!ffa_dev) + if (!ffa_dev) { + ida_free(&ffa_bus_id, id); return NULL; + } dev = &ffa_dev->dev; dev->bus = &ffa_bus_type; dev->release = ffa_release_device; - dev_set_name(&ffa_dev->dev, "arm-ffa-%04x", vm_id); + dev_set_name(&ffa_dev->dev, "arm-ffa-%d", id); ffa_dev->vm_id = vm_id; ffa_dev->ops = ops; @@ -218,4 +227,5 @@ void arm_ffa_bus_exit(void) { ffa_devices_unregister(); bus_unregister(&ffa_bus_type); + ida_destroy(&ffa_bus_id); } diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h index c87aeecaa9b2..583fe3b49a49 100644 --- a/include/linux/arm_ffa.h +++ b/include/linux/arm_ffa.h @@ -96,6 +96,7 @@ /* FFA Bus/Device/Driver related */ struct ffa_device { + u32 id; int vm_id; bool mode_32bit; uuid_t uuid; -- cgit v1.2.3 From 162bd18eb55adf464a0fa2b4144b8d61c75ff7c2 Mon Sep 17 00:00:00 2001 From: Roy Novich Date: Sun, 7 May 2023 16:57:43 +0300 Subject: linux/dim: Do nothing if no time delta between samples Add return value for dim_calc_stats. This is an indication for the caller if curr_stats was assigned by the function. Avoid using curr_stats uninitialized over {rdma/net}_dim, when no time delta between samples. Coverity reported this potential use of an uninitialized variable. Fixes: 4c4dbb4a7363 ("net/mlx5e: Move dynamic interrupt coalescing code to include/linux") Fixes: cb3c7fd4f839 ("net/mlx5e: Support adaptive RX coalescing") Signed-off-by: Roy Novich Reviewed-by: Aya Levin Reviewed-by: Saeed Mahameed Signed-off-by: Tariq Toukan Reviewed-by: Leon Romanovsky Reviewed-by: Michal Kubiak Link: https://lore.kernel.org/r/20230507135743.138993-1-tariqt@nvidia.com Signed-off-by: Paolo Abeni --- include/linux/dim.h | 3 ++- lib/dim/dim.c | 5 +++-- lib/dim/net_dim.c | 3 ++- lib/dim/rdma_dim.c | 3 ++- 4 files changed, 9 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dim.h b/include/linux/dim.h index 6c5733981563..f343bc9aa2ec 100644 --- a/include/linux/dim.h +++ b/include/linux/dim.h @@ -236,8 +236,9 @@ void dim_park_tired(struct dim *dim); * * Calculate the delta between two samples (in data rates). * Takes into consideration counter wrap-around. + * Returned boolean indicates whether curr_stats are reliable. */ -void dim_calc_stats(struct dim_sample *start, struct dim_sample *end, +bool dim_calc_stats(struct dim_sample *start, struct dim_sample *end, struct dim_stats *curr_stats); /** diff --git a/lib/dim/dim.c b/lib/dim/dim.c index 38045d6d0538..e89aaf07bde5 100644 --- a/lib/dim/dim.c +++ b/lib/dim/dim.c @@ -54,7 +54,7 @@ void dim_park_tired(struct dim *dim) } EXPORT_SYMBOL(dim_park_tired); -void dim_calc_stats(struct dim_sample *start, struct dim_sample *end, +bool dim_calc_stats(struct dim_sample *start, struct dim_sample *end, struct dim_stats *curr_stats) { /* u32 holds up to 71 minutes, should be enough */ @@ -66,7 +66,7 @@ void dim_calc_stats(struct dim_sample *start, struct dim_sample *end, start->comp_ctr); if (!delta_us) - return; + return false; curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us); curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us); @@ -79,5 +79,6 @@ void dim_calc_stats(struct dim_sample *start, struct dim_sample *end, else curr_stats->cpe_ratio = 0; + return true; } EXPORT_SYMBOL(dim_calc_stats); diff --git a/lib/dim/net_dim.c b/lib/dim/net_dim.c index 53f6b9c6e936..4e32f7aaac86 100644 --- a/lib/dim/net_dim.c +++ b/lib/dim/net_dim.c @@ -227,7 +227,8 @@ void net_dim(struct dim *dim, struct dim_sample end_sample) dim->start_sample.event_ctr); if (nevents < DIM_NEVENTS) break; - dim_calc_stats(&dim->start_sample, &end_sample, &curr_stats); + if (!dim_calc_stats(&dim->start_sample, &end_sample, &curr_stats)) + break; if (net_dim_decision(&curr_stats, dim)) { dim->state = DIM_APPLY_NEW_PROFILE; schedule_work(&dim->work); diff --git a/lib/dim/rdma_dim.c b/lib/dim/rdma_dim.c index 15462d54758d..88f779486707 100644 --- a/lib/dim/rdma_dim.c +++ b/lib/dim/rdma_dim.c @@ -88,7 +88,8 @@ void rdma_dim(struct dim *dim, u64 completions) nevents = curr_sample->event_ctr - dim->start_sample.event_ctr; if (nevents < DIM_NEVENTS) break; - dim_calc_stats(&dim->start_sample, curr_sample, &curr_stats); + if (!dim_calc_stats(&dim->start_sample, curr_sample, &curr_stats)) + break; if (rdma_dim_decision(&curr_stats, dim)) { dim->state = DIM_APPLY_NEW_PROFILE; schedule_work(&dim->work); -- cgit v1.2.3 From 293007b033418c8c9d1b35d68dec49a500750fde Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 8 May 2023 12:13:27 -0600 Subject: io_uring: make io_uring_sqe_cmd() unconditionally available If CONFIG_IO_URING isn't set, then io_uring_sqe_cmd() is not defined. As the nvme driver uses this helper, it causes a compilation issue: drivers/nvme/host/ioctl.c: In function 'nvme_uring_cmd_io': drivers/nvme/host/ioctl.c:555:44: error: implicit declaration of function 'io_uring_sqe_cmd'; did you mean 'io_uring_free'? [-Werror=implicit-function-declaration] 555 | const struct nvme_uring_cmd *cmd = io_uring_sqe_cmd(ioucmd->sqe); | ^~~~~~~~~~~~~~~~ | io_uring_free Fix it by just making io_uring_sqe_cmd() generally available - the types are known, and there's no reason to hide it under CONFIG_IO_URING. Fixes: fd9b8547bc5c ("io_uring: Pass whole sqe to commands") Reported-by: kernel test robot Reported-by: Chen-Yu Tsai Tested-by: Chen-Yu Tsai Signed-off-by: Jens Axboe --- include/linux/io_uring.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 3399d979ee1c..7fe31b2cd02f 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -36,6 +36,11 @@ struct io_uring_cmd { u8 pdu[32]; /* available inline for free use */ }; +static inline const void *io_uring_sqe_cmd(const struct io_uring_sqe *sqe) +{ + return sqe->cmd; +} + #if defined(CONFIG_IO_URING) int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, struct iov_iter *iter, void *ioucmd); @@ -66,11 +71,6 @@ static inline void io_uring_free(struct task_struct *tsk) if (tsk->io_uring) __io_uring_free(tsk); } - -static inline const void *io_uring_sqe_cmd(const struct io_uring_sqe *sqe) -{ - return sqe->cmd; -} #else static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, struct iov_iter *iter, void *ioucmd) -- cgit v1.2.3 From 4063384ef762cc5946fc7a3f89879e76c6ec51e2 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 9 May 2023 13:18:57 +0000 Subject: net: add vlan_get_protocol_and_depth() helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before blamed commit, pskb_may_pull() was used instead of skb_header_pointer() in __vlan_get_protocol() and friends. Few callers depended on skb->head being populated with MAC header, syzbot caught one of them (skb_mac_gso_segment()) Add vlan_get_protocol_and_depth() to make the intent clearer and use it where sensible. This is a more generic fix than commit e9d3f80935b6 ("net/af_packet: make sure to pull mac header") which was dealing with a similar issue. kernel BUG at include/linux/skbuff.h:2655 ! invalid opcode: 0000 [#1] SMP KASAN CPU: 0 PID: 1441 Comm: syz-executor199 Not tainted 6.1.24-syzkaller #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 04/14/2023 RIP: 0010:__skb_pull include/linux/skbuff.h:2655 [inline] RIP: 0010:skb_mac_gso_segment+0x68f/0x6a0 net/core/gro.c:136 Code: fd 48 8b 5c 24 10 44 89 6b 70 48 c7 c7 c0 ae 0d 86 44 89 e6 e8 a1 91 d0 00 48 c7 c7 00 af 0d 86 48 89 de 31 d2 e8 d1 4a e9 ff <0f> 0b 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 41 RSP: 0018:ffffc90001bd7520 EFLAGS: 00010286 RAX: ffffffff8469736a RBX: ffff88810f31dac0 RCX: ffff888115a18b00 RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000 RBP: ffffc90001bd75e8 R08: ffffffff84697183 R09: fffff5200037adf9 R10: 0000000000000000 R11: dffffc0000000001 R12: 0000000000000012 R13: 000000000000fee5 R14: 0000000000005865 R15: 000000000000fed7 FS: 000055555633f300(0000) GS:ffff8881f6a00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000020000000 CR3: 0000000116fea000 CR4: 00000000003506f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: [] __skb_gso_segment+0x32d/0x4c0 net/core/dev.c:3419 [] skb_gso_segment include/linux/netdevice.h:4819 [inline] [] validate_xmit_skb+0x3aa/0xee0 net/core/dev.c:3725 [] __dev_queue_xmit+0x1332/0x3300 net/core/dev.c:4313 [] dev_queue_xmit+0x17/0x20 include/linux/netdevice.h:3029 [] packet_snd net/packet/af_packet.c:3111 [inline] [] packet_sendmsg+0x49d2/0x6470 net/packet/af_packet.c:3142 [] sock_sendmsg_nosec net/socket.c:716 [inline] [] sock_sendmsg net/socket.c:736 [inline] [] __sys_sendto+0x472/0x5f0 net/socket.c:2139 [] __do_sys_sendto net/socket.c:2151 [inline] [] __se_sys_sendto net/socket.c:2147 [inline] [] __x64_sys_sendto+0xe5/0x100 net/socket.c:2147 [] do_syscall_x64 arch/x86/entry/common.c:50 [inline] [] do_syscall_64+0x2f/0x50 arch/x86/entry/common.c:80 [] entry_SYSCALL_64_after_hwframe+0x63/0xcd Fixes: 469aceddfa3e ("vlan: consolidate VLAN parsing code and limit max parsing depth") Reported-by: syzbot Signed-off-by: Eric Dumazet Cc: Toke Høiland-Jørgensen Cc: Willem de Bruijn Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- drivers/net/tap.c | 4 ++-- include/linux/if_vlan.h | 17 +++++++++++++++++ net/bridge/br_forward.c | 2 +- net/core/dev.c | 2 +- net/packet/af_packet.c | 6 ++---- 5 files changed, 23 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/tap.c b/drivers/net/tap.c index ce993cc75bf3..d30d730ed5a7 100644 --- a/drivers/net/tap.c +++ b/drivers/net/tap.c @@ -742,7 +742,7 @@ static ssize_t tap_get_user(struct tap_queue *q, void *msg_control, /* Move network header to the right position for VLAN tagged packets */ if (eth_type_vlan(skb->protocol) && - __vlan_get_protocol(skb, skb->protocol, &depth) != 0) + vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0) skb_set_network_header(skb, depth); /* copy skb_ubuf_info for callback when skb has no error */ @@ -1197,7 +1197,7 @@ static int tap_get_user_xdp(struct tap_queue *q, struct xdp_buff *xdp) /* Move network header to the right position for VLAN tagged packets */ if (eth_type_vlan(skb->protocol) && - __vlan_get_protocol(skb, skb->protocol, &depth) != 0) + vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0) skb_set_network_header(skb, depth); rcu_read_lock(); diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 0f40f379d75c..6ba71957851e 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -637,6 +637,23 @@ static inline __be16 vlan_get_protocol(const struct sk_buff *skb) return __vlan_get_protocol(skb, skb->protocol, NULL); } +/* This version of __vlan_get_protocol() also pulls mac header in skb->head */ +static inline __be16 vlan_get_protocol_and_depth(struct sk_buff *skb, + __be16 type, int *depth) +{ + int maclen; + + type = __vlan_get_protocol(skb, type, &maclen); + + if (type) { + if (!pskb_may_pull(skb, maclen)) + type = 0; + else if (depth) + *depth = maclen; + } + return type; +} + /* A getter for the SKB protocol field which will handle VLAN tags consistently * whether VLAN acceleration is enabled or not. */ diff --git a/net/bridge/br_forward.c b/net/bridge/br_forward.c index 57744704ff69..84d6dd5e5b1a 100644 --- a/net/bridge/br_forward.c +++ b/net/bridge/br_forward.c @@ -42,7 +42,7 @@ int br_dev_queue_push_xmit(struct net *net, struct sock *sk, struct sk_buff *skb eth_type_vlan(skb->protocol)) { int depth; - if (!__vlan_get_protocol(skb, skb->protocol, &depth)) + if (!vlan_get_protocol_and_depth(skb, skb->protocol, &depth)) goto drop; skb_set_network_header(skb, depth); diff --git a/net/core/dev.c b/net/core/dev.c index 735096d42c1d..b3c13e041935 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3335,7 +3335,7 @@ __be16 skb_network_protocol(struct sk_buff *skb, int *depth) type = eth->h_proto; } - return __vlan_get_protocol(skb, type, depth); + return vlan_get_protocol_and_depth(skb, type, depth); } /* openvswitch calls this on rx path, so we need a different check. diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 640d94e34635..94c6a1ffa459 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1934,10 +1934,8 @@ static void packet_parse_headers(struct sk_buff *skb, struct socket *sock) /* Move network header to the right position for VLAN tagged packets */ if (likely(skb->dev->type == ARPHRD_ETHER) && eth_type_vlan(skb->protocol) && - __vlan_get_protocol(skb, skb->protocol, &depth) != 0) { - if (pskb_may_pull(skb, depth)) - skb_set_network_header(skb, depth); - } + vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0) + skb_set_network_header(skb, depth); skb_probe_transport_header(skb); } -- cgit v1.2.3 From 8432a15cd810c99db464b7e7858d559f3e1920e3 Mon Sep 17 00:00:00 2001 From: Jó Ágila Bitsch Date: Sat, 22 Apr 2023 18:05:32 +0200 Subject: usb: gadget: drop superfluous ':' in doc string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There was one superfluous ':' that kernel-doc complained about. Reported-by: Randy Dunlap Closes: https://lore.kernel.org/all/c718a490-028d-2682-9ad7-8256d16504bf@infradead.org/ Fixes: fb6211f1584a ("usb: gadget: add doc to struct usb_composite_dev") Signed-off-by: Jó Ágila Bitsch Reviewed-by: Randy Dunlap Link: https://lore.kernel.org/r/ZEQFzMntIrwvZl4+@jo-einhundert Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/composite.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/usb/composite.h b/include/linux/usb/composite.h index a2448e98854f..07531c4f4350 100644 --- a/include/linux/usb/composite.h +++ b/include/linux/usb/composite.h @@ -443,7 +443,7 @@ static inline struct usb_composite_driver *to_cdriver( * @bcd_webusb_version: 0x0100 by default, WebUSB specification version * @b_webusb_vendor_code: 0x0 by default, vendor code for WebUSB * @landing_page: empty by default, landing page to announce in WebUSB - * @use_webusb:: false by default, interested gadgets set it + * @use_webusb: false by default, interested gadgets set it * @os_desc_config: the configuration to be used with OS descriptors * @setup_pending: true when setup request is queued but not completed * @os_desc_pending: true when os_desc request is queued but not completed -- cgit v1.2.3 From 672cde9ef80ffde9e76d38f7aa2b287c4a18de9a Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Fri, 21 Apr 2023 08:46:11 +0300 Subject: iio: fix doc for iio_gts_find_sel_by_int_time The kerneldoc for iio_gts_find_sel_by_int_time() has an error. Documentation states that function is searching a selector for a HW-gain while it is searching a selector for an integration time. Fix the documentation by saying the function is looking for a selector for an integration time. Fixes: 38416c28e168 ("iio: light: Add gain-time-scale helpers") Signed-off-by: Matti Vaittinen Link: https://lore.kernel.org/r/ZEIjI4YUzqPZk/9X@fedora Signed-off-by: Jonathan Cameron --- include/linux/iio/iio-gts-helper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/iio/iio-gts-helper.h b/include/linux/iio/iio-gts-helper.h index dd64e544a3da..9cb6c80dea71 100644 --- a/include/linux/iio/iio-gts-helper.h +++ b/include/linux/iio/iio-gts-helper.h @@ -135,7 +135,7 @@ static inline int iio_gts_find_int_time_by_sel(struct iio_gts *gts, int sel) /** * iio_gts_find_sel_by_int_time - find selector matching integration time * @gts: Gain time scale descriptor - * @gain: HW-gain for which matching selector is searched for + * @time: Integration time for which matching selector is searched for * * Return: a selector matching given integration time or -EINVAL if * selector was not found. -- cgit v1.2.3 From 948f072ada23e0a504c5e4d7d71d4c83bd0785ec Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Tue, 9 May 2023 09:42:47 +1000 Subject: SUNRPC: always free ctxt when freeing deferred request Since the ->xprt_ctxt pointer was added to svc_deferred_req, it has not been sufficient to use kfree() to free a deferred request. We may need to free the ctxt as well. As freeing the ctxt is all that ->xpo_release_rqst() does, we repurpose it to explicit do that even when the ctxt is not stored in an rqst. So we now have ->xpo_release_ctxt() which is given an xprt and a ctxt, which may have been taken either from an rqst or from a dreq. The caller is now responsible for clearing that pointer after the call to ->xpo_release_ctxt. We also clear dr->xprt_ctxt when the ctxt is moved into a new rqst when revisiting a deferred request. This ensures there is only one pointer to the ctxt, so the risk of double freeing in future is reduced. The new code in svc_xprt_release which releases both the ctxt and any rq_deferred depends on this. Fixes: 773f91b2cf3f ("SUNRPC: Fix NFSD's request deferral on RDMA transports") Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 2 +- include/linux/sunrpc/svc_xprt.h | 2 +- net/sunrpc/svc_xprt.c | 23 +++++++++++++++++------ net/sunrpc/svcsock.c | 30 ++++++++++++++++-------------- net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 11 +++++------ net/sunrpc/xprtrdma/svc_rdma_transport.c | 2 +- 6 files changed, 41 insertions(+), 29 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 24aa159d29a7..fbc4bd423b35 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -176,7 +176,7 @@ extern struct svc_rdma_recv_ctxt * extern void svc_rdma_recv_ctxt_put(struct svcxprt_rdma *rdma, struct svc_rdma_recv_ctxt *ctxt); extern void svc_rdma_flush_recv_queues(struct svcxprt_rdma *rdma); -extern void svc_rdma_release_rqst(struct svc_rqst *rqstp); +extern void svc_rdma_release_ctxt(struct svc_xprt *xprt, void *ctxt); extern int svc_rdma_recvfrom(struct svc_rqst *); /* svc_rdma_rw.c */ diff --git a/include/linux/sunrpc/svc_xprt.h b/include/linux/sunrpc/svc_xprt.h index 867479204840..a6b12631db21 100644 --- a/include/linux/sunrpc/svc_xprt.h +++ b/include/linux/sunrpc/svc_xprt.h @@ -23,7 +23,7 @@ struct svc_xprt_ops { int (*xpo_sendto)(struct svc_rqst *); int (*xpo_result_payload)(struct svc_rqst *, unsigned int, unsigned int); - void (*xpo_release_rqst)(struct svc_rqst *); + void (*xpo_release_ctxt)(struct svc_xprt *xprt, void *ctxt); void (*xpo_detach)(struct svc_xprt *); void (*xpo_free)(struct svc_xprt *); void (*xpo_kill_temp_xprt)(struct svc_xprt *); diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 5fd94f6bdc75..13a14897bc17 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -532,13 +532,23 @@ void svc_reserve(struct svc_rqst *rqstp, int space) } EXPORT_SYMBOL_GPL(svc_reserve); +static void free_deferred(struct svc_xprt *xprt, struct svc_deferred_req *dr) +{ + if (!dr) + return; + + xprt->xpt_ops->xpo_release_ctxt(xprt, dr->xprt_ctxt); + kfree(dr); +} + static void svc_xprt_release(struct svc_rqst *rqstp) { struct svc_xprt *xprt = rqstp->rq_xprt; - xprt->xpt_ops->xpo_release_rqst(rqstp); + xprt->xpt_ops->xpo_release_ctxt(xprt, rqstp->rq_xprt_ctxt); + rqstp->rq_xprt_ctxt = NULL; - kfree(rqstp->rq_deferred); + free_deferred(xprt, rqstp->rq_deferred); rqstp->rq_deferred = NULL; svc_rqst_release_pages(rqstp); @@ -1054,7 +1064,7 @@ static void svc_delete_xprt(struct svc_xprt *xprt) spin_unlock_bh(&serv->sv_lock); while ((dr = svc_deferred_dequeue(xprt)) != NULL) - kfree(dr); + free_deferred(xprt, dr); call_xpt_users(xprt); svc_xprt_put(xprt); @@ -1176,8 +1186,8 @@ static void svc_revisit(struct cache_deferred_req *dreq, int too_many) if (too_many || test_bit(XPT_DEAD, &xprt->xpt_flags)) { spin_unlock(&xprt->xpt_lock); trace_svc_defer_drop(dr); + free_deferred(xprt, dr); svc_xprt_put(xprt); - kfree(dr); return; } dr->xprt = NULL; @@ -1222,14 +1232,13 @@ static struct cache_deferred_req *svc_defer(struct cache_req *req) dr->addrlen = rqstp->rq_addrlen; dr->daddr = rqstp->rq_daddr; dr->argslen = rqstp->rq_arg.len >> 2; - dr->xprt_ctxt = rqstp->rq_xprt_ctxt; /* back up head to the start of the buffer and copy */ skip = rqstp->rq_arg.len - rqstp->rq_arg.head[0].iov_len; memcpy(dr->args, rqstp->rq_arg.head[0].iov_base - skip, dr->argslen << 2); } - WARN_ON_ONCE(rqstp->rq_xprt_ctxt != dr->xprt_ctxt); + dr->xprt_ctxt = rqstp->rq_xprt_ctxt; rqstp->rq_xprt_ctxt = NULL; trace_svc_defer(rqstp); svc_xprt_get(rqstp->rq_xprt); @@ -1263,6 +1272,8 @@ static noinline int svc_deferred_recv(struct svc_rqst *rqstp) rqstp->rq_daddr = dr->daddr; rqstp->rq_respages = rqstp->rq_pages; rqstp->rq_xprt_ctxt = dr->xprt_ctxt; + + dr->xprt_ctxt = NULL; svc_xprt_received(rqstp->rq_xprt); return dr->argslen << 2; } diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index 9989194446a5..63fe7a338992 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -121,27 +121,27 @@ static void svc_reclassify_socket(struct socket *sock) #endif /** - * svc_tcp_release_rqst - Release transport-related resources - * @rqstp: request structure with resources to be released + * svc_tcp_release_ctxt - Release transport-related resources + * @xprt: the transport which owned the context + * @ctxt: the context from rqstp->rq_xprt_ctxt or dr->xprt_ctxt * */ -static void svc_tcp_release_rqst(struct svc_rqst *rqstp) +static void svc_tcp_release_ctxt(struct svc_xprt *xprt, void *ctxt) { } /** - * svc_udp_release_rqst - Release transport-related resources - * @rqstp: request structure with resources to be released + * svc_udp_release_ctxt - Release transport-related resources + * @xprt: the transport which owned the context + * @ctxt: the context from rqstp->rq_xprt_ctxt or dr->xprt_ctxt * */ -static void svc_udp_release_rqst(struct svc_rqst *rqstp) +static void svc_udp_release_ctxt(struct svc_xprt *xprt, void *ctxt) { - struct sk_buff *skb = rqstp->rq_xprt_ctxt; + struct sk_buff *skb = ctxt; - if (skb) { - rqstp->rq_xprt_ctxt = NULL; + if (skb) consume_skb(skb); - } } union svc_pktinfo_u { @@ -696,7 +696,8 @@ static int svc_udp_sendto(struct svc_rqst *rqstp) unsigned int sent; int err; - svc_udp_release_rqst(rqstp); + svc_udp_release_ctxt(xprt, rqstp->rq_xprt_ctxt); + rqstp->rq_xprt_ctxt = NULL; svc_set_cmsg_data(rqstp, cmh); @@ -768,7 +769,7 @@ static const struct svc_xprt_ops svc_udp_ops = { .xpo_recvfrom = svc_udp_recvfrom, .xpo_sendto = svc_udp_sendto, .xpo_result_payload = svc_sock_result_payload, - .xpo_release_rqst = svc_udp_release_rqst, + .xpo_release_ctxt = svc_udp_release_ctxt, .xpo_detach = svc_sock_detach, .xpo_free = svc_sock_free, .xpo_has_wspace = svc_udp_has_wspace, @@ -1301,7 +1302,8 @@ static int svc_tcp_sendto(struct svc_rqst *rqstp) unsigned int sent; int err; - svc_tcp_release_rqst(rqstp); + svc_tcp_release_ctxt(xprt, rqstp->rq_xprt_ctxt); + rqstp->rq_xprt_ctxt = NULL; atomic_inc(&svsk->sk_sendqlen); mutex_lock(&xprt->xpt_mutex); @@ -1346,7 +1348,7 @@ static const struct svc_xprt_ops svc_tcp_ops = { .xpo_recvfrom = svc_tcp_recvfrom, .xpo_sendto = svc_tcp_sendto, .xpo_result_payload = svc_sock_result_payload, - .xpo_release_rqst = svc_tcp_release_rqst, + .xpo_release_ctxt = svc_tcp_release_ctxt, .xpo_detach = svc_tcp_sock_detach, .xpo_free = svc_sock_free, .xpo_has_wspace = svc_tcp_has_wspace, diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index 1c658fa43063..a22fe7587fa6 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -239,21 +239,20 @@ void svc_rdma_recv_ctxt_put(struct svcxprt_rdma *rdma, } /** - * svc_rdma_release_rqst - Release transport-specific per-rqst resources - * @rqstp: svc_rqst being released + * svc_rdma_release_ctxt - Release transport-specific per-rqst resources + * @xprt: the transport which owned the context + * @vctxt: the context from rqstp->rq_xprt_ctxt or dr->xprt_ctxt * * Ensure that the recv_ctxt is released whether or not a Reply * was sent. For example, the client could close the connection, * or svc_process could drop an RPC, before the Reply is sent. */ -void svc_rdma_release_rqst(struct svc_rqst *rqstp) +void svc_rdma_release_ctxt(struct svc_xprt *xprt, void *vctxt) { - struct svc_rdma_recv_ctxt *ctxt = rqstp->rq_xprt_ctxt; - struct svc_xprt *xprt = rqstp->rq_xprt; + struct svc_rdma_recv_ctxt *ctxt = vctxt; struct svcxprt_rdma *rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt); - rqstp->rq_xprt_ctxt = NULL; if (ctxt) svc_rdma_recv_ctxt_put(rdma, ctxt); } diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 416b298f74dd..ca04f7a6a085 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -80,7 +80,7 @@ static const struct svc_xprt_ops svc_rdma_ops = { .xpo_recvfrom = svc_rdma_recvfrom, .xpo_sendto = svc_rdma_sendto, .xpo_result_payload = svc_rdma_result_payload, - .xpo_release_rqst = svc_rdma_release_rqst, + .xpo_release_ctxt = svc_rdma_release_ctxt, .xpo_detach = svc_rdma_detach, .xpo_free = svc_rdma_free, .xpo_has_wspace = svc_rdma_has_wspace, -- cgit v1.2.3 From 99d46450625590d410f86fe4660a5eff7d3b8343 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Wed, 26 Apr 2023 20:29:28 +0300 Subject: tpm: Prevent hwrng from activating during resume Set TPM_CHIP_FLAG_SUSPENDED in tpm_pm_suspend() and reset in tpm_pm_resume(). While the flag is set, tpm_hwrng() gives back zero bytes. This prevents hwrng from racing during resume. Cc: stable@vger.kernel.org Fixes: 6e592a065d51 ("tpm: Move Linux RNG connection to hwrng") Reviewed-by: Jerry Snitselaar Signed-off-by: Jarkko Sakkinen --- drivers/char/tpm/tpm-chip.c | 4 ++++ drivers/char/tpm/tpm-interface.c | 10 ++++++++++ include/linux/tpm.h | 1 + 3 files changed, 15 insertions(+) (limited to 'include/linux') diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c index c10a4aa97373..cd48033b804a 100644 --- a/drivers/char/tpm/tpm-chip.c +++ b/drivers/char/tpm/tpm-chip.c @@ -571,6 +571,10 @@ static int tpm_hwrng_read(struct hwrng *rng, void *data, size_t max, bool wait) { struct tpm_chip *chip = container_of(rng, struct tpm_chip, hwrng); + /* Give back zero bytes, as TPM chip has not yet fully resumed: */ + if (chip->flags & TPM_CHIP_FLAG_SUSPENDED) + return 0; + return tpm_get_random(chip, data, max); } diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c index 4463d0018290..586ca10b0d72 100644 --- a/drivers/char/tpm/tpm-interface.c +++ b/drivers/char/tpm/tpm-interface.c @@ -412,6 +412,8 @@ int tpm_pm_suspend(struct device *dev) } suspended: + chip->flags |= TPM_CHIP_FLAG_SUSPENDED; + if (rc) dev_err(dev, "Ignoring error %d while suspending\n", rc); return 0; @@ -429,6 +431,14 @@ int tpm_pm_resume(struct device *dev) if (chip == NULL) return -ENODEV; + chip->flags &= ~TPM_CHIP_FLAG_SUSPENDED; + + /* + * Guarantee that SUSPENDED is written last, so that hwrng does not + * activate before the chip has been fully resumed. + */ + wmb(); + return 0; } EXPORT_SYMBOL_GPL(tpm_pm_resume); diff --git a/include/linux/tpm.h b/include/linux/tpm.h index 77693389c3f9..6a1e8f157255 100644 --- a/include/linux/tpm.h +++ b/include/linux/tpm.h @@ -282,6 +282,7 @@ enum tpm_chip_flags { TPM_CHIP_FLAG_ALWAYS_POWERED = BIT(5), TPM_CHIP_FLAG_FIRMWARE_POWER_MANAGED = BIT(6), TPM_CHIP_FLAG_FIRMWARE_UPGRADE = BIT(7), + TPM_CHIP_FLAG_SUSPENDED = BIT(8), }; #define to_tpm_chip(d) container_of(d, struct tpm_chip, dev) -- cgit v1.2.3 From f15afbd34d8fadbd375f1212e97837e32bc170cc Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Mon, 24 Apr 2023 13:18:35 +0800 Subject: fs: fix undefined behavior in bit shift for SB_NOUSER Shifting signed 32-bit value by 31 bits is undefined, so changing significant bit to unsigned. It was spotted by UBSAN. So let's just fix this by using the BIT() helper for all SB_* flags. Fixes: e462ec50cb5f ("VFS: Differentiate mount flags (MS_*) from internal superblock flags") Signed-off-by: Hao Ge Message-Id: <20230424051835.374204-1-gehao@kylinos.cn> [brauner@kernel.org: use BIT() for all SB_* flags] Signed-off-by: Christian Brauner --- include/linux/fs.h | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 21a981680856..133f0640fb24 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1076,29 +1076,29 @@ extern int send_sigurg(struct fown_struct *fown); * sb->s_flags. Note that these mirror the equivalent MS_* flags where * represented in both. */ -#define SB_RDONLY 1 /* Mount read-only */ -#define SB_NOSUID 2 /* Ignore suid and sgid bits */ -#define SB_NODEV 4 /* Disallow access to device special files */ -#define SB_NOEXEC 8 /* Disallow program execution */ -#define SB_SYNCHRONOUS 16 /* Writes are synced at once */ -#define SB_MANDLOCK 64 /* Allow mandatory locks on an FS */ -#define SB_DIRSYNC 128 /* Directory modifications are synchronous */ -#define SB_NOATIME 1024 /* Do not update access times. */ -#define SB_NODIRATIME 2048 /* Do not update directory access times */ -#define SB_SILENT 32768 -#define SB_POSIXACL (1<<16) /* VFS does not apply the umask */ -#define SB_INLINECRYPT (1<<17) /* Use blk-crypto for encrypted files */ -#define SB_KERNMOUNT (1<<22) /* this is a kern_mount call */ -#define SB_I_VERSION (1<<23) /* Update inode I_version field */ -#define SB_LAZYTIME (1<<25) /* Update the on-disk [acm]times lazily */ +#define SB_RDONLY BIT(0) /* Mount read-only */ +#define SB_NOSUID BIT(1) /* Ignore suid and sgid bits */ +#define SB_NODEV BIT(2) /* Disallow access to device special files */ +#define SB_NOEXEC BIT(3) /* Disallow program execution */ +#define SB_SYNCHRONOUS BIT(4) /* Writes are synced at once */ +#define SB_MANDLOCK BIT(6) /* Allow mandatory locks on an FS */ +#define SB_DIRSYNC BIT(7) /* Directory modifications are synchronous */ +#define SB_NOATIME BIT(10) /* Do not update access times. */ +#define SB_NODIRATIME BIT(11) /* Do not update directory access times */ +#define SB_SILENT BIT(15) +#define SB_POSIXACL BIT(16) /* VFS does not apply the umask */ +#define SB_INLINECRYPT BIT(17) /* Use blk-crypto for encrypted files */ +#define SB_KERNMOUNT BIT(22) /* this is a kern_mount call */ +#define SB_I_VERSION BIT(23) /* Update inode I_version field */ +#define SB_LAZYTIME BIT(25) /* Update the on-disk [acm]times lazily */ /* These sb flags are internal to the kernel */ -#define SB_SUBMOUNT (1<<26) -#define SB_FORCE (1<<27) -#define SB_NOSEC (1<<28) -#define SB_BORN (1<<29) -#define SB_ACTIVE (1<<30) -#define SB_NOUSER (1<<31) +#define SB_SUBMOUNT BIT(26) +#define SB_FORCE BIT(27) +#define SB_NOSEC BIT(28) +#define SB_BORN BIT(29) +#define SB_ACTIVE BIT(30) +#define SB_NOUSER BIT(31) /* These flags relate to encoding and casefolding */ #define SB_ENC_STRICT_MODE_FL (1 << 0) -- cgit v1.2.3 From a18ef64fe1e4558b14a6e0ca9fbe8264475b7013 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 14:47:12 +0200 Subject: tracing: make ftrace_likely_update() declaration visible This function is only used when CONFIG_TRACE_BRANCH_PROFILING is set and DISABLE_BRANCH_PROFILING is not set, and the declaration is hidden behind this combination of tests. But that causes a warning when building with CONFIG_TRACING_BRANCHES, since that sets DISABLE_BRANCH_PROFILING for the tracing code, and the declaration is thus hidden: kernel/trace/trace_branch.c:205:6: error: no previous prototype for 'ftrace_likely_update' [-Werror=missing-prototypes] Move the declaration out of the #ifdef to avoid the warning. Signed-off-by: Arnd Bergmann Signed-off-by: Linus Torvalds --- include/linux/compiler.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 947a60b801db..d7779a18b24f 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -12,11 +12,10 @@ * Note: DISABLE_BRANCH_PROFILING can be used by special lowlevel code * to disable branch tracing on a per file basis. */ -#if defined(CONFIG_TRACE_BRANCH_PROFILING) \ - && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__) void ftrace_likely_update(struct ftrace_likely_data *f, int val, int expect, int is_constant); - +#if defined(CONFIG_TRACE_BRANCH_PROFILING) \ + && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__) #define likely_notrace(x) __builtin_expect(!!(x), 1) #define unlikely_notrace(x) __builtin_expect(!!(x), 0) -- cgit v1.2.3 From 26e239b37ebdfd189a2e6f94d3407e70313348bc Mon Sep 17 00:00:00 2001 From: Joan Bruguera Micó Date: Wed, 3 May 2023 01:32:32 +0000 Subject: mm: shrinkers: fix race condition on debugfs cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When something registers and unregisters many shrinkers, such as: for x in $(seq 10000); do unshare -Ui true; done Sometimes the following error is printed to the kernel log: debugfs: Directory '...' with parent 'shrinker' already present! This occurs since commit badc28d4924b ("mm: shrinkers: fix deadlock in shrinker debugfs") / v6.2: Since the call to `debugfs_remove_recursive` was moved outside the `shrinker_rwsem`/`shrinker_mutex` lock, but the call to `ida_free` stayed inside, a newly registered shrinker can be re-assigned that ID and attempt to create the debugfs directory before the directory from the previous shrinker has been removed. The locking changes in commit f95bdb700bc6 ("mm: vmscan: make global slab shrink lockless") made the race condition more likely, though it existed before then. Commit badc28d4924b ("mm: shrinkers: fix deadlock in shrinker debugfs") could be reverted since the issue is addressed should no longer occur since the count and scan operations are lockless since commit 20cd1892fcc3 ("mm: shrinkers: make count and scan in shrinker debugfs lockless"). However, since this is a contended lock, prefer instead moving `ida_free` outside the lock to avoid the race. Link: https://lkml.kernel.org/r/20230503013232.299211-1-joanbrugueram@gmail.com Fixes: badc28d4924b ("mm: shrinkers: fix deadlock in shrinker debugfs") Signed-off-by: Joan Bruguera Micó Cc: Qi Zheng Cc: Roman Gushchin Signed-off-by: Andrew Morton --- include/linux/shrinker.h | 13 +++++++++++-- mm/shrinker_debug.c | 15 ++++++++++----- mm/vmscan.c | 5 +++-- 3 files changed, 24 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/shrinker.h b/include/linux/shrinker.h index 7bde8e1c228a..224293b2dd06 100644 --- a/include/linux/shrinker.h +++ b/include/linux/shrinker.h @@ -107,7 +107,10 @@ extern void synchronize_shrinkers(void); #ifdef CONFIG_SHRINKER_DEBUG extern int shrinker_debugfs_add(struct shrinker *shrinker); -extern struct dentry *shrinker_debugfs_remove(struct shrinker *shrinker); +extern struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker, + int *debugfs_id); +extern void shrinker_debugfs_remove(struct dentry *debugfs_entry, + int debugfs_id); extern int __printf(2, 3) shrinker_debugfs_rename(struct shrinker *shrinker, const char *fmt, ...); #else /* CONFIG_SHRINKER_DEBUG */ @@ -115,10 +118,16 @@ static inline int shrinker_debugfs_add(struct shrinker *shrinker) { return 0; } -static inline struct dentry *shrinker_debugfs_remove(struct shrinker *shrinker) +static inline struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker, + int *debugfs_id) { + *debugfs_id = -1; return NULL; } +static inline void shrinker_debugfs_remove(struct dentry *debugfs_entry, + int debugfs_id) +{ +} static inline __printf(2, 3) int shrinker_debugfs_rename(struct shrinker *shrinker, const char *fmt, ...) { diff --git a/mm/shrinker_debug.c b/mm/shrinker_debug.c index 3f83b10c5031..fe10436d9911 100644 --- a/mm/shrinker_debug.c +++ b/mm/shrinker_debug.c @@ -237,7 +237,8 @@ int shrinker_debugfs_rename(struct shrinker *shrinker, const char *fmt, ...) } EXPORT_SYMBOL(shrinker_debugfs_rename); -struct dentry *shrinker_debugfs_remove(struct shrinker *shrinker) +struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker, + int *debugfs_id) { struct dentry *entry = shrinker->debugfs_entry; @@ -246,14 +247,18 @@ struct dentry *shrinker_debugfs_remove(struct shrinker *shrinker) kfree_const(shrinker->name); shrinker->name = NULL; - if (entry) { - ida_free(&shrinker_debugfs_ida, shrinker->debugfs_id); - shrinker->debugfs_entry = NULL; - } + *debugfs_id = entry ? shrinker->debugfs_id : -1; + shrinker->debugfs_entry = NULL; return entry; } +void shrinker_debugfs_remove(struct dentry *debugfs_entry, int debugfs_id) +{ + debugfs_remove_recursive(debugfs_entry); + ida_free(&shrinker_debugfs_ida, debugfs_id); +} + static int __init shrinker_debugfs_init(void) { struct shrinker *shrinker; diff --git a/mm/vmscan.c b/mm/vmscan.c index d257916f39e5..6d0cd2840cf0 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -805,6 +805,7 @@ EXPORT_SYMBOL(register_shrinker); void unregister_shrinker(struct shrinker *shrinker) { struct dentry *debugfs_entry; + int debugfs_id; if (!(shrinker->flags & SHRINKER_REGISTERED)) return; @@ -814,13 +815,13 @@ void unregister_shrinker(struct shrinker *shrinker) shrinker->flags &= ~SHRINKER_REGISTERED; if (shrinker->flags & SHRINKER_MEMCG_AWARE) unregister_memcg_shrinker(shrinker); - debugfs_entry = shrinker_debugfs_remove(shrinker); + debugfs_entry = shrinker_debugfs_detach(shrinker, &debugfs_id); mutex_unlock(&shrinker_mutex); atomic_inc(&shrinker_srcu_generation); synchronize_srcu(&shrinker_srcu); - debugfs_remove_recursive(debugfs_entry); + shrinker_debugfs_remove(debugfs_entry, debugfs_id); kfree(shrinker->nr_deferred); shrinker->nr_deferred = NULL; -- cgit v1.2.3 From 2e9f8ab68f42b059e80db71266c1675c07c664bd Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 21:45:36 +0200 Subject: mdio_bus: unhide mdio_bus_init prototype mdio_bus_init() is either used as a local module_init() entry, or it gets called in phy_device.c. In the former case, there is no declaration, which causes a warning: drivers/net/phy/mdio_bus.c:1371:12: error: no previous prototype for 'mdio_bus_init' [-Werror=missing-prototypes] Remove the #ifdef around the declaration to avoid the warning.. Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20230516194625.549249-4-arnd@kernel.org Signed-off-by: Jakub Kicinski --- include/linux/phy.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/phy.h b/include/linux/phy.h index c5a0dc829714..6478838405a0 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -1900,10 +1900,8 @@ void phy_package_leave(struct phy_device *phydev); int devm_phy_package_join(struct device *dev, struct phy_device *phydev, int addr, size_t priv_size); -#if IS_ENABLED(CONFIG_PHYLIB) int __init mdio_bus_init(void); void mdio_bus_exit(void); -#endif int phy_ethtool_get_strings(struct phy_device *phydev, u8 *data); int phy_ethtool_get_sset_count(struct phy_device *phydev); -- cgit v1.2.3 From 14c4be92ebb3e36e392aa9dd8f314038a9f96f3c Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Tue, 16 May 2023 18:50:38 -0700 Subject: tls: rx: strp: force mixed decrypted records into copy mode If a record is partially decrypted we'll have to CoW it, anyway, so go into copy mode and allocate a writable skb right away. This will make subsequent fix simpler because we won't have to teach tls_strp_msg_make_copy() how to copy skbs while preserving decrypt status. Tested-by: Shai Amiram Signed-off-by: Jakub Kicinski Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- include/linux/skbuff.h | 10 ++++++++++ net/tls/tls_strp.c | 16 +++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 738776ab8838..0b40417457cd 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1587,6 +1587,16 @@ static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from) to->l4_hash = from->l4_hash; }; +static inline int skb_cmp_decrypted(const struct sk_buff *skb1, + const struct sk_buff *skb2) +{ +#ifdef CONFIG_TLS_DEVICE + return skb2->decrypted - skb1->decrypted; +#else + return 0; +#endif +} + static inline void skb_copy_decrypted(struct sk_buff *to, const struct sk_buff *from) { diff --git a/net/tls/tls_strp.c b/net/tls/tls_strp.c index 24016c865e00..2b6fa9855999 100644 --- a/net/tls/tls_strp.c +++ b/net/tls/tls_strp.c @@ -317,15 +317,19 @@ static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort) return 0; } -static bool tls_strp_check_no_dup(struct tls_strparser *strp) +static bool tls_strp_check_queue_ok(struct tls_strparser *strp) { unsigned int len = strp->stm.offset + strp->stm.full_len; - struct sk_buff *skb; + struct sk_buff *first, *skb; u32 seq; - skb = skb_shinfo(strp->anchor)->frag_list; - seq = TCP_SKB_CB(skb)->seq; + first = skb_shinfo(strp->anchor)->frag_list; + skb = first; + seq = TCP_SKB_CB(first)->seq; + /* Make sure there's no duplicate data in the queue, + * and the decrypted status matches. + */ while (skb->len < len) { seq += skb->len; len -= skb->len; @@ -333,6 +337,8 @@ static bool tls_strp_check_no_dup(struct tls_strparser *strp) if (TCP_SKB_CB(skb)->seq != seq) return false; + if (skb_cmp_decrypted(first, skb)) + return false; } return true; @@ -413,7 +419,7 @@ static int tls_strp_read_sock(struct tls_strparser *strp) return tls_strp_read_copy(strp, true); } - if (!tls_strp_check_no_dup(strp)) + if (!tls_strp_check_queue_ok(strp)) return tls_strp_read_copy(strp, false); strp->msg_ready = 1; -- cgit v1.2.3 From ddaf098ea779b3c1302c7843f6ff01e89b1fd380 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Tue, 16 May 2023 21:20:14 +0200 Subject: driver core: class: properly reference count class_dev_iter() When class_dev_iter is initialized, the reference count for the subsys private structure is incremented, but never decremented, causing a memory leak over time. To resolve this, save off a pointer to the internal structure into the class_dev_iter structure and then when the iterator is finished, drop the reference count. Reported-and-tested-by: syzbot+e7afd76ad060fa0d2605@syzkaller.appspotmail.com Fixes: 7b884b7f24b4 ("driver core: class.c: convert to only use class_to_subsys") Reported-by: Mirsad Goran Todorovac Cc: Alan Stern Acked-by: Rafael J. Wysocki Tested-by: Mirsad Goran Todorovac Link: https://lore.kernel.org/r/2023051610-stove-condense-9a77@gregkh Signed-off-by: Greg Kroah-Hartman --- drivers/base/class.c | 2 ++ include/linux/device/class.h | 1 + 2 files changed, 3 insertions(+) (limited to 'include/linux') diff --git a/drivers/base/class.c b/drivers/base/class.c index ac1808d1a2e8..05d9df90f621 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -320,6 +320,7 @@ void class_dev_iter_init(struct class_dev_iter *iter, const struct class *class, start_knode = &start->p->knode_class; klist_iter_init_node(&sp->klist_devices, &iter->ki, start_knode); iter->type = type; + iter->sp = sp; } EXPORT_SYMBOL_GPL(class_dev_iter_init); @@ -361,6 +362,7 @@ EXPORT_SYMBOL_GPL(class_dev_iter_next); void class_dev_iter_exit(struct class_dev_iter *iter) { klist_iter_exit(&iter->ki); + subsys_put(iter->sp); } EXPORT_SYMBOL_GPL(class_dev_iter_exit); diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 9deeaeb457bb..abf3d3bfb6fe 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -74,6 +74,7 @@ struct class { struct class_dev_iter { struct klist_iter ki; const struct device_type *type; + struct subsys_private *sp; }; int __must_check class_register(const struct class *class); -- cgit v1.2.3 From ae9b15fbe63447bc1d3bba3769f409d17ca6fdf6 Mon Sep 17 00:00:00 2001 From: Taehee Yoo Date: Wed, 17 May 2023 14:30:10 +0000 Subject: net: fix stack overflow when LRO is disabled for virtual interfaces When the virtual interface's feature is updated, it synchronizes the updated feature for its own lower interface. This propagation logic should be worked as the iteration, not recursively. But it works recursively due to the netdev notification unexpectedly. This problem occurs when it disables LRO only for the team and bonding interface type. team0 | +------+------+-----+-----+ | | | | | team1 team2 team3 ... team200 If team0's LRO feature is updated, it generates the NETDEV_FEAT_CHANGE event to its own lower interfaces(team1 ~ team200). It is worked by netdev_sync_lower_features(). So, the NETDEV_FEAT_CHANGE notification logic of each lower interface work iteratively. But generated NETDEV_FEAT_CHANGE event is also sent to the upper interface too. upper interface(team0) generates the NETDEV_FEAT_CHANGE event for its own lower interfaces again. lower and upper interfaces receive this event and generate this event again and again. So, the stack overflow occurs. But it is not the infinite loop issue. Because the netdev_sync_lower_features() updates features before generating the NETDEV_FEAT_CHANGE event. Already synchronized lower interfaces skip notification logic. So, it is just the problem that iteration logic is changed to the recursive unexpectedly due to the notification mechanism. Reproducer: ip link add team0 type team ethtool -K team0 lro on for i in {1..200} do ip link add team$i master team0 type team ethtool -K team$i lro on done ethtool -K team0 lro off In order to fix it, the notifier_ctx member of bonding/team is introduced. Reported-by: syzbot+60748c96cf5c6df8e581@syzkaller.appspotmail.com Fixes: fd867d51f889 ("net/core: generic support for disabling netdev features down stack") Signed-off-by: Taehee Yoo Reviewed-by: Eric Dumazet Reviewed-by: Nikolay Aleksandrov Link: https://lore.kernel.org/r/20230517143010.3596250-1-ap420073@gmail.com Signed-off-by: Jakub Kicinski --- drivers/net/bonding/bond_main.c | 8 +++++++- drivers/net/team/team.c | 7 ++++++- include/linux/if_team.h | 1 + include/net/bonding.h | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 3fed888629f7..edbaa1444f8e 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -3947,7 +3947,11 @@ static int bond_slave_netdev_event(unsigned long event, unblock_netpoll_tx(); break; case NETDEV_FEAT_CHANGE: - bond_compute_features(bond); + if (!bond->notifier_ctx) { + bond->notifier_ctx = true; + bond_compute_features(bond); + bond->notifier_ctx = false; + } break; case NETDEV_RESEND_IGMP: /* Propagate to master device */ @@ -6342,6 +6346,8 @@ static int bond_init(struct net_device *bond_dev) if (!bond->wq) return -ENOMEM; + bond->notifier_ctx = false; + spin_lock_init(&bond->stats_lock); netdev_lockdep_set_classes(bond_dev); diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c index d10606f257c4..555b0b1e9a78 100644 --- a/drivers/net/team/team.c +++ b/drivers/net/team/team.c @@ -1629,6 +1629,7 @@ static int team_init(struct net_device *dev) team->dev = dev; team_set_no_mode(team); + team->notifier_ctx = false; team->pcpu_stats = netdev_alloc_pcpu_stats(struct team_pcpu_stats); if (!team->pcpu_stats) @@ -3022,7 +3023,11 @@ static int team_device_event(struct notifier_block *unused, team_del_slave(port->team->dev, dev); break; case NETDEV_FEAT_CHANGE: - team_compute_features(port->team); + if (!port->team->notifier_ctx) { + port->team->notifier_ctx = true; + team_compute_features(port->team); + port->team->notifier_ctx = false; + } break; case NETDEV_PRECHANGEMTU: /* Forbid to change mtu of underlaying device */ diff --git a/include/linux/if_team.h b/include/linux/if_team.h index fc985e5c739d..8de6b6e67829 100644 --- a/include/linux/if_team.h +++ b/include/linux/if_team.h @@ -208,6 +208,7 @@ struct team { bool queue_override_enabled; struct list_head *qom_lists; /* array of queue override mapping lists */ bool port_mtu_change_allowed; + bool notifier_ctx; struct { unsigned int count; unsigned int interval; /* in ms */ diff --git a/include/net/bonding.h b/include/net/bonding.h index 0efef2a952b7..59955ac33157 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -221,6 +221,7 @@ struct bonding { struct bond_up_slave __rcu *usable_slaves; struct bond_up_slave __rcu *all_slaves; bool force_primary; + bool notifier_ctx; s32 slave_cnt; /* never change this value outside the attach/detach wrappers */ int (*recv_probe)(const struct sk_buff *, struct bonding *, struct slave *); -- cgit v1.2.3 From e3afec91aad23c52dfe426c7d7128e4839c3eed8 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 20 May 2023 11:00:10 +0200 Subject: block: remove NFL4_UFLG_MASK The NFL4_UFLG_MASK define slipped in in commit 9208d4149758 ("block: add a ->get_unique_id method") and should never have been added, as NFSD as the only user of it already has it's copy. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20230520090010.527046-1-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b441e633f4dd..c0ffe203a602 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1376,8 +1376,6 @@ enum blk_unique_id { BLK_UID_NAA = 3, }; -#define NFL4_UFLG_MASK 0x0000003F - struct block_device_operations { void (*submit_bio)(struct bio *bio); int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob, -- cgit v1.2.3 From c7dd225bc224726c22db08e680bf787f60ebdee3 Mon Sep 17 00:00:00 2001 From: Yevgeny Kliteynik Date: Sun, 2 Apr 2023 17:14:10 +0300 Subject: net/mlx5: DR, Check force-loopback RC QP capability independently from RoCE SW Steering uses RC QP for writing STEs to ICM. This writingis done in LB (loopback), and FL (force-loopback) QP is preferred for performance. FL is available when RoCE is enabled or disabled based on RoCE caps. This patch adds reading of FL capability from HCA caps in addition to the existing reading from RoCE caps, thus fixing the case where we didn't have loopback enabled when RoCE was disabled. Fixes: 7304d603a57a ("net/mlx5: DR, Add support for force-loopback QP") Signed-off-by: Itamar Gozlan Signed-off-by: Yevgeny Kliteynik Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c | 4 +++- include/linux/mlx5/mlx5_ifc.h | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c index 3835ba3f4dda..1aa525e509f1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c @@ -117,6 +117,8 @@ int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev, caps->gvmi = MLX5_CAP_GEN(mdev, vhca_id); caps->flex_protocols = MLX5_CAP_GEN(mdev, flex_parser_protocols); caps->sw_format_ver = MLX5_CAP_GEN(mdev, steering_format_version); + caps->roce_caps.fl_rc_qp_when_roce_disabled = + MLX5_CAP_GEN(mdev, fl_rc_qp_when_roce_disabled); if (MLX5_CAP_GEN(mdev, roce)) { err = dr_cmd_query_nic_vport_roce_en(mdev, 0, &roce_en); @@ -124,7 +126,7 @@ int mlx5dr_cmd_query_device(struct mlx5_core_dev *mdev, return err; caps->roce_caps.roce_en = roce_en; - caps->roce_caps.fl_rc_qp_when_roce_disabled = + caps->roce_caps.fl_rc_qp_when_roce_disabled |= MLX5_CAP_ROCE(mdev, fl_rc_qp_when_roce_disabled); caps->roce_caps.fl_rc_qp_when_roce_enabled = MLX5_CAP_ROCE(mdev, fl_rc_qp_when_roce_enabled); diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index dc5e2cb302a5..b89778d0d326 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1705,7 +1705,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 rc[0x1]; u8 uar_4k[0x1]; - u8 reserved_at_241[0x9]; + u8 reserved_at_241[0x7]; + u8 fl_rc_qp_when_roce_disabled[0x1]; + u8 regexp_params[0x1]; u8 uar_sz[0x6]; u8 port_selection_cap[0x1]; u8 reserved_at_248[0x1]; -- cgit v1.2.3 From 29173d07f79883ac94f5570294f98af3d4287382 Mon Sep 17 00:00:00 2001 From: John Fastabend Date: Mon, 22 May 2023 19:56:06 -0700 Subject: bpf, sockmap: Convert schedule_work into delayed_work Sk_buffs are fed into sockmap verdict programs either from a strparser (when the user might want to decide how framing of skb is done by attaching another parser program) or directly through tcp_read_sock. The tcp_read_sock is the preferred method for performance when the BPF logic is a stream parser. The flow for Cilium's common use case with a stream parser is, tcp_read_sock() sk_psock_verdict_recv ret = bpf_prog_run_pin_on_cpu() sk_psock_verdict_apply(sock, skb, ret) // if system is under memory pressure or app is slow we may // need to queue skb. Do this queuing through ingress_skb and // then kick timer to wake up handler skb_queue_tail(ingress_skb, skb) schedule_work(work); The work queue is wired up to sk_psock_backlog(). This will then walk the ingress_skb skb list that holds our sk_buffs that could not be handled, but should be OK to run at some later point. However, its possible that the workqueue doing this work still hits an error when sending the skb. When this happens the skbuff is requeued on a temporary 'state' struct kept with the workqueue. This is necessary because its possible to partially send an skbuff before hitting an error and we need to know how and where to restart when the workqueue runs next. Now for the trouble, we don't rekick the workqueue. This can cause a stall where the skbuff we just cached on the state variable might never be sent. This happens when its the last packet in a flow and no further packets come along that would cause the system to kick the workqueue from that side. To fix we could do simple schedule_work(), but while under memory pressure it makes sense to back off some instead of continue to retry repeatedly. So instead to fix convert schedule_work to schedule_delayed_work and add backoff logic to reschedule from backlog queue on errors. Its not obvious though what a good backoff is so use '1'. To test we observed some flakes whil running NGINX compliance test with sockmap we attributed these failed test to this bug and subsequent issue. >From on list discussion. This commit bec217197b41("skmsg: Schedule psock work if the cached skb exists on the psock") was intended to address similar race, but had a couple cases it missed. Most obvious it only accounted for receiving traffic on the local socket so if redirecting into another socket we could still get an sk_buff stuck here. Next it missed the case where copied=0 in the recv() handler and then we wouldn't kick the scheduler. Also its sub-optimal to require userspace to kick the internal mechanisms of sockmap to wake it up and copy data to user. It results in an extra syscall and requires the app to actual handle the EAGAIN correctly. Fixes: 04919bed948dc ("tcp: Introduce tcp_read_skb()") Signed-off-by: John Fastabend Signed-off-by: Daniel Borkmann Tested-by: William Findlay Reviewed-by: Jakub Sitnicki Link: https://lore.kernel.org/bpf/20230523025618.113937-3-john.fastabend@gmail.com --- include/linux/skmsg.h | 2 +- net/core/skmsg.c | 21 ++++++++++++++------- net/core/sock_map.c | 3 ++- 3 files changed, 17 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 84f787416a54..904ff9a32ad6 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -105,7 +105,7 @@ struct sk_psock { struct proto *sk_proto; struct mutex work_mutex; struct sk_psock_work_state work_state; - struct work_struct work; + struct delayed_work work; struct rcu_work rwork; }; diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 4a3dc8d27295..0a9ee2acac0b 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -482,7 +482,7 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, } out: if (psock->work_state.skb && copied > 0) - schedule_work(&psock->work); + schedule_delayed_work(&psock->work, 0); return copied; } EXPORT_SYMBOL_GPL(sk_msg_recvmsg); @@ -640,7 +640,8 @@ static void sk_psock_skb_state(struct sk_psock *psock, static void sk_psock_backlog(struct work_struct *work) { - struct sk_psock *psock = container_of(work, struct sk_psock, work); + struct delayed_work *dwork = to_delayed_work(work); + struct sk_psock *psock = container_of(dwork, struct sk_psock, work); struct sk_psock_work_state *state = &psock->work_state; struct sk_buff *skb = NULL; bool ingress; @@ -680,6 +681,12 @@ start: if (ret == -EAGAIN) { sk_psock_skb_state(psock, state, skb, len, off); + + /* Delay slightly to prioritize any + * other work that might be here. + */ + if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) + schedule_delayed_work(&psock->work, 1); goto end; } /* Hard errors break pipe and stop xmit. */ @@ -734,7 +741,7 @@ struct sk_psock *sk_psock_init(struct sock *sk, int node) INIT_LIST_HEAD(&psock->link); spin_lock_init(&psock->link_lock); - INIT_WORK(&psock->work, sk_psock_backlog); + INIT_DELAYED_WORK(&psock->work, sk_psock_backlog); mutex_init(&psock->work_mutex); INIT_LIST_HEAD(&psock->ingress_msg); spin_lock_init(&psock->ingress_lock); @@ -823,7 +830,7 @@ static void sk_psock_destroy(struct work_struct *work) sk_psock_done_strp(psock); - cancel_work_sync(&psock->work); + cancel_delayed_work_sync(&psock->work); mutex_destroy(&psock->work_mutex); psock_progs_drop(&psock->progs); @@ -938,7 +945,7 @@ static int sk_psock_skb_redirect(struct sk_psock *from, struct sk_buff *skb) } skb_queue_tail(&psock_other->ingress_skb, skb); - schedule_work(&psock_other->work); + schedule_delayed_work(&psock_other->work, 0); spin_unlock_bh(&psock_other->ingress_lock); return 0; } @@ -1018,7 +1025,7 @@ static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb, spin_lock_bh(&psock->ingress_lock); if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { skb_queue_tail(&psock->ingress_skb, skb); - schedule_work(&psock->work); + schedule_delayed_work(&psock->work, 0); err = 0; } spin_unlock_bh(&psock->ingress_lock); @@ -1049,7 +1056,7 @@ static void sk_psock_write_space(struct sock *sk) psock = sk_psock(sk); if (likely(psock)) { if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) - schedule_work(&psock->work); + schedule_delayed_work(&psock->work, 0); write_space = psock->saved_write_space; } rcu_read_unlock(); diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 7c189c2e2fbf..00afb66cd095 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1644,9 +1644,10 @@ void sock_map_close(struct sock *sk, long timeout) rcu_read_unlock(); sk_psock_stop(psock); release_sock(sk); - cancel_work_sync(&psock->work); + cancel_delayed_work_sync(&psock->work); sk_psock_put(sk, psock); } + /* Make sure we do not recurse. This is a bug. * Leak the socket instead of crashing on a stack overflow. */ -- cgit v1.2.3 From 405df89dd52cbcd69a3cd7d9a10d64de38f854b2 Mon Sep 17 00:00:00 2001 From: John Fastabend Date: Mon, 22 May 2023 19:56:08 -0700 Subject: bpf, sockmap: Improved check for empty queue We noticed some rare sk_buffs were stepping past the queue when system was under memory pressure. The general theory is to skip enqueueing sk_buffs when its not necessary which is the normal case with a system that is properly provisioned for the task, no memory pressure and enough cpu assigned. But, if we can't allocate memory due to an ENOMEM error when enqueueing the sk_buff into the sockmap receive queue we push it onto a delayed workqueue to retry later. When a new sk_buff is received we then check if that queue is empty. However, there is a problem with simply checking the queue length. When a sk_buff is being processed from the ingress queue but not yet on the sockmap msg receive queue its possible to also recv a sk_buff through normal path. It will check the ingress queue which is zero and then skip ahead of the pkt being processed. Previously we used sock lock from both contexts which made the problem harder to hit, but not impossible. To fix instead of popping the skb from the queue entirely we peek the skb from the queue and do the copy there. This ensures checks to the queue length are non-zero while skb is being processed. Then finally when the entire skb has been copied to user space queue or another socket we pop it off the queue. This way the queue length check allows bypassing the queue only after the list has been completely processed. To reproduce issue we run NGINX compliance test with sockmap running and observe some flakes in our testing that we attributed to this issue. Fixes: 04919bed948dc ("tcp: Introduce tcp_read_skb()") Suggested-by: Jakub Sitnicki Signed-off-by: John Fastabend Signed-off-by: Daniel Borkmann Tested-by: William Findlay Reviewed-by: Jakub Sitnicki Link: https://lore.kernel.org/bpf/20230523025618.113937-5-john.fastabend@gmail.com --- include/linux/skmsg.h | 1 - net/core/skmsg.c | 32 ++++++++------------------------ 2 files changed, 8 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 904ff9a32ad6..054d7911bfc9 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -71,7 +71,6 @@ struct sk_psock_link { }; struct sk_psock_work_state { - struct sk_buff *skb; u32 len; u32 off; }; diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 76ff15f8bb06..bcd45a99a3db 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -622,16 +622,12 @@ static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb, static void sk_psock_skb_state(struct sk_psock *psock, struct sk_psock_work_state *state, - struct sk_buff *skb, int len, int off) { spin_lock_bh(&psock->ingress_lock); if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { - state->skb = skb; state->len = len; state->off = off; - } else { - sock_drop(psock->sk, skb); } spin_unlock_bh(&psock->ingress_lock); } @@ -642,23 +638,17 @@ static void sk_psock_backlog(struct work_struct *work) struct sk_psock *psock = container_of(dwork, struct sk_psock, work); struct sk_psock_work_state *state = &psock->work_state; struct sk_buff *skb = NULL; + u32 len = 0, off = 0; bool ingress; - u32 len, off; int ret; mutex_lock(&psock->work_mutex); - if (unlikely(state->skb)) { - spin_lock_bh(&psock->ingress_lock); - skb = state->skb; + if (unlikely(state->len)) { len = state->len; off = state->off; - state->skb = NULL; - spin_unlock_bh(&psock->ingress_lock); } - if (skb) - goto start; - while ((skb = skb_dequeue(&psock->ingress_skb))) { + while ((skb = skb_peek(&psock->ingress_skb))) { len = skb->len; off = 0; if (skb_bpf_strparser(skb)) { @@ -667,7 +657,6 @@ static void sk_psock_backlog(struct work_struct *work) off = stm->offset; len = stm->full_len; } -start: ingress = skb_bpf_ingress(skb); skb_bpf_redirect_clear(skb); do { @@ -677,8 +666,7 @@ start: len, ingress); if (ret <= 0) { if (ret == -EAGAIN) { - sk_psock_skb_state(psock, state, skb, - len, off); + sk_psock_skb_state(psock, state, len, off); /* Delay slightly to prioritize any * other work that might be here. @@ -690,15 +678,16 @@ start: /* Hard errors break pipe and stop xmit. */ sk_psock_report_error(psock, ret ? -ret : EPIPE); sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); - sock_drop(psock->sk, skb); goto end; } off += ret; len -= ret; } while (len); - if (!ingress) + skb = skb_dequeue(&psock->ingress_skb); + if (!ingress) { kfree_skb(skb); + } } end: mutex_unlock(&psock->work_mutex); @@ -791,11 +780,6 @@ static void __sk_psock_zap_ingress(struct sk_psock *psock) skb_bpf_redirect_clear(skb); sock_drop(psock->sk, skb); } - kfree_skb(psock->work_state.skb); - /* We null the skb here to ensure that calls to sk_psock_backlog - * do not pick up the free'd skb. - */ - psock->work_state.skb = NULL; __sk_psock_purge_ingress_msg(psock); } @@ -814,7 +798,6 @@ void sk_psock_stop(struct sk_psock *psock) spin_lock_bh(&psock->ingress_lock); sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); sk_psock_cork_free(psock); - __sk_psock_zap_ingress(psock); spin_unlock_bh(&psock->ingress_lock); } @@ -829,6 +812,7 @@ static void sk_psock_destroy(struct work_struct *work) sk_psock_done_strp(psock); cancel_delayed_work_sync(&psock->work); + __sk_psock_zap_ingress(psock); mutex_destroy(&psock->work_mutex); psock_progs_drop(&psock->progs); -- cgit v1.2.3 From dcbd1ac2668b5fa02069ea96d581ca3f70a7543c Mon Sep 17 00:00:00 2001 From: Beau Belgrave Date: Fri, 19 May 2023 16:07:40 -0700 Subject: tracing/user_events: Rename link fields for clarity Currently most list_head fields of various structs within user_events are simply named link. This causes folks to keep additional context in their head when working with the code, which can be confusing. Instead of using link, describe what the actual link is, for example: list_del_rcu(&mm->link); Changes into: list_del_rcu(&mm->mms_link); The reader now is given a hint the link is to the mms global list instead of having to remember or spot check within the code. Link: https://lkml.kernel.org/r/20230519230741.669-4-beaub@linux.microsoft.com Link: https://lore.kernel.org/linux-trace-kernel/CAHk-=wicngggxVpbnrYHjRTwGE0WYscPRM+L2HO2BF8ia1EXgQ@mail.gmail.com/ Suggested-by: Linus Torvalds Signed-off-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- include/linux/user_events.h | 2 +- kernel/trace/trace_events_user.c | 40 ++++++++++++++++++++++------------------ 2 files changed, 23 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/user_events.h b/include/linux/user_events.h index 2847f5a18a86..17d452b389de 100644 --- a/include/linux/user_events.h +++ b/include/linux/user_events.h @@ -17,7 +17,7 @@ #ifdef CONFIG_USER_EVENTS struct user_event_mm { - struct list_head link; + struct list_head mms_link; struct list_head enablers; struct mm_struct *mm; struct user_event_mm *next; diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index d34a59630e70..238c7a0615fa 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -96,7 +96,7 @@ struct user_event { * these to track enablement sites that are tied to an event. */ struct user_event_enabler { - struct list_head link; + struct list_head mm_enablers_link; struct user_event *event; unsigned long addr; @@ -155,7 +155,7 @@ struct user_event_file_info { #define VALIDATOR_REL (1 << 1) struct user_event_validator { - struct list_head link; + struct list_head user_event_link; int offset; int flags; }; @@ -261,7 +261,7 @@ error: static void user_event_enabler_destroy(struct user_event_enabler *enabler) { - list_del_rcu(&enabler->link); + list_del_rcu(&enabler->mm_enablers_link); /* No longer tracking the event via the enabler */ refcount_dec(&enabler->event->refcnt); @@ -440,7 +440,7 @@ static bool user_event_enabler_exists(struct user_event_mm *mm, { struct user_event_enabler *enabler; - list_for_each_entry(enabler, &mm->enablers, link) { + list_for_each_entry(enabler, &mm->enablers, mm_enablers_link) { if (enabler->addr == uaddr && ENABLE_BIT(enabler) == bit) return true; } @@ -461,7 +461,7 @@ static void user_event_enabler_update(struct user_event *user) next = mm->next; mmap_read_lock(mm->mm); - list_for_each_entry(enabler, &mm->enablers, link) { + list_for_each_entry(enabler, &mm->enablers, mm_enablers_link) { if (enabler->event == user) { attempt = 0; user_event_enabler_write(mm, enabler, true, &attempt); @@ -497,7 +497,7 @@ static bool user_event_enabler_dup(struct user_event_enabler *orig, refcount_inc(&enabler->event->refcnt); /* Enablers not exposed yet, RCU not required */ - list_add(&enabler->link, &mm->enablers); + list_add(&enabler->mm_enablers_link, &mm->enablers); return true; } @@ -527,13 +527,15 @@ static struct user_event_mm *user_event_mm_get_all(struct user_event *user) */ rcu_read_lock(); - list_for_each_entry_rcu(mm, &user_event_mms, link) - list_for_each_entry_rcu(enabler, &mm->enablers, link) + list_for_each_entry_rcu(mm, &user_event_mms, mms_link) { + list_for_each_entry_rcu(enabler, &mm->enablers, mm_enablers_link) { if (enabler->event == user) { mm->next = found; found = user_event_mm_get(mm); break; } + } + } rcu_read_unlock(); @@ -572,7 +574,7 @@ static void user_event_mm_attach(struct user_event_mm *user_mm, struct task_stru unsigned long flags; spin_lock_irqsave(&user_event_mms_lock, flags); - list_add_rcu(&user_mm->link, &user_event_mms); + list_add_rcu(&user_mm->mms_link, &user_event_mms); spin_unlock_irqrestore(&user_event_mms_lock, flags); t->user_event_mm = user_mm; @@ -601,7 +603,7 @@ static void user_event_mm_destroy(struct user_event_mm *mm) { struct user_event_enabler *enabler, *next; - list_for_each_entry_safe(enabler, next, &mm->enablers, link) + list_for_each_entry_safe(enabler, next, &mm->enablers, mm_enablers_link) user_event_enabler_destroy(enabler); mmdrop(mm->mm); @@ -638,7 +640,7 @@ void user_event_mm_remove(struct task_struct *t) /* Remove the mm from the list, so it can no longer be enabled */ spin_lock_irqsave(&user_event_mms_lock, flags); - list_del_rcu(&mm->link); + list_del_rcu(&mm->mms_link); spin_unlock_irqrestore(&user_event_mms_lock, flags); /* @@ -686,9 +688,10 @@ void user_event_mm_dup(struct task_struct *t, struct user_event_mm *old_mm) rcu_read_lock(); - list_for_each_entry_rcu(enabler, &old_mm->enablers, link) + list_for_each_entry_rcu(enabler, &old_mm->enablers, mm_enablers_link) { if (!user_event_enabler_dup(enabler, mm)) goto error; + } rcu_read_unlock(); @@ -757,7 +760,7 @@ retry: */ if (!*write_result) { refcount_inc(&enabler->event->refcnt); - list_add_rcu(&enabler->link, &user_mm->enablers); + list_add_rcu(&enabler->mm_enablers_link, &user_mm->enablers); } mutex_unlock(&event_mutex); @@ -913,8 +916,8 @@ static void user_event_destroy_validators(struct user_event *user) struct user_event_validator *validator, *next; struct list_head *head = &user->validators; - list_for_each_entry_safe(validator, next, head, link) { - list_del(&validator->link); + list_for_each_entry_safe(validator, next, head, user_event_link) { + list_del(&validator->user_event_link); kfree(validator); } } @@ -968,7 +971,7 @@ add_validator: validator->offset = offset; /* Want sequential access when validating */ - list_add_tail(&validator->link, &user->validators); + list_add_tail(&validator->user_event_link, &user->validators); add_field: field->type = type; @@ -1358,7 +1361,7 @@ static int user_event_validate(struct user_event *user, void *data, int len) void *pos, *end = data + len; u32 loc, offset, size; - list_for_each_entry(validator, head, link) { + list_for_each_entry(validator, head, user_event_link) { pos = data + validator->offset; /* Already done min_size check, no bounds check here */ @@ -2279,7 +2282,7 @@ static long user_events_ioctl_unreg(unsigned long uarg) */ mutex_lock(&event_mutex); - list_for_each_entry_safe(enabler, next, &mm->enablers, link) + list_for_each_entry_safe(enabler, next, &mm->enablers, mm_enablers_link) { if (enabler->addr == reg.disable_addr && ENABLE_BIT(enabler) == reg.disable_bit) { set_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler)); @@ -2290,6 +2293,7 @@ static long user_events_ioctl_unreg(unsigned long uarg) /* Removed at least one */ ret = 0; } + } mutex_unlock(&event_mutex); -- cgit v1.2.3 From ff9e1632d69e596d8ca256deb07433a8f3565038 Mon Sep 17 00:00:00 2001 From: Beau Belgrave Date: Fri, 19 May 2023 16:07:41 -0700 Subject: tracing/user_events: Document user_event_mm one-shot list usage During 6.4 development it became clear that the one-shot list used by the user_event_mm's next field was confusing to others. It is not clear how this list is protected or what the next field usage is for unless you are familiar with the code. Add comments into the user_event_mm struct indicating lock requirement and usage. Also document how and why this approach was used via comments in both user_event_enabler_update() and user_event_mm_get_all() and the rules to properly use it. Link: https://lkml.kernel.org/r/20230519230741.669-5-beaub@linux.microsoft.com Link: https://lore.kernel.org/linux-trace-kernel/CAHk-=wicngggxVpbnrYHjRTwGE0WYscPRM+L2HO2BF8ia1EXgQ@mail.gmail.com/ Suggested-by: Linus Torvalds Signed-off-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- include/linux/user_events.h | 1 + kernel/trace/trace_events_user.c | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/user_events.h b/include/linux/user_events.h index 17d452b389de..8afa8c3a0973 100644 --- a/include/linux/user_events.h +++ b/include/linux/user_events.h @@ -20,6 +20,7 @@ struct user_event_mm { struct list_head mms_link; struct list_head enablers; struct mm_struct *mm; + /* Used for one-shot lists, protected by event_mutex */ struct user_event_mm *next; refcount_t refcnt; refcount_t tasks; diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 238c7a0615fa..dbb14705d0d3 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -451,12 +451,25 @@ static bool user_event_enabler_exists(struct user_event_mm *mm, static void user_event_enabler_update(struct user_event *user) { struct user_event_enabler *enabler; - struct user_event_mm *mm = user_event_mm_get_all(user); struct user_event_mm *next; + struct user_event_mm *mm; int attempt; lockdep_assert_held(&event_mutex); + /* + * We need to build a one-shot list of all the mms that have an + * enabler for the user_event passed in. This list is only valid + * while holding the event_mutex. The only reason for this is due + * to the global mm list being RCU protected and we use methods + * which can wait (mmap_read_lock and pin_user_pages_remote). + * + * NOTE: user_event_mm_get_all() increments the ref count of each + * mm that is added to the list to prevent removal timing windows. + * We must always put each mm after they are used, which may wait. + */ + mm = user_event_mm_get_all(user); + while (mm) { next = mm->next; mmap_read_lock(mm->mm); @@ -515,6 +528,14 @@ static struct user_event_mm *user_event_mm_get_all(struct user_event *user) struct user_event_enabler *enabler; struct user_event_mm *mm; + /* + * We use the mm->next field to build a one-shot list from the global + * RCU protected list. To build this list the event_mutex must be held. + * This lets us build a list without requiring allocs that could fail + * when user based events are most wanted for diagnostics. + */ + lockdep_assert_held(&event_mutex); + /* * We do not want to block fork/exec while enablements are being * updated, so we use RCU to walk the current tasks that have used -- cgit v1.2.3 From 4b512860bdbdddcf41467ebd394f27cb8dfb528c Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 23 May 2023 23:09:13 -0400 Subject: tracing: Rename stacktrace field to common_stacktrace The histogram and synthetic events can use a pseudo event called "stacktrace" that will create a stacktrace at the time of the event and use it just like it was a normal field. We have other pseudo events such as "common_cpu" and "common_timestamp". To stay consistent with that, convert "stacktrace" to "common_stacktrace". As this was used in older kernels, to keep backward compatibility, this will act just like "common_cpu" did with "cpu". That is, "cpu" will be the same as "common_cpu" unless the event has a "cpu" field. In which case, the event's field is used. The same is true with "stacktrace". Also update the documentation to reflect this change. Link: https://lore.kernel.org/linux-trace-kernel/20230523230913.6860e28d@rorschach.local.home Cc: Masami Hiramatsu Cc: Tom Zanussi Cc: Mark Rutland Signed-off-by: Steven Rostedt (Google) --- Documentation/trace/histogram.rst | 64 +++++++++++++++++++-------------------- include/linux/trace_events.h | 1 + kernel/trace/trace.c | 2 +- kernel/trace/trace_events.c | 2 ++ kernel/trace/trace_events_hist.c | 16 ++++++---- 5 files changed, 46 insertions(+), 39 deletions(-) (limited to 'include/linux') diff --git a/Documentation/trace/histogram.rst b/Documentation/trace/histogram.rst index 479c9eac6335..3c9b263de9c2 100644 --- a/Documentation/trace/histogram.rst +++ b/Documentation/trace/histogram.rst @@ -35,7 +35,7 @@ Documentation written by Tom Zanussi in place of an explicit value field - this is simply a count of event hits. If 'values' isn't specified, an implicit 'hitcount' value will be automatically created and used as the only value. - Keys can be any field, or the special string 'stacktrace', which + Keys can be any field, or the special string 'common_stacktrace', which will use the event's kernel stacktrace as the key. The keywords 'keys' or 'key' can be used to specify keys, and the keywords 'values', 'vals', or 'val' can be used to specify values. Compound @@ -54,7 +54,7 @@ Documentation written by Tom Zanussi 'compatible' if the fields named in the trigger share the same number and type of fields and those fields also have the same names. Note that any two events always share the compatible 'hitcount' and - 'stacktrace' fields and can therefore be combined using those + 'common_stacktrace' fields and can therefore be combined using those fields, however pointless that may be. 'hist' triggers add a 'hist' file to each event's subdirectory. @@ -547,9 +547,9 @@ Extended error information the hist trigger display symbolic call_sites, we can have the hist trigger additionally display the complete set of kernel stack traces that led to each call_site. To do that, we simply use the special - value 'stacktrace' for the key parameter:: + value 'common_stacktrace' for the key parameter:: - # echo 'hist:keys=stacktrace:values=bytes_req,bytes_alloc:sort=bytes_alloc' > \ + # echo 'hist:keys=common_stacktrace:values=bytes_req,bytes_alloc:sort=bytes_alloc' > \ /sys/kernel/tracing/events/kmem/kmalloc/trigger The above trigger will use the kernel stack trace in effect when an @@ -561,9 +561,9 @@ Extended error information every callpath to a kmalloc for a kernel compile):: # cat /sys/kernel/tracing/events/kmem/kmalloc/hist - # trigger info: hist:keys=stacktrace:vals=bytes_req,bytes_alloc:sort=bytes_alloc:size=2048 [active] + # trigger info: hist:keys=common_stacktrace:vals=bytes_req,bytes_alloc:sort=bytes_alloc:size=2048 [active] - { stacktrace: + { common_stacktrace: __kmalloc_track_caller+0x10b/0x1a0 kmemdup+0x20/0x50 hidraw_report_event+0x8a/0x120 [hid] @@ -581,7 +581,7 @@ Extended error information cpu_startup_entry+0x315/0x3e0 rest_init+0x7c/0x80 } hitcount: 3 bytes_req: 21 bytes_alloc: 24 - { stacktrace: + { common_stacktrace: __kmalloc_track_caller+0x10b/0x1a0 kmemdup+0x20/0x50 hidraw_report_event+0x8a/0x120 [hid] @@ -596,7 +596,7 @@ Extended error information do_IRQ+0x5a/0xf0 ret_from_intr+0x0/0x30 } hitcount: 3 bytes_req: 21 bytes_alloc: 24 - { stacktrace: + { common_stacktrace: kmem_cache_alloc_trace+0xeb/0x150 aa_alloc_task_context+0x27/0x40 apparmor_cred_prepare+0x1f/0x50 @@ -608,7 +608,7 @@ Extended error information . . . - { stacktrace: + { common_stacktrace: __kmalloc+0x11b/0x1b0 i915_gem_execbuffer2+0x6c/0x2c0 [i915] drm_ioctl+0x349/0x670 [drm] @@ -616,7 +616,7 @@ Extended error information SyS_ioctl+0x81/0xa0 system_call_fastpath+0x12/0x6a } hitcount: 17726 bytes_req: 13944120 bytes_alloc: 19593808 - { stacktrace: + { common_stacktrace: __kmalloc+0x11b/0x1b0 load_elf_phdrs+0x76/0xa0 load_elf_binary+0x102/0x1650 @@ -625,7 +625,7 @@ Extended error information SyS_execve+0x3a/0x50 return_from_execve+0x0/0x23 } hitcount: 33348 bytes_req: 17152128 bytes_alloc: 20226048 - { stacktrace: + { common_stacktrace: kmem_cache_alloc_trace+0xeb/0x150 apparmor_file_alloc_security+0x27/0x40 security_file_alloc+0x16/0x20 @@ -636,7 +636,7 @@ Extended error information SyS_open+0x1e/0x20 system_call_fastpath+0x12/0x6a } hitcount: 4766422 bytes_req: 9532844 bytes_alloc: 38131376 - { stacktrace: + { common_stacktrace: __kmalloc+0x11b/0x1b0 seq_buf_alloc+0x1b/0x50 seq_read+0x2cc/0x370 @@ -1026,7 +1026,7 @@ Extended error information First we set up an initially paused stacktrace trigger on the netif_receive_skb event:: - # echo 'hist:key=stacktrace:vals=len:pause' > \ + # echo 'hist:key=common_stacktrace:vals=len:pause' > \ /sys/kernel/tracing/events/net/netif_receive_skb/trigger Next, we set up an 'enable_hist' trigger on the sched_process_exec @@ -1060,9 +1060,9 @@ Extended error information $ wget https://www.kernel.org/pub/linux/kernel/v3.x/patch-3.19.xz # cat /sys/kernel/tracing/events/net/netif_receive_skb/hist - # trigger info: hist:keys=stacktrace:vals=len:sort=hitcount:size=2048 [paused] + # trigger info: hist:keys=common_stacktrace:vals=len:sort=hitcount:size=2048 [paused] - { stacktrace: + { common_stacktrace: __netif_receive_skb_core+0x46d/0x990 __netif_receive_skb+0x18/0x60 netif_receive_skb_internal+0x23/0x90 @@ -1079,7 +1079,7 @@ Extended error information kthread+0xd2/0xf0 ret_from_fork+0x42/0x70 } hitcount: 85 len: 28884 - { stacktrace: + { common_stacktrace: __netif_receive_skb_core+0x46d/0x990 __netif_receive_skb+0x18/0x60 netif_receive_skb_internal+0x23/0x90 @@ -1097,7 +1097,7 @@ Extended error information irq_thread+0x11f/0x150 kthread+0xd2/0xf0 } hitcount: 98 len: 664329 - { stacktrace: + { common_stacktrace: __netif_receive_skb_core+0x46d/0x990 __netif_receive_skb+0x18/0x60 process_backlog+0xa8/0x150 @@ -1115,7 +1115,7 @@ Extended error information inet_sendmsg+0x64/0xa0 sock_sendmsg+0x3d/0x50 } hitcount: 115 len: 13030 - { stacktrace: + { common_stacktrace: __netif_receive_skb_core+0x46d/0x990 __netif_receive_skb+0x18/0x60 netif_receive_skb_internal+0x23/0x90 @@ -1142,14 +1142,14 @@ Extended error information into the histogram. In order to avoid having to set everything up again, we can just clear the histogram first:: - # echo 'hist:key=stacktrace:vals=len:clear' >> \ + # echo 'hist:key=common_stacktrace:vals=len:clear' >> \ /sys/kernel/tracing/events/net/netif_receive_skb/trigger Just to verify that it is in fact cleared, here's what we now see in the hist file:: # cat /sys/kernel/tracing/events/net/netif_receive_skb/hist - # trigger info: hist:keys=stacktrace:vals=len:sort=hitcount:size=2048 [paused] + # trigger info: hist:keys=common_stacktrace:vals=len:sort=hitcount:size=2048 [paused] Totals: Hits: 0 @@ -1485,12 +1485,12 @@ Extended error information And here's an example that shows how to combine histogram data from any two events even if they don't share any 'compatible' fields - other than 'hitcount' and 'stacktrace'. These commands create a + other than 'hitcount' and 'common_stacktrace'. These commands create a couple of triggers named 'bar' using those fields:: - # echo 'hist:name=bar:key=stacktrace:val=hitcount' > \ + # echo 'hist:name=bar:key=common_stacktrace:val=hitcount' > \ /sys/kernel/tracing/events/sched/sched_process_fork/trigger - # echo 'hist:name=bar:key=stacktrace:val=hitcount' > \ + # echo 'hist:name=bar:key=common_stacktrace:val=hitcount' > \ /sys/kernel/tracing/events/net/netif_rx/trigger And displaying the output of either shows some interesting if @@ -1501,16 +1501,16 @@ Extended error information # event histogram # - # trigger info: hist:name=bar:keys=stacktrace:vals=hitcount:sort=hitcount:size=2048 [active] + # trigger info: hist:name=bar:keys=common_stacktrace:vals=hitcount:sort=hitcount:size=2048 [active] # - { stacktrace: + { common_stacktrace: kernel_clone+0x18e/0x330 kernel_thread+0x29/0x30 kthreadd+0x154/0x1b0 ret_from_fork+0x3f/0x70 } hitcount: 1 - { stacktrace: + { common_stacktrace: netif_rx_internal+0xb2/0xd0 netif_rx_ni+0x20/0x70 dev_loopback_xmit+0xaa/0xd0 @@ -1528,7 +1528,7 @@ Extended error information call_cpuidle+0x3b/0x60 cpu_startup_entry+0x22d/0x310 } hitcount: 1 - { stacktrace: + { common_stacktrace: netif_rx_internal+0xb2/0xd0 netif_rx_ni+0x20/0x70 dev_loopback_xmit+0xaa/0xd0 @@ -1543,7 +1543,7 @@ Extended error information SyS_sendto+0xe/0x10 entry_SYSCALL_64_fastpath+0x12/0x6a } hitcount: 2 - { stacktrace: + { common_stacktrace: netif_rx_internal+0xb2/0xd0 netif_rx+0x1c/0x60 loopback_xmit+0x6c/0xb0 @@ -1561,7 +1561,7 @@ Extended error information sock_sendmsg+0x38/0x50 ___sys_sendmsg+0x14e/0x270 } hitcount: 76 - { stacktrace: + { common_stacktrace: netif_rx_internal+0xb2/0xd0 netif_rx+0x1c/0x60 loopback_xmit+0x6c/0xb0 @@ -1579,7 +1579,7 @@ Extended error information sock_sendmsg+0x38/0x50 ___sys_sendmsg+0x269/0x270 } hitcount: 77 - { stacktrace: + { common_stacktrace: netif_rx_internal+0xb2/0xd0 netif_rx+0x1c/0x60 loopback_xmit+0x6c/0xb0 @@ -1597,7 +1597,7 @@ Extended error information sock_sendmsg+0x38/0x50 SYSC_sendto+0xef/0x170 } hitcount: 88 - { stacktrace: + { common_stacktrace: kernel_clone+0x18e/0x330 SyS_clone+0x19/0x20 entry_SYSCALL_64_fastpath+0x12/0x6a @@ -1949,7 +1949,7 @@ uninterruptible state:: # cd /sys/kernel/tracing # echo 's:block_lat pid_t pid; u64 delta; unsigned long[] stack;' > dynamic_events - # echo 'hist:keys=next_pid:ts=common_timestamp.usecs,st=stacktrace if prev_state == 2' >> events/sched/sched_switch/trigger + # echo 'hist:keys=next_pid:ts=common_timestamp.usecs,st=common_stacktrace if prev_state == 2' >> events/sched/sched_switch/trigger # echo 'hist:keys=prev_pid:delta=common_timestamp.usecs-$ts,s=$st:onmax($delta).trace(block_lat,prev_pid,$delta,$s)' >> events/sched/sched_switch/trigger # echo 1 > events/synthetic/block_lat/enable # cat trace diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 0e373222a6df..7c4a0b72334e 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -806,6 +806,7 @@ enum { FILTER_TRACE_FN, FILTER_COMM, FILTER_CPU, + FILTER_STACKTRACE, }; extern int trace_event_raw_init(struct trace_event_call *call); diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index ebc59781456a..81801dc31784 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -5752,7 +5752,7 @@ static const char readme_msg[] = "\t table using the key(s) and value(s) named, and the value of a\n" "\t sum called 'hitcount' is incremented. Keys and values\n" "\t correspond to fields in the event's format description. Keys\n" - "\t can be any field, or the special string 'stacktrace'.\n" + "\t can be any field, or the special string 'common_stacktrace'.\n" "\t Compound keys consisting of up to two fields can be specified\n" "\t by the 'keys' keyword. Values must correspond to numeric\n" "\t fields. Sort keys consisting of up to two fields can be\n" diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 654ffa40457a..57e539d47989 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -194,6 +194,8 @@ static int trace_define_generic_fields(void) __generic_field(int, common_cpu, FILTER_CPU); __generic_field(char *, COMM, FILTER_COMM); __generic_field(char *, comm, FILTER_COMM); + __generic_field(char *, stacktrace, FILTER_STACKTRACE); + __generic_field(char *, STACKTRACE, FILTER_STACKTRACE); return ret; } diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 543cb7dc84ad..b97d3ad832f1 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -1364,7 +1364,7 @@ static const char *hist_field_name(struct hist_field *field, if (field->field) field_name = field->field->name; else - field_name = "stacktrace"; + field_name = "common_stacktrace"; } else if (field->flags & HIST_FIELD_FL_HITCOUNT) field_name = "hitcount"; @@ -2367,7 +2367,7 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, hist_data->enable_timestamps = true; if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) hist_data->attrs->ts_in_usecs = true; - } else if (strcmp(field_name, "stacktrace") == 0) { + } else if (strcmp(field_name, "common_stacktrace") == 0) { *flags |= HIST_FIELD_FL_STACKTRACE; } else if (strcmp(field_name, "common_cpu") == 0) *flags |= HIST_FIELD_FL_CPU; @@ -2378,11 +2378,15 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, if (!field || !field->size) { /* * For backward compatibility, if field_name - * was "cpu", then we treat this the same as - * common_cpu. This also works for "CPU". + * was "cpu" or "stacktrace", then we treat this + * the same as common_cpu and common_stacktrace + * respectively. This also works for "CPU", and + * "STACKTRACE". */ if (field && field->filter_type == FILTER_CPU) { *flags |= HIST_FIELD_FL_CPU; + } else if (field && field->filter_type == FILTER_STACKTRACE) { + *flags |= HIST_FIELD_FL_STACKTRACE; } else { hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name)); @@ -5394,7 +5398,7 @@ static void hist_trigger_print_key(struct seq_file *m, if (key_field->field) seq_printf(m, "%s.stacktrace", key_field->field->name); else - seq_puts(m, "stacktrace:\n"); + seq_puts(m, "common_stacktrace:\n"); hist_trigger_stacktrace_print(m, key + key_field->offset, HIST_STACKTRACE_DEPTH); @@ -5977,7 +5981,7 @@ static int event_hist_trigger_print(struct seq_file *m, if (field->field) seq_printf(m, "%s.stacktrace", field->field->name); else - seq_puts(m, "stacktrace"); + seq_puts(m, "common_stacktrace"); } else hist_field_print(m, field); } -- cgit v1.2.3 From 335b4223466dd75f9f3ea4918187afbadd22e5c8 Mon Sep 17 00:00:00 2001 From: Maximilian Heyne Date: Wed, 3 May 2023 13:16:53 +0000 Subject: x86/pci/xen: populate MSI sysfs entries Commit bf5e758f02fc ("genirq/msi: Simplify sysfs handling") reworked the creation of sysfs entries for MSI IRQs. The creation used to be in msi_domain_alloc_irqs_descs_locked after calling ops->domain_alloc_irqs. Then it moved into __msi_domain_alloc_irqs which is an implementation of domain_alloc_irqs. However, Xen comes with the only other implementation of domain_alloc_irqs and hence doesn't run the sysfs population code anymore. Commit 6c796996ee70 ("x86/pci/xen: Fixup fallout from the PCI/MSI overhaul") set the flag MSI_FLAG_DEV_SYSFS for the xen msi_domain_info but that doesn't actually have an effect because Xen uses it's own domain_alloc_irqs implementation. Fix this by making use of the fallback functions for sysfs population. Fixes: bf5e758f02fc ("genirq/msi: Simplify sysfs handling") Signed-off-by: Maximilian Heyne Reviewed-by: Juergen Gross Link: https://lore.kernel.org/r/20230503131656.15928-1-mheyne@amazon.de Signed-off-by: Juergen Gross --- arch/x86/pci/xen.c | 8 +++++--- include/linux/msi.h | 9 ++++++++- kernel/irq/msi.c | 4 ++-- 3 files changed, 15 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index 8babce71915f..014c508e914d 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -198,7 +198,7 @@ static int xen_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) i++; } kfree(v); - return 0; + return msi_device_populate_sysfs(&dev->dev); error: if (ret == -ENOSYS) @@ -254,7 +254,7 @@ static int xen_hvm_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) dev_dbg(&dev->dev, "xen: msi --> pirq=%d --> irq=%d\n", pirq, irq); } - return 0; + return msi_device_populate_sysfs(&dev->dev); error: dev_err(&dev->dev, "Failed to create MSI%s! ret=%d!\n", @@ -346,7 +346,7 @@ static int xen_initdom_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) if (ret < 0) goto out; } - ret = 0; + ret = msi_device_populate_sysfs(&dev->dev); out: return ret; } @@ -394,6 +394,8 @@ static void xen_teardown_msi_irqs(struct pci_dev *dev) xen_destroy_irq(msidesc->irq + i); msidesc->irq = 0; } + + msi_device_destroy_sysfs(&dev->dev); } static void xen_pv_teardown_msi_irqs(struct pci_dev *dev) diff --git a/include/linux/msi.h b/include/linux/msi.h index cdb14a1ef268..a50ea79522f8 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -383,6 +383,13 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc); void arch_teardown_msi_irq(unsigned int irq); int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); void arch_teardown_msi_irqs(struct pci_dev *dev); +#endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS */ + +/* + * Xen uses non-default msi_domain_ops and hence needs a way to populate sysfs + * entries of MSI IRQs. + */ +#if defined(CONFIG_PCI_XEN) || defined(CONFIG_PCI_MSI_ARCH_FALLBACKS) #ifdef CONFIG_SYSFS int msi_device_populate_sysfs(struct device *dev); void msi_device_destroy_sysfs(struct device *dev); @@ -390,7 +397,7 @@ void msi_device_destroy_sysfs(struct device *dev); static inline int msi_device_populate_sysfs(struct device *dev) { return 0; } static inline void msi_device_destroy_sysfs(struct device *dev) { } #endif /* !CONFIG_SYSFS */ -#endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS */ +#endif /* CONFIG_PCI_XEN || CONFIG_PCI_MSI_ARCH_FALLBACKS */ /* * The restore hook is still available even for fully irq domain based diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index 7a97bcb086bf..b4c31a5c1147 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -542,7 +542,7 @@ fail: return ret; } -#ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS +#if defined(CONFIG_PCI_MSI_ARCH_FALLBACKS) || defined(CONFIG_PCI_XEN) /** * msi_device_populate_sysfs - Populate msi_irqs sysfs entries for a device * @dev: The device (PCI, platform etc) which will get sysfs entries @@ -574,7 +574,7 @@ void msi_device_destroy_sysfs(struct device *dev) msi_for_each_desc(desc, dev, MSI_DESC_ALL) msi_sysfs_remove_desc(dev, desc); } -#endif /* CONFIG_PCI_MSI_ARCH_FALLBACK */ +#endif /* CONFIG_PCI_MSI_ARCH_FALLBACK || CONFIG_PCI_XEN */ #else /* CONFIG_SYSFS */ static inline int msi_sysfs_create_group(struct device *dev) { return 0; } static inline int msi_sysfs_populate_desc(struct device *dev, struct msi_desc *desc) { return 0; } -- cgit v1.2.3 From 1db1f21caebbb1b6e9b1e7657df613616be3fb49 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Thu, 13 Apr 2023 15:48:30 +0300 Subject: net/mlx5e: Use query_special_contexts cmd only once per mdev Don't query the firmware so many times (num rqs * num wqes * wqe frags) because it slows down linearly the interface creation time when the product is larger. Do it only once per mdev and store the result in mlx5e_param. Due to helper function being called from different files, move it to an appropriate location. Rename the function with a proper prefix and add a small cleanup. This fix applies only for legacy rq. Fixes: 1b1e4868836a ("net/mlx5e: Use query_special_contexts for mkeys") Signed-off-by: Dragos Tatulea Reviewed-by: Or Har-Toov Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 1 + drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 24 +++-------------------- drivers/net/ethernet/mellanox/mlx5/core/mr.c | 21 ++++++++++++++++++++ include/linux/mlx5/driver.h | 1 + 4 files changed, 26 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index b8987a404d75..8e999f238194 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -327,6 +327,7 @@ struct mlx5e_params { unsigned int sw_mtu; int hard_mtu; bool ptp_rx; + __be32 terminate_lkey_be; }; static inline u8 mlx5e_get_dcb_num_tc(struct mlx5e_params *params) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 2944691f06ad..0235adcbc609 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -727,26 +727,6 @@ static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) mlx5e_rq_shampo_hd_free(rq); } -static __be32 mlx5e_get_terminate_scatter_list_mkey(struct mlx5_core_dev *dev) -{ - u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {}; - u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {}; - int res; - - if (!MLX5_CAP_GEN(dev, terminate_scatter_list_mkey)) - return MLX5_TERMINATE_SCATTER_LIST_LKEY; - - MLX5_SET(query_special_contexts_in, in, opcode, - MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS); - res = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out); - if (res) - return MLX5_TERMINATE_SCATTER_LIST_LKEY; - - res = MLX5_GET(query_special_contexts_out, out, - terminate_scatter_list_mkey); - return cpu_to_be32(res); -} - static int mlx5e_alloc_rq(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, struct mlx5e_rq_param *rqp, @@ -908,7 +888,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, /* check if num_frags is not a pow of two */ if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { wqe->data[f].byte_count = 0; - wqe->data[f].lkey = mlx5e_get_terminate_scatter_list_mkey(mdev); + wqe->data[f].lkey = params->terminate_lkey_be; wqe->data[f].addr = 0; } } @@ -5007,6 +4987,8 @@ void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 /* RQ */ mlx5e_build_rq_params(mdev, params); + params->terminate_lkey_be = mlx5_core_get_terminate_scatter_list_mkey(mdev); + params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); /* CQ moderation params */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mr.c b/drivers/net/ethernet/mellanox/mlx5/core/mr.c index 9d735c343a3b..678f0be81375 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mr.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/mr.c @@ -32,6 +32,7 @@ #include #include +#include #include "mlx5_core.h" int mlx5_core_create_mkey(struct mlx5_core_dev *dev, u32 *mkey, u32 *in, @@ -122,3 +123,23 @@ int mlx5_core_destroy_psv(struct mlx5_core_dev *dev, int psv_num) return mlx5_cmd_exec_in(dev, destroy_psv, in); } EXPORT_SYMBOL(mlx5_core_destroy_psv); + +__be32 mlx5_core_get_terminate_scatter_list_mkey(struct mlx5_core_dev *dev) +{ + u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {}; + u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {}; + u32 mkey; + + if (!MLX5_CAP_GEN(dev, terminate_scatter_list_mkey)) + return MLX5_TERMINATE_SCATTER_LIST_LKEY; + + MLX5_SET(query_special_contexts_in, in, opcode, + MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS); + if (mlx5_cmd_exec_inout(dev, query_special_contexts, in, out)) + return MLX5_TERMINATE_SCATTER_LIST_LKEY; + + mkey = MLX5_GET(query_special_contexts_out, out, + terminate_scatter_list_mkey); + return cpu_to_be32(mkey); +} +EXPORT_SYMBOL(mlx5_core_get_terminate_scatter_list_mkey); diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index a4c4f737f9c1..94d2be5848ae 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1093,6 +1093,7 @@ void mlx5_cmdif_debugfs_cleanup(struct mlx5_core_dev *dev); int mlx5_core_create_psv(struct mlx5_core_dev *dev, u32 pdn, int npsvs, u32 *sig_index); int mlx5_core_destroy_psv(struct mlx5_core_dev *dev, int psv_num); +__be32 mlx5_core_get_terminate_scatter_list_mkey(struct mlx5_core_dev *dev); void mlx5_core_put_rsc(struct mlx5_core_rsc_common *common); int mlx5_query_odp_caps(struct mlx5_core_dev *dev, struct mlx5_odp_caps *odp_caps); -- cgit v1.2.3 From fd936fd8ac105ba3eb764185e8ba483c789c893e Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 23 May 2023 21:01:30 +0200 Subject: efi: fix missing prototype warnings The cper.c file needs to include an extra header, and efi_zboot_entry needs an extern declaration to avoid these 'make W=1' warnings: drivers/firmware/efi/libstub/zboot.c:65:1: error: no previous prototype for 'efi_zboot_entry' [-Werror=missing-prototypes] drivers/firmware/efi/efi.c:176:16: error: no previous prototype for 'efi_attr_is_visible' [-Werror=missing-prototypes] drivers/firmware/efi/cper.c:626:6: error: no previous prototype for 'cper_estatus_print' [-Werror=missing-prototypes] drivers/firmware/efi/cper.c:649:5: error: no previous prototype for 'cper_estatus_check_header' [-Werror=missing-prototypes] drivers/firmware/efi/cper.c:662:5: error: no previous prototype for 'cper_estatus_check' [-Werror=missing-prototypes] To make this easier, move the cper specific declarations to include/linux/cper.h. Signed-off-by: Arnd Bergmann Acked-by: Rafael J. Wysocki Signed-off-by: Ard Biesheuvel --- drivers/acpi/apei/apei-internal.h | 6 ------ drivers/acpi/apei/bert.c | 1 + drivers/firmware/efi/libstub/efistub.h | 3 +++ include/linux/cper.h | 6 ++++++ include/linux/efi.h | 2 ++ 5 files changed, 12 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/apei/apei-internal.h b/drivers/acpi/apei/apei-internal.h index 1d6ef9654725..67c2c3b959e1 100644 --- a/drivers/acpi/apei/apei-internal.h +++ b/drivers/acpi/apei/apei-internal.h @@ -7,7 +7,6 @@ #ifndef APEI_INTERNAL_H #define APEI_INTERNAL_H -#include #include struct apei_exec_context; @@ -130,10 +129,5 @@ static inline u32 cper_estatus_len(struct acpi_hest_generic_status *estatus) return sizeof(*estatus) + estatus->data_length; } -void cper_estatus_print(const char *pfx, - const struct acpi_hest_generic_status *estatus); -int cper_estatus_check_header(const struct acpi_hest_generic_status *estatus); -int cper_estatus_check(const struct acpi_hest_generic_status *estatus); - int apei_osc_setup(void); #endif diff --git a/drivers/acpi/apei/bert.c b/drivers/acpi/apei/bert.c index c23eb75866d0..7514e38d5640 100644 --- a/drivers/acpi/apei/bert.c +++ b/drivers/acpi/apei/bert.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "apei-internal.h" diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index 67d5a20802e0..54a2822cae77 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -1133,4 +1133,7 @@ const u8 *__efi_get_smbios_string(const struct efi_smbios_record *record, void efi_remap_image(unsigned long image_base, unsigned alloc_size, unsigned long code_size); +asmlinkage efi_status_t __efiapi +efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab); + #endif diff --git a/include/linux/cper.h b/include/linux/cper.h index eacb7dd7b3af..c1a7dc325121 100644 --- a/include/linux/cper.h +++ b/include/linux/cper.h @@ -572,4 +572,10 @@ void cper_print_proc_ia(const char *pfx, int cper_mem_err_location(struct cper_mem_err_compact *mem, char *msg); int cper_dimm_err_location(struct cper_mem_err_compact *mem, char *msg); +struct acpi_hest_generic_status; +void cper_estatus_print(const char *pfx, + const struct acpi_hest_generic_status *estatus); +int cper_estatus_check_header(const struct acpi_hest_generic_status *estatus); +int cper_estatus_check(const struct acpi_hest_generic_status *estatus); + #endif diff --git a/include/linux/efi.h b/include/linux/efi.h index 7aa62c92185f..571d1a6e1b74 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1338,4 +1338,6 @@ bool efi_config_table_is_usable(const efi_guid_t *guid, unsigned long table) return xen_efi_config_table_is_usable(guid, table); } +umode_t efi_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n); + #endif /* _LINUX_EFI_H */ -- cgit v1.2.3 From 9828ed3f695a138f7add89fa2a186ababceb8006 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Thu, 25 May 2023 09:32:25 -0700 Subject: module: error out early on concurrent load of the same module file It turns out that udev under certain circumstances will concurrently try to load the same modules over-and-over excessively. This isn't a kernel bug, but it ends up affecting the kernel, to the point that under certain circumstances we can fail to boot, because the kernel uses a lot of memory to read all the module data all at once. Note that it isn't a memory leak, it's just basically a thundering herd problem happening at bootup with a lot of CPUs, with the worst cases then being pretty bad. Admittedly the worst situations are somewhat contrived: lots and lots of CPUs, not a lot of memory, and KASAN enabled to make it all slower and as such (unintentionally) exacerbate the problem. Luis explains: [1] "My best assessment of the situation is that each CPU in udev ends up triggering a load of duplicate set of modules, not just one, but *a lot*. Not sure what heuristics udev uses to load a set of modules per CPU." Petr Pavlu chimes in: [2] "My understanding is that udev workers are forked. An initial kmod context is created by the main udevd process but no sharing happens after the fork. It means that the mentioned memory pool logic doesn't really kick in. Multiple parallel load requests come from multiple udev workers, for instance, each handling an udev event for one CPU device and making the exactly same requests as all others are doing at the same time. The optimization idea would be to recognize these duplicate requests at the udevd/kmod level and converge them" Note that module loading has tried to mitigate this issue before, see for example commit 064f4536d139 ("module: avoid allocation if module is already present and ready"), which has a few ASCII graphs on memory use due to this same issue. However, while that noticed that the module was already loaded, and exited with an error early before spending any more time on setting up the module, it didn't handle the case of multiple concurrent module loads all being active - but not complete - at the same time. Yes, one of them will eventually win the race and finalize its copy, and the others will then notice that the module already exists and error out, but while this all happens, we have tons of unnecessary concurrent work being done. Again, the real fix is for udev to not do that (maybe it should use threads instead of fork, and have actual shared data structures and not cause duplicate work). That real fix is apparently not trivial. But it turns out that the kernel already has a pretty good model for dealing with concurrent access to the same file: the i_writecount of the inode. In fact, the module loading already indirectly uses 'i_writecount' , because 'kernel_file_read()' will in fact do ret = deny_write_access(file); if (ret) return ret; ... allow_write_access(file); around the read of the file data. We do not allow concurrent writes to the file, and return -ETXTBUSY if the file was open for writing at the same time as the module data is loaded from it. And the solution to the reader concurrency problem is to simply extend this "no concurrent writers" logic to simply be "exclusive access". Note that "exclusive" in this context isn't really some absolute thing: it's only exclusion from writers and from other "special readers" that do this writer denial. So we simply introduce a variation of that "deny_write_access()" logic that not only denies write access, but also requires that this is the _only_ such access that denies write access. Which means that you can't start loading a module that is already being loaded as a module by somebody else, or you will get the same -ETXTBSY error that you would get if there were writers around. [ It also means that you can't try to load a currently executing executable as a module, for the same reason: executables do that same "deny_write_access()" thing, and that's obviously where the whole ETXTBSY logic traditionally came from. This is not a problem for kernel modules, since the set of normal executable files and kernel module files is entirely disjoint. ] This new function is called "exclusive_deny_write_access()", and the implementation is trivial, in that it's just an atomic decrement of i_writecount if it was 0 before. To use that new exclusivity check, all we then do is wrap the module loading with that exclusive_deny_write_access()() / allow_write_access() pair. The actual patch is a bit bigger than that, because we want to surround not just the "load file data" part, but the whole module setup, to get maximum exclusion. So this ends up splitting up "finit_module()" into a few helper functions to make it all very clear and legible. In Luis' test-case (bringing up 255 vcpu's in a virtual machine [3]), the "wasted vmalloc" space (ie module data read into a vmalloc'ed area in order to be loaded as a module, but then discarded because somebody else loaded the same module instead) dropped from 1.8GiB to 474kB. Yes, that's gigabytes to kilobytes. It doesn't drop completely to zero, because even with this change, you can still end up having completely serial pointless module loads, where one udev process has loaded a module fully (and thus the kernel has released that exclusive lock on the module file), and then another udev process tries to load the same module again. So while we cannot fully get rid of the fundamental bug in user space, we _can_ get rid of the excessive concurrent thundering herd effect. A couple of final side notes on this all: - This tweak only affects the "finit_module()" system call, which gives the kernel a file descriptor with the module data. You can also just feed the module data as raw data from user space with "init_module()" (note the lack of 'f' at the beginning), and obviously for that case we do _not_ have any "exclusive read" logic. So if you absolutely want to do things wrong in user space, and try to load the same module multiple times, and error out only later when the kernel ends up saying "you can't load the same module name twice", you can still do that. And in fact, some distros will do exactly that, because they will uncompress the kernel module data in user space before feeding it to the kernel (mainly because they haven't started using the new kernel side decompression yet). So this is not some absolute "you can't do concurrent loads of the same module". It's literally just a very simple heuristic that will catch it early in case you try to load the exact same module file at the same time, and in that case avoid a potentially nasty situation. - There is another user of "deny_write_access()": the verity code that enables fs-verity on a file (the FS_IOC_ENABLE_VERITY ioctl). If you use fs-verity and you care about verifying the kernel modules (which does make sense), you should do it *before* loading said kernel module. That may sound obvious, but now the implementation basically requires it. Because if you try to do it concurrently, the kernel may refuse to load the module file that is being set up by the fs-verity code. - This all will obviously mean that if you insist on loading the same module in parallel, only one module load will succeed, and the others will return with an error. That was true before too, but what is different is that the -ETXTBSY error can be returned *before* the success case of another process fully loading and instantiating the module. Again, that might sound obvious, and it is indeed the whole point of the whole change: we are much quicker to notice the whole "you're already in the process of loading this module". So it's very much intentional, but it does mean that if you just spray the kernel with "finit_module()", and expect that the module is immediately loaded afterwards without checking the return value, you are doing something horribly horribly wrong. I'd like to say that that would never happen, but the whole _reason_ for this commit is that udev is currently doing something horribly horribly wrong, so ... Link: https://lore.kernel.org/all/ZEGopJ8VAYnE7LQ2@bombadil.infradead.org/ [1] Link: https://lore.kernel.org/all/23bd0ce6-ef78-1cd8-1f21-0e706a00424a@suse.com/ [2] Link: https://lore.kernel.org/lkml/ZG%2Fa+nrt4%2FAAUi5z@bombadil.infradead.org/ [3] Cc: Greg Kroah-Hartman Cc: Lucas De Marchi Cc: Petr Pavlu Tested-by: Luis Chamberlain Signed-off-by: Linus Torvalds --- include/linux/fs.h | 6 ++++++ kernel/module/main.c | 58 ++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 133f0640fb24..86b50271b4f7 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2566,6 +2566,12 @@ static inline int deny_write_access(struct file *file) struct inode *inode = file_inode(file); return atomic_dec_unless_positive(&inode->i_writecount) ? 0 : -ETXTBSY; } +static inline int exclusive_deny_write_access(struct file *file) +{ + int old = 0; + struct inode *inode = file_inode(file); + return atomic_try_cmpxchg(&inode->i_writecount, &old, -1) ? 0 : -ETXTBSY; +} static inline void put_write_access(struct inode * inode) { atomic_dec(&inode->i_writecount); diff --git a/kernel/module/main.c b/kernel/module/main.c index 044aa2c9e3cb..b4c7e925fdb0 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -3057,25 +3057,13 @@ SYSCALL_DEFINE3(init_module, void __user *, umod, return load_module(&info, uargs, 0); } -SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags) +static int file_init_module(struct file *file, const char __user * uargs, int flags) { struct load_info info = { }; void *buf = NULL; int len; - int err; - - err = may_init_module(); - if (err) - return err; - - pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags); - if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS - |MODULE_INIT_IGNORE_VERMAGIC - |MODULE_INIT_COMPRESSED_FILE)) - return -EINVAL; - - len = kernel_read_file_from_fd(fd, 0, &buf, INT_MAX, NULL, + len = kernel_read_file(file, 0, &buf, INT_MAX, NULL, READING_MODULE); if (len < 0) { mod_stat_inc(&failed_kreads); @@ -3084,7 +3072,7 @@ SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags) } if (flags & MODULE_INIT_COMPRESSED_FILE) { - err = module_decompress(&info, buf, len); + int err = module_decompress(&info, buf, len); vfree(buf); /* compressed data is no longer needed */ if (err) { mod_stat_inc(&failed_decompress); @@ -3099,6 +3087,46 @@ SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags) return load_module(&info, uargs, flags); } +/* + * kernel_read_file() will already deny write access, but module + * loading wants _exclusive_ access to the file, so we do that + * here, along with basic sanity checks. + */ +static int prepare_file_for_module_load(struct file *file) +{ + if (!file || !(file->f_mode & FMODE_READ)) + return -EBADF; + if (!S_ISREG(file_inode(file)->i_mode)) + return -EINVAL; + return exclusive_deny_write_access(file); +} + +SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags) +{ + struct fd f; + int err; + + err = may_init_module(); + if (err) + return err; + + pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags); + + if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS + |MODULE_INIT_IGNORE_VERMAGIC + |MODULE_INIT_COMPRESSED_FILE)) + return -EINVAL; + + f = fdget(fd); + err = prepare_file_for_module_load(f.file); + if (!err) { + err = file_init_module(f.file, uargs, flags); + allow_write_access(f.file); + } + fdput(f); + return err; +} + /* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */ char *module_flags(struct module *mod, char *buf, bool show_state) { -- cgit v1.2.3 From 36e4fc57fc1619f462e669e939209c45763bc8f5 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sun, 28 May 2023 19:36:02 +0200 Subject: efi: Bump stub image version for macOS HVF compatibility The macOS hypervisor framework includes a host-side VMM called VZLinuxBootLoader [1] which implements native support for booting the Linux kernel inside a guest directly (instead of, e.g., via GRUB installed inside the guest). On x86, it incorporates a BIOS style loader that does not implement or expose EFI to the loaded kernel. However, this loader appears to fail when the 'image minor version' field in the kernel image's PE/COFF header (which is generally only used by EFI based bootloaders) is set to any value other than 0x0. [2] Commit e346bebbd36b1576 ("efi: libstub: Always enable initrd command line loader and bump version") incremented the EFI stub image minor version to convey that all EFI stub kernels now implement support for the initrd= command line option, and do so in a way where it can load initrd images from any filesystem known to the EFI firmware (as opposed to prior implementations that could only load initrds from the same volume that the kernel image was loaded from). Unfortunately, bumping the version to v1.1 triggers this issue in VZLinuxBootLoader, breaking the boot on x86. So let's keep the image minor version at 0x0, and bump the image major version instead. While at it, convert this field to a bit field, so that individual features are discoverable from it, as suggested by Linus. So let's bump the major version to v3, and document the initrd= command line loading feature as being represented by bit 1 in the mask. Note that, due to the prior interpretation as a monotonically increasing version field, loaders are still permitted to assume that the LoadFile2 initrd loading feature is supported for any major version value >= 1, even if bit 0 is not set. [1] https://developer.apple.com/documentation/virtualization/vzlinuxbootloader [2] https://lore.kernel.org/linux-efi/CAG8fp8Teu4G9JuenQrqGndFt2Gy+V4YgJ=hN1xX7AD940YKf3A@mail.gmail.com/ Fixes: e346bebbd36b1576 ("efi: libstub: Always enable initrd command ...") Closes: https://bugzilla.kernel.org/show_bug.cgi?id=217485 Signed-off-by: Akihiro Suda [ardb: rewrite comment and commit log] Signed-off-by: Ard Biesheuvel --- include/linux/pe.h | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pe.h b/include/linux/pe.h index 5e1e11540870..fdf9c95709ba 100644 --- a/include/linux/pe.h +++ b/include/linux/pe.h @@ -11,25 +11,26 @@ #include /* - * Linux EFI stub v1.0 adds the following functionality: - * - Loading initrd from the LINUX_EFI_INITRD_MEDIA_GUID device path, - * - Loading/starting the kernel from firmware that targets a different - * machine type, via the entrypoint exposed in the .compat PE/COFF section. + * Starting from version v3.0, the major version field should be interpreted as + * a bit mask of features supported by the kernel's EFI stub: + * - 0x1: initrd loading from the LINUX_EFI_INITRD_MEDIA_GUID device path, + * - 0x2: initrd loading using the initrd= command line option, where the file + * may be specified using device path notation, and is not required to + * reside on the same volume as the loaded kernel image. * * The recommended way of loading and starting v1.0 or later kernels is to use * the LoadImage() and StartImage() EFI boot services, and expose the initrd * via the LINUX_EFI_INITRD_MEDIA_GUID device path. * - * Versions older than v1.0 support initrd loading via the image load options - * (using initrd=, limited to the volume from which the kernel itself was - * loaded), or via arch specific means (bootparams, DT, etc). + * Versions older than v1.0 may support initrd loading via the image load + * options (using initrd=, limited to the volume from which the kernel itself + * was loaded), or only via arch specific means (bootparams, DT, etc). * - * On x86, LoadImage() and StartImage() can be omitted if the EFI handover - * protocol is implemented, which can be inferred from the version, - * handover_offset and xloadflags fields in the bootparams structure. + * The minor version field must remain 0x0. + * (https://lore.kernel.org/all/efd6f2d4-547c-1378-1faa-53c044dbd297@gmail.com/) */ -#define LINUX_EFISTUB_MAJOR_VERSION 0x1 -#define LINUX_EFISTUB_MINOR_VERSION 0x1 +#define LINUX_EFISTUB_MAJOR_VERSION 0x3 +#define LINUX_EFISTUB_MINOR_VERSION 0x0 /* * LINUX_PE_MAGIC appears at offset 0x38 into the MS-DOS header of EFI bootable -- cgit v1.2.3 From ac2263b588dffd3a1efd7ed0b156ea6c5aea200d Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Mon, 29 May 2023 06:40:33 -0400 Subject: Revert "module: error out early on concurrent load of the same module file" This reverts commit 9828ed3f695a138f7add89fa2a186ababceb8006. Sadly, it does seem to cause failures to load modules. Johan Hovold reports: "This change breaks module loading during boot on the Lenovo Thinkpad X13s (aarch64). Specifically it results in indefinite probe deferral of the display and USB (ethernet) which makes it a pain to debug. Typing in the dark to acquire some logs reveals that other modules are missing as well" Since this was applied late as a "let's try this", I'm reverting it asap, and we can try to figure out what goes wrong later. The excessive parallel module loading problem is annoying, but not noticeable in normal situations, and this was only meant as an optimistic workaround for a user-space bug. One possible solution may be to do the optimistic exclusive open first, and then use a lock to serialize loading if that fails. Reported-by: Johan Hovold Link: https://lore.kernel.org/lkml/ZHRpH-JXAxA6DnzR@hovoldconsulting.com/ Signed-off-by: Linus Torvalds --- include/linux/fs.h | 6 ------ kernel/module/main.c | 58 ++++++++++++++-------------------------------------- 2 files changed, 15 insertions(+), 49 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 86b50271b4f7..133f0640fb24 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2566,12 +2566,6 @@ static inline int deny_write_access(struct file *file) struct inode *inode = file_inode(file); return atomic_dec_unless_positive(&inode->i_writecount) ? 0 : -ETXTBSY; } -static inline int exclusive_deny_write_access(struct file *file) -{ - int old = 0; - struct inode *inode = file_inode(file); - return atomic_try_cmpxchg(&inode->i_writecount, &old, -1) ? 0 : -ETXTBSY; -} static inline void put_write_access(struct inode * inode) { atomic_dec(&inode->i_writecount); diff --git a/kernel/module/main.c b/kernel/module/main.c index b4c7e925fdb0..044aa2c9e3cb 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -3057,13 +3057,25 @@ SYSCALL_DEFINE3(init_module, void __user *, umod, return load_module(&info, uargs, 0); } -static int file_init_module(struct file *file, const char __user * uargs, int flags) +SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags) { struct load_info info = { }; void *buf = NULL; int len; + int err; + + err = may_init_module(); + if (err) + return err; + + pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags); - len = kernel_read_file(file, 0, &buf, INT_MAX, NULL, + if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS + |MODULE_INIT_IGNORE_VERMAGIC + |MODULE_INIT_COMPRESSED_FILE)) + return -EINVAL; + + len = kernel_read_file_from_fd(fd, 0, &buf, INT_MAX, NULL, READING_MODULE); if (len < 0) { mod_stat_inc(&failed_kreads); @@ -3072,7 +3084,7 @@ static int file_init_module(struct file *file, const char __user * uargs, int fl } if (flags & MODULE_INIT_COMPRESSED_FILE) { - int err = module_decompress(&info, buf, len); + err = module_decompress(&info, buf, len); vfree(buf); /* compressed data is no longer needed */ if (err) { mod_stat_inc(&failed_decompress); @@ -3087,46 +3099,6 @@ static int file_init_module(struct file *file, const char __user * uargs, int fl return load_module(&info, uargs, flags); } -/* - * kernel_read_file() will already deny write access, but module - * loading wants _exclusive_ access to the file, so we do that - * here, along with basic sanity checks. - */ -static int prepare_file_for_module_load(struct file *file) -{ - if (!file || !(file->f_mode & FMODE_READ)) - return -EBADF; - if (!S_ISREG(file_inode(file)->i_mode)) - return -EINVAL; - return exclusive_deny_write_access(file); -} - -SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags) -{ - struct fd f; - int err; - - err = may_init_module(); - if (err) - return err; - - pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags); - - if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS - |MODULE_INIT_IGNORE_VERMAGIC - |MODULE_INIT_COMPRESSED_FILE)) - return -EINVAL; - - f = fdget(fd); - err = prepare_file_for_module_load(f.file); - if (!err) { - err = file_init_module(f.file, uargs, flags); - allow_write_access(f.file); - } - fdput(f); - return err; -} - /* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */ char *module_flags(struct module *mod, char *buf, bool show_state) { -- cgit v1.2.3 From 0143d148d1e882fb1538dc9974c94d63961719b9 Mon Sep 17 00:00:00 2001 From: Ruihan Li Date: Mon, 15 May 2023 21:09:55 +0800 Subject: usb: usbfs: Enforce page requirements for mmap The current implementation of usbdev_mmap uses usb_alloc_coherent to allocate memory pages that will later be mapped into the user space. Meanwhile, usb_alloc_coherent employs three different methods to allocate memory, as outlined below: * If hcd->localmem_pool is non-null, it uses gen_pool_dma_alloc to allocate memory; * If DMA is not available, it uses kmalloc to allocate memory; * Otherwise, it uses dma_alloc_coherent. However, it should be noted that gen_pool_dma_alloc does not guarantee that the resulting memory will be page-aligned. Furthermore, trying to map slab pages (i.e., memory allocated by kmalloc) into the user space is not resonable and can lead to problems, such as a type confusion bug when PAGE_TABLE_CHECK=y [1]. To address these issues, this patch introduces hcd_alloc_coherent_pages, which addresses the above two problems. Specifically, hcd_alloc_coherent_pages uses gen_pool_dma_alloc_align instead of gen_pool_dma_alloc to ensure that the memory is page-aligned. To replace kmalloc, hcd_alloc_coherent_pages directly allocates pages by calling __get_free_pages. Reported-by: syzbot+fcf1a817ceb50935ce99@syzkaller.appspotmail.comm Closes: https://lore.kernel.org/lkml/000000000000258e5e05fae79fc1@google.com/ [1] Fixes: f7d34b445abc ("USB: Add support for usbfs zerocopy.") Fixes: ff2437befd8f ("usb: host: Fix excessive alignment restriction for local memory allocations") Cc: stable@vger.kernel.org Signed-off-by: Ruihan Li Acked-by: Alan Stern Link: https://lore.kernel.org/r/20230515130958.32471-2-lrh2000@pku.edu.cn Signed-off-by: Greg Kroah-Hartman --- drivers/usb/core/buffer.c | 41 +++++++++++++++++++++++++++++++++++++++++ drivers/usb/core/devio.c | 9 +++++---- include/linux/usb/hcd.h | 5 +++++ 3 files changed, 51 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/core/buffer.c b/drivers/usb/core/buffer.c index fbb087b728dc..268ccbec88f9 100644 --- a/drivers/usb/core/buffer.c +++ b/drivers/usb/core/buffer.c @@ -172,3 +172,44 @@ void hcd_buffer_free( } dma_free_coherent(hcd->self.sysdev, size, addr, dma); } + +void *hcd_buffer_alloc_pages(struct usb_hcd *hcd, + size_t size, gfp_t mem_flags, dma_addr_t *dma) +{ + if (size == 0) + return NULL; + + if (hcd->localmem_pool) + return gen_pool_dma_alloc_align(hcd->localmem_pool, + size, dma, PAGE_SIZE); + + /* some USB hosts just use PIO */ + if (!hcd_uses_dma(hcd)) { + *dma = DMA_MAPPING_ERROR; + return (void *)__get_free_pages(mem_flags, + get_order(size)); + } + + return dma_alloc_coherent(hcd->self.sysdev, + size, dma, mem_flags); +} + +void hcd_buffer_free_pages(struct usb_hcd *hcd, + size_t size, void *addr, dma_addr_t dma) +{ + if (!addr) + return; + + if (hcd->localmem_pool) { + gen_pool_free(hcd->localmem_pool, + (unsigned long)addr, size); + return; + } + + if (!hcd_uses_dma(hcd)) { + free_pages((unsigned long)addr, get_order(size)); + return; + } + + dma_free_coherent(hcd->self.sysdev, size, addr, dma); +} diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c index e501a03d6c70..3936ca7f7d2f 100644 --- a/drivers/usb/core/devio.c +++ b/drivers/usb/core/devio.c @@ -186,6 +186,7 @@ static int connected(struct usb_dev_state *ps) static void dec_usb_memory_use_count(struct usb_memory *usbm, int *count) { struct usb_dev_state *ps = usbm->ps; + struct usb_hcd *hcd = bus_to_hcd(ps->dev->bus); unsigned long flags; spin_lock_irqsave(&ps->lock, flags); @@ -194,8 +195,8 @@ static void dec_usb_memory_use_count(struct usb_memory *usbm, int *count) list_del(&usbm->memlist); spin_unlock_irqrestore(&ps->lock, flags); - usb_free_coherent(ps->dev, usbm->size, usbm->mem, - usbm->dma_handle); + hcd_buffer_free_pages(hcd, usbm->size, + usbm->mem, usbm->dma_handle); usbfs_decrease_memory_usage( usbm->size + sizeof(struct usb_memory)); kfree(usbm); @@ -247,8 +248,8 @@ static int usbdev_mmap(struct file *file, struct vm_area_struct *vma) goto error_decrease_mem; } - mem = usb_alloc_coherent(ps->dev, size, GFP_USER | __GFP_NOWARN, - &dma_handle); + mem = hcd_buffer_alloc_pages(hcd, + size, GFP_USER | __GFP_NOWARN, &dma_handle); if (!mem) { ret = -ENOMEM; goto error_free_usbm; diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 094c77eaf455..0c7eff91adf4 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -501,6 +501,11 @@ void *hcd_buffer_alloc(struct usb_bus *bus, size_t size, void hcd_buffer_free(struct usb_bus *bus, size_t size, void *addr, dma_addr_t dma); +void *hcd_buffer_alloc_pages(struct usb_hcd *hcd, + size_t size, gfp_t mem_flags, dma_addr_t *dma); +void hcd_buffer_free_pages(struct usb_hcd *hcd, + size_t size, void *addr, dma_addr_t dma); + /* generic bus glue, needed for host controllers that don't use PCI */ extern irqreturn_t usb_hcd_irq(int irq, void *__hcd); -- cgit v1.2.3 From 44d0fb387b53e56c8a050bac5c7d460e21eb226f Mon Sep 17 00:00:00 2001 From: Ruihan Li Date: Mon, 15 May 2023 21:09:58 +0800 Subject: mm: page_table_check: Ensure user pages are not slab pages The current uses of PageAnon in page table check functions can lead to type confusion bugs between struct page and slab [1], if slab pages are accidentally mapped into the user space. This is because slab reuses the bits in struct page to store its internal states, which renders PageAnon ineffective on slab pages. Since slab pages are not expected to be mapped into the user space, this patch adds BUG_ON(PageSlab(page)) checks to make sure that slab pages are not inadvertently mapped. Otherwise, there must be some bugs in the kernel. Reported-by: syzbot+fcf1a817ceb50935ce99@syzkaller.appspotmail.com Closes: https://lore.kernel.org/lkml/000000000000258e5e05fae79fc1@google.com/ [1] Fixes: df4e817b7108 ("mm: page table check") Cc: # 5.17 Signed-off-by: Ruihan Li Acked-by: Pasha Tatashin Link: https://lore.kernel.org/r/20230515130958.32471-5-lrh2000@pku.edu.cn Signed-off-by: Greg Kroah-Hartman --- include/linux/page-flags.h | 6 ++++++ mm/page_table_check.c | 6 ++++++ 2 files changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 1c68d67b832f..92a2063a0a23 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -617,6 +617,12 @@ PAGEFLAG_FALSE(VmemmapSelfHosted, vmemmap_self_hosted) * Please note that, confusingly, "page_mapping" refers to the inode * address_space which maps the page from disk; whereas "page_mapped" * refers to user virtual address space into which the page is mapped. + * + * For slab pages, since slab reuses the bits in struct page to store its + * internal states, the page->mapping does not exist as such, nor do these + * flags below. So in order to avoid testing non-existent bits, please + * make sure that PageSlab(page) actually evaluates to false before calling + * the following functions (e.g., PageAnon). See mm/slab.h. */ #define PAGE_MAPPING_ANON 0x1 #define PAGE_MAPPING_MOVABLE 0x2 diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 25d8610c0042..f2baf97d5f38 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -71,6 +71,8 @@ static void page_table_check_clear(struct mm_struct *mm, unsigned long addr, page = pfn_to_page(pfn); page_ext = page_ext_get(page); + + BUG_ON(PageSlab(page)); anon = PageAnon(page); for (i = 0; i < pgcnt; i++) { @@ -107,6 +109,8 @@ static void page_table_check_set(struct mm_struct *mm, unsigned long addr, page = pfn_to_page(pfn); page_ext = page_ext_get(page); + + BUG_ON(PageSlab(page)); anon = PageAnon(page); for (i = 0; i < pgcnt; i++) { @@ -133,6 +137,8 @@ void __page_table_check_zero(struct page *page, unsigned int order) struct page_ext *page_ext; unsigned long i; + BUG_ON(PageSlab(page)); + page_ext = page_ext_get(page); BUG_ON(!page_ext); for (i = 0; i < (1ul << order); i++) { -- cgit v1.2.3 From c034203b6a9dae6751ef4371c18cb77983e30c28 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 29 May 2023 14:35:55 +0300 Subject: nfsd: fix double fget() bug in __write_ports_addfd() The bug here is that you cannot rely on getting the same socket from multiple calls to fget() because userspace can influence that. This is a kind of double fetch bug. The fix is to delete the svc_alien_sock() function and instead do the checking inside the svc_addsock() function. Fixes: 3064639423c4 ("nfsd: check passed socket's net matches NFSd superblock's one") Signed-off-by: Dan Carpenter Reviewed-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfsctl.c | 7 +------ include/linux/sunrpc/svcsock.h | 7 +++---- net/sunrpc/svcsock.c | 24 ++++++------------------ 3 files changed, 10 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index c159817d1282..b4fd7a7062d5 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -690,16 +690,11 @@ static ssize_t __write_ports_addfd(char *buf, struct net *net, const struct cred if (err != 0 || fd < 0) return -EINVAL; - if (svc_alien_sock(net, fd)) { - printk(KERN_ERR "%s: socket net is different to NFSd's one\n", __func__); - return -EINVAL; - } - err = nfsd_create_serv(net); if (err != 0) return err; - err = svc_addsock(nn->nfsd_serv, fd, buf, SIMPLE_TRANSACTION_LIMIT, cred); + err = svc_addsock(nn->nfsd_serv, net, fd, buf, SIMPLE_TRANSACTION_LIMIT, cred); if (err >= 0 && !nn->nfsd_serv->sv_nrthreads && !xchg(&nn->keep_active, 1)) diff --git a/include/linux/sunrpc/svcsock.h b/include/linux/sunrpc/svcsock.h index d16ae621782c..a7116048a4d4 100644 --- a/include/linux/sunrpc/svcsock.h +++ b/include/linux/sunrpc/svcsock.h @@ -61,10 +61,9 @@ int svc_recv(struct svc_rqst *, long); void svc_send(struct svc_rqst *rqstp); void svc_drop(struct svc_rqst *); void svc_sock_update_bufs(struct svc_serv *serv); -bool svc_alien_sock(struct net *net, int fd); -int svc_addsock(struct svc_serv *serv, const int fd, - char *name_return, const size_t len, - const struct cred *cred); +int svc_addsock(struct svc_serv *serv, struct net *net, + const int fd, char *name_return, const size_t len, + const struct cred *cred); void svc_init_xprt_sock(void); void svc_cleanup_xprt_sock(void); struct svc_xprt *svc_sock_create(struct svc_serv *serv, int prot); diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index 63fe7a338992..f77cebe2c071 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -1480,25 +1480,10 @@ static struct svc_sock *svc_setup_socket(struct svc_serv *serv, return svsk; } -bool svc_alien_sock(struct net *net, int fd) -{ - int err; - struct socket *sock = sockfd_lookup(fd, &err); - bool ret = false; - - if (!sock) - goto out; - if (sock_net(sock->sk) != net) - ret = true; - sockfd_put(sock); -out: - return ret; -} -EXPORT_SYMBOL_GPL(svc_alien_sock); - /** * svc_addsock - add a listener socket to an RPC service * @serv: pointer to RPC service to which to add a new listener + * @net: caller's network namespace * @fd: file descriptor of the new listener * @name_return: pointer to buffer to fill in with name of listener * @len: size of the buffer @@ -1508,8 +1493,8 @@ EXPORT_SYMBOL_GPL(svc_alien_sock); * Name is terminated with '\n'. On error, returns a negative errno * value. */ -int svc_addsock(struct svc_serv *serv, const int fd, char *name_return, - const size_t len, const struct cred *cred) +int svc_addsock(struct svc_serv *serv, struct net *net, const int fd, + char *name_return, const size_t len, const struct cred *cred) { int err = 0; struct socket *so = sockfd_lookup(fd, &err); @@ -1520,6 +1505,9 @@ int svc_addsock(struct svc_serv *serv, const int fd, char *name_return, if (!so) return err; + err = -EINVAL; + if (sock_net(so->sk) != net) + goto out; err = -EAFNOSUPPORT; if ((so->sk->sk_family != PF_INET) && (so->sk->sk_family != PF_INET6)) goto out; -- cgit v1.2.3 From 4420528254153189c70b6267593e445dc8654e37 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 29 May 2023 12:52:07 -0600 Subject: firewire: Replace zero-length array with flexible-array member MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zero-length and one-element arrays are deprecated, and we are moving towards adopting C99 flexible-array members, instead. Address the following warnings found with GCC-13 and -fstrict-flex-arrays=3 enabled: sound/firewire/amdtp-stream.c: In function ‘build_it_pkt_header’: sound/firewire/amdtp-stream.c:694:17: warning: ‘generate_cip_header’ accessing 8 bytes in a region of size 0 [-Wstringop-overflow=] 694 | generate_cip_header(s, cip_header, data_block_counter, syt); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ sound/firewire/amdtp-stream.c:694:17: note: referencing argument 2 of type ‘__be32[2]’ {aka ‘unsigned int[2]’} sound/firewire/amdtp-stream.c:667:13: note: in a call to function ‘generate_cip_header’ 667 | static void generate_cip_header(struct amdtp_stream *s, __be32 cip_header[2], | ^~~~~~~~~~~~~~~~~~~ This helps with the ongoing efforts to tighten the FORTIFY_SOURCE routines on memcpy() and help us make progress towards globally enabling -fstrict-flex-arrays=3 [1]. Link: https://github.com/KSPP/linux/issues/21 Link: https://github.com/KSPP/linux/issues/303 Link: https://gcc.gnu.org/pipermail/gcc-patches/2022-October/602902.html [1] Signed-off-by: Gustavo A. R. Silva Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/ZHT0V3SpvHyxCv5W@work Signed-off-by: Takashi Sakamoto --- include/linux/firewire.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/firewire.h b/include/linux/firewire.h index 1716c01c4e54..efb6e2cf2034 100644 --- a/include/linux/firewire.h +++ b/include/linux/firewire.h @@ -391,7 +391,7 @@ struct fw_iso_packet { u32 tag:2; /* tx: Tag in packet header */ u32 sy:4; /* tx: Sy in packet header */ u32 header_length:8; /* Length of immediate header */ - u32 header[0]; /* tx: Top of 1394 isoch. data_block */ + u32 header[]; /* tx: Top of 1394 isoch. data_block */ }; #define FW_ISO_CONTEXT_TRANSMIT 0 -- cgit v1.2.3 From f9010dbdce911ee1f1af1398a24b1f9f992e0080 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Thu, 1 Jun 2023 13:32:32 -0500 Subject: fork, vhost: Use CLONE_THREAD to fix freezer/ps regression When switching from kthreads to vhost_tasks two bugs were added: 1. The vhost worker tasks's now show up as processes so scripts doing ps or ps a would not incorrectly detect the vhost task as another process. 2. kthreads disabled freeze by setting PF_NOFREEZE, but vhost tasks's didn't disable or add support for them. To fix both bugs, this switches the vhost task to be thread in the process that does the VHOST_SET_OWNER ioctl, and has vhost_worker call get_signal to support SIGKILL/SIGSTOP and freeze signals. Note that SIGKILL/STOP support is required because CLONE_THREAD requires CLONE_SIGHAND which requires those 2 signals to be supported. This is a modified version of the patch written by Mike Christie which was a modified version of patch originally written by Linus. Much of what depended upon PF_IO_WORKER now depends on PF_USER_WORKER. Including ignoring signals, setting up the register state, and having get_signal return instead of calling do_group_exit. Tidied up the vhost_task abstraction so that the definition of vhost_task only needs to be visible inside of vhost_task.c. Making it easier to review the code and tell what needs to be done where. As part of this the main loop has been moved from vhost_worker into vhost_task_fn. vhost_worker now returns true if work was done. The main loop has been updated to call get_signal which handles SIGSTOP, freezing, and collects the message that tells the thread to exit as part of process exit. This collection clears __fatal_signal_pending. This collection is not guaranteed to clear signal_pending() so clear that explicitly so the schedule() sleeps. For now the vhost thread continues to exist and run work until the last file descriptor is closed and the release function is called as part of freeing struct file. To avoid hangs in the coredump rendezvous and when killing threads in a multi-threaded exec. The coredump code and de_thread have been modified to ignore vhost threads. Remvoing the special case for exec appears to require teaching vhost_dev_flush how to directly complete transactions in case the vhost thread is no longer running. Removing the special case for coredump rendezvous requires either the above fix needed for exec or moving the coredump rendezvous into get_signal. Fixes: 6e890c5d5021 ("vhost: use vhost_tasks for worker threads") Signed-off-by: Eric W. Biederman Co-developed-by: Mike Christie Signed-off-by: Mike Christie Acked-by: Michael S. Tsirkin Signed-off-by: Linus Torvalds --- arch/x86/include/asm/fpu/sched.h | 2 +- arch/x86/kernel/fpu/context.h | 2 +- arch/x86/kernel/fpu/core.c | 2 +- drivers/vhost/vhost.c | 22 +++------- fs/coredump.c | 4 +- include/linux/sched/task.h | 1 - include/linux/sched/vhost_task.h | 15 ++----- kernel/exit.c | 5 ++- kernel/fork.c | 13 +++--- kernel/signal.c | 8 ++-- kernel/vhost_task.c | 92 ++++++++++++++++++++++++++-------------- 11 files changed, 89 insertions(+), 77 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/fpu/sched.h b/arch/x86/include/asm/fpu/sched.h index c2d6cd78ed0c..78fcde7b1f07 100644 --- a/arch/x86/include/asm/fpu/sched.h +++ b/arch/x86/include/asm/fpu/sched.h @@ -39,7 +39,7 @@ extern void fpu_flush_thread(void); static inline void switch_fpu_prepare(struct fpu *old_fpu, int cpu) { if (cpu_feature_enabled(X86_FEATURE_FPU) && - !(current->flags & (PF_KTHREAD | PF_IO_WORKER))) { + !(current->flags & (PF_KTHREAD | PF_USER_WORKER))) { save_fpregs_to_fpstate(old_fpu); /* * The save operation preserved register state, so the diff --git a/arch/x86/kernel/fpu/context.h b/arch/x86/kernel/fpu/context.h index 9fcfa5c4dad7..af5cbdd9bd29 100644 --- a/arch/x86/kernel/fpu/context.h +++ b/arch/x86/kernel/fpu/context.h @@ -57,7 +57,7 @@ static inline void fpregs_restore_userregs(void) struct fpu *fpu = ¤t->thread.fpu; int cpu = smp_processor_id(); - if (WARN_ON_ONCE(current->flags & (PF_KTHREAD | PF_IO_WORKER))) + if (WARN_ON_ONCE(current->flags & (PF_KTHREAD | PF_USER_WORKER))) return; if (!fpregs_state_valid(fpu, cpu)) { diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index caf33486dc5e..1015af1ae562 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -426,7 +426,7 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask) this_cpu_write(in_kernel_fpu, true); - if (!(current->flags & (PF_KTHREAD | PF_IO_WORKER)) && + if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) && !test_thread_flag(TIF_NEED_FPU_LOAD)) { set_thread_flag(TIF_NEED_FPU_LOAD); save_fpregs_to_fpstate(¤t->thread.fpu); diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index a92af08e7864..074273020849 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -256,7 +256,7 @@ void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work) * test_and_set_bit() implies a memory barrier. */ llist_add(&work->node, &dev->worker->work_list); - wake_up_process(dev->worker->vtsk->task); + vhost_task_wake(dev->worker->vtsk); } } EXPORT_SYMBOL_GPL(vhost_work_queue); @@ -333,31 +333,19 @@ static void vhost_vq_reset(struct vhost_dev *dev, __vhost_vq_meta_reset(vq); } -static int vhost_worker(void *data) +static bool vhost_worker(void *data) { struct vhost_worker *worker = data; struct vhost_work *work, *work_next; struct llist_node *node; - for (;;) { - /* mb paired w/ kthread_stop */ - set_current_state(TASK_INTERRUPTIBLE); - - if (vhost_task_should_stop(worker->vtsk)) { - __set_current_state(TASK_RUNNING); - break; - } - - node = llist_del_all(&worker->work_list); - if (!node) - schedule(); - + node = llist_del_all(&worker->work_list); + if (node) { node = llist_reverse_order(node); /* make sure flag is seen after deletion */ smp_wmb(); llist_for_each_entry_safe(work, work_next, node, node) { clear_bit(VHOST_WORK_QUEUED, &work->flags); - __set_current_state(TASK_RUNNING); kcov_remote_start_common(worker->kcov_handle); work->fn(work); kcov_remote_stop(); @@ -365,7 +353,7 @@ static int vhost_worker(void *data) } } - return 0; + return !!node; } static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq) diff --git a/fs/coredump.c b/fs/coredump.c index ece7badf701b..88740c51b942 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -371,7 +371,9 @@ static int zap_process(struct task_struct *start, int exit_code) if (t != current && !(t->flags & PF_POSTCOREDUMP)) { sigaddset(&t->pending.signal, SIGKILL); signal_wake_up(t, 1); - nr++; + /* The vhost_worker does not particpate in coredumps */ + if ((t->flags & (PF_USER_WORKER | PF_IO_WORKER)) != PF_USER_WORKER) + nr++; } } diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index 537cbf9a2ade..e0f5ac90a228 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -29,7 +29,6 @@ struct kernel_clone_args { u32 io_thread:1; u32 user_worker:1; u32 no_files:1; - u32 ignore_signals:1; unsigned long stack; unsigned long stack_size; unsigned long tls; diff --git a/include/linux/sched/vhost_task.h b/include/linux/sched/vhost_task.h index 6123c10b99cf..837a23624a66 100644 --- a/include/linux/sched/vhost_task.h +++ b/include/linux/sched/vhost_task.h @@ -2,22 +2,13 @@ #ifndef _LINUX_VHOST_TASK_H #define _LINUX_VHOST_TASK_H -#include -struct task_struct; +struct vhost_task; -struct vhost_task { - int (*fn)(void *data); - void *data; - struct completion exited; - unsigned long flags; - struct task_struct *task; -}; - -struct vhost_task *vhost_task_create(int (*fn)(void *), void *arg, +struct vhost_task *vhost_task_create(bool (*fn)(void *), void *arg, const char *name); void vhost_task_start(struct vhost_task *vtsk); void vhost_task_stop(struct vhost_task *vtsk); -bool vhost_task_should_stop(struct vhost_task *vtsk); +void vhost_task_wake(struct vhost_task *vtsk); #endif diff --git a/kernel/exit.c b/kernel/exit.c index 34b90e2e7cf7..edb50b4c9972 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -411,7 +411,10 @@ static void coredump_task_exit(struct task_struct *tsk) tsk->flags |= PF_POSTCOREDUMP; core_state = tsk->signal->core_state; spin_unlock_irq(&tsk->sighand->siglock); - if (core_state) { + + /* The vhost_worker does not particpate in coredumps */ + if (core_state && + ((tsk->flags & (PF_IO_WORKER | PF_USER_WORKER)) != PF_USER_WORKER)) { struct core_thread self; self.task = current; diff --git a/kernel/fork.c b/kernel/fork.c index ed4e01daccaa..81cba91f30bb 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2336,16 +2336,16 @@ __latent_entropy struct task_struct *copy_process( p->flags &= ~PF_KTHREAD; if (args->kthread) p->flags |= PF_KTHREAD; - if (args->user_worker) - p->flags |= PF_USER_WORKER; - if (args->io_thread) { + if (args->user_worker) { /* - * Mark us an IO worker, and block any signal that isn't + * Mark us a user worker, and block any signal that isn't * fatal or STOP */ - p->flags |= PF_IO_WORKER; + p->flags |= PF_USER_WORKER; siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP)); } + if (args->io_thread) + p->flags |= PF_IO_WORKER; if (args->name) strscpy_pad(p->comm, args->name, sizeof(p->comm)); @@ -2517,9 +2517,6 @@ __latent_entropy struct task_struct *copy_process( if (retval) goto bad_fork_cleanup_io; - if (args->ignore_signals) - ignore_signals(p); - stackleak_task_init(p); if (pid != &init_struct_pid) { diff --git a/kernel/signal.c b/kernel/signal.c index 8f6330f0e9ca..2547fa73bde5 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1368,7 +1368,9 @@ int zap_other_threads(struct task_struct *p) while_each_thread(p, t) { task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK); - count++; + /* Don't require de_thread to wait for the vhost_worker */ + if ((t->flags & (PF_IO_WORKER | PF_USER_WORKER)) != PF_USER_WORKER) + count++; /* Don't bother with already dead threads */ if (t->exit_state) @@ -2861,11 +2863,11 @@ relock: } /* - * PF_IO_WORKER threads will catch and exit on fatal signals + * PF_USER_WORKER threads will catch and exit on fatal signals * themselves. They have cleanup that must be performed, so * we cannot call do_exit() on their behalf. */ - if (current->flags & PF_IO_WORKER) + if (current->flags & PF_USER_WORKER) goto out; /* diff --git a/kernel/vhost_task.c b/kernel/vhost_task.c index b7cbd66f889e..f80d5c51ae67 100644 --- a/kernel/vhost_task.c +++ b/kernel/vhost_task.c @@ -12,58 +12,88 @@ enum vhost_task_flags { VHOST_TASK_FLAGS_STOP, }; +struct vhost_task { + bool (*fn)(void *data); + void *data; + struct completion exited; + unsigned long flags; + struct task_struct *task; +}; + static int vhost_task_fn(void *data) { struct vhost_task *vtsk = data; - int ret; + bool dead = false; + + for (;;) { + bool did_work; + + /* mb paired w/ vhost_task_stop */ + if (test_bit(VHOST_TASK_FLAGS_STOP, &vtsk->flags)) + break; + + if (!dead && signal_pending(current)) { + struct ksignal ksig; + /* + * Calling get_signal will block in SIGSTOP, + * or clear fatal_signal_pending, but remember + * what was set. + * + * This thread won't actually exit until all + * of the file descriptors are closed, and + * the release function is called. + */ + dead = get_signal(&ksig); + if (dead) + clear_thread_flag(TIF_SIGPENDING); + } + + did_work = vtsk->fn(vtsk->data); + if (!did_work) { + set_current_state(TASK_INTERRUPTIBLE); + schedule(); + } + } - ret = vtsk->fn(vtsk->data); complete(&vtsk->exited); - do_exit(ret); + do_exit(0); +} + +/** + * vhost_task_wake - wakeup the vhost_task + * @vtsk: vhost_task to wake + * + * wake up the vhost_task worker thread + */ +void vhost_task_wake(struct vhost_task *vtsk) +{ + wake_up_process(vtsk->task); } +EXPORT_SYMBOL_GPL(vhost_task_wake); /** * vhost_task_stop - stop a vhost_task * @vtsk: vhost_task to stop * - * Callers must call vhost_task_should_stop and return from their worker - * function when it returns true; + * vhost_task_fn ensures the worker thread exits after + * VHOST_TASK_FLAGS_SOP becomes true. */ void vhost_task_stop(struct vhost_task *vtsk) { - pid_t pid = vtsk->task->pid; - set_bit(VHOST_TASK_FLAGS_STOP, &vtsk->flags); - wake_up_process(vtsk->task); + vhost_task_wake(vtsk); /* * Make sure vhost_task_fn is no longer accessing the vhost_task before - * freeing it below. If userspace crashed or exited without closing, - * then the vhost_task->task could already be marked dead so - * kernel_wait will return early. + * freeing it below. */ wait_for_completion(&vtsk->exited); - /* - * If we are just closing/removing a device and the parent process is - * not exiting then reap the task. - */ - kernel_wait4(pid, NULL, __WCLONE, NULL); kfree(vtsk); } EXPORT_SYMBOL_GPL(vhost_task_stop); /** - * vhost_task_should_stop - should the vhost task return from the work function - * @vtsk: vhost_task to stop - */ -bool vhost_task_should_stop(struct vhost_task *vtsk) -{ - return test_bit(VHOST_TASK_FLAGS_STOP, &vtsk->flags); -} -EXPORT_SYMBOL_GPL(vhost_task_should_stop); - -/** - * vhost_task_create - create a copy of a process to be used by the kernel - * @fn: thread stack + * vhost_task_create - create a copy of a task to be used by the kernel + * @fn: vhost worker function * @arg: data to be passed to fn * @name: the thread's name * @@ -71,17 +101,17 @@ EXPORT_SYMBOL_GPL(vhost_task_should_stop); * failure. The returned task is inactive, and the caller must fire it up * through vhost_task_start(). */ -struct vhost_task *vhost_task_create(int (*fn)(void *), void *arg, +struct vhost_task *vhost_task_create(bool (*fn)(void *), void *arg, const char *name) { struct kernel_clone_args args = { - .flags = CLONE_FS | CLONE_UNTRACED | CLONE_VM, + .flags = CLONE_FS | CLONE_UNTRACED | CLONE_VM | + CLONE_THREAD | CLONE_SIGHAND, .exit_signal = 0, .fn = vhost_task_fn, .name = name, .user_worker = 1, .no_files = 1, - .ignore_signals = 1, }; struct vhost_task *vtsk; struct task_struct *tsk; -- cgit v1.2.3