From 1dff4156d1f63b525c54aea7f097a657cbbbf837 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 7 Feb 2020 13:38:50 +0100 Subject: lib/vdso: Allow the high resolution parts to be compiled out If the architecture knows at compile time that there is no VDSO capable clocksource supported it makes sense to optimize the guts of the high resolution parts of the VDSO out at build time. Add a helper function to check whether the VDSO should be high resolution capable and provide a stub which can be overridden by an architecture. Signed-off-by: Thomas Gleixner Tested-by: Vincenzo Frascino Reviewed-by: Vincenzo Frascino Link: https://lkml.kernel.org/r/20200207124402.530143168@linutronix.de --- lib/vdso/gettimeofday.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index f8b8ec5e63ac..5804e4e168e7 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -38,6 +38,13 @@ u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) } #endif +#ifndef __arch_vdso_hres_capable +static inline bool __arch_vdso_hres_capable(void) +{ + return true; +} +#endif + #ifdef CONFIG_TIME_NS static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, struct __kernel_timespec *ts) @@ -101,6 +108,10 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, u64 cycles, last, sec, ns; u32 seq; + /* Allows to compile the high resolution parts out */ + if (!__arch_vdso_hres_capable()) + return -1; + do { /* * Open coded to handle VCLOCK_TIMENS. Time namespace -- cgit v1.2.3 From 5d51bee725cc1497352d6b0b604e42a90c680540 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 7 Feb 2020 13:38:55 +0100 Subject: clocksource: Add common vdso clock mode storage All architectures which use the generic VDSO code have their own storage for the VDSO clock mode. That's pointless and just requires duplicate code. Provide generic storage for it. The new Kconfig symbol is intermediate and will be removed once all architectures are converted over. Signed-off-by: Thomas Gleixner Tested-by: Vincenzo Frascino Reviewed-by: Vincenzo Frascino Link: https://lkml.kernel.org/r/20200207124403.028046322@linutronix.de --- include/linux/clocksource.h | 12 +++++++++++- kernel/time/clocksource.c | 9 +++++++++ kernel/time/vsyscall.c | 10 ++++++++-- lib/vdso/Kconfig | 3 +++ lib/vdso/gettimeofday.c | 13 +++++++++++-- 5 files changed, 42 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index 2c4574b517d2..6d5ed1b4d24d 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -23,10 +23,19 @@ struct clocksource; struct module; -#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA +#if defined(CONFIG_ARCH_CLOCKSOURCE_DATA) || \ + defined(CONFIG_GENERIC_VDSO_CLOCK_MODE) #include #endif +enum vdso_clock_mode { + VDSO_CLOCKMODE_NONE, +#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE + VDSO_ARCH_CLOCKMODES, +#endif + VDSO_CLOCKMODE_MAX, +}; + /** * struct clocksource - hardware abstraction for a free running counter * Provides mostly state-free accessors to the underlying hardware. @@ -97,6 +106,7 @@ struct clocksource { const char *name; struct list_head list; int rating; + enum vdso_clock_mode vdso_clock_mode; unsigned long flags; int (*enable)(struct clocksource *cs); diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 428beb69426a..7cb09c4cf21c 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -928,6 +928,15 @@ int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq) clocksource_arch_init(cs); +#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE + if (cs->vdso_clock_mode < 0 || + cs->vdso_clock_mode >= VDSO_CLOCKMODE_MAX) { + pr_warn("clocksource %s registered with invalid VDSO mode %d. Disabling VDSO support.\n", + cs->name, cs->vdso_clock_mode); + cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; + } +#endif + /* Initialize mult/shift and max_idle_ns */ __clocksource_update_freq_scale(cs, scale, freq); diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c index 9577c89179cd..f9a5178c69bb 100644 --- a/kernel/time/vsyscall.c +++ b/kernel/time/vsyscall.c @@ -71,13 +71,19 @@ void update_vsyscall(struct timekeeper *tk) { struct vdso_data *vdata = __arch_get_k_vdso_data(); struct vdso_timestamp *vdso_ts; + s32 clock_mode; u64 nsec; /* copy vsyscall data */ vdso_write_begin(vdata); - vdata[CS_HRES_COARSE].clock_mode = __arch_get_clock_mode(tk); - vdata[CS_RAW].clock_mode = __arch_get_clock_mode(tk); +#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE + clock_mode = tk->tkr_mono.clock->vdso_clock_mode; +#else + clock_mode = __arch_get_clock_mode(tk); +#endif + vdata[CS_HRES_COARSE].clock_mode = clock_mode; + vdata[CS_RAW].clock_mode = clock_mode; /* CLOCK_REALTIME also required for time() */ vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_REALTIME]; diff --git a/lib/vdso/Kconfig b/lib/vdso/Kconfig index d883ac299508..d9f43c84fcc6 100644 --- a/lib/vdso/Kconfig +++ b/lib/vdso/Kconfig @@ -30,4 +30,7 @@ config GENERIC_VDSO_TIME_NS Selected by architectures which support time namespaces in the VDSO +config GENERIC_VDSO_CLOCK_MODE + bool + endif diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 5804e4e168e7..3f2d8b859130 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -64,10 +65,14 @@ static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, do { seq = vdso_read_begin(vd); + if (IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && + vd->clock_mode == VDSO_CLOCKMODE_NONE) + return -1; cycles = __arch_get_hw_counter(vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; - if (unlikely((s64)cycles < 0)) + if (!IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && + unlikely((s64)cycles < 0)) return -1; ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); @@ -132,10 +137,14 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, } smp_rmb(); + if (IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && + vd->clock_mode == VDSO_CLOCKMODE_NONE) + return -1; cycles = __arch_get_hw_counter(vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; - if (unlikely((s64)cycles < 0)) + if (!IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && + unlikely((s64)cycles < 0)) return -1; ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); -- cgit v1.2.3 From f86fd32db706613fe8d0104057efa6e83e0d7e8f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 7 Feb 2020 13:38:59 +0100 Subject: lib/vdso: Cleanup clock mode storage leftovers Now that all architectures are converted to use the generic storage the helpers and conditionals can be removed. Signed-off-by: Thomas Gleixner Tested-by: Vincenzo Frascino Reviewed-by: Vincenzo Frascino Link: https://lkml.kernel.org/r/20200207124403.470699892@linutronix.de --- arch/arm/mm/Kconfig | 1 - arch/arm64/Kconfig | 1 - arch/mips/Kconfig | 1 - arch/x86/Kconfig | 1 - include/asm-generic/vdso/vsyscall.h | 7 ------- include/linux/clocksource.h | 4 ++-- kernel/time/vsyscall.c | 4 ---- lib/vdso/Kconfig | 3 --- lib/vdso/gettimeofday.c | 17 +++++------------ 9 files changed, 7 insertions(+), 32 deletions(-) (limited to 'lib') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 865e888bb84f..65e4482e3849 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -900,7 +900,6 @@ config VDSO select GENERIC_TIME_VSYSCALL select GENERIC_VDSO_32 select GENERIC_GETTIMEOFDAY - select GENERIC_VDSO_CLOCK_MODE help Place in the process address space an ELF shared object providing fast implementations of gettimeofday and diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 7809d4976269..c6c32fb7f546 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -110,7 +110,6 @@ config ARM64 select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL select GENERIC_GETTIMEOFDAY - select GENERIC_VDSO_CLOCK_MODE select HANDLE_DOMAIN_IRQ select HARDIRQS_SW_RESEND select HAVE_PCI diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 23b5c0578776..654369a7209d 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -37,7 +37,6 @@ config MIPS select GENERIC_SCHED_CLOCK if !CAVIUM_OCTEON_SOC select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL - select GENERIC_VDSO_CLOCK_MODE select GUP_GET_PTE_LOW_HIGH if CPU_MIPS32 && PHYS_ADDR_T_64BIT select HANDLE_DOMAIN_IRQ select HAVE_ARCH_COMPILER_H diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 698e9c835cc5..8b995db3d10f 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -125,7 +125,6 @@ config X86 select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL select GENERIC_GETTIMEOFDAY - select GENERIC_VDSO_CLOCK_MODE select GENERIC_VDSO_TIME_NS select GUP_GET_PTE_LOW_HIGH if X86_PAE select HARDLOCKUP_CHECK_TIMESTAMP if X86_64 diff --git a/include/asm-generic/vdso/vsyscall.h b/include/asm-generic/vdso/vsyscall.h index cec543d9e87b..4a28797495d7 100644 --- a/include/asm-generic/vdso/vsyscall.h +++ b/include/asm-generic/vdso/vsyscall.h @@ -18,13 +18,6 @@ static __always_inline bool __arch_update_vdso_data(void) } #endif /* __arch_update_vdso_data */ -#ifndef __arch_get_clock_mode -static __always_inline int __arch_get_clock_mode(struct timekeeper *tk) -{ - return 0; -} -#endif /* __arch_get_clock_mode */ - #ifndef __arch_update_vsyscall static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk) diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index 6d5ed1b4d24d..7fefe0b21a14 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -24,13 +24,13 @@ struct clocksource; struct module; #if defined(CONFIG_ARCH_CLOCKSOURCE_DATA) || \ - defined(CONFIG_GENERIC_VDSO_CLOCK_MODE) + defined(CONFIG_GENERIC_GETTIMEOFDAY) #include #endif enum vdso_clock_mode { VDSO_CLOCKMODE_NONE, -#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE +#ifdef CONFIG_GENERIC_GETTIMEOFDAY VDSO_ARCH_CLOCKMODES, #endif VDSO_CLOCKMODE_MAX, diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c index f9a5178c69bb..d31a5ef4ade5 100644 --- a/kernel/time/vsyscall.c +++ b/kernel/time/vsyscall.c @@ -77,11 +77,7 @@ void update_vsyscall(struct timekeeper *tk) /* copy vsyscall data */ vdso_write_begin(vdata); -#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE clock_mode = tk->tkr_mono.clock->vdso_clock_mode; -#else - clock_mode = __arch_get_clock_mode(tk); -#endif vdata[CS_HRES_COARSE].clock_mode = clock_mode; vdata[CS_RAW].clock_mode = clock_mode; diff --git a/lib/vdso/Kconfig b/lib/vdso/Kconfig index d9f43c84fcc6..d883ac299508 100644 --- a/lib/vdso/Kconfig +++ b/lib/vdso/Kconfig @@ -30,7 +30,4 @@ config GENERIC_VDSO_TIME_NS Selected by architectures which support time namespaces in the VDSO -config GENERIC_VDSO_CLOCK_MODE - bool - endif diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 3f2d8b859130..00f8d1f1405b 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -65,16 +65,13 @@ static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, do { seq = vdso_read_begin(vd); - if (IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && - vd->clock_mode == VDSO_CLOCKMODE_NONE) + + if (unlikely(vd->clock_mode == VDSO_CLOCKMODE_NONE)) return -1; + cycles = __arch_get_hw_counter(vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; - if (!IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && - unlikely((s64)cycles < 0)) - return -1; - ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); ns >>= vd->shift; sec = vdso_ts->sec; @@ -137,16 +134,12 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, } smp_rmb(); - if (IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && - vd->clock_mode == VDSO_CLOCKMODE_NONE) + if (unlikely(vd->clock_mode == VDSO_CLOCKMODE_NONE)) return -1; + cycles = __arch_get_hw_counter(vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; - if (!IS_ENABLED(CONFIG_GENERIC_VDSO_CLOCK_MODE) && - unlikely((s64)cycles < 0)) - return -1; - ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); ns >>= vd->shift; sec = vdso_ts->sec; -- cgit v1.2.3 From 2d6b01bd88ccabba06d342ef80eaab6b39d12497 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 7 Feb 2020 13:39:01 +0100 Subject: lib/vdso: Move VCLOCK_TIMENS to vdso_clock_modes Move the time namespace indicator clock mode to the other ones for consistency sake. Signed-off-by: Thomas Gleixner Reviewed-by: Vincenzo Frascino Link: https://lkml.kernel.org/r/20200207124403.656097274@linutronix.de --- include/linux/clocksource.h | 3 +++ include/vdso/datapage.h | 2 -- kernel/time/namespace.c | 7 ++++--- lib/vdso/gettimeofday.c | 18 ++++++++++-------- 4 files changed, 17 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index 7fefe0b21a14..02e3282719bd 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -34,6 +34,9 @@ enum vdso_clock_mode { VDSO_ARCH_CLOCKMODES, #endif VDSO_CLOCKMODE_MAX, + + /* Indicator for time namespace VDSO */ + VDSO_CLOCKMODE_TIMENS = INT_MAX }; /** diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index c5f347cc5e55..30c4cb0428d1 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -21,8 +21,6 @@ #define CS_RAW 1 #define CS_BASES (CS_RAW + 1) -#define VCLOCK_TIMENS UINT_MAX - /** * struct vdso_timestamp - basetime per clock_id * @sec: seconds diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index 12858507d75a..e6ba064ce773 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -172,8 +173,8 @@ static struct timens_offset offset_from_ts(struct timespec64 off) * for vdso_data->clock_mode is a non-issue. The task is spin waiting for the * update to finish and for 'seq' to become even anyway. * - * Timens page has vdso_data->clock_mode set to VCLOCK_TIMENS which enforces - * the time namespace handling path. + * Timens page has vdso_data->clock_mode set to VDSO_CLOCKMODE_TIMENS which + * enforces the time namespace handling path. */ static void timens_setup_vdso_data(struct vdso_data *vdata, struct time_namespace *ns) @@ -183,7 +184,7 @@ static void timens_setup_vdso_data(struct vdso_data *vdata, struct timens_offset boottime = offset_from_ts(ns->offsets.boottime); vdata->seq = 1; - vdata->clock_mode = VCLOCK_TIMENS; + vdata->clock_mode = VDSO_CLOCKMODE_TIMENS; offset[CLOCK_MONOTONIC] = monotonic; offset[CLOCK_MONOTONIC_RAW] = monotonic; offset[CLOCK_MONOTONIC_COARSE] = monotonic; diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 00f8d1f1405b..a76ac8d17c5f 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -116,10 +116,10 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, do { /* - * Open coded to handle VCLOCK_TIMENS. Time namespace + * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace * enabled tasks have a special VVAR page installed which * has vd->seq set to 1 and vd->clock_mode set to - * VCLOCK_TIMENS. For non time namespace affected tasks + * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks * this does not affect performance because if vd->seq is * odd, i.e. a concurrent update is in progress the extra * check for vd->clock_mode is just a few extra @@ -128,7 +128,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, */ while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { if (IS_ENABLED(CONFIG_TIME_NS) && - vd->clock_mode == VCLOCK_TIMENS) + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) return do_hres_timens(vd, clk, ts); cpu_relax(); } @@ -200,12 +200,12 @@ static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, do { /* - * Open coded to handle VCLOCK_TIMENS. See comment in + * Open coded to handle VDSO_CLOCK_TIMENS. See comment in * do_hres(). */ while ((seq = READ_ONCE(vd->seq)) & 1) { if (IS_ENABLED(CONFIG_TIME_NS) && - vd->clock_mode == VCLOCK_TIMENS) + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) return do_coarse_timens(vd, clk, ts); cpu_relax(); } @@ -292,7 +292,7 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) if (unlikely(tz != NULL)) { if (IS_ENABLED(CONFIG_TIME_NS) && - vd->clock_mode == VCLOCK_TIMENS) + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) vd = __arch_get_timens_vdso_data(); tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; @@ -308,7 +308,8 @@ static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time const struct vdso_data *vd = __arch_get_vdso_data(); __kernel_old_time_t t; - if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) vd = __arch_get_timens_vdso_data(); t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); @@ -332,7 +333,8 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) if (unlikely((u32) clock >= MAX_CLOCKS)) return -1; - if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) vd = __arch_get_timens_vdso_data(); /* -- cgit v1.2.3 From ae12e08539de6717502c2f9f83bd60df939b5c08 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 7 Feb 2020 13:39:02 +0100 Subject: lib/vdso: Allow fixed clock mode Some architectures have a fixed clocksource which is known at compile time and cannot be replaced or disabled at runtime, e.g. timebase on PowerPC. For such cases the clock mode check in the VDSO code is pointless. Move the check for a VDSO capable clocksource into an inline function and allow architectures to redefine it via a macro. [ tglx: Removed the #ifdef mess ] Signed-off-by: Christophe Leroy Signed-off-by: Thomas Gleixner Tested-by: Vincenzo Frascino Reviewed-by: Vincenzo Frascino Link: https://lkml.kernel.org/r/20200207124403.748756829@linutronix.de --- lib/vdso/gettimeofday.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index a76ac8d17c5f..8eb6d1e9a8ff 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -46,6 +46,13 @@ static inline bool __arch_vdso_hres_capable(void) } #endif +#ifndef vdso_clocksource_ok +static inline bool vdso_clocksource_ok(const struct vdso_data *vd) +{ + return vd->clock_mode != VDSO_CLOCKMODE_NONE; +} +#endif + #ifdef CONFIG_TIME_NS static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, struct __kernel_timespec *ts) @@ -66,7 +73,7 @@ static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, do { seq = vdso_read_begin(vd); - if (unlikely(vd->clock_mode == VDSO_CLOCKMODE_NONE)) + if (unlikely(!vdso_clocksource_ok(vd))) return -1; cycles = __arch_get_hw_counter(vd->clock_mode); @@ -134,7 +141,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, } smp_rmb(); - if (unlikely(vd->clock_mode == VDSO_CLOCKMODE_NONE)) + if (unlikely(!vdso_clocksource_ok(vd))) return -1; cycles = __arch_get_hw_counter(vd->clock_mode); -- cgit v1.2.3 From 8345228ccf31f94e3ff7ec5458ac7cc13cb323fa Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 7 Feb 2020 13:39:03 +0100 Subject: lib/vdso: Allow architectures to override the ns shift operation On powerpc/32, GCC (8.1) generates pretty bad code for the ns >>= vd->shift operation taking into account that the shift is always <= 32 and the upper part of the result is likely to be zero. GCC makes reversed assumptions considering the shift to be likely >= 32 and the upper part to be like not zero. unsigned long long shift(unsigned long long x, unsigned char s) { return x >> s; } results in: 00000018 : 18: 35 25 ff e0 addic. r9,r5,-32 1c: 41 80 00 10 blt 2c 20: 7c 64 4c 30 srw r4,r3,r9 24: 38 60 00 00 li r3,0 28: 4e 80 00 20 blr 2c: 54 69 08 3c rlwinm r9,r3,1,0,30 30: 21 45 00 1f subfic r10,r5,31 34: 7c 84 2c 30 srw r4,r4,r5 38: 7d 29 50 30 slw r9,r9,r10 3c: 7c 63 2c 30 srw r3,r3,r5 40: 7d 24 23 78 or r4,r9,r4 44: 4e 80 00 20 blr Even when forcing the shift to be smaller than 32 with an &= 31, it still considers the shift as likely >= 32. Move the default shift implementation into an inline which can be redefined in architecture code via a macro. [ tglx: Made the shift argument u32 and removed the __arch prefix ] Signed-off-by: Christophe Leroy Signed-off-by: Thomas Gleixner Tested-by: Vincenzo Frascino Reviewed-by: Vincenzo Frascino Link: https://lore.kernel.org/r/b3d449de856982ed060a71e6ace8eeca4654e685.1580399657.git.christophe.leroy@c-s.fr Link: https://lkml.kernel.org/r/20200207124403.857649978@linutronix.de --- lib/vdso/gettimeofday.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 8eb6d1e9a8ff..b95aef97501e 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -39,6 +39,13 @@ u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) } #endif +#ifndef vdso_shift_ns +static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) +{ + return ns >> shift; +} +#endif + #ifndef __arch_vdso_hres_capable static inline bool __arch_vdso_hres_capable(void) { @@ -80,7 +87,7 @@ static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, ns = vdso_ts->nsec; last = vd->cycle_last; ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); - ns >>= vd->shift; + ns = vdso_shift_ns(ns, vd->shift); sec = vdso_ts->sec; } while (unlikely(vdso_read_retry(vd, seq))); @@ -148,7 +155,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, ns = vdso_ts->nsec; last = vd->cycle_last; ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); - ns >>= vd->shift; + ns = vdso_shift_ns(ns, vd->shift); sec = vdso_ts->sec; } while (unlikely(vdso_read_retry(vd, seq))); -- cgit v1.2.3 From e876f0b69dc993e86ca7795e63e98385aa9a7ef3 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 7 Feb 2020 13:39:04 +0100 Subject: lib/vdso: Allow architectures to provide the vdso data pointer On powerpc, __arch_get_vdso_data() clobbers the link register, requiring the caller to save it. As the parent function already has to set a stack frame and saves the link register before calling the C vdso function, retrieving the vdso data pointer there is less overhead. Split out the functional code from the __cvdso.*() interfaces into new static functions which can either be called from the existing interfaces with the vdso data pointer supplied via __arch_get_vdso_data() or directly from ASM code. Signed-off-by: Christophe Leroy Signed-off-by: Thomas Gleixner Tested-by: Vincenzo Frascino Reviewed-by: Vincenzo Frascino Link: https://lore.kernel.org/r/abf97996602ef07223fec30c005df78e5ed41b2e.1580399657.git.christophe.leroy@c-s.fr Link: https://lkml.kernel.org/r/20200207124403.965789141@linutronix.de --- lib/vdso/gettimeofday.c | 72 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index b95aef97501e..72d282ffd156 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -233,9 +233,9 @@ static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, } static __maybe_unused int -__cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) +__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *ts) { - const struct vdso_data *vd = __arch_get_vdso_data(); u32 msk; /* Check for negative values or invalid clocks */ @@ -260,23 +260,31 @@ __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) } static __maybe_unused int -__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *ts) { - int ret = __cvdso_clock_gettime_common(clock, ts); + int ret = __cvdso_clock_gettime_common(vd, clock, ts); if (unlikely(ret)) return clock_gettime_fallback(clock, ts); return 0; } +static __maybe_unused int +__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts); +} + #ifdef BUILD_VDSO32 static __maybe_unused int -__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) +__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, + struct old_timespec32 *res) { struct __kernel_timespec ts; int ret; - ret = __cvdso_clock_gettime_common(clock, &ts); + ret = __cvdso_clock_gettime_common(vd, clock, &ts); if (unlikely(ret)) return clock_gettime32_fallback(clock, res); @@ -287,12 +295,18 @@ __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) return ret; } + +static __maybe_unused int +__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) +{ + return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res); +} #endif /* BUILD_VDSO32 */ static __maybe_unused int -__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +__cvdso_gettimeofday_data(const struct vdso_data *vd, + struct __kernel_old_timeval *tv, struct timezone *tz) { - const struct vdso_data *vd = __arch_get_vdso_data(); if (likely(tv != NULL)) { struct __kernel_timespec ts; @@ -316,10 +330,16 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) return 0; } +static __maybe_unused int +__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz); +} + #ifdef VDSO_HAS_TIME -static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) +static __maybe_unused __kernel_old_time_t +__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) { - const struct vdso_data *vd = __arch_get_vdso_data(); __kernel_old_time_t t; if (IS_ENABLED(CONFIG_TIME_NS) && @@ -333,13 +353,18 @@ static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time return t; } + +static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) +{ + return __cvdso_time_data(__arch_get_vdso_data(), time); +} #endif /* VDSO_HAS_TIME */ #ifdef VDSO_HAS_CLOCK_GETRES static __maybe_unused -int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) +int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *res) { - const struct vdso_data *vd = __arch_get_vdso_data(); u32 msk; u64 ns; @@ -378,23 +403,31 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) } static __maybe_unused -int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) +int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *res) { - int ret = __cvdso_clock_getres_common(clock, res); + int ret = __cvdso_clock_getres_common(vd, clock, res); if (unlikely(ret)) return clock_getres_fallback(clock, res); return 0; } +static __maybe_unused +int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) +{ + return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res); +} + #ifdef BUILD_VDSO32 static __maybe_unused int -__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) +__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, + struct old_timespec32 *res) { struct __kernel_timespec ts; int ret; - ret = __cvdso_clock_getres_common(clock, &ts); + ret = __cvdso_clock_getres_common(vd, clock, &ts); if (unlikely(ret)) return clock_getres32_fallback(clock, res); @@ -405,5 +438,12 @@ __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) } return ret; } + +static __maybe_unused int +__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) +{ + return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(), + clock, res); +} #endif /* BUILD_VDSO32 */ #endif /* VDSO_HAS_CLOCK_GETRES */ -- cgit v1.2.3 From 8c59ab839f526437831ff6d1405c9a6d93f475eb Mon Sep 17 00:00:00 2001 From: Vincenzo Frascino Date: Fri, 20 Mar 2020 14:53:50 +0000 Subject: lib/vdso: Enable common headers The vDSO library should only include the necessary headers required for a userspace library (UAPI and a minimal set of kernel headers). To make this possible it is necessary to isolate from the kernel headers the common parts that are strictly necessary to build the library. Refactor the unified vdso code to use the common headers. Signed-off-by: Vincenzo Frascino Signed-off-by: Thomas Gleixner Link: https://lkml.kernel.org/r/20200320145351.32292-26-vincenzo.frascino@arm.com --- include/vdso/datapage.h | 33 ++++++++++++++++++++++++++++++--- lib/vdso/gettimeofday.c | 22 ---------------------- 2 files changed, 30 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index 30c4cb0428d1..5cbc9fcbfd45 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -4,9 +4,20 @@ #ifndef __ASSEMBLY__ -#include -#include -#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include #define VDSO_BASES (CLOCK_TAI + 1) #define VDSO_HRES (BIT(CLOCK_REALTIME) | \ @@ -99,6 +110,22 @@ struct vdso_data { */ extern struct vdso_data _vdso_data[CS_BASES] __attribute__((visibility("hidden"))); +/* + * The generic vDSO implementation requires that gettimeofday.h + * provides: + * - __arch_get_vdso_data(): to get the vdso datapage. + * - __arch_get_hw_counter(): to get the hw counter based on the + * clock_mode. + * - gettimeofday_fallback(): fallback for gettimeofday. + * - clock_gettime_fallback(): fallback for clock_gettime. + * - clock_getres_fallback(): fallback for clock_getres. + */ +#ifdef ENABLE_COMPAT_VDSO +#include +#else +#include +#endif /* ENABLE_COMPAT_VDSO */ + #endif /* !__ASSEMBLY__ */ #endif /* __VDSO_DATAPAGE_H */ diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 72d282ffd156..a2909af4b924 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -2,31 +2,9 @@ /* * Generic userspace implementations of gettimeofday() and similar. */ -#include -#include -#include -#include -#include -#include #include #include -/* - * The generic vDSO implementation requires that gettimeofday.h - * provides: - * - __arch_get_vdso_data(): to get the vdso datapage. - * - __arch_get_hw_counter(): to get the hw counter based on the - * clock_mode. - * - gettimeofday_fallback(): fallback for gettimeofday. - * - clock_gettime_fallback(): fallback for clock_gettime. - * - clock_getres_fallback(): fallback for clock_getres. - */ -#ifdef ENABLE_COMPAT_VDSO -#include -#else -#include -#endif /* ENABLE_COMPAT_VDSO */ - #ifndef vdso_calc_delta /* * Default implementation which works for all sane clocksources. That -- cgit v1.2.3