diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2020-03-31 04:51:47 +0300 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2020-03-31 04:51:47 +0300 |
| commit | dbb381b619aa5242c9cb1a8fd54d71c4d79c91eb (patch) | |
| tree | e2bcd337c4eb7f4f3a44e0f0f05a19e775f18f90 | |
| parent | 336622e9fce7a0b8243e1cd9f6e1c24d96f13cd8 (diff) | |
| parent | 4479730e9263befbb9ce9563a09563db2acb8f7c (diff) | |
| download | linux-dbb381b619aa5242c9cb1a8fd54d71c4d79c91eb.tar.xz | |
Merge tag 'timers-core-2020-03-30' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull timekeeping and timer updates from Thomas Gleixner:
"Core:
- Consolidation of the vDSO build infrastructure to address the
difficulties of cross-builds for ARM64 compat vDSO libraries by
restricting the exposure of header content to the vDSO build.
This is achieved by splitting out header content into separate
headers. which contain only the minimaly required information which
is necessary to build the vDSO. These new headers are included from
the kernel headers and the vDSO specific files.
- Enhancements to the generic vDSO library allowing more fine grained
control over the compiled in code, further reducing architecture
specific storage and preparing for adopting the generic library by
PPC.
- Cleanup and consolidation of the exit related code in posix CPU
timers.
- Small cleanups and enhancements here and there
Drivers:
- The obligatory new drivers: Ingenic JZ47xx and X1000 TCU support
- Correct the clock rate of PIT64b global clock
- setup_irq() cleanup
- Preparation for PWM and suspend support for the TI DM timer
- Expand the fttmr010 driver to support ast2600 systems
- The usual small fixes, enhancements and cleanups all over the
place"
* tag 'timers-core-2020-03-30' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (80 commits)
Revert "clocksource/drivers/timer-probe: Avoid creating dead devices"
vdso: Fix clocksource.h macro detection
um: Fix header inclusion
arm64: vdso32: Enable Clang Compilation
lib/vdso: Enable common headers
arm: vdso: Enable arm to use common headers
x86/vdso: Enable x86 to use common headers
mips: vdso: Enable mips to use common headers
arm64: vdso32: Include common headers in the vdso library
arm64: vdso: Include common headers in the vdso library
arm64: Introduce asm/vdso/processor.h
arm64: vdso32: Code clean up
linux/elfnote.h: Replace elf.h with UAPI equivalent
scripts: Fix the inclusion order in modpost
common: Introduce processor.h
linux/ktime.h: Extract common header for vDSO
linux/jiffies.h: Extract common header for vDSO
linux/time64.h: Extract common header for vDSO
linux/time32.h: Extract common header for vDSO
linux/time.h: Extract common header for vDSO
...
123 files changed, 1289 insertions, 927 deletions
diff --git a/Documentation/devicetree/bindings/timer/faraday,fttmr010.txt b/Documentation/devicetree/bindings/timer/faraday,fttmr010.txt index 195792270414..3cb2f4c98d64 100644 --- a/Documentation/devicetree/bindings/timer/faraday,fttmr010.txt +++ b/Documentation/devicetree/bindings/timer/faraday,fttmr010.txt @@ -11,6 +11,7 @@ Required properties: "moxa,moxart-timer", "faraday,fttmr010" "aspeed,ast2400-timer" "aspeed,ast2500-timer" + "aspeed,ast2600-timer" - reg : Should contain registers location and length - interrupts : Should contain the three timer interrupts usually with diff --git a/Documentation/devicetree/bindings/timer/ingenic,tcu.txt b/Documentation/devicetree/bindings/timer/ingenic,tcu.txt index 0b63cebc5f45..91f704951845 100644 --- a/Documentation/devicetree/bindings/timer/ingenic,tcu.txt +++ b/Documentation/devicetree/bindings/timer/ingenic,tcu.txt @@ -10,6 +10,7 @@ Required properties: * ingenic,jz4740-tcu * ingenic,jz4725b-tcu * ingenic,jz4770-tcu + * ingenic,x1000-tcu followed by "simple-mfd". - reg: Should be the offset/length value corresponding to the TCU registers - clocks: List of phandle & clock specifiers for clocks external to the TCU. diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 97864aabc2a6..03bbfc312fe7 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -3,7 +3,6 @@ config ARM bool default y select ARCH_32BIT_OFF_T - select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_BINFMT_FLAT select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEVMEM_IS_ALLOWED diff --git a/arch/arm/include/asm/clocksource.h b/arch/arm/include/asm/clocksource.h index 0b350a7e26f3..13651c731a81 100644 --- a/arch/arm/include/asm/clocksource.h +++ b/arch/arm/include/asm/clocksource.h @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_CLOCKSOURCE_H #define _ASM_CLOCKSOURCE_H -struct arch_clocksource_data { - bool vdso_direct; /* Usable for direct VDSO access? */ -}; +#include <asm/vdso/clocksource.h> -#endif +#endif /* _ASM_CLOCKSOURCE_H */ diff --git a/arch/arm/include/asm/cp15.h b/arch/arm/include/asm/cp15.h index d2453e2d3f1f..a54230e65647 100644 --- a/arch/arm/include/asm/cp15.h +++ b/arch/arm/include/asm/cp15.h @@ -50,25 +50,7 @@ #ifdef CONFIG_CPU_CP15 -#define __ACCESS_CP15(CRn, Op1, CRm, Op2) \ - "mrc", "mcr", __stringify(p15, Op1, %0, CRn, CRm, Op2), u32 -#define __ACCESS_CP15_64(Op1, CRm) \ - "mrrc", "mcrr", __stringify(p15, Op1, %Q0, %R0, CRm), u64 - -#define __read_sysreg(r, w, c, t) ({ \ - t __val; \ - asm volatile(r " " c : "=r" (__val)); \ - __val; \ -}) -#define read_sysreg(...) __read_sysreg(__VA_ARGS__) - -#define __write_sysreg(v, r, w, c, t) asm volatile(w " " c : : "r" ((t)(v))) -#define write_sysreg(v, ...) __write_sysreg(v, __VA_ARGS__) - -#define BPIALL __ACCESS_CP15(c7, 0, c5, 6) -#define ICIALLU __ACCESS_CP15(c7, 0, c5, 0) - -#define CNTVCT __ACCESS_CP15_64(1, c14) +#include <asm/vdso/cp15.h> extern unsigned long cr_alignment; /* defined in entry-armv.S */ diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h index 614bf829e454..b9241051e5cb 100644 --- a/arch/arm/include/asm/processor.h +++ b/arch/arm/include/asm/processor.h @@ -14,6 +14,7 @@ #include <asm/ptrace.h> #include <asm/types.h> #include <asm/unified.h> +#include <asm/vdso/processor.h> #ifdef __KERNEL__ #define STACK_TOP ((current->personality & ADDR_LIMIT_32BIT) ? \ @@ -85,16 +86,6 @@ extern void release_thread(struct task_struct *); unsigned long get_wchan(struct task_struct *p); -#if __LINUX_ARM_ARCH__ == 6 || defined(CONFIG_ARM_ERRATA_754327) -#define cpu_relax() \ - do { \ - smp_mb(); \ - __asm__ __volatile__("nop; nop; nop; nop; nop; nop; nop; nop; nop; nop;"); \ - } while (0) -#else -#define cpu_relax() barrier() -#endif - #define task_pt_regs(p) \ ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1) diff --git a/arch/arm/include/asm/vdso/clocksource.h b/arch/arm/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..50c0b19fb755 --- /dev/null +++ b/arch/arm/include/asm/vdso/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif /* __ASM_VDSOCLOCKSOURCE_H */ diff --git a/arch/arm/include/asm/vdso/cp15.h b/arch/arm/include/asm/vdso/cp15.h new file mode 100644 index 000000000000..bed16fa1865e --- /dev/null +++ b/arch/arm/include/asm/vdso/cp15.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 ARM Ltd. + */ +#ifndef __ASM_VDSO_CP15_H +#define __ASM_VDSO_CP15_H + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_CPU_CP15 + +#include <linux/stringify.h> + +#define __ACCESS_CP15(CRn, Op1, CRm, Op2) \ + "mrc", "mcr", __stringify(p15, Op1, %0, CRn, CRm, Op2), u32 +#define __ACCESS_CP15_64(Op1, CRm) \ + "mrrc", "mcrr", __stringify(p15, Op1, %Q0, %R0, CRm), u64 + +#define __read_sysreg(r, w, c, t) ({ \ + t __val; \ + asm volatile(r " " c : "=r" (__val)); \ + __val; \ +}) +#define read_sysreg(...) __read_sysreg(__VA_ARGS__) + +#define __write_sysreg(v, r, w, c, t) asm volatile(w " " c : : "r" ((t)(v))) +#define write_sysreg(v, ...) __write_sysreg(v, __VA_ARGS__) + +#define BPIALL __ACCESS_CP15(c7, 0, c5, 6) +#define ICIALLU __ACCESS_CP15(c7, 0, c5, 0) + +#define CNTVCT __ACCESS_CP15_64(1, c14) + +#endif /* CONFIG_CPU_CP15 */ + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_CP15_H */ diff --git a/arch/arm/include/asm/vdso/gettimeofday.h b/arch/arm/include/asm/vdso/gettimeofday.h index fe6e1f65932d..36dc18553ed8 100644 --- a/arch/arm/include/asm/vdso/gettimeofday.h +++ b/arch/arm/include/asm/vdso/gettimeofday.h @@ -7,9 +7,9 @@ #ifndef __ASSEMBLY__ -#include <asm/barrier.h> -#include <asm/cp15.h> +#include <asm/errno.h> #include <asm/unistd.h> +#include <asm/vdso/cp15.h> #include <uapi/linux/time.h> #define VDSO_HAS_CLOCK_GETRES 1 @@ -106,20 +106,32 @@ static __always_inline int clock_getres32_fallback( return ret; } +static inline bool arm_vdso_hres_capable(void) +{ + return IS_ENABLED(CONFIG_ARM_ARCH_TIMER); +} +#define __arch_vdso_hres_capable arm_vdso_hres_capable + static __always_inline u64 __arch_get_hw_counter(int clock_mode) { #ifdef CONFIG_ARM_ARCH_TIMER u64 cycle_now; - if (!clock_mode) - return -EINVAL; + /* + * Core checks for mode already, so this raced against a concurrent + * update. Return something. Core will do another round and then + * see the mode change and fallback to the syscall. + */ + if (clock_mode == VDSO_CLOCKMODE_NONE) + return 0; isb(); cycle_now = read_sysreg(CNTVCT); return cycle_now; #else - return -EINVAL; /* use fallback */ + /* Make GCC happy. This is compiled out anyway */ + return 0; #endif } diff --git a/arch/arm/include/asm/vdso/processor.h b/arch/arm/include/asm/vdso/processor.h new file mode 100644 index 000000000000..45efb3ff511c --- /dev/null +++ b/arch/arm/include/asm/vdso/processor.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 ARM Ltd. + */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +#if __LINUX_ARM_ARCH__ == 6 || defined(CONFIG_ARM_ERRATA_754327) +#define cpu_relax() \ + do { \ + smp_mb(); \ + __asm__ __volatile__("nop; nop; nop; nop; nop; nop; nop; nop; nop; nop;"); \ + } while (0) +#else +#define cpu_relax() barrier() +#endif + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/arm/include/asm/vdso/vsyscall.h b/arch/arm/include/asm/vdso/vsyscall.h index cff87d8d30da..47e41ae8ccd0 100644 --- a/arch/arm/include/asm/vdso/vsyscall.h +++ b/arch/arm/include/asm/vdso/vsyscall.h @@ -11,18 +11,6 @@ extern struct vdso_data *vdso_data; extern bool cntvct_ok; -static __always_inline -bool tk_is_cntvct(const struct timekeeper *tk) -{ - if (!IS_ENABLED(CONFIG_ARM_ARCH_TIMER)) - return false; - - if (!tk->tkr_mono.clock->archdata.vdso_direct) - return false; - - return true; -} - /* * Update the vDSO data page to keep in sync with kernel timekeeping. */ @@ -34,29 +22,6 @@ struct vdso_data *__arm_get_k_vdso_data(void) #define __arch_get_k_vdso_data __arm_get_k_vdso_data static __always_inline -bool __arm_update_vdso_data(void) -{ - return cntvct_ok; -} -#define __arch_update_vdso_data __arm_update_vdso_data - -static __always_inline -int __arm_get_clock_mode(struct timekeeper *tk) -{ - u32 __tk_is_cntvct = tk_is_cntvct(tk); - - return __tk_is_cntvct; -} -#define __arch_get_clock_mode __arm_get_clock_mode - -static __always_inline -int __arm_use_vsyscall(struct vdso_data *vdata) -{ - return vdata[CS_HRES_COARSE].clock_mode; -} -#define __arch_use_vsyscall __arm_use_vsyscall - -static __always_inline void __arm_sync_vdso_data(struct vdso_data *vdata) { flush_dcache_page(virt_to_page(vdata)); diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 0b30e884e088..c6c32fb7f546 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -9,7 +9,6 @@ config ARM64 select ACPI_MCFG if (ACPI && PCI) select ACPI_SPCR_TABLE if ACPI select ACPI_PPTT if ACPI - select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEVMEM_IS_ALLOWED select ARCH_HAS_DMA_PREP_COHERENT diff --git a/arch/arm64/include/asm/clocksource.h b/arch/arm64/include/asm/clocksource.h index 0ece64a26c8c..482185566b0c 100644 --- a/arch/arm64/include/asm/clocksource.h +++ b/arch/arm64/include/asm/clocksource.h @@ -2,8 +2,6 @@ #ifndef _ASM_CLOCKSOURCE_H #define _ASM_CLOCKSOURCE_H -struct arch_clocksource_data { - bool vdso_direct; /* Usable for direct VDSO access? */ -}; +#include <asm/vdso/clocksource.h> #endif diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 5ba63204d078..e51ef2dc5749 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -28,6 +28,8 @@ #include <linux/string.h> #include <linux/thread_info.h> +#include <vdso/processor.h> + #include <asm/alternative.h> #include <asm/cpufeature.h> #include <asm/hw_breakpoint.h> @@ -256,11 +258,6 @@ extern void release_thread(struct task_struct *); unsigned long get_wchan(struct task_struct *p); -static inline void cpu_relax(void) -{ - asm volatile("yield" ::: "memory"); -} - /* Thread switching */ extern struct task_struct *cpu_switch_to(struct task_struct *prev, struct task_struct *next); diff --git a/arch/arm64/include/asm/vdso/clocksource.h b/arch/arm64/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..df6ea65c1dec --- /dev/null +++ b/arch/arm64/include/asm/vdso/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif diff --git a/arch/arm64/include/asm/vdso/compat_gettimeofday.h b/arch/arm64/include/asm/vdso/compat_gettimeofday.h index 537b1e695365..b6907ae78e53 100644 --- a/arch/arm64/include/asm/vdso/compat_gettimeofday.h +++ b/arch/arm64/include/asm/vdso/compat_gettimeofday.h @@ -8,12 +8,10 @@ #ifndef __ASSEMBLY__ #include <asm/unistd.h> -#include <uapi/linux/time.h> +#include <asm/errno.h> #include <asm/vdso/compat_barrier.h> -#define __VDSO_USE_SYSCALL ULLONG_MAX - #define VDSO_HAS_CLOCK_GETRES 1 #define BUILD_VDSO32 1 @@ -78,10 +76,6 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) register long ret asm ("r0"); register long nr asm("r7") = __NR_compat_clock_getres_time64; - /* The checks below are required for ABI consistency with arm */ - if ((_clkid >= MAX_CLOCKS) && (_ts == NULL)) - return -EINVAL; - asm volatile( " swi #0\n" : "=r" (ret) @@ -99,10 +93,6 @@ int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) register long ret asm ("r0"); register long nr asm("r7") = __NR_compat_clock_getres; - /* The checks below are required for ABI consistency with arm */ - if ((_clkid >= MAX_CLOCKS) && (_ts == NULL)) - return -EINVAL; - asm volatile( " swi #0\n" : "=r" (ret) @@ -117,11 +107,12 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode) u64 res; /* - * clock_mode == 0 implies that vDSO are enabled otherwise - * fallback on syscall. + * Core checks for mode already, so this raced against a concurrent + * update. Return something. Core will do another round and then + * see the mode change and fallback to the syscall. */ - if (clock_mode) - return __VDSO_USE_SYSCALL; + if (clock_mode == VDSO_CLOCKMODE_NONE) + return 0; /* * This isb() is required to prevent that the counter value diff --git a/arch/arm64/include/asm/vdso/gettimeofday.h b/arch/arm64/include/asm/vdso/gettimeofday.h index b08f476b72b4..afba6ba332f8 100644 --- a/arch/arm64/include/asm/vdso/gettimeofday.h +++ b/arch/arm64/include/asm/vdso/gettimeofday.h @@ -8,9 +8,6 @@ #ifndef __ASSEMBLY__ #include <asm/unistd.h> -#include <uapi/linux/time.h> - -#define __VDSO_USE_SYSCALL ULLONG_MAX #define VDSO_HAS_CLOCK_GETRES 1 @@ -71,11 +68,12 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode) u64 res; /* - * clock_mode == 0 implies that vDSO are enabled otherwise - * fallback on syscall. + * Core checks for mode already, so this raced against a concurrent + * update. Return something. Core will do another round and then + * see the mode change and fallback to the syscall. */ - if (clock_mode) - return __VDSO_USE_SYSCALL; + if (clock_mode == VDSO_CLOCKMODE_NONE) + return 0; /* * This isb() is required to prevent that the counter value diff --git a/arch/arm64/include/asm/vdso/processor.h b/arch/arm64/include/asm/vdso/processor.h new file mode 100644 index 000000000000..ff830b766ad2 --- /dev/null +++ b/arch/arm64/include/asm/vdso/processor.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 ARM Ltd. + */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +static inline void cpu_relax(void) +{ + asm volatile("yield" ::: "memory"); +} + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/arm64/include/asm/vdso/vsyscall.h b/arch/arm64/include/asm/vdso/vsyscall.h index 0c20a7c1bee5..f94b1457c117 100644 --- a/arch/arm64/include/asm/vdso/vsyscall.h +++ b/arch/arm64/include/asm/vdso/vsyscall.h @@ -22,15 +22,6 @@ struct vdso_data *__arm64_get_k_vdso_data(void) #define __arch_get_k_vdso_data __arm64_get_k_vdso_data static __always_inline -int __arm64_get_clock_mode(struct timekeeper *tk) -{ - u32 use_syscall = !tk->tkr_mono.clock->archdata.vdso_direct; - - return use_syscall; -} -#define __arch_get_clock_mode __arm64_get_clock_mode - -static __always_inline void __arm64_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk) { vdata[CS_HRES_COARSE].mask = VDSO_PRECISION_MASK; diff --git a/arch/arm64/kernel/vdso/vgettimeofday.c b/arch/arm64/kernel/vdso/vgettimeofday.c index 747635501a14..4236cf34d7d9 100644 --- a/arch/arm64/kernel/vdso/vgettimeofday.c +++ b/arch/arm64/kernel/vdso/vgettimeofday.c @@ -5,8 +5,6 @@ * Copyright (C) 2018 ARM Limited * */ -#include <linux/time.h> -#include <linux/types.h> int __kernel_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile index 04df57b43cb1..3964738ebbde 100644 --- a/arch/arm64/kernel/vdso32/Makefile +++ b/arch/arm64/kernel/vdso32/Makefile @@ -10,7 +10,18 @@ include $(srctree)/lib/vdso/Makefile # Same as cc-*option, but using CC_COMPAT instead of CC ifeq ($(CONFIG_CC_IS_CLANG), y) +COMPAT_GCC_TOOLCHAIN_DIR := $(dir $(shell which $(CROSS_COMPILE_COMPAT)elfedit)) +COMPAT_GCC_TOOLCHAIN := $(realpath $(COMPAT_GCC_TOOLCHAIN_DIR)/..) + +CC_COMPAT_CLANG_FLAGS := --target=$(notdir $(CROSS_COMPILE_COMPAT:%-=%)) +CC_COMPAT_CLANG_FLAGS += --prefix=$(COMPAT_GCC_TOOLCHAIN_DIR) +CC_COMPAT_CLANG_FLAGS += -no-integrated-as -Qunused-arguments +ifneq ($(COMPAT_GCC_TOOLCHAIN),) +CC_COMPAT_CLANG_FLAGS += --gcc-toolchain=$(COMPAT_GCC_TOOLCHAIN) +endif + CC_COMPAT ?= $(CC) +CC_COMPAT += $(CC_COMPAT_CLANG_FLAGS) else CC_COMPAT ?= $(CROSS_COMPILE_COMPAT)gcc endif diff --git a/arch/arm64/kernel/vdso32/vgettimeofday.c b/arch/arm64/kernel/vdso32/vgettimeofday.c index 54fc1c2ce93f..5acff29c5991 100644 --- a/arch/arm64/kernel/vdso32/vgettimeofday.c +++ b/arch/arm64/kernel/vdso32/vgettimeofday.c @@ -5,26 +5,16 @@ * Copyright (C) 2018 ARM Limited * */ -#include <linux/time.h> -#include <linux/types.h> int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts) { - /* The checks below are required for ABI consistency with arm */ - if ((u32)ts >= TASK_SIZE_32) - return -EFAULT; - return __cvdso_clock_gettime32(clock, ts); } int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts) { - /* The checks below are required for ABI consistency with arm */ - if ((u32)ts >= TASK_SIZE_32) - return -EFAULT; - return __cvdso_clock_gettime(clock, ts); } @@ -37,10 +27,6 @@ int __vdso_gettimeofday(struct __kernel_old_timeval *tv, int __vdso_clock_getres(clockid_t clock_id, struct old_timespec32 *res) { - /* The checks below are required for ABI consistency with arm */ - if ((u32)res >= TASK_SIZE_32) - return -EFAULT; - return __cvdso_clock_getres_time32(clock_id, res); } diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 2589d4760e45..66e94278bd10 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -4,7 +4,6 @@ config MIPS default y select ARCH_32BIT_OFF_T if !64BIT select ARCH_BINFMT_ELF_STATE if MIPS_FP_SUPPORT - select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_KCOV select ARCH_HAS_PTE_SPECIAL if !(32BIT && CPU_HAS_RIXI) diff --git a/arch/mips/include/asm/clocksource.h b/arch/mips/include/asm/clocksource.h index cab9ae9f1e14..2f1ebbea3d72 100644 --- a/arch/mips/include/asm/clocksource.h +++ b/arch/mips/include/asm/clocksource.h @@ -3,23 +3,9 @@ * Copyright (C) 2015 Imagination Technologies * Author: Alex Smith <alex.smith@imgtec.com> */ - #ifndef __ASM_CLOCKSOURCE_H #define __ASM_CLOCKSOURCE_H -#include <linux/types.h> - -/* VDSO clocksources. */ -#define VDSO_CLOCK_NONE 0 /* No suitable clocksource. */ -#define VDSO_CLOCK_R4K 1 /* Use the coprocessor 0 count. */ -#define VDSO_CLOCK_GIC 2 /* Use the GIC. */ - -/** - * struct arch_clocksource_data - Architecture-specific clocksource information. - * @vdso_clock_mode: Method the VDSO should use to access the clocksource. - */ -struct arch_clocksource_data { - u8 vdso_clock_mode; -}; +#include <asm/vdso/clocksource.h> #endif /* __ASM_CLOCKSOURCE_H */ diff --git a/arch/mips/include/asm/processor.h b/arch/mips/include/asm/processor.h index 7619ad319400..4c9cc667f3ed 100644 --- a/arch/mips/include/asm/processor.h +++ b/arch/mips/include/asm/processor.h @@ -22,6 +22,7 @@ #include <asm/dsemul.h> #include <asm/mipsregs.h> #include <asm/prefetch.h> +#include <asm/vdso/processor.h> /* * System setup and hardware flags.. @@ -385,21 +386,6 @@ unsigned long get_wchan(struct task_struct *p); #define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[29]) #define KSTK_STATUS(tsk) (task_pt_regs(tsk)->cp0_status) -#ifdef CONFIG_CPU_LOONGSON64 -/* - * Loongson-3's SFB (Store-Fill-Buffer) may buffer writes indefinitely when a - * tight read loop is executed, because reads take priority over writes & the - * hardware (incorrectly) doesn't ensure that writes will eventually occur. - * - * Since spin loops of any kind should have a cpu_relax() in them, force an SFB - * flush from cpu_relax() such that any pending writes will become visible as - * expected. - */ -#define cpu_relax() smp_mb() -#else -#define cpu_relax() barrier() -#endif - /* * Return_address is a replacement for __builtin_return_address(count) * which on certain architectures cannot reasonably be implemented in GCC diff --git a/arch/mips/include/asm/vdso/clocksource.h b/arch/mips/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..510e1671d898 --- /dev/null +++ b/arch/mips/include/asm/vdso/clocksource.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_R4K, \ + VDSO_CLOCKMODE_GIC + +#endif /* __ASM_VDSOCLOCKSOURCE_H */ diff --git a/arch/mips/include/asm/vdso/gettimeofday.h b/arch/mips/include/asm/vdso/gettimeofday.h index a58687e26c5d..c63ddcaea54c 100644 --- a/arch/mips/include/asm/vdso/gettimeofday.h +++ b/arch/mips/include/asm/vdso/gettimeofday.h @@ -13,19 +13,13 @@ #ifndef __ASSEMBLY__ -#include <linux/compiler.h> -#include <linux/time.h> - #include <asm/vdso/vdso.h> #include <asm/clocksource.h> -#include <asm/io.h> #include <asm/unistd.h> #include <asm/vdso.h> #define VDSO_HAS_CLOCK_GETRES 1 -#define __VDSO_USE_SYSCALL ULLONG_MAX - static __always_inline long gettimeofday_fallback( struct __kernel_old_timeval *_tv, struct timezone *_tz) @@ -175,29 +169,28 @@ static __always_inline u64 read_gic_count(const struct vdso_data *data) static __always_inline u64 __arch_get_hw_counter(s32 clock_mode) { -#ifdef CONFIG_CLKSRC_MIPS_GIC - const struct vdso_data *data = get_vdso_data(); -#endif - u64 cycle_now; - - switch (clock_mode) { #ifdef CONFIG_CSRC_R4K - case VDSO_CLOCK_R4K: - cycle_now = read_r4k_count(); - break; + if (clock_mode == VDSO_CLOCKMODE_R4K) + return read_r4k_count(); #endif #ifdef CONFIG_CLKSRC_MIPS_GIC - case VDSO_CLOCK_GIC: - cycle_now = read_gic_count(data); - break; + if (clock_mode == VDSO_CLOCKMODE_GIC) + return read_gic_count(get_vdso_data()); #endif - default: - cycle_now = __VDSO_USE_SYSCALL; - break; - } + /* + * Core checks mode already. So this raced against a concurrent + * update. Return something. Core will do another round see the + * change and fallback to syscall. + */ + return 0; +} - return cycle_now; +static inline bool mips_vdso_hres_capable(void) +{ + return IS_ENABLED(CONFIG_CSRC_R4K) || + IS_ENABLED(CONFIG_CLKSRC_MIPS_GIC); } +#define __arch_vdso_hres_capable mips_vdso_hres_capable static __always_inline const struct vdso_data *__arch_get_vdso_data(void) { diff --git a/arch/mips/include/asm/vdso/processor.h b/arch/mips/include/asm/vdso/processor.h new file mode 100644 index 000000000000..511c95d735e6 --- /dev/null +++ b/arch/mips/include/asm/vdso/processor.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 ARM Ltd. + */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_CPU_LOONGSON64 +/* + * Loongson-3's SFB (Store-Fill-Buffer) may buffer writes indefinitely when a + * tight read loop is executed, because reads take priority over writes & the + * hardware (incorrectly) doesn't ensure that writes will eventually occur. + * + * Since spin loops of any kind should have a cpu_relax() in them, force an SFB + * flush from cpu_relax() such that any pending writes will become visible as + * expected. + */ +#define cpu_relax() smp_mb() +#else +#define cpu_relax() barrier() +#endif + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/mips/include/asm/vdso/vsyscall.h b/arch/mips/include/asm/vdso/vsyscall.h index 00d41b94ba31..47168aaf1eff 100644 --- a/arch/mips/include/asm/vdso/vsyscall.h +++ b/arch/mips/include/asm/vdso/vsyscall.h @@ -19,15 +19,6 @@ struct vdso_data *__mips_get_k_vdso_data(void) } #define __arch_get_k_vdso_data __mips_get_k_vdso_data -static __always_inline -int __mips_get_clock_mode(struct timekeeper *tk) -{ - u32 clock_mode = tk->tkr_mono.clock->archdata.vdso_clock_mode; - - return clock_mode; -} -#define __arch_get_clock_mode __mips_get_clock_mode - /* The asm-generic header needs to be included after the definitions above */ #include <asm-generic/vdso/vsyscall.h> diff --git a/arch/mips/kernel/csrc-r4k.c b/arch/mips/kernel/csrc-r4k.c index eed099f35bf1..437dda64fd7a 100644 --- a/arch/mips/kernel/csrc-r4k.c +++ b/arch/mips/kernel/csrc-r4k.c @@ -78,7 +78,7 @@ int __init init_r4k_clocksource(void) * by the VDSO (HWREna is configured by configure_hwrena()). */ if (cpu_has_mips_r2_r6 && rdhwr_count_usable()) - clocksource_mips.archdata.vdso_clock_mode = VDSO_CLOCK_R4K; + clocksource_mips.vdso_clock_mode = VDSO_CLOCKMODE_R4K; clocksource_register_hz(&clocksource_mips, mips_hpt_frequency); diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 51da546e0745..e2e326a96788 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -57,7 +57,6 @@ config X86 select ACPI_LEGACY_TABLES_LOOKUP if ACPI select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI select ARCH_32BIT_OFF_T if X86_32 - select ARCH_CLOCKSOURCE_DATA select ARCH_CLOCKSOURCE_INIT select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_DEBUG_VIRTUAL diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index c1b8496b5606..43428cc514c8 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -38,6 +38,8 @@ struct vdso_data *arch_get_vdso_data(void *vvar_page) } #undef EMIT_VVAR +unsigned int vclocks_used __read_mostly; + #if defined(CONFIG_X86_64) unsigned int __read_mostly vdso64_enabled = 1; #endif @@ -219,7 +221,7 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, } else if (sym_offset == image->sym_pvclock_page) { struct pvclock_vsyscall_time_info *pvti = pvclock_get_pvti_cpu0_va(); - if (pvti && vclock_was_used(VCLOCK_PVCLOCK)) { + if (pvti && vclock_was_used(VDSO_CLOCKMODE_PVCLOCK)) { return vmf_insert_pfn_prot(vma, vmf->address, __pa(pvti) >> PAGE_SHIFT, pgprot_decrypted(vma->vm_page_prot)); @@ -227,7 +229,7 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, } else if (sym_offset == image->sym_hvclock_page) { struct ms_hyperv_tsc_page *tsc_pg = hv_get_tsc_page(); - if (tsc_pg && vclock_was_used(VCLOCK_HVCLOCK)) + if (tsc_pg && vclock_was_used(VDSO_CLOCKMODE_HVCLOCK)) return vmf_insert_pfn(vma, vmf->address, virt_to_phys(tsc_pg) >> PAGE_SHIFT); } else if (sym_offset == image->sym_timens_page) { @@ -445,6 +447,8 @@ __setup("vdso=", vdso_setup); static int __init init_vdso(void) { + BUILD_BUG_ON(VDSO_CLOCKMODE_MAX >= 32); + init_vdso_image(&vdso_image_64); #ifdef CONFIG_X86_X32_ABI diff --git a/arch/x86/include/asm/clocksource.h b/arch/x86/include/asm/clocksource.h index dc4cfc888d6d..dc9dc7b3911a 100644 --- a/arch/x86/include/asm/clocksource.h +++ b/arch/x86/include/asm/clocksource.h @@ -4,14 +4,18 @@ #ifndef _ASM_X86_CLOCKSOURCE_H #define _ASM_X86_CLOCKSOURCE_H -#define VCLOCK_NONE 0 /* No vDSO clock available. */ -#define VCLOCK_TSC 1 /* vDSO should use vread_tsc. */ -#define VCLOCK_PVCLOCK 2 /* vDSO should use vread_pvclock. */ -#define VCLOCK_HVCLOCK 3 /* vDSO should use vread_hvclock. */ -#define VCLOCK_MAX 3 +#include <asm/vdso/clocksource.h> -struct arch_clocksource_data { - int vclock_mode; -}; +extern unsigned int vclocks_used; + +static inline bool vclock_was_used(int vclock) +{ + return READ_ONCE(vclocks_used) & (1U << vclock); +} + +static inline void vclocks_set_used(unsigned int which) +{ + WRITE_ONCE(vclocks_used, READ_ONCE(vclocks_used) | (1 << which)); +} #endif /* _ASM_X86_CLOCKSOURCE_H */ diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index 6b79515abb82..edc2c581704a 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -46,7 +46,9 @@ typedef int (*hyperv_fill_flush_list_func)( #define hv_set_reference_tsc(val) \ wrmsrl(HV_X64_MSR_REFERENCE_TSC, val) #define hv_set_clocksource_vdso(val) \ - ((val).archdata.vclock_mode = VCLOCK_HVCLOCK) + ((val).vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK) +#define hv_enable_vdso_clocksource() \ + vclocks_set_used(VDSO_CLOCKMODE_HVCLOCK); #define hv_get_raw_timer() rdtsc_ordered() void hyperv_callback_vector(void); diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 09705ccc393c..94789db550df 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -26,6 +26,7 @@ struct vm86; #include <asm/fpu/types.h> #include <asm/unwind_hints.h> #include <asm/vmxfeatures.h> +#include <asm/vdso/processor.h> #include <linux/personality.h> #include <linux/cache.h> @@ -677,17 +678,6 @@ static inline unsigned int cpuid_edx(unsigned int op) return edx; } -/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ -static __always_inline void rep_nop(void) -{ - asm volatile("rep; nop" ::: "memory"); -} - -static __always_inline void cpu_relax(void) -{ - rep_nop(); -} - /* * This function forces the icache and prefetched instruction stream to * catch up with reality in two very specific cases: diff --git a/arch/x86/include/asm/vdso/clocksource.h b/arch/x86/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..119ac8612d89 --- /dev/null +++ b/arch/x86/include/asm/vdso/clocksource.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_CLOCKSOURCE_H +#define __ASM_VDSO_CLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_TSC, \ + VDSO_CLOCKMODE_PVCLOCK, \ + VDSO_CLOCKMODE_HVCLOCK + +#endif /* __ASM_VDSO_CLOCKSOURCE_H */ diff --git a/arch/x86/include/asm/vdso/gettimeofday.h b/arch/x86/include/asm/vdso/gettimeofday.h index 6ee1f7dba34b..9a6dc9b4ec99 100644 --- a/arch/x86/include/asm/vdso/gettimeofday.h +++ b/arch/x86/include/asm/vdso/gettimeofday.h @@ -243,7 +243,7 @@ static u64 vread_hvclock(void) static inline u64 __arch_get_hw_counter(s32 clock_mode) { - if (clock_mode == VCLOCK_TSC) + if (likely(clock_mode == VDSO_CLOCKMODE_TSC)) return (u64)rdtsc_ordered(); /* * For any memory-mapped vclock type, we need to make sure that gcc @@ -252,13 +252,13 @@ static inline u64 __arch_get_hw_counter(s32 clock_mode) * question isn't enabled, which will segfault. Hence the barriers. */ #ifdef CONFIG_PARAVIRT_CLOCK - if (clock_mode == VCLOCK_PVCLOCK) { + if (clock_mode == VDSO_CLOCKMODE_PVCLOCK) { barrier(); return vread_pvclock(); } #endif #ifdef CONFIG_HYPERV_TIMER - if (clock_mode == VCLOCK_HVCLOCK) { + if (clock_mode == VDSO_CLOCKMODE_HVCLOCK) { barrier(); return vread_hvclock(); } diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h new file mode 100644 index 000000000000..57b1a7034c64 --- /dev/null +++ b/arch/x86/include/asm/vdso/processor.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 ARM Ltd. + */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ +static __always_inline void rep_nop(void) +{ + asm volatile("rep; nop" ::: "memory"); +} + +static __always_inline void cpu_relax(void) +{ + rep_nop(); +} + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/x86/include/asm/vdso/vsyscall.h b/arch/x86/include/asm/vdso/vsyscall.h index 0026ab2123ce..be199a9b2676 100644 --- a/arch/x86/include/asm/vdso/vsyscall.h +++ b/arch/x86/include/asm/vdso/vsyscall.h @@ -10,8 +10,6 @@ #include <asm/vgtod.h> #include <asm/vvar.h> -int vclocks_used __read_mostly; - DEFINE_VVAR(struct vdso_data, _vdso_data); /* * Update the vDSO data page to keep in sync with kernel timekeeping. @@ -23,19 +21,6 @@ struct vdso_data *__x86_get_k_vdso_data(void) } #define __arch_get_k_vdso_data __x86_get_k_vdso_data -static __always_inline -int __x86_get_clock_mode(struct timekeeper *tk) -{ - int vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode; - - /* Mark the new vclock used. */ - BUILD_BUG_ON(VCLOCK_MAX >= 32); - WRITE_ONCE(vclocks_used, READ_ONCE(vclocks_used) | (1 << vclock_mode)); - - return vclock_mode; -} -#define __arch_get_clock_mode __x86_get_clock_mode - /* The asm-generic header needs to be included after the definitions above */ #include <asm-generic/vdso/vsyscall.h> diff --git a/arch/x86/include/asm/vgtod.h b/arch/x86/include/asm/vgtod.h index a2638c6124ed..7aa38b2ad8a9 100644 --- a/arch/x86/include/asm/vgtod.h +++ b/arch/x86/include/asm/vgtod.h @@ -2,6 +2,11 @@ #ifndef _ASM_X86_VGTOD_H #define _ASM_X86_VGTOD_H +/* + * This check is required to prevent ARCH=um to include + * unwanted headers. + */ +#ifdef CONFIG_GENERIC_GETTIMEOFDAY #include <linux/compiler.h> #include <asm/clocksource.h> #include <vdso/datapage.h> @@ -14,11 +19,6 @@ typedef u64 gtod_long_t; #else typedef unsigned long gtod_long_t; #endif - -extern int vclocks_used; -static inline bool vclock_was_used(int vclock) -{ - return READ_ONCE(vclocks_used) & (1 << vclock); -} +#endif /* CONFIG_GENERIC_GETTIMEOFDAY */ #endif /* _ASM_X86_VGTOD_H */ diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index 904494b924c1..34b18f6eeb2c 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -159,12 +159,19 @@ bool kvm_check_and_clear_guest_paused(void) return ret; } +static int kvm_cs_enable(struct clocksource *cs) +{ + vclocks_set_used(VDSO_CLOCKMODE_PVCLOCK); + return 0; +} + struct clocksource kvm_clock = { .name = "kvm-clock", .read = kvm_clock_get_cycles, .rating = 400, .mask = CLOCKSOURCE_MASK(64), .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .enable = kvm_cs_enable, }; EXPORT_SYMBOL_GPL(kvm_clock); @@ -272,7 +279,7 @@ static int __init kvm_setup_vsyscall_timeinfo(void) if (!(flags & PVCLOCK_TSC_STABLE_BIT)) return 0; - kvm_clock.archdata.vclock_mode = VCLOCK_PVCLOCK; + kvm_clock.vdso_clock_mode = VDSO_CLOCKMODE_PVCLOCK; #endif kvmclock_init_mem(); diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c index 10125358b9c4..11065dc03f5b 100644 --- a/arch/x86/kernel/pvclock.c +++ b/arch/x86/kernel/pvclock.c @@ -145,7 +145,7 @@ void pvclock_read_wallclock(struct pvclock_wall_clock *wall_clock, void pvclock_set_pvti_cpu0_va(struct pvclock_vsyscall_time_info *pvti) { - WARN_ON(vclock_was_used(VCLOCK_PVCLOCK)); + WARN_ON(vclock_was_used(VDSO_CLOCKMODE_PVCLOCK)); pvti_cpu0_va = pvti; } diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c index d8673d8a779b..4d545db52efc 100644 --- a/arch/x86/kernel/time.c +++ b/arch/x86/kernel/time.c @@ -122,18 +122,12 @@ void __init time_init(void) */ void clocksource_arch_init(struct clocksource *cs) { - if (cs->archdata.vclock_mode == VCLOCK_NONE) + if (cs->vdso_clock_mode == VDSO_CLOCKMODE_NONE) return; - if (cs->archdata.vclock_mode > VCLOCK_MAX) { - pr_warn("clocksource %s registered with invalid vclock_mode %d. Disabling vclock.\n", - cs->name, cs->archdata.vclock_mode); - cs->archdata.vclock_mode = VCLOCK_NONE; - } - if (cs->mask != CLOCKSOURCE_MASK(64)) { - pr_warn("clocksource %s registered with invalid mask %016llx. Disabling vclock.\n", + pr_warn("clocksource %s registered with invalid mask %016llx for VDSO. Disabling VDSO support.\n", cs->name, cs->mask); - cs->archdata.vclock_mode = VCLOCK_NONE; + cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; } } diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 7e322e2daaf5..971d6f0216df 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -1108,17 +1108,24 @@ static void tsc_cs_tick_stable(struct clocksource *cs) sched_clock_tick_stable(); } +static int tsc_cs_enable(struct clocksource *cs) +{ + vclocks_set_used(VDSO_CLOCKMODE_TSC); + return 0; +} + /* * .mask MUST be CLOCKSOURCE_MASK(64). See comment above read_tsc() */ static struct clocksource clocksource_tsc_early = { - .name = "tsc-early", - .rating = 299, - .read = read_tsc, - .mask = CLOCKSOURCE_MASK(64), - .flags = CLOCK_SOURCE_IS_CONTINUOUS | + .name = "tsc-early", + .rating = 299, + .read = read_tsc, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_MUST_VERIFY, - .archdata = { .vclock_mode = VCLOCK_TSC }, + .vdso_clock_mode = VDSO_CLOCKMODE_TSC, + .enable = tsc_cs_enable, .resume = tsc_resume, .mark_unstable = tsc_cs_mark_unstable, .tick_stable = tsc_cs_tick_stable, @@ -1131,14 +1138,15 @@ static struct clocksource clocksource_tsc_early = { * been found good. */ static struct clocksource clocksource_tsc = { - .name = "tsc", - .rating = 300, - .read = read_tsc, - .mask = CLOCKSOURCE_MASK(64), - .flags = CLOCK_SOURCE_IS_CONTINUOUS | + .name = "tsc", + .rating = 300, + .read = read_tsc, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_MUST_VERIFY, - .archdata = { .vclock_mode = VCLOCK_TSC }, + .vdso_clock_mode = VDSO_CLOCKMODE_TSC, + .enable = tsc_cs_enable, .resume = tsc_resume, .mark_unstable = tsc_cs_mark_unstable, .tick_stable = tsc_cs_tick_stable, diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index f194dd058470..cef5a344fedb 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -815,8 +815,8 @@ TRACE_EVENT(kvm_write_tsc_offset, #ifdef CONFIG_X86_64 #define host_clocks \ - {VCLOCK_NONE, "none"}, \ - {VCLOCK_TSC, "tsc"} \ + {VDSO_CLOCKMODE_NONE, "none"}, \ + {VDSO_CLOCKMODE_TSC, "tsc"} \ TRACE_EVENT(kvm_update_master_clock, TP_PROTO(bool use_master_clock, unsigned int host_clock, bool offset_matched), diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index cf95c36cb4f4..bf8564d73fc3 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1634,7 +1634,7 @@ static void update_pvclock_gtod(struct timekeeper *tk) write_seqcount_begin(&vdata->seq); /* copy pvclock gtod data */ - vdata->clock.vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode; + vdata->clock.vclock_mode = tk->tkr_mono.clock->vdso_clock_mode; vdata->clock.cycle_last = tk->tkr_mono.cycle_last; vdata->clock.mask = tk->tkr_mono.mask; vdata->clock.mult = tk->tkr_mono.mult; @@ -1642,7 +1642,7 @@ static void update_pvclock_gtod(struct timekeeper *tk) vdata->clock.base_cycles = tk->tkr_mono.xtime_nsec; vdata->clock.offset = tk->tkr_mono.base; - vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->archdata.vclock_mode; + vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->vdso_clock_mode; vdata->raw_clock.cycle_last = tk->tkr_raw.cycle_last; vdata->raw_clock.mask = tk->tkr_raw.mask; vdata->raw_clock.mult = tk->tkr_raw.mult; @@ -1843,7 +1843,7 @@ static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) static inline int gtod_is_based_on_tsc(int mode) { - return mode == VCLOCK_TSC || mode == VCLOCK_HVCLOCK; + return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK; } static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu) @@ -1936,7 +1936,7 @@ static inline bool kvm_check_tsc_unstable(void) * TSC is marked unstable when we're running on Hyper-V, * 'TSC page' clocksource is good. */ - if (pvclock_gtod_data.clock.vclock_mode == VCLOCK_HVCLOCK) + if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK) return false; #endif return check_tsc_unstable(); @@ -2091,30 +2091,30 @@ static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp, u64 tsc_pg_val; switch (clock->vclock_mode) { - case VCLOCK_HVCLOCK: + case VDSO_CLOCKMODE_HVCLOCK: tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(), tsc_timestamp); if (tsc_pg_val != U64_MAX) { /* TSC page valid */ - *mode = VCLOCK_HVCLOCK; + *mode = VDSO_CLOCKMODE_HVCLOCK; v = (tsc_pg_val - clock->cycle_last) & clock->mask; } else { /* TSC page invalid */ - *mode = VCLOCK_NONE; + *mode = VDSO_CLOCKMODE_NONE; } break; - case VCLOCK_TSC: - *mode = VCLOCK_TSC; + case VDSO_CLOCKMODE_TSC: + *mode = VDSO_CLOCKMODE_TSC; *tsc_timestamp = read_tsc(); v = (*tsc_timestamp - clock->cycle_last) & clock->mask; break; default: - *mode = VCLOCK_NONE; + *mode = VDSO_CLOCKMODE_NONE; } - if (*mode == VCLOCK_NONE) + if (*mode == VDSO_CLOCKMODE_NONE) *tsc_timestamp = v = 0; return v * clock->mult; diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index befbdd8b17f0..c8897aad13cd 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -145,12 +145,19 @@ static struct notifier_block xen_pvclock_gtod_notifier = { .notifier_call = xen_pvclock_gtod_notify, }; +static int xen_cs_enable(struct clocksource *cs) +{ + vclocks_set_used(VDSO_CLOCKMODE_PVCLOCK); + return 0; +} + static struct clocksource xen_clocksource __read_mostly = { - .name = "xen", - .rating = 400, - .read = xen_clocksource_get_cycles, - .mask = ~0, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .name = "xen", + .rating = 400, + .read = xen_clocksource_get_cycles, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .enable = xen_cs_enable, }; /* @@ -412,12 +419,13 @@ void xen_restore_time_memory_area(void) ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t); /* - * We don't disable VCLOCK_PVCLOCK entirely if it fails to register the - * secondary time info with Xen or if we migrated to a host without the - * necessary flags. On both of these cases what happens is either - * process seeing a zeroed out pvti or seeing no PVCLOCK_TSC_STABLE_BIT - * bit set. Userspace checks the latter and if 0, it discards the data - * in pvti and fallbacks to a system call for a reliable timestamp. + * We don't disable VDSO_CLOCKMODE_PVCLOCK entirely if it fails to + * register the secondary time info with Xen or if we migrated to a + * host without the necessary flags. On both of these cases what + * happens is either process seeing a zeroed out pvti or seeing no + * PVCLOCK_TSC_STABLE_BIT bit set. Userspace checks the latter and + * if 0, it discards the data in pvti and fallbacks to a system + * call for a reliable timestamp. */ if (ret != 0) pr_notice("Cannot restore secondary vcpu_time_info (err %d)", @@ -443,7 +451,7 @@ static void xen_setup_vsyscall_time_info(void) ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t); if (ret) { - pr_notice("xen: VCLOCK_PVCLOCK not supported (err %d)\n", ret); + pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (err %d)\n", ret); free_page((unsigned long)ti); return; } @@ -460,14 +468,14 @@ static void xen_setup_vsyscall_time_info(void) if (!ret) free_page((unsigned long)ti); - pr_notice("xen: VCLOCK_PVCLOCK not supported (tsc unstable)\n"); + pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (tsc unstable)\n"); return; } xen_clock = ti; pvclock_set_pvti_cpu0_va(xen_clock); - xen_clocksource.archdata.vclock_mode = VCLOCK_PVCLOCK; + xen_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_PVCLOCK; } static void __init xen_time_init(void) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index cc909e465823..f2142e6bbea3 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -697,6 +697,14 @@ config INGENIC_TIMER help Support for the timer/counter unit of the Ingenic JZ SoCs. +config INGENIC_OST + bool "Clocksource for Ingenic OS Timer" + depends on MIPS || COMPILE_TEST + depends on COMMON_CLK + select MFD_SYSCON + help + Support for the Operating System Timer of the Ingenic JZ SoCs. + config MICROCHIP_PIT64B bool "Microchip PIT64B support" depends on OF || COMPILE_TEST diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index 713686faa549..641ba5383ab5 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -80,6 +80,7 @@ obj-$(CONFIG_ASM9260_TIMER) += asm9260_timer.o obj-$(CONFIG_H8300_TMR8) += h8300_timer8.o obj-$(CONFIG_H8300_TMR16) += h8300_timer16.o obj-$(CONFIG_H8300_TPU) += h8300_tpu.o +obj-$(CONFIG_INGENIC_OST) += ingenic-ost.o obj-$(CONFIG_INGENIC_TIMER) += ingenic-timer.o obj-$(CONFIG_CLKSRC_ST_LPC) += clksrc_st_lpc.o obj-$(CONFIG_X86_NUMACHIP) += numachip.o diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 9a5464c625b4..d53f4c7ccaae 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -69,7 +69,11 @@ static enum arch_timer_ppi_nr arch_timer_uses_ppi = ARCH_TIMER_VIRT_PPI; static bool arch_timer_c3stop; static bool arch_timer_mem_use_virtual; static bool arch_counter_suspend_stop; -static bool vdso_default = true; +#ifdef CONFIG_GENERIC_GETTIMEOFDAY +static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_ARCHTIMER; +#else +static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_NONE; +#endif /* CONFIG_GENERIC_GETTIMEOFDAY */ static cpumask_t evtstrm_available = CPU_MASK_NONE; static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); @@ -560,8 +564,8 @@ void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa * change both the default value and the vdso itself. */ if (wa->read_cntvct_el0) { - clocksource_counter.archdata.vdso_direct = false; - vdso_default = false; + clocksource_counter.vdso_clock_mode = VDSO_CLOCKMODE_NONE; + vdso_default = VDSO_CLOCKMODE_NONE; } } @@ -979,7 +983,7 @@ static void __init arch_counter_register(unsigned type) } arch_timer_read_counter = rd; - clocksource_counter.archdata.vdso_direct = vdso_default; + clocksource_counter.vdso_clock_mode = vdso_default; } else { arch_timer_read_counter = arch_counter_get_cntvct_mem; } diff --git a/drivers/clocksource/bcm2835_timer.c b/drivers/clocksource/bcm2835_timer.c index b235f446ee50..1592650b2c92 100644 --- a/drivers/clocksource/bcm2835_timer.c +++ b/drivers/clocksource/bcm2835_timer.c @@ -31,7 +31,6 @@ struct bcm2835_timer { void __iomem *compare; int match_mask; struct clock_event_device evt; - struct irqaction act; }; static void __iomem *system_clock __read_mostly; @@ -113,12 +112,9 @@ static int __init bcm2835_timer_init(struct device_node *node) timer->evt.features = CLOCK_EVT_FEAT_ONESHOT; timer->evt.set_next_event = bcm2835_time_set_next_event; timer->evt.cpumask = cpumask_of(0); - timer->act.name = node->name; - timer->act.flags = IRQF_TIMER | IRQF_SHARED; - timer->act.dev_id = timer; - timer->act.handler = bcm2835_time_interrupt; - ret = setup_irq(irq, &timer->act); + ret = request_irq(irq, bcm2835_time_interrupt, IRQF_TIMER | IRQF_SHARED, + node->name, timer); if (ret) { pr_err("Can't set up timer IRQ\n"); goto err_timer_free; diff --git a/drivers/clocksource/bcm_kona_timer.c b/drivers/clocksource/bcm_kona_timer.c index 5c40be9880f5..a50ab5c2154f 100644 --- a/drivers/clocksource/bcm_kona_timer.c +++ b/drivers/clocksource/bcm_kona_timer.c @@ -160,12 +160,6 @@ static irqreturn_t kona_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction kona_timer_irq = { - .name = "Kona Timer Tick", - .flags = IRQF_TIMER, - .handler = kona_timer_interrupt, -}; - static int __init kona_timer_init(struct device_node *node) { u32 freq; @@ -192,7 +186,9 @@ static int __init kona_timer_init(struct device_node *node) kona_timer_disable_and_clear(timers.tmr_regs); kona_timer_clockevents_init(); - setup_irq(timers.tmr_irq, &kona_timer_irq); + if (request_irq(timers.tmr_irq, kona_timer_interrupt, IRQF_TIMER, + "Kona Timer Tick", NULL)) + pr_err("%s: request_irq() failed\n", "Kona Timer Tick"); kona_timer_set_next_event((arch_timer_rate / HZ), NULL); return 0; diff --git a/drivers/clocksource/dw_apb_timer.c b/drivers/clocksource/dw_apb_timer.c index 654766538f93..b207a77b0831 100644 --- a/drivers/clocksource/dw_apb_timer.c +++ b/drivers/clocksource/dw_apb_timer.c @@ -270,15 +270,10 @@ dw_apb_clockevent_init(int cpu, const char *name, unsigned rating, dw_ced->ced.rating = rating; dw_ced->ced.name = name; - dw_ced->irqaction.name = dw_ced->ced.name; - dw_ced->irqaction.handler = dw_apb_clockevent_irq; - dw_ced->irqaction.dev_id = &dw_ced->ced; - dw_ced->irqaction.irq = irq; - dw_ced->irqaction.flags = IRQF_TIMER | IRQF_IRQPOLL | - IRQF_NOBALANCING; - dw_ced->eoi = apbt_eoi; - err = setup_irq(irq, &dw_ced->irqaction); + err = request_irq(irq, dw_apb_clockevent_irq, + IRQF_TIMER | IRQF_IRQPOLL | IRQF_NOBALANCING, + dw_ced->ced.name, &dw_ced->ced); if (err) { pr_err("failed to request timer irq\n"); kfree(dw_ced); diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index a267fe31ef13..fabad79baafc 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -329,19 +329,15 @@ static irqreturn_t exynos4_mct_comp_isr(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction mct_comp_event_irq = { - .name = "mct_comp_irq", - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = exynos4_mct_comp_isr, - .dev_id = &mct_comp_device, -}; - static int exynos4_clockevent_init(void) { mct_comp_device.cpumask = cpumask_of(0); clockevents_config_and_register(&mct_comp_device, clk_rate, 0xf, 0xffffffff); - setup_irq(mct_irqs[MCT_G0_IRQ], &mct_comp_event_irq); + if (request_irq(mct_irqs[MCT_G0_IRQ], exynos4_mct_comp_isr, + IRQF_TIMER | IRQF_IRQPOLL, "mct_comp_irq", + &mct_comp_device)) + pr_err("%s: request_irq() failed\n", "mct_comp_irq"); return 0; } diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c index eb0ba7818eb0..09aa44cb8a91 100644 --- a/drivers/clocksource/hyperv_timer.c +++ b/drivers/clocksource/hyperv_timer.c @@ -370,6 +370,12 @@ static void resume_hv_clock_tsc(struct clocksource *arg) hv_set_reference_tsc(tsc_msr); } +static int hv_cs_enable(struct clocksource *cs) +{ + hv_enable_vdso_clocksource(); + return 0; +} + static struct clocksource hyperv_cs_tsc = { .name = "hyperv_clocksource_tsc_page", .rating = 250, @@ -378,6 +384,7 @@ static struct clocksource hyperv_cs_tsc = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, .suspend= suspend_hv_clock_tsc, .resume = resume_hv_clock_tsc, + .enable = hv_cs_enable, }; static u64 notrace read_hv_clock_msr(void) diff --git a/drivers/clocksource/ingenic-ost.c b/drivers/clocksource/ingenic-ost.c new file mode 100644 index 000000000000..029efc2731b4 --- /dev/null +++ b/drivers/clocksource/ingenic-ost.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * JZ47xx SoCs TCU Operating System Timer driver + * + * Copyright (C) 2016 Maarten ter Huurne <maarten@treewalker.org> + * Copyright (C) 2020 Paul Cercueil <paul@crapouillou.net> + */ + +#include <linux/clk.h> +#include <linux/clocksource.h> +#include <linux/mfd/ingenic-tcu.h> +#include <linux/mfd/syscon.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm.h> +#include <linux/regmap.h> +#include <linux/sched_clock.h> + +#define TCU_OST_TCSR_MASK 0xffc0 +#define TCU_OST_TCSR_CNT_MD BIT(15) + +#define TCU_OST_CHANNEL 15 + +/* + * The TCU_REG_OST_CNT{L,R} from <linux/mfd/ingenic-tcu.h> are only for the + * regmap; these are for use with the __iomem pointer. + */ +#define OST_REG_CNTL 0x4 +#define OST_REG_CNTH 0x8 + +struct ingenic_ost_soc_info { + bool is64bit; +}; + +struct ingenic_ost { + void __iomem *regs; + struct clk *clk; + + struct clocksource cs; +}; + +static struct ingenic_ost *ingenic_ost; + +static u64 notrace ingenic_ost_read_cntl(void) +{ + /* Read using __iomem pointer instead of regmap to avoid locking */ + return readl(ingenic_ost->regs + OST_REG_CNTL); +} + +static u64 notrace ingenic_ost_read_cnth(void) +{ + /* Read using __iomem pointer instead of regmap to avoid locking */ + return readl(ingenic_ost->regs + OST_REG_CNTH); +} + +static u64 notrace ingenic_ost_clocksource_readl(struct clocksource *cs) +{ + return ingenic_ost_read_cntl(); +} + +static u64 notrace ingenic_ost_clocksource_readh(struct clocksource *cs) +{ + return ingenic_ost_read_cnth(); +} + +static int __init ingenic_ost_probe(struct platform_device *pdev) +{ + const struct ingenic_ost_soc_info *soc_info; + struct device *dev = &pdev->dev; + struct ingenic_ost *ost; + struct clocksource *cs; + struct regmap *map; + unsigned long rate; + int err; + + soc_info = device_get_match_data(dev); + if (!soc_info) + return -EINVAL; + + ost = devm_kzalloc(dev, sizeof(*ost), GFP_KERNEL); + if (!ost) + return -ENOMEM; + + ingenic_ost = ost; + + ost->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ost->regs)) + return PTR_ERR(ost->regs); + + map = device_node_to_regmap(dev->parent->of_node); + if (!map) { + dev_err(dev, "regmap not found"); + return -EINVAL; + } + + ost->clk = devm_clk_get(dev, "ost"); + if (IS_ERR(ost->clk)) + return PTR_ERR(ost->clk); + + err = clk_prepare_enable(ost->clk); + if (err) + return err; + + /* Clear counter high/low registers */ + if (soc_info->is64bit) + regmap_write(map, TCU_REG_OST_CNTL, 0); + regmap_write(map, TCU_REG_OST_CNTH, 0); + + /* Don't reset counter at compare value. */ + regmap_update_bits(map, TCU_REG_OST_TCSR, + TCU_OST_TCSR_MASK, TCU_OST_TCSR_CNT_MD); + + rate = clk_get_rate(ost->clk); + + /* Enable OST TCU channel */ + regmap_write(map, TCU_REG_TESR, BIT(TCU_OST_CHANNEL)); + + cs = &ost->cs; + cs->name = "ingenic-ost"; + cs->rating = 320; + cs->flags = CLOCK_SOURCE_IS_CONTINUOUS; + cs->mask = CLOCKSOURCE_MASK(32); + + if (soc_info->is64bit) + cs->read = ingenic_ost_clocksource_readl; + else + cs->read = ingenic_ost_clocksource_readh; + + err = clocksource_register_hz(cs, rate); + if (err) { + dev_err(dev, "clocksource registration failed"); + clk_disable_unprepare(ost->clk); + return err; + } + + if (soc_info->is64bit) + sched_clock_register(ingenic_ost_read_cntl, 32, rate); + else + sched_clock_register(ingenic_ost_read_cnth, 32, rate); + + return 0; +} + +static int __maybe_unused ingenic_ost_suspend(struct device *dev) +{ + struct ingenic_ost *ost = dev_get_drvdata(dev); + + clk_disable(ost->clk); + + return 0; +} + +static int __maybe_unused ingenic_ost_resume(struct device *dev) +{ + struct ingenic_ost *ost = dev_get_drvdata(dev); + + return clk_enable(ost->clk); +} + +static const struct dev_pm_ops __maybe_unused ingenic_ost_pm_ops = { + /* _noirq: We want the OST clock to be gated last / ungated first */ + .suspend_noirq = ingenic_ost_suspend, + .resume_noirq = ingenic_ost_resume, +}; + +static const struct ingenic_ost_soc_info jz4725b_ost_soc_info = { + .is64bit = false, +}; + +static const struct ingenic_ost_soc_info jz4770_ost_soc_info = { + .is64bit = true, +}; + +static const struct of_device_id ingenic_ost_of_match[] = { + { .compatible = "ingenic,jz4725b-ost", .data = &jz4725b_ost_soc_info, }, + { .compatible = "ingenic,jz4770-ost", .data = &jz4770_ost_soc_info, }, + { } +}; + +static struct platform_driver ingenic_ost_driver = { + .driver = { + .name = "ingenic-ost", +#ifdef CONFIG_PM_SUSPEND + .pm = &ingenic_ost_pm_ops, +#endif + .of_match_table = ingenic_ost_of_match, + }, +}; +builtin_platform_driver_probe(ingenic_ost_driver, ingenic_ost_probe); diff --git a/drivers/clocksource/ingenic-timer.c b/drivers/clocksource/ingenic-timer.c index 4bbdb3d3d0c6..496333650de2 100644 --- a/drivers/clocksource/ingenic-timer.c +++ b/drivers/clocksource/ingenic-timer.c @@ -230,6 +230,7 @@ static const struct of_device_id ingenic_tcu_of_match[] = { { .compatible = "ingenic,jz4740-tcu", .data = &jz4740_soc_info, }, { .compatible = "ingenic,jz4725b-tcu", .data = &jz4725b_soc_info, }, { .compatible = "ingenic,jz4770-tcu", .data = &jz4740_soc_info, }, + { .compatible = "ingenic,x1000-tcu", .data = &jz4740_soc_info, }, { /* sentinel */ } }; @@ -302,7 +303,7 @@ err_free_ingenic_tcu: TIMER_OF_DECLARE(jz4740_tcu_intc, "ingenic,jz4740-tcu", ingenic_tcu_init); TIMER_OF_DECLARE(jz4725b_tcu_intc, "ingenic,jz4725b-tcu", ingenic_tcu_init); TIMER_OF_DECLARE(jz4770_tcu_intc, "ingenic,jz4770-tcu", ingenic_tcu_init); - +TIMER_OF_DECLARE(x1000_tcu_intc, "ingenic,x1000-tcu", ingenic_tcu_init); static int __init ingenic_tcu_probe(struct platform_device *pdev) { diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c index 37671a5d4ed9..8b5f8ae723cb 100644 --- a/drivers/clocksource/mips-gic-timer.c +++ b/drivers/clocksource/mips-gic-timer.c @@ -155,10 +155,10 @@ static u64 gic_hpt_read(struct clocksource *cs) } static struct clocksource gic_clocksource = { - .name = "GIC", - .read = gic_hpt_read, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .archdata = { .vdso_clock_mode = VDSO_CLOCK_GIC }, + .name = "GIC", + .read = gic_hpt_read, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .vdso_clock_mode = VDSO_CLOCKMODE_GIC, }; static int __init __gic_clocksource_init(void) diff --git a/drivers/clocksource/mxs_timer.c b/drivers/clocksource/mxs_timer.c index f6ddae30933f..bc96a4cbf26c 100644 --- a/drivers/clocksource/mxs_timer.c +++ b/drivers/clocksource/mxs_timer.c @@ -117,13 +117,6 @@ static irqreturn_t mxs_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction mxs_timer_irq = { - .name = "MXS Timer Tick", - .dev_id = &mxs_clockevent_device, - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = mxs_timer_interrupt, -}; - static void mxs_irq_clear(char *state) { /* Disable interrupt in timer module */ @@ -277,6 +270,7 @@ static int __init mxs_timer_init(struct device_node *np) if (irq <= 0) return -EINVAL; - return setup_irq(irq, &mxs_timer_irq); + return request_irq(irq, mxs_timer_interrupt, IRQF_TIMER | IRQF_IRQPOLL, + "MXS Timer Tick", &mxs_clockevent_device); } TIMER_OF_DECLARE(mxs, "fsl,timrot", mxs_timer_init); diff --git a/drivers/clocksource/nomadik-mtu.c b/drivers/clocksource/nomadik-mtu.c index 3f7fa8c01367..f49a631d8f58 100644 --- a/drivers/clocksource/nomadik-mtu.c +++ b/drivers/clocksource/nomadik-mtu.c @@ -181,13 +181,6 @@ static irqreturn_t nmdk_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction nmdk_timer_irq = { - .name = "Nomadik Timer Tick", - .flags = IRQF_TIMER, - .handler = nmdk_timer_interrupt, - .dev_id = &nmdk_clkevt, -}; - static int __init nmdk_timer_init(void __iomem *base, int irq, struct clk *pclk, struct clk *clk) { @@ -232,7 +225,9 @@ static int __init nmdk_timer_init(void __iomem *base, int irq, sched_clock_register(nomadik_read_sched_clock, 32, rate); /* Timer 1 is used for events, register irq and clockevents */ - setup_irq(irq, &nmdk_timer_irq); + if (request_irq(irq, nmdk_timer_interrupt, IRQF_TIMER, + "Nomadik Timer Tick", &nmdk_clkevt)) + pr_err("%s: request_irq() failed\n", "Nomadik Timer Tick"); nmdk_clkevt.cpumask = cpumask_of(0); nmdk_clkevt.irq = irq; clockevents_config_and_register(&nmdk_clkevt, rate, 2, 0xffffffffU); diff --git a/drivers/clocksource/samsung_pwm_timer.c b/drivers/clocksource/samsung_pwm_timer.c index dae1b2b5a0c5..f760229d0c7f 100644 --- a/drivers/clocksource/samsung_pwm_timer.c +++ b/drivers/clocksource/samsung_pwm_timer.c @@ -256,13 +256,6 @@ static irqreturn_t samsung_clock_event_isr(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction samsung_clock_event_irq = { - .name = "samsung_time_irq", - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = samsung_clock_event_isr, - .dev_id = &time_event_device, -}; - static void __init samsung_clockevent_init(void) { unsigned long pclk; @@ -282,7 +275,10 @@ static void __init samsung_clockevent_init(void) clock_rate, 1, pwm.tcnt_max); irq_number = pwm.irq[pwm.event_id]; - setup_irq(irq_number, &samsung_clock_event_irq); + if (request_irq(irq_number, samsung_clock_event_isr, + IRQF_TIMER | IRQF_IRQPOLL, "samsung_time_irq", + &time_event_device)) + pr_err("%s: request_irq() failed\n", "samsung_time_irq"); if (pwm.variant.has_tint_cstat) { u32 mask = (1 << pwm.event_id); diff --git a/drivers/clocksource/timer-atlas7.c b/drivers/clocksource/timer-atlas7.c index 93c3ac6d72bd..c21c91c2bc56 100644 --- a/drivers/clocksource/timer-atlas7.c +++ b/drivers/clocksource/timer-atlas7.c @@ -159,29 +159,23 @@ static struct clocksource sirfsoc_clocksource = { .resume = sirfsoc_clocksource_resume, }; -static struct irqaction sirfsoc_timer_irq = { - .name = "sirfsoc_timer0", - .flags = IRQF_TIMER | IRQF_NOBALANCING, - .handler = sirfsoc_timer_interrupt, -}; - -static struct irqaction sirfsoc_timer1_irq = { - .name = "sirfsoc_timer1", - .flags = IRQF_TIMER | IRQF_NOBALANCING, - .handler = sirfsoc_timer_interrupt, -}; +static unsigned int sirfsoc_timer_irq, sirfsoc_timer1_irq; static int sirfsoc_local_timer_starting_cpu(unsigned int cpu) { struct clock_event_device *ce = per_cpu_ptr(sirfsoc_clockevent, cpu); - struct irqaction *action; - - if (cpu == 0) - action = &sirfsoc_timer_irq; - else - action = &sirfsoc_timer1_irq; + unsigned int irq; + const char *name; + + if (cpu == 0) { + irq = sirfsoc_timer_irq; + name = "sirfsoc_timer0"; + } else { + irq = sirfsoc_timer1_irq; + name = "sirfsoc_timer1"; + } - ce->irq = action->irq; + ce->irq = irq; ce->name = "local_timer"; ce->features = CLOCK_EVT_FEAT_ONESHOT; ce->rating = 200; @@ -196,9 +190,9 @@ static int sirfsoc_local_timer_starting_cpu(unsigned int cpu) ce->min_delta_ticks = 2; ce->cpumask = cpumask_of(cpu); - action->dev_id = ce; - BUG_ON(setup_irq(ce->irq, action)); - irq_force_affinity(action->irq, cpumask_of(cpu)); + BUG_ON(request_irq(ce->irq, sirfsoc_timer_interrupt, + IRQF_TIMER | IRQF_NOBALANCING, name, ce)); + irq_force_affinity(ce->irq, cpumask_of(cpu)); clockevents_register_device(ce); return 0; @@ -206,12 +200,14 @@ static int sirfsoc_local_timer_starting_cpu(unsigned int cpu) static int sirfsoc_local_timer_dying_cpu(unsigned int cpu) { + struct clock_event_device *ce = per_cpu_ptr(sirfsoc_clockevent, cpu); + sirfsoc_timer_count_disable(1); if (cpu == 0) - remove_irq(sirfsoc_timer_irq.irq, &sirfsoc_timer_irq); + free_irq(sirfsoc_timer_irq, ce); else - remove_irq(sirfsoc_timer1_irq.irq, &sirfsoc_timer1_irq); + free_irq(sirfsoc_timer1_irq, ce); return 0; } @@ -268,14 +264,14 @@ static int __init sirfsoc_of_timer_init(struct device_node *np) return -ENXIO; } - sirfsoc_timer_irq.irq = irq_of_parse_and_map(np, 0); - if (!sirfsoc_timer_irq.irq) { + sirfsoc_timer_irq = irq_of_parse_and_map(np, 0); + if (!sirfsoc_timer_irq) { pr_err("No irq passed for timer0 via DT\n"); return -EINVAL; } - sirfsoc_timer1_irq.irq = irq_of_parse_and_map(np, 1); - if (!sirfsoc_timer1_irq.irq) { + sirfsoc_timer1_irq = irq_of_parse_and_map(np, 1); + if (!sirfsoc_timer1_irq) { pr_err("No irq passed for timer1 via DT\n"); return -EINVAL; } diff --git a/drivers/clocksource/timer-cs5535.c b/drivers/clocksource/timer-cs5535.c index 8f6bc536bef2..d47acfe848ae 100644 --- a/drivers/clocksource/timer-cs5535.c +++ b/drivers/clocksource/timer-cs5535.c @@ -131,14 +131,9 @@ static irqreturn_t mfgpt_tick(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction mfgptirq = { - .handler = mfgpt_tick, - .flags = IRQF_NOBALANCING | IRQF_TIMER | IRQF_SHARED, - .name = DRV_NAME, -}; - static int __init cs5535_mfgpt_init(void) { + unsigned long flags = IRQF_NOBALANCING | IRQF_TIMER | IRQF_SHARED; struct cs5535_mfgpt_timer *timer; int ret; uint16_t val; @@ -158,7 +153,7 @@ static int __init cs5535_mfgpt_init(void) } /* And register it with the kernel */ - ret = setup_irq(timer_irq, &mfgptirq); + ret = request_irq(timer_irq, mfgpt_tick, flags, DRV_NAME, timer); if (ret) { printk(KERN_ERR DRV_NAME ": Unable to set up the interrupt.\n"); goto err_irq; diff --git a/drivers/clocksource/timer-efm32.c b/drivers/clocksource/timer-efm32.c index 5a22cb079ad3..441a4b916841 100644 --- a/drivers/clocksource/timer-efm32.c +++ b/drivers/clocksource/timer-efm32.c @@ -119,13 +119,6 @@ static struct efm32_clock_event_ddata clock_event_ddata = { }, }; -static struct irqaction efm32_clock_event_irq = { - .name = "efm32 clockevent", - .flags = IRQF_TIMER, - .handler = efm32_clock_event_handler, - .dev_id = &clock_event_ddata, -}; - static int __init efm32_clocksource_init(struct device_node *np) { struct clk *clk; @@ -230,7 +223,8 @@ static int __init efm32_clockevent_init(struct device_node *np) DIV_ROUND_CLOSEST(rate, 1024), 0xf, 0xffff); - ret = setup_irq(irq, &efm32_clock_event_irq); + ret = request_irq(irq, efm32_clock_event_handler, IRQF_TIMER, + "efm32 clockevent", &clock_event_ddata); if (ret) { pr_err("Failed setup irq\n"); goto err_setup_irq; diff --git a/drivers/clocksource/timer-fsl-ftm.c b/drivers/clocksource/timer-fsl-ftm.c index a9d9a3ca5996..12a2ed7cfaff 100644 --- a/drivers/clocksource/timer-fsl-ftm.c +++ b/drivers/clocksource/timer-fsl-ftm.c @@ -176,13 +176,6 @@ static struct clock_event_device ftm_clockevent = { .rating = 300, }; -static struct irqaction ftm_timer_irq = { - .name = "Freescale ftm timer", - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = ftm_evt_interrupt, - .dev_id = &ftm_clockevent, -}; - static int __init ftm_clockevent_init(unsigned long freq, int irq) { int err; @@ -192,7 +185,8 @@ static int __init ftm_clockevent_init(unsigned long freq, int irq) ftm_reset_counter(priv->clkevt_base); - err = setup_irq(irq, &ftm_timer_irq); + err = request_irq(irq, ftm_evt_interrupt, IRQF_TIMER | IRQF_IRQPOLL, + "Freescale ftm timer", &ftm_clockevent); if (err) { pr_err("ftm: setup irq failed: %d\n", err); return err; diff --git a/drivers/clocksource/timer-fttmr010.c b/drivers/clocksource/timer-fttmr010.c index fadff7915dd9..edb1d5f193f5 100644 --- a/drivers/clocksource/timer-fttmr010.c +++ b/drivers/clocksource/timer-fttmr010.c @@ -38,6 +38,11 @@ #define TIMER_CR (0x30) /* + * Control register set to clear for ast2600 only. + */ +#define AST2600_TIMER_CR_CLR (0x3c) + +/* * Control register (TMC30) bit fields for fttmr010/gemini/moxart timers. */ #define TIMER_1_CR_ENABLE BIT(0) @@ -97,6 +102,7 @@ struct fttmr010 { bool is_aspeed; u32 t1_enable_val; struct clock_event_device clkevt; + int (*timer_shutdown)(struct clock_event_device *evt); #ifdef CONFIG_ARM struct delay_timer delay_timer; #endif @@ -140,9 +146,7 @@ static int fttmr010_timer_set_next_event(unsigned long cycles, u32 cr; /* Stop */ - cr = readl(fttmr010->base + TIMER_CR); - cr &= ~fttmr010->t1_enable_val; - writel(cr, fttmr010->base + TIMER_CR); + fttmr010->timer_shutdown(evt); if (fttmr010->is_aspeed) { /* @@ -164,6 +168,16 @@ static int fttmr010_timer_set_next_event(unsigned long cycles, return 0; } +static int ast2600_timer_shutdown(struct clock_event_device *evt) +{ + struct fttmr010 *fttmr010 = to_fttmr010(evt); + + /* Stop */ + writel(fttmr010->t1_enable_val, fttmr010->base + AST2600_TIMER_CR_CLR); + + return 0; +} + static int fttmr010_timer_shutdown(struct clock_event_device *evt) { struct fttmr010 *fttmr010 = to_fttmr010(evt); @@ -183,9 +197,7 @@ static int fttmr010_timer_set_oneshot(struct clock_event_device *evt) u32 cr; /* Stop */ - cr = readl(fttmr010->base + TIMER_CR); - cr &= ~fttmr010->t1_enable_val; - writel(cr, fttmr010->base + TIMER_CR); + fttmr010->timer_shutdown(evt); /* Setup counter start from 0 or ~0 */ writel(0, fttmr010->base + TIMER1_COUNT); @@ -211,9 +223,7 @@ static int fttmr010_timer_set_periodic(struct clock_event_device *evt) u32 cr; /* Stop */ - cr = readl(fttmr010->base + TIMER_CR); - cr &= ~fttmr010->t1_enable_val; - writel(cr, fttmr010->base + TIMER_CR); + fttmr010->timer_shutdown(evt); /* Setup timer to fire at 1/HZ intervals. */ if (fttmr010->is_aspeed) { @@ -249,7 +259,21 @@ static irqreturn_t fttmr010_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static int __init fttmr010_common_init(struct device_node *np, bool is_aspeed) +static irqreturn_t ast2600_timer_interrupt(int irq, void *dev_id) +{ + struct clock_event_device *evt = dev_id; + struct fttmr010 *fttmr010 = to_fttmr010(evt); + + writel(0x1, fttmr010->base + TIMER_INTR_STATE); + + evt->event_handler(evt); + return IRQ_HANDLED; +} + +static int __init fttmr010_common_init(struct device_node *np, + bool is_aspeed, + int (*timer_shutdown)(struct clock_event_device *), + irq_handler_t irq_handler) { struct fttmr010 *fttmr010; int irq; @@ -350,6 +374,8 @@ static int __init fttmr010_common_init(struct device_node *np, bool is_aspeed) fttmr010->tick_rate); } + fttmr010->timer_shutdown = timer_shutdown; + /* * Setup clockevent timer (interrupt-driven) on timer 1. */ @@ -357,7 +383,7 @@ static int __init fttmr010_common_init(struct device_node *np, bool is_aspeed) writel(0, fttmr010->base + TIMER1_LOAD); writel(0, fttmr010->base + TIMER1_MATCH1); writel(0, fttmr010->base + TIMER1_MATCH2); - ret = request_irq(irq, fttmr010_timer_interrupt, IRQF_TIMER, + ret = request_irq(irq, irq_handler, IRQF_TIMER, "FTTMR010-TIMER1", &fttmr010->clkevt); if (ret) { pr_err("FTTMR010-TIMER1 no IRQ\n"); @@ -370,10 +396,10 @@ static int __init fttmr010_common_init(struct device_node *np, bool is_aspeed) fttmr010->clkevt.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; fttmr010->clkevt.set_next_event = fttmr010_timer_set_next_event; - fttmr010->clkevt.set_state_shutdown = fttmr010_timer_shutdown; + fttmr010->clkevt.set_state_shutdown = fttmr010->timer_shutdown; fttmr010->clkevt.set_state_periodic = fttmr010_timer_set_periodic; fttmr010->clkevt.set_state_oneshot = fttmr010_timer_set_oneshot; - fttmr010->clkevt.tick_resume = fttmr010_timer_shutdown; + fttmr010->clkevt.tick_resume = fttmr010->timer_shutdown; fttmr010->clkevt.cpumask = cpumask_of(0); fttmr010->clkevt.irq = irq; clockevents_config_and_register(&fttmr010->clkevt, @@ -404,14 +430,25 @@ out_disable_clock: return ret; } +static __init int ast2600_timer_init(struct device_node *np) +{ + return fttmr010_common_init(np, true, + ast2600_timer_shutdown, + ast2600_timer_interrupt); +} + static __init int aspeed_timer_init(struct device_node *np) { - return fttmr010_common_init(np, true); + return fttmr010_common_init(np, true, + fttmr010_timer_shutdown, + fttmr010_timer_interrupt); } static __init int fttmr010_timer_init(struct device_node *np) { - return fttmr010_common_init(np, false); + return fttmr010_common_init(np, false, + fttmr010_timer_shutdown, + fttmr010_timer_interrupt); } TIMER_OF_DECLARE(fttmr010, "faraday,fttmr010", fttmr010_timer_init); @@ -419,3 +456,4 @@ TIMER_OF_DECLARE(gemini, "cortina,gemini-timer", fttmr010_timer_init); TIMER_OF_DECLARE(moxart, "moxa,moxart-timer", fttmr010_timer_init); TIMER_OF_DECLARE(ast2400, "aspeed,ast2400-timer", aspeed_timer_init); TIMER_OF_DECLARE(ast2500, "aspeed,ast2500-timer", aspeed_timer_init); +TIMER_OF_DECLARE(ast2600, "aspeed,ast2600-timer", ast2600_timer_init); diff --git a/drivers/clocksource/timer-imx-gpt.c b/drivers/clocksource/timer-imx-gpt.c index 706c0d0ff56c..7b2c70f2f353 100644 --- a/drivers/clocksource/timer-imx-gpt.c +++ b/drivers/clocksource/timer-imx-gpt.c @@ -67,7 +67,6 @@ struct imx_timer { struct clk *clk_ipg; const struct imx_gpt_data *gpt; struct clock_event_device ced; - struct irqaction act; }; struct imx_gpt_data { @@ -273,7 +272,6 @@ static irqreturn_t mxc_timer_interrupt(int irq, void *dev_id) static int __init mxc_clockevent_init(struct imx_timer *imxtm) { struct clock_event_device *ced = &imxtm->ced; - struct irqaction *act = &imxtm->act; ced->name = "mxc_timer1"; ced->features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_DYNIRQ; @@ -287,12 +285,8 @@ static int __init mxc_clockevent_init(struct imx_timer *imxtm) clockevents_config_and_register(ced, clk_get_rate(imxtm->clk_per), 0xff, 0xfffffffe); - act->name = "i.MX Timer Tick"; - act->flags = IRQF_TIMER | IRQF_IRQPOLL; - act->handler = mxc_timer_interrupt; - act->dev_id = ced; - - return setup_irq(imxtm->irq, act); + return request_irq(imxtm->irq, mxc_timer_interrupt, + IRQF_TIMER | IRQF_IRQPOLL, "i.MX Timer Tick", ced); } static void imx1_gpt_setup_tctl(struct imx_timer *imxtm) diff --git a/drivers/clocksource/timer-imx-sysctr.c b/drivers/clocksource/timer-imx-sysctr.c index b7c80a368a1b..18b90fc56bfc 100644 --- a/drivers/clocksource/timer-imx-sysctr.c +++ b/drivers/clocksource/timer-imx-sysctr.c @@ -4,8 +4,6 @@ #include <linux/interrupt.h> #include <linux/clockchips.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> #include "timer-of.h" diff --git a/drivers/clocksource/timer-imx-tpm.c b/drivers/clocksource/timer-imx-tpm.c index c1d52d5264c2..6334a35fdc2f 100644 --- a/drivers/clocksource/timer-imx-tpm.c +++ b/drivers/clocksource/timer-imx-tpm.c @@ -8,8 +8,6 @@ #include <linux/clocksource.h> #include <linux/delay.h> #include <linux/interrupt.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> #include <linux/sched_clock.h> #include "timer-of.h" diff --git a/drivers/clocksource/timer-integrator-ap.c b/drivers/clocksource/timer-integrator-ap.c index c90a69c7b5fa..b0fcbaac58b0 100644 --- a/drivers/clocksource/timer-integrator-ap.c +++ b/drivers/clocksource/timer-integrator-ap.c @@ -123,13 +123,6 @@ static struct clock_event_device integrator_clockevent = { .rating = 300, }; -static struct irqaction integrator_timer_irq = { - .name = "timer", - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = integrator_timer_interrupt, - .dev_id = &integrator_clockevent, -}; - static int integrator_clockevent_init(unsigned long inrate, void __iomem *base, int irq) { @@ -149,7 +142,9 @@ static int integrator_clockevent_init(unsigned long inrate, timer_reload = rate / HZ; writel(ctrl, clkevt_base + TIMER_CTRL); - ret = setup_irq(irq, &integrator_timer_irq); + ret = request_irq(irq, integrator_timer_interrupt, + IRQF_TIMER | IRQF_IRQPOLL, "timer", + &integrator_clockevent); if (ret) return ret; diff --git a/drivers/clocksource/timer-meson6.c b/drivers/clocksource/timer-meson6.c index 9e8b467c71da..99f5510a2b56 100644 --- a/drivers/clocksource/timer-meson6.c +++ b/drivers/clocksource/timer-meson6.c @@ -150,13 +150,6 @@ static irqreturn_t meson6_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction meson6_timer_irq = { - .name = "meson6_timer", - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = meson6_timer_interrupt, - .dev_id = &meson6_clockevent, -}; - static int __init meson6_timer_init(struct device_node *node) { u32 val; @@ -194,7 +187,9 @@ static int __init meson6_timer_init(struct device_node *node) /* Stop the timer A */ meson6_clkevt_time_stop(); - ret = setup_irq(irq, &meson6_timer_irq); + ret = request_irq(irq, meson6_timer_interrupt, + IRQF_TIMER | IRQF_IRQPOLL, "meson6_timer", + &meson6_clockevent); if (ret) { pr_warn("failed to setup irq %d\n", irq); return ret; diff --git a/drivers/clocksource/timer-microchip-pit64b.c b/drivers/clocksource/timer-microchip-pit64b.c index bd63d3484838..59e11ca8ee73 100644 --- a/drivers/clocksource/timer-microchip-pit64b.c +++ b/drivers/clocksource/timer-microchip-pit64b.c @@ -264,6 +264,7 @@ static int __init mchp_pit64b_init_mode(struct mchp_pit64b_timer *timer, if (!best_diff) { timer->mode |= MCHP_PIT64B_MR_SGCLK; + clk_set_rate(timer->gclk, gclk_round); goto done; } diff --git a/drivers/clocksource/timer-orion.c b/drivers/clocksource/timer-orion.c index 7d487107e3cd..d01ff4181867 100644 --- a/drivers/clocksource/timer-orion.c +++ b/drivers/clocksource/timer-orion.c @@ -114,12 +114,6 @@ static irqreturn_t orion_clkevt_irq_handler(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction orion_clkevt_irq = { - .name = "orion_event", - .flags = IRQF_TIMER, - .handler = orion_clkevt_irq_handler, -}; - static int __init orion_timer_init(struct device_node *np) { unsigned long rate; @@ -172,7 +166,8 @@ static int __init orion_timer_init(struct device_node *np) sched_clock_register(orion_read_sched_clock, 32, rate); /* setup timer1 as clockevent timer */ - ret = setup_irq(irq, &orion_clkevt_irq); + ret = request_irq(irq, orion_clkevt_irq_handler, IRQF_TIMER, + "orion_event", NULL); if (ret) { pr_err("%pOFn: unable to setup irq\n", np); return ret; diff --git a/drivers/clocksource/timer-owl.c b/drivers/clocksource/timer-owl.c index 900fe736145d..ac97420bfa7c 100644 --- a/drivers/clocksource/timer-owl.c +++ b/drivers/clocksource/timer-owl.c @@ -135,8 +135,11 @@ static int __init owl_timer_init(struct device_node *node) } clk = of_clk_get(node, 0); - if (IS_ERR(clk)) - return PTR_ERR(clk); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + pr_err("Failed to get clock for clocksource (%d)\n", ret); + return ret; + } rate = clk_get_rate(clk); @@ -144,8 +147,12 @@ static int __init owl_timer_init(struct device_node *node) owl_timer_set_enabled(owl_clksrc_base, true); sched_clock_register(owl_timer_sched_read, 32, rate); - clocksource_mmio_init(owl_clksrc_base + OWL_Tx_VAL, node->name, - rate, 200, 32, clocksource_mmio_readl_up); + ret = clocksource_mmio_init(owl_clksrc_base + OWL_Tx_VAL, node->name, + rate, 200, 32, clocksource_mmio_readl_up); + if (ret) { + pr_err("Failed to register clocksource (%d)\n", ret); + return ret; + } owl_timer_reset(owl_clkevt_base); diff --git a/drivers/clocksource/timer-prima2.c b/drivers/clocksource/timer-prima2.c index d4a9dcf5fba2..c5d469342a9d 100644 --- a/drivers/clocksource/timer-prima2.c +++ b/drivers/clocksource/timer-prima2.c @@ -165,14 +165,6 @@ static struct clocksource sirfsoc_clocksource = { .resume = sirfsoc_clocksource_resume, }; -static struct irqaction sirfsoc_timer_irq = { - .name = "sirfsoc_timer0", - .flags = IRQF_TIMER, - .irq = 0, - .handler = sirfsoc_timer_interrupt, - .dev_id = &sirfsoc_clockevent, -}; - /* Overwrite weak default sched_clock with more precise one */ static u64 notrace sirfsoc_read_sched_clock(void) { @@ -190,6 +182,7 @@ static void __init sirfsoc_clockevent_init(void) static int __init sirfsoc_prima2_timer_init(struct device_node *np) { unsigned long rate; + unsigned int irq; struct clk *clk; int ret; @@ -218,7 +211,7 @@ static int __init sirfsoc_prima2_timer_init(struct device_node *np) return -ENXIO; } - sirfsoc_timer_irq.irq = irq_of_parse_and_map(np, 0); + irq = irq_of_parse_and_map(np, 0); writel_relaxed(rate / PRIMA2_CLOCK_FREQ / 2 - 1, sirfsoc_timer_base + SIRFSOC_TIMER_DIV); @@ -234,7 +227,8 @@ static int __init sirfsoc_prima2_timer_init(struct device_node *np) sched_clock_register(sirfsoc_read_sched_clock, 64, PRIMA2_CLOCK_FREQ); - ret = setup_irq(sirfsoc_timer_irq.irq, &sirfsoc_timer_irq); + ret = request_irq(irq, sirfsoc_timer_interrupt, IRQF_TIMER, + "sirfsoc_timer0", &sirfsoc_clockevent); if (ret) { pr_err("Failed to setup irq\n"); return ret; diff --git a/drivers/clocksource/timer-pxa.c b/drivers/clocksource/timer-pxa.c index 913a5d354a1f..7ad0e5adb2ff 100644 --- a/drivers/clocksource/timer-pxa.c +++ b/drivers/clocksource/timer-pxa.c @@ -143,13 +143,6 @@ static struct clock_event_device ckevt_pxa_osmr0 = { .resume = pxa_timer_resume, }; -static struct irqaction pxa_ost0_irq = { - .name = "ost0", - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = pxa_ost0_interrupt, - .dev_id = &ckevt_pxa_osmr0, -}; - static int __init pxa_timer_common_init(int irq, unsigned long clock_tick_rate) { int ret; @@ -161,7 +154,8 @@ static int __init pxa_timer_common_init(int irq, unsigned long clock_tick_rate) ckevt_pxa_osmr0.cpumask = cpumask_of(0); - ret = setup_irq(irq, &pxa_ost0_irq); + ret = request_irq(irq, pxa_ost0_interrupt, IRQF_TIMER | IRQF_IRQPOLL, + "ost0", &ckevt_pxa_osmr0); if (ret) { pr_err("Failed to setup irq\n"); return ret; diff --git a/drivers/clocksource/timer-sp804.c b/drivers/clocksource/timer-sp804.c index 9c841980eed1..5cd0abf9b396 100644 --- a/drivers/clocksource/timer-sp804.c +++ b/drivers/clocksource/timer-sp804.c @@ -168,13 +168,6 @@ static struct clock_event_device sp804_clockevent = { .rating = 300, }; -static struct irqaction sp804_timer_irq = { - .name = "timer", - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = sp804_timer_interrupt, - .dev_id = &sp804_clockevent, -}; - int __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struct clk *clk, const char *name) { struct clock_event_device *evt = &sp804_clockevent; @@ -200,7 +193,9 @@ int __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struct writel(0, base + TIMER_CTRL); - setup_irq(irq, &sp804_timer_irq); + if (request_irq(irq, sp804_timer_interrupt, IRQF_TIMER | IRQF_IRQPOLL, + "timer", &sp804_clockevent)) + pr_err("%s: request_irq() failed\n", "timer"); clockevents_config_and_register(evt, rate, 0xf, 0xffffffff); return 0; diff --git a/drivers/clocksource/timer-ti-dm.c b/drivers/clocksource/timer-ti-dm.c index 269a994d6a99..2531eab3d6d7 100644 --- a/drivers/clocksource/timer-ti-dm.c +++ b/drivers/clocksource/timer-ti-dm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0+ /* * linux/arch/arm/plat-omap/dmtimer.c * @@ -15,28 +16,11 @@ * * Copyright (C) 2009 Texas Instruments * Added OMAP4 support - Santosh Shilimkar <santosh.shilimkar@ti.com> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. */ #include <linux/clk.h> #include <linux/clk-provider.h> +#include <linux/cpu_pm.h> #include <linux/module.h> #include <linux/io.h> #include <linux/device.h> @@ -109,6 +93,47 @@ static void omap_timer_restore_context(struct omap_dm_timer *timer) timer->context.tclr); } +static void omap_timer_save_context(struct omap_dm_timer *timer) +{ + timer->context.tclr = + omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); + timer->context.twer = + omap_dm_timer_read_reg(timer, OMAP_TIMER_WAKEUP_EN_REG); + timer->context.tldr = + omap_dm_timer_read_reg(timer, OMAP_TIMER_LOAD_REG); + timer->context.tmar = + omap_dm_timer_read_reg(timer, OMAP_TIMER_MATCH_REG); + timer->context.tier = readl_relaxed(timer->irq_ena); + timer->context.tsicr = + omap_dm_timer_read_reg(timer, OMAP_TIMER_IF_CTRL_REG); +} + +static int omap_timer_context_notifier(struct notifier_block *nb, + unsigned long cmd, void *v) +{ + struct omap_dm_timer *timer; + + timer = container_of(nb, struct omap_dm_timer, nb); + + switch (cmd) { + case CPU_CLUSTER_PM_ENTER: + if ((timer->capability & OMAP_TIMER_ALWON) || + !atomic_read(&timer->enabled)) + break; + omap_timer_save_context(timer); + break; + case CPU_CLUSTER_PM_ENTER_FAILED: + case CPU_CLUSTER_PM_EXIT: + if ((timer->capability & OMAP_TIMER_ALWON) || + !atomic_read(&timer->enabled)) + break; + omap_timer_restore_context(timer); + break; + } + + return NOTIFY_OK; +} + static int omap_dm_timer_reset(struct omap_dm_timer *timer) { u32 l, timeout = 100000; @@ -138,35 +163,6 @@ static int omap_dm_timer_reset(struct omap_dm_timer *timer) return 0; } -static int omap_dm_timer_of_set_source(struct omap_dm_timer *timer) -{ - int ret; - struct clk *parent; - - /* - * FIXME: OMAP1 devices do not use the clock framework for dmtimers so - * do not call clk_get() for these devices. - */ - if (!timer->fclk) - return -ENODEV; - - parent = clk_get(&timer->pdev->dev, NULL); - if (IS_ERR(parent)) - return -ENODEV; - - /* Bail out if both clocks point to fck */ - if (clk_is_match(parent, timer->fclk)) - return 0; - - ret = clk_set_parent(timer->fclk, parent); - if (ret < 0) - pr_err("%s: failed to set parent\n", __func__); - - clk_put(parent); - - return ret; -} - static int omap_dm_timer_set_source(struct omap_dm_timer *timer, int source) { int ret; @@ -225,21 +221,7 @@ static int omap_dm_timer_set_source(struct omap_dm_timer *timer, int source) static void omap_dm_timer_enable(struct omap_dm_timer *timer) { - int c; - pm_runtime_get_sync(&timer->pdev->dev); - - if (!(timer->capability & OMAP_TIMER_ALWON)) { - if (timer->get_context_loss_count) { - c = timer->get_context_loss_count(&timer->pdev->dev); - if (c != timer->ctx_loss_count) { - omap_timer_restore_context(timer); - timer->ctx_loss_count = c; - } - } else { - omap_timer_restore_context(timer); - } - } } static void omap_dm_timer_disable(struct omap_dm_timer *timer) @@ -276,9 +258,7 @@ static int omap_dm_timer_prepare(struct omap_dm_timer *timer) __omap_dm_timer_enable_posted(timer); omap_dm_timer_disable(timer); - rc = omap_dm_timer_of_set_source(timer); - if (rc == -ENODEV) - return omap_dm_timer_set_source(timer, OMAP_TIMER_SRC_32_KHZ); + rc = omap_dm_timer_set_source(timer, OMAP_TIMER_SRC_32_KHZ); return rc; } @@ -508,7 +488,7 @@ __u32 omap_dm_timer_modify_idlect_mask(__u32 inputmask) int omap_dm_timer_trigger(struct omap_dm_timer *timer) { - if (unlikely(!timer || pm_runtime_suspended(&timer->pdev->dev))) { + if (unlikely(!timer || !atomic_read(&timer->enabled))) { pr_err("%s: timer not available or enabled.\n", __func__); return -EINVAL; } @@ -532,8 +512,6 @@ static int omap_dm_timer_start(struct omap_dm_timer *timer) omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, l); } - /* Save the context */ - timer->context.tclr = l; return 0; } @@ -549,38 +527,19 @@ static int omap_dm_timer_stop(struct omap_dm_timer *timer) __omap_dm_timer_stop(timer, timer->posted, rate); - /* - * Since the register values are computed and written within - * __omap_dm_timer_stop, we need to use read to retrieve the - * context. - */ - timer->context.tclr = - omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); omap_dm_timer_disable(timer); return 0; } -static int omap_dm_timer_set_load(struct omap_dm_timer *timer, int autoreload, +static int omap_dm_timer_set_load(struct omap_dm_timer *timer, unsigned int load) { - u32 l; - if (unlikely(!timer)) return -EINVAL; omap_dm_timer_enable(timer); - l = omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); - if (autoreload) - l |= OMAP_TIMER_CTRL_AR; - else - l &= ~OMAP_TIMER_CTRL_AR; - omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, l); omap_dm_timer_write_reg(timer, OMAP_TIMER_LOAD_REG, load); - omap_dm_timer_write_reg(timer, OMAP_TIMER_TRIGGER_REG, 0); - /* Save the context */ - timer->context.tclr = l; - timer->context.tldr = load; omap_dm_timer_disable(timer); return 0; } @@ -602,15 +561,12 @@ static int omap_dm_timer_set_match(struct omap_dm_timer *timer, int enable, omap_dm_timer_write_reg(timer, OMAP_TIMER_MATCH_REG, match); omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, l); - /* Save the context */ - timer->context.tclr = l; - timer->context.tmar = match; omap_dm_timer_disable(timer); return 0; } static int omap_dm_timer_set_pwm(struct omap_dm_timer *timer, int def_on, - int toggle, int trigger) + int toggle, int trigger, int autoreload) { u32 l; @@ -620,20 +576,34 @@ static int omap_dm_timer_set_pwm(struct omap_dm_timer *timer, int def_on, omap_dm_timer_enable(timer); l = omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); l &= ~(OMAP_TIMER_CTRL_GPOCFG | OMAP_TIMER_CTRL_SCPWM | - OMAP_TIMER_CTRL_PT | (0x03 << 10)); + OMAP_TIMER_CTRL_PT | (0x03 << 10) | OMAP_TIMER_CTRL_AR); if (def_on) l |= OMAP_TIMER_CTRL_SCPWM; if (toggle) l |= OMAP_TIMER_CTRL_PT; l |= trigger << 10; + if (autoreload) + l |= OMAP_TIMER_CTRL_AR; omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, l); - /* Save the context */ - timer->context.tclr = l; omap_dm_timer_disable(timer); return 0; } +static int omap_dm_timer_get_pwm_status(struct omap_dm_timer *timer) +{ + u32 l; + + if (unlikely(!timer)) + return -EINVAL; + + omap_dm_timer_enable(timer); + l = omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); + omap_dm_timer_disable(timer); + + return l; +} + static int omap_dm_timer_set_prescaler(struct omap_dm_timer *timer, int prescaler) { @@ -651,8 +621,6 @@ static int omap_dm_timer_set_prescaler(struct omap_dm_timer *timer, } omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, l); - /* Save the context */ - timer->context.tclr = l; omap_dm_timer_disable(timer); return 0; } @@ -666,9 +634,6 @@ static int omap_dm_timer_set_int_enable(struct omap_dm_timer *timer, omap_dm_timer_enable(timer); __omap_dm_timer_int_enable(timer, value); - /* Save the context */ - timer->context.tier = value; - timer->context.twer = value; omap_dm_timer_disable(timer); return 0; } @@ -696,9 +661,6 @@ static int omap_dm_timer_set_int_disable(struct omap_dm_timer *timer, u32 mask) l = omap_dm_timer_read_reg(timer, OMAP_TIMER_WAKEUP_EN_REG) & ~mask; omap_dm_timer_write_reg(timer, OMAP_TIMER_WAKEUP_EN_REG, l); - /* Save the context */ - timer->context.tier &= ~mask; - timer->context.twer &= ~mask; omap_dm_timer_disable(timer); return 0; } @@ -707,7 +669,7 @@ static unsigned int omap_dm_timer_read_status(struct omap_dm_timer *timer) { unsigned int l; - if (unlikely(!timer || pm_runtime_suspended(&timer->pdev->dev))) { + if (unlikely(!timer || !atomic_read(&timer->enabled))) { pr_err("%s: timer not available or enabled.\n", __func__); return 0; } @@ -719,7 +681,7 @@ static unsigned int omap_dm_timer_read_status(struct omap_dm_timer *timer) static int omap_dm_timer_write_status(struct omap_dm_timer *timer, unsigned int value) { - if (unlikely(!timer || pm_runtime_suspended(&timer->pdev->dev))) + if (unlikely(!timer || !atomic_read(&timer->enabled))) return -EINVAL; __omap_dm_timer_write_status(timer, value); @@ -729,7 +691,7 @@ static int omap_dm_timer_write_status(struct omap_dm_timer *timer, unsigned int static unsigned int omap_dm_timer_read_counter(struct omap_dm_timer *timer) { - if (unlikely(!timer || pm_runtime_suspended(&timer->pdev->dev))) { + if (unlikely(!timer || !atomic_read(&timer->enabled))) { pr_err("%s: timer not iavailable or enabled.\n", __func__); return 0; } @@ -739,7 +701,7 @@ static unsigned int omap_dm_timer_read_counter(struct omap_dm_timer *timer) static int omap_dm_timer_write_counter(struct omap_dm_timer *timer, unsigned int value) { - if (unlikely(!timer || pm_runtime_suspended(&timer->pdev->dev))) { + if (unlikely(!timer || !atomic_read(&timer->enabled))) { pr_err("%s: timer not available or enabled.\n", __func__); return -EINVAL; } @@ -767,6 +729,37 @@ int omap_dm_timers_active(void) return 0; } +static int __maybe_unused omap_dm_timer_runtime_suspend(struct device *dev) +{ + struct omap_dm_timer *timer = dev_get_drvdata(dev); + + atomic_set(&timer->enabled, 0); + + if (timer->capability & OMAP_TIMER_ALWON || !timer->func_base) + return 0; + + omap_timer_save_context(timer); + + return 0; +} + +static int __maybe_unused omap_dm_timer_runtime_resume(struct device *dev) +{ + struct omap_dm_timer *timer = dev_get_drvdata(dev); + + if (!(timer->capability & OMAP_TIMER_ALWON) && timer->func_base) + omap_timer_restore_context(timer); + + atomic_set(&timer->enabled, 1); + + return 0; +} + +static const struct dev_pm_ops omap_dm_timer_pm_ops = { + SET_RUNTIME_PM_OPS(omap_dm_timer_runtime_suspend, + omap_dm_timer_runtime_resume, NULL) +}; + static const struct of_device_id omap_timer_match[]; /** @@ -808,6 +801,8 @@ static int omap_dm_timer_probe(struct platform_device *pdev) if (IS_ERR(timer->io_base)) return PTR_ERR(timer->io_base); + platform_set_drvdata(pdev, timer); + if (dev->of_node) { if (of_find_property(dev->of_node, "ti,timer-alwon", NULL)) timer->capability |= OMAP_TIMER_ALWON; @@ -821,7 +816,11 @@ static int omap_dm_timer_probe(struct platform_device *pdev) timer->id = pdev->id; timer->capability = pdata->timer_capability; timer->reserved = omap_dm_timer_reserved_systimer(timer->id); - timer->get_context_loss_count = pdata->get_context_loss_count; + } + + if (!(timer->capability & OMAP_TIMER_ALWON)) { + timer->nb.notifier_call = omap_timer_context_notifier; + cpu_pm_register_notifier(&timer->nb); } if (pdata) @@ -875,6 +874,8 @@ static int omap_dm_timer_remove(struct platform_device *pdev) list_for_each_entry(timer, &omap_timer_list, node) if (!strcmp(dev_name(&timer->pdev->dev), dev_name(&pdev->dev))) { + if (!(timer->capability & OMAP_TIMER_ALWON)) + cpu_pm_unregister_notifier(&timer->nb); list_del(&timer->node); ret = 0; break; @@ -903,6 +904,7 @@ static const struct omap_dm_timer_ops dmtimer_ops = { .set_load = omap_dm_timer_set_load, .set_match = omap_dm_timer_set_match, .set_pwm = omap_dm_timer_set_pwm, + .get_pwm_status = omap_dm_timer_get_pwm_status, .set_prescaler = omap_dm_timer_set_prescaler, .read_counter = omap_dm_timer_read_counter, .write_counter = omap_dm_timer_write_counter, @@ -953,6 +955,7 @@ static struct platform_driver omap_dm_timer_driver = { .driver = { .name = "omap_timer", .of_match_table = of_match_ptr(omap_timer_match), + .pm = &omap_dm_timer_pm_ops, }, }; diff --git a/drivers/clocksource/timer-u300.c b/drivers/clocksource/timer-u300.c index 32adc3057dda..37cba8dfd45f 100644 --- a/drivers/clocksource/timer-u300.c +++ b/drivers/clocksource/timer-u300.c @@ -330,12 +330,6 @@ static irqreturn_t u300_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction u300_timer_irq = { - .name = "U300 Timer Tick", - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = u300_timer_interrupt, -}; - /* * Override the global weak sched_clock symbol with this * local implementation which uses the clocksource to get some @@ -420,7 +414,8 @@ static int __init u300_timer_init_of(struct device_node *np) u300_timer_base + U300_TIMER_APP_RGPT1); /* Set up the IRQ handler */ - ret = setup_irq(irq, &u300_timer_irq); + ret = request_irq(irq, u300_timer_interrupt, + IRQF_TIMER | IRQF_IRQPOLL, "U300 Timer Tick", NULL); if (ret) return ret; diff --git a/drivers/clocksource/timer-vf-pit.c b/drivers/clocksource/timer-vf-pit.c index fef0bb4e0c8c..7ad4a8b008c2 100644 --- a/drivers/clocksource/timer-vf-pit.c +++ b/drivers/clocksource/timer-vf-pit.c @@ -123,19 +123,13 @@ static struct clock_event_device clockevent_pit = { .rating = 300, }; -static struct irqaction pit_timer_irq = { - .name = "VF pit timer", - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = pit_timer_interrupt, - .dev_id = &clockevent_pit, -}; - static int __init pit_clockevent_init(unsigned long rate, int irq) { __raw_writel(0, clkevt_base + PITTCTRL); __raw_writel(PITTFLG_TIF, clkevt_base + PITTFLG); - BUG_ON(setup_irq(irq, &pit_timer_irq)); + BUG_ON(request_irq(irq, pit_timer_interrupt, IRQF_TIMER | IRQF_IRQPOLL, + "VF pit timer", &clockevent_pit); clockevent_pit.cpumask = cpumask_of(0); clockevent_pit.irq = irq; diff --git a/drivers/clocksource/timer-vt8500.c b/drivers/clocksource/timer-vt8500.c index bb424bcefbb3..a469b1b5f972 100644 --- a/drivers/clocksource/timer-vt8500.c +++ b/drivers/clocksource/timer-vt8500.c @@ -101,13 +101,6 @@ static irqreturn_t vt8500_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction irq = { - .name = "vt8500_timer", - .flags = IRQF_TIMER | IRQF_IRQPOLL, - .handler = vt8500_timer_interrupt, - .dev_id = &clockevent, -}; - static int __init vt8500_timer_init(struct device_node *np) { int timer_irq, ret; @@ -139,7 +132,9 @@ static int __init vt8500_timer_init(struct device_node *np) clockevent.cpumask = cpumask_of(0); - ret = setup_irq(timer_irq, &irq); + ret = request_irq(timer_irq, vt8500_timer_interrupt, + IRQF_TIMER | IRQF_IRQPOLL, "vt8500_timer", + &clockevent); if (ret) { pr_err("%s: setup_irq failed for %s\n", __func__, clockevent.name); diff --git a/drivers/clocksource/timer-zevio.c b/drivers/clocksource/timer-zevio.c index c0041561f1be..ecaa3568841c 100644 --- a/drivers/clocksource/timer-zevio.c +++ b/drivers/clocksource/timer-zevio.c @@ -53,7 +53,6 @@ struct zevio_timer { struct clk *clk; struct clock_event_device clkevt; - struct irqaction clkevt_irq; char clocksource_name[64]; char clockevent_name[64]; @@ -172,12 +171,12 @@ static int __init zevio_timer_add(struct device_node *node) /* Interrupt to occur when timer value matches 0 */ writel(0, timer->base + IO_MATCH(TIMER_MATCH)); - timer->clkevt_irq.name = timer->clockevent_name; - timer->clkevt_irq.handler = zevio_timer_interrupt; - timer->clkevt_irq.dev_id = timer; - timer->clkevt_irq.flags = IRQF_TIMER | IRQF_IRQPOLL; - - setup_irq(irqnr, &timer->clkevt_irq); + if (request_irq(irqnr, zevio_timer_interrupt, + IRQF_TIMER | IRQF_IRQPOLL, + timer->clockevent_name, timer)) { + pr_err("%s: request_irq() failed\n", + timer->clockevent_name); + } clockevents_config_and_register(&timer->clkevt, clk_get_rate(timer->clk), 0x0001, 0xffff); diff --git a/drivers/pwm/pwm-omap-dmtimer.c b/drivers/pwm/pwm-omap-dmtimer.c index 88a3c5690fea..9e4378dc6897 100644 --- a/drivers/pwm/pwm-omap-dmtimer.c +++ b/drivers/pwm/pwm-omap-dmtimer.c @@ -183,7 +183,7 @@ static int pwm_omap_dmtimer_config(struct pwm_chip *chip, if (timer_active) omap->pdata->stop(omap->dm_timer); - omap->pdata->set_load(omap->dm_timer, true, load_value); + omap->pdata->set_load(omap->dm_timer, load_value); omap->pdata->set_match(omap->dm_timer, true, match_value); dev_dbg(chip->dev, "load value: %#08x (%d), match value: %#08x (%d)\n", @@ -192,7 +192,8 @@ static int pwm_omap_dmtimer_config(struct pwm_chip *chip, omap->pdata->set_pwm(omap->dm_timer, pwm_get_polarity(pwm) == PWM_POLARITY_INVERSED, true, - PWM_OMAP_DMTIMER_TRIGGER_OVERFLOW_AND_COMPARE); + PWM_OMAP_DMTIMER_TRIGGER_OVERFLOW_AND_COMPARE, + true); /* If config was called while timer was running it must be reenabled. */ if (timer_active) @@ -222,7 +223,8 @@ static int pwm_omap_dmtimer_set_polarity(struct pwm_chip *chip, omap->pdata->set_pwm(omap->dm_timer, polarity == PWM_POLARITY_INVERSED, true, - PWM_OMAP_DMTIMER_TRIGGER_OVERFLOW_AND_COMPARE); + PWM_OMAP_DMTIMER_TRIGGER_OVERFLOW_AND_COMPARE, + true); mutex_unlock(&omap->mutex); return 0; diff --git a/include/asm-generic/vdso/vsyscall.h b/include/asm-generic/vdso/vsyscall.h index cec543d9e87b..c835607f78ae 100644 --- a/include/asm-generic/vdso/vsyscall.h +++ b/include/asm-generic/vdso/vsyscall.h @@ -11,20 +11,6 @@ static __always_inline struct vdso_data *__arch_get_k_vdso_data(void) } #endif /* __arch_get_k_vdso_data */ -#ifndef __arch_update_vdso_data -static __always_inline bool __arch_update_vdso_data(void) -{ - return true; -} -#endif /* __arch_update_vdso_data */ - -#ifndef __arch_get_clock_mode -static __always_inline int __arch_get_clock_mode(struct timekeeper *tk) -{ - return 0; -} -#endif /* __arch_get_clock_mode */ - #ifndef __arch_update_vsyscall static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk) diff --git a/include/clocksource/timer-ti-dm.h b/include/clocksource/timer-ti-dm.h index 7d9598dc578d..25f05235866e 100644 --- a/include/clocksource/timer-ti-dm.h +++ b/include/clocksource/timer-ti-dm.h @@ -105,17 +105,17 @@ struct omap_dm_timer { void __iomem *pend; /* write pending */ void __iomem *func_base; /* function register base */ + atomic_t enabled; unsigned long rate; unsigned reserved:1; unsigned posted:1; struct timer_regs context; - int (*get_context_loss_count)(struct device *); - int ctx_loss_count; int revision; u32 capability; u32 errata; struct platform_device *pdev; struct list_head node; + struct notifier_block nb; }; int omap_dm_timer_reserve_systimer(int id); diff --git a/include/linux/bits.h b/include/linux/bits.h index 669d69441a62..a740bbcf3cd2 100644 --- a/include/linux/bits.h +++ b/include/linux/bits.h @@ -3,9 +3,9 @@ #define __LINUX_BITS_H #include <linux/const.h> +#include <vdso/bits.h> #include <asm/bitsperlong.h> -#define BIT(nr) (UL(1) << (nr)) #define BIT_ULL(nr) (ULL(1) << (nr)) #define BIT_MASK(nr) (UL(1) << ((nr) % BITS_PER_LONG)) #define BIT_WORD(nr) ((nr) / BITS_PER_LONG) diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index b21db536fd52..86d143db6523 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -23,18 +23,31 @@ struct clocksource; struct module; -#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA +#if defined(CONFIG_ARCH_CLOCKSOURCE_DATA) || \ + defined(CONFIG_GENERIC_GETTIMEOFDAY) #include <asm/clocksource.h> #endif +#include <vdso/clocksource.h> + /** * struct clocksource - hardware abstraction for a free running counter * Provides mostly state-free accessors to the underlying hardware. * This is the structure used for system time. * - * @name: ptr to clocksource name - * @list: list head for registration - * @rating: rating value for selection (higher is better) + * @read: Returns a cycle value, passes clocksource as argument + * @mask: Bitmask for two's complement + * subtraction of non 64 bit counters + * @mult: Cycle to nanosecond multiplier + * @shift: Cycle to nanosecond divisor (power of two) + * @max_idle_ns: Maximum idle time permitted by the clocksource (nsecs) + * @maxadj: Maximum adjustment value to mult (~11%) + * @archdata: Optional arch-specific data + * @max_cycles: Maximum safe cycle value which won't overflow on + * multiplication + * @name: Pointer to clocksource name + * @list: List head for registration (internal) + * @rating: Rating value for selection (higher is better) * To avoid rating inflation the following * list should give you a guide as to how * to assign your clocksource a rating @@ -49,27 +62,23 @@ struct module; * 400-499: Perfect * The ideal clocksource. A must-use where * available. - * @read: returns a cycle value, passes clocksource as argument - * @enable: optional function to enable the clocksource - * @disable: optional function to disable the clocksource - * @mask: bitmask for two's complement - * subtraction of non 64 bit counters - * @mult: cycle to nanosecond multiplier - * @shift: cycle to nanosecond divisor (power of two) - * @max_idle_ns: max idle time permitted by the clocksource (nsecs) - * @maxadj: maximum adjustment value to mult (~11%) - * @max_cycles: maximum safe cycle value which won't overflow on multiplication - * @flags: flags describing special properties - * @archdata: arch-specific data - * @suspend: suspend function for the clocksource, if necessary - * @resume: resume function for the clocksource, if necessary + * @flags: Flags describing special properties + * @enable: Optional function to enable the clocksource + * @disable: Optional function to disable the clocksource + * @suspend: Optional suspend function for the clocksource + * @resume: Optional resume function for the clocksource * @mark_unstable: Optional function to inform the clocksource driver that * the watchdog marked the clocksource unstable - * @owner: module reference, must be set by clocksource in modules + * @tick_stable: Optional function called periodically from the watchdog + * code to provide stable syncrhonization points + * @wd_list: List head to enqueue into the watchdog list (internal) + * @cs_last: Last clocksource value for clocksource watchdog + * @wd_last: Last watchdog value corresponding to @cs_last + * @owner: Module reference, must be set by clocksource in modules * * Note: This struct is not used in hotpathes of the timekeeping code * because the timekeeper caches the hot path fields in its own data - * structure, so no line cache alignment is required, + * structure, so no cache line alignment is required, * * The pointer to the clocksource itself is handed to the read * callback. If you need extra information there you can wrap struct @@ -78,35 +87,37 @@ struct module; * structure. */ struct clocksource { - u64 (*read)(struct clocksource *cs); - u64 mask; - u32 mult; - u32 shift; - u64 max_idle_ns; - u32 maxadj; + u64 (*read)(struct clocksource *cs); + u64 mask; + u32 mult; + u32 shift; + u64 max_idle_ns; + u32 maxadj; #ifdef CONFIG_ARCH_CLOCKSOURCE_DATA struct arch_clocksource_data archdata; #endif - u64 max_cycles; - const char *name; - struct list_head list; - int rating; - int (*enable)(struct clocksource *cs); - void (*disable)(struct clocksource *cs); - unsigned long flags; - void (*suspend)(struct clocksource *cs); - void (*resume)(struct clocksource *cs); - void (*mark_unstable)(struct clocksource *cs); - void (*tick_stable)(struct clocksource *cs); + u64 max_cycles; + const char *name; + struct list_head list; + int rating; + enum vdso_clock_mode vdso_clock_mode; + unsigned long flags; + + int (*enable)(struct clocksource *cs); + void (*disable)(struct clocksource *cs); + void (*suspend)(struct clocksource *cs); + void (*resume)(struct clocksource *cs); + void (*mark_unstable)(struct clocksource *cs); + void (*tick_stable)(struct clocksource *cs); /* private: */ #ifdef CONFIG_CLOCKSOURCE_WATCHDOG /* Watchdog related data, used by the framework */ - struct list_head wd_list; - u64 cs_last; - u64 wd_last; + struct list_head wd_list; + u64 cs_last; + u64 wd_last; #endif - struct module *owner; + struct module *owner; }; /* diff --git a/include/linux/const.h b/include/linux/const.h index 7b55a55f5911..81b8aae5a855 100644 --- a/include/linux/const.h +++ b/include/linux/const.h @@ -1,9 +1,6 @@ #ifndef _LINUX_CONST_H #define _LINUX_CONST_H -#include <uapi/linux/const.h> - -#define UL(x) (_UL(x)) -#define ULL(x) (_ULL(x)) +#include <vdso/const.h> #endif /* _LINUX_CONST_H */ diff --git a/include/linux/dw_apb_timer.h b/include/linux/dw_apb_timer.h index 14f072edbca5..82ebf9223948 100644 --- a/include/linux/dw_apb_timer.h +++ b/include/linux/dw_apb_timer.h @@ -25,7 +25,6 @@ struct dw_apb_timer { struct dw_apb_clock_event_device { struct clock_event_device ced; struct dw_apb_timer timer; - struct irqaction irqaction; void (*eoi)(struct dw_apb_timer *); }; diff --git a/include/linux/elfnote.h b/include/linux/elfnote.h index f236f5b931b2..594d4e78654f 100644 --- a/include/linux/elfnote.h +++ b/include/linux/elfnote.h @@ -59,7 +59,7 @@ ELFNOTE_END #else /* !__ASSEMBLER__ */ -#include <linux/elf.h> +#include <uapi/linux/elf.h> /* * Use an anonymous structure which matches the shape of * Elf{32,64}_Nhdr, but includes the name and desc data. The size and diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h index e3279ef24d28..fed6ba96c527 100644 --- a/include/linux/jiffies.h +++ b/include/linux/jiffies.h @@ -8,6 +8,7 @@ #include <linux/types.h> #include <linux/time.h> #include <linux/timex.h> +#include <vdso/jiffies.h> #include <asm/param.h> /* for HZ */ #include <generated/timeconst.h> @@ -59,9 +60,6 @@ extern int register_refined_jiffies(long clock_tick_rate); -/* TICK_NSEC is the time between ticks in nsec assuming SHIFTED_HZ */ -#define TICK_NSEC ((NSEC_PER_SEC+HZ/2)/HZ) - /* TICK_USEC is the time between ticks in usec assuming SHIFTED_HZ */ #define TICK_USEC ((USEC_PER_SEC + HZ/2) / HZ) diff --git a/include/linux/ktime.h b/include/linux/ktime.h index d1fb05135665..42d2e6ac35f2 100644 --- a/include/linux/ktime.h +++ b/include/linux/ktime.h @@ -216,14 +216,7 @@ static inline __must_check bool ktime_to_timespec64_cond(const ktime_t kt, } } -/* - * The resolution of the clocks. The resolution value is returned in - * the clock_getres() system call to give application programmers an - * idea of the (in)accuracy of timers. Timer values are rounded up to - * this resolution values. - */ -#define LOW_RES_NSEC TICK_NSEC -#define KTIME_LOW_RES (LOW_RES_NSEC) +#include <vdso/ktime.h> static inline ktime_t ns_to_ktime(u64 ns) { diff --git a/include/linux/limits.h b/include/linux/limits.h index 76afcd24ff8c..7fc497ee1393 100644 --- a/include/linux/limits.h +++ b/include/linux/limits.h @@ -4,19 +4,8 @@ #include <uapi/linux/limits.h> #include <linux/types.h> +#include <vdso/limits.h> -#define USHRT_MAX ((unsigned short)~0U) -#define SHRT_MAX ((short)(USHRT_MAX >> 1)) -#define SHRT_MIN ((short)(-SHRT_MAX - 1)) -#define INT_MAX ((int)(~0U >> 1)) -#define INT_MIN (-INT_MAX - 1) -#define UINT_MAX (~0U) -#define LONG_MAX ((long)(~0UL >> 1)) -#define LONG_MIN (-LONG_MAX - 1) -#define ULONG_MAX (~0UL) -#define LLONG_MAX ((long long)(~0ULL >> 1)) -#define LLONG_MIN (-LLONG_MAX - 1) -#define ULLONG_MAX (~0ULL) #define SIZE_MAX (~(size_t)0) #define PHYS_ADDR_MAX (~(phys_addr_t)0) diff --git a/include/linux/math64.h b/include/linux/math64.h index 65bef21cdddb..11a267413e8e 100644 --- a/include/linux/math64.h +++ b/include/linux/math64.h @@ -3,6 +3,7 @@ #define _LINUX_MATH64_H #include <linux/types.h> +#include <vdso/math64.h> #include <asm/div64.h> #if BITS_PER_LONG == 64 @@ -142,25 +143,6 @@ static inline s64 div_s64(s64 dividend, s32 divisor) u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder); -static __always_inline u32 -__iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder) -{ - u32 ret = 0; - - while (dividend >= divisor) { - /* The following asm() prevents the compiler from - optimising this loop into a modulo operation. */ - asm("" : "+rm"(dividend)); - - dividend -= divisor; - ret++; - } - - *remainder = dividend; - - return ret; -} - #ifndef mul_u32_u32 /* * Many a GCC version messes this up and generates a 64x64 mult :-( diff --git a/include/linux/platform_data/dmtimer-omap.h b/include/linux/platform_data/dmtimer-omap.h index bdaaf537604a..95d852aef130 100644 --- a/include/linux/platform_data/dmtimer-omap.h +++ b/include/linux/platform_data/dmtimer-omap.h @@ -30,12 +30,12 @@ struct omap_dm_timer_ops { int (*stop)(struct omap_dm_timer *timer); int (*set_source)(struct omap_dm_timer *timer, int source); - int (*set_load)(struct omap_dm_timer *timer, int autoreload, - unsigned int value); + int (*set_load)(struct omap_dm_timer *timer, unsigned int value); int (*set_match)(struct omap_dm_timer *timer, int enable, unsigned int match); int (*set_pwm)(struct omap_dm_timer *timer, int def_on, - int toggle, int trigger); + int toggle, int trigger, int autoreload); + int (*get_pwm_status)(struct omap_dm_timer *timer); int (*set_prescaler)(struct omap_dm_timer *timer, int prescaler); unsigned int (*read_counter)(struct omap_dm_timer *timer); diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index 3d10c84a97a9..e3f0f8585da4 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -69,7 +69,7 @@ static inline int clockid_to_fd(const clockid_t clk) struct cpu_timer { struct timerqueue_node node; struct timerqueue_head *head; - struct task_struct *task; + struct pid *pid; struct list_head elist; int firing; }; diff --git a/include/linux/time.h b/include/linux/time.h index 8ef5e5cc9f57..4c325bf44ce0 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -111,9 +111,6 @@ static inline bool itimerspec64_valid(const struct itimerspec64 *its) */ #define time_between32(t, l, h) ((u32)(h) - (u32)(l) >= (u32)(t) - (u32)(l)) -struct timens_offset { - s64 sec; - u64 nsec; -}; +# include <vdso/time.h> #endif diff --git a/include/linux/time32.h b/include/linux/time32.h index cf9320cd2d0b..83a400b6ba99 100644 --- a/include/linux/time32.h +++ b/include/linux/time32.h @@ -12,17 +12,7 @@ #include <linux/time64.h> #include <linux/timex.h> -typedef s32 old_time32_t; - -struct old_timespec32 { - old_time32_t tv_sec; - s32 tv_nsec; -}; - -struct old_timeval32 { - old_time32_t tv_sec; - s32 tv_usec; -}; +#include <vdso/time32.h> struct old_itimerspec32 { struct old_timespec32 it_interval; diff --git a/include/linux/time64.h b/include/linux/time64.h index 19125489ae94..c9dcb3e5781f 100644 --- a/include/linux/time64.h +++ b/include/linux/time64.h @@ -3,6 +3,7 @@ #define _LINUX_TIME64_H #include <linux/math64.h> +#include <vdso/time64.h> typedef __s64 time64_t; typedef __u64 timeu64_t; @@ -19,15 +20,6 @@ struct itimerspec64 { struct timespec64 it_value; }; -/* Parameters used to convert the timespec values: */ -#define MSEC_PER_SEC 1000L -#define USEC_PER_MSEC 1000L -#define NSEC_PER_USEC 1000L -#define NSEC_PER_MSEC 1000000L -#define USEC_PER_SEC 1000000L -#define NSEC_PER_SEC 1000000000L -#define FSEC_PER_SEC 1000000000000000LL - /* Located here for timespec[64]_valid_strict */ #define TIME64_MAX ((s64)~((u64)1 << 63)) #define TIME64_MIN (-TIME64_MAX - 1) diff --git a/include/vdso/bits.h b/include/vdso/bits.h new file mode 100644 index 000000000000..6d005a1f5d94 --- /dev/null +++ b/include/vdso/bits.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_BITS_H +#define __VDSO_BITS_H + +#include <vdso/const.h> + +#define BIT(nr) (UL(1) << (nr)) + +#endif /* __VDSO_BITS_H */ diff --git a/include/vdso/clocksource.h b/include/vdso/clocksource.h new file mode 100644 index 000000000000..c682e7c60273 --- /dev/null +++ b/include/vdso/clocksource.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_CLOCKSOURCE_H +#define __VDSO_CLOCKSOURCE_H + +#include <vdso/limits.h> + +#ifdef CONFIG_GENERIC_GETTIMEOFDAY +#include <asm/vdso/clocksource.h> +#endif /* CONFIG_GENERIC_GETTIMEOFDAY */ + +enum vdso_clock_mode { + VDSO_CLOCKMODE_NONE, +#ifdef CONFIG_GENERIC_GETTIMEOFDAY + VDSO_ARCH_CLOCKMODES, +#endif + VDSO_CLOCKMODE_MAX, + + /* Indicator for time namespace VDSO */ + VDSO_CLOCKMODE_TIMENS = INT_MAX +}; + +#endif /* __VDSO_CLOCKSOURCE_H */ diff --git a/include/vdso/const.h b/include/vdso/const.h new file mode 100644 index 000000000000..94b385ad438d --- /dev/null +++ b/include/vdso/const.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_CONST_H +#define __VDSO_CONST_H + +#include <uapi/linux/const.h> + +#define UL(x) (_UL(x)) +#define ULL(x) (_ULL(x)) + +#endif /* __VDSO_CONST_H */ diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index c5f347cc5e55..5cbc9fcbfd45 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -4,9 +4,20 @@ #ifndef __ASSEMBLY__ -#include <linux/bits.h> -#include <linux/time.h> -#include <linux/types.h> +#include <linux/compiler.h> +#include <uapi/linux/time.h> +#include <uapi/linux/types.h> +#include <uapi/asm-generic/errno-base.h> + +#include <vdso/bits.h> +#include <vdso/clocksource.h> +#include <vdso/ktime.h> +#include <vdso/limits.h> +#include <vdso/math64.h> +#include <vdso/processor.h> +#include <vdso/time.h> +#include <vdso/time32.h> +#include <vdso/time64.h> #define VDSO_BASES (CLOCK_TAI + 1) #define VDSO_HRES (BIT(CLOCK_REALTIME) | \ @@ -21,8 +32,6 @@ #define CS_RAW 1 #define CS_BASES (CS_RAW + 1) -#define VCLOCK_TIMENS UINT_MAX - /** * struct vdso_timestamp - basetime per clock_id * @sec: seconds @@ -101,6 +110,22 @@ struct vdso_data { */ extern struct vdso_data _vdso_data[CS_BASES] __attribute__((visibility("hidden"))); +/* + * The generic vDSO implementation requires that gettimeofday.h + * provides: + * - __arch_get_vdso_data(): to get the vdso datapage. + * - __arch_get_hw_counter(): to get the hw counter based on the + * clock_mode. + * - gettimeofday_fallback(): fallback for gettimeofday. + * - clock_gettime_fallback(): fallback for clock_gettime. + * - clock_getres_fallback(): fallback for clock_getres. + */ +#ifdef ENABLE_COMPAT_VDSO +#include <asm/vdso/compat_gettimeofday.h> +#else +#include <asm/vdso/gettimeofday.h> +#endif /* ENABLE_COMPAT_VDSO */ + #endif /* !__ASSEMBLY__ */ #endif /* __VDSO_DATAPAGE_H */ diff --git a/include/vdso/jiffies.h b/include/vdso/jiffies.h new file mode 100644 index 000000000000..2f9d596c8b29 --- /dev/null +++ b/include/vdso/jiffies.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_JIFFIES_H +#define __VDSO_JIFFIES_H + +#include <asm/param.h> /* for HZ */ +#include <vdso/time64.h> + +/* TICK_NSEC is the time between ticks in nsec assuming SHIFTED_HZ */ +#define TICK_NSEC ((NSEC_PER_SEC+HZ/2)/HZ) + +#endif /* __VDSO_JIFFIES_H */ diff --git a/include/vdso/ktime.h b/include/vdso/ktime.h new file mode 100644 index 000000000000..a0fd07239e0e --- /dev/null +++ b/include/vdso/ktime.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_KTIME_H +#define __VDSO_KTIME_H + +#include <vdso/jiffies.h> + +/* + * The resolution of the clocks. The resolution value is returned in + * the clock_getres() system call to give application programmers an + * idea of the (in)accuracy of timers. Timer values are rounded up to + * this resolution values. + */ +#define LOW_RES_NSEC TICK_NSEC +#define KTIME_LOW_RES (LOW_RES_NSEC) + +#endif /* __VDSO_KTIME_H */ diff --git a/include/vdso/limits.h b/include/vdso/limits.h new file mode 100644 index 000000000000..0197888ad0e0 --- /dev/null +++ b/include/vdso/limits.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_LIMITS_H +#define __VDSO_LIMITS_H + +#define USHRT_MAX ((unsigned short)~0U) +#define SHRT_MAX ((short)(USHRT_MAX >> 1)) +#define SHRT_MIN ((short)(-SHRT_MAX - 1)) +#define INT_MAX ((int)(~0U >> 1)) +#define INT_MIN (-INT_MAX - 1) +#define UINT_MAX (~0U) +#define LONG_MAX ((long)(~0UL >> 1)) +#define LONG_MIN (-LONG_MAX - 1) +#define ULONG_MAX (~0UL) +#define LLONG_MAX ((long long)(~0ULL >> 1)) +#define LLONG_MIN (-LLONG_MAX - 1) +#define ULLONG_MAX (~0ULL) +#define UINTPTR_MAX ULONG_MAX + +#endif /* __VDSO_LIMITS_H */ diff --git a/include/vdso/math64.h b/include/vdso/math64.h new file mode 100644 index 000000000000..7da703ee5561 --- /dev/null +++ b/include/vdso/math64.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_MATH64_H +#define __VDSO_MATH64_H + +static __always_inline u32 +__iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder) +{ + u32 ret = 0; + + while (dividend >= divisor) { + /* The following asm() prevents the compiler from + optimising this loop into a modulo operation. */ + asm("" : "+rm"(dividend)); + + dividend -= divisor; + ret++; + } + + *remainder = dividend; + + return ret; +} + +#endif /* __VDSO_MATH64_H */ diff --git a/include/vdso/processor.h b/include/vdso/processor.h new file mode 100644 index 000000000000..fbe8265ea3c4 --- /dev/null +++ b/include/vdso/processor.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 ARM Ltd. + */ +#ifndef __VDSO_PROCESSOR_H +#define __VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +#include <asm/vdso/processor.h> + +#endif /* __ASSEMBLY__ */ + +#endif /* __VDSO_PROCESSOR_H */ diff --git a/include/vdso/time.h b/include/vdso/time.h new file mode 100644 index 000000000000..739f53cd2949 --- /dev/null +++ b/include/vdso/time.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_TIME_H +#define __VDSO_TIME_H + +#include <uapi/linux/types.h> + +struct timens_offset { + s64 sec; + u64 nsec; +}; + +#endif /* __VDSO_TIME_H */ diff --git a/include/vdso/time32.h b/include/vdso/time32.h new file mode 100644 index 000000000000..fdf56f932f67 --- /dev/null +++ b/include/vdso/time32.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_TIME32_H +#define __VDSO_TIME32_H + +typedef s32 old_time32_t; + +struct old_timespec32 { + old_time32_t tv_sec; + s32 tv_nsec; +}; + +struct old_timeval32 { + old_time32_t tv_sec; + s32 tv_usec; +}; + +#endif /* __VDSO_TIME32_H */ diff --git a/include/vdso/time64.h b/include/vdso/time64.h new file mode 100644 index 000000000000..9d43c3f5e89d --- /dev/null +++ b/include/vdso/time64.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_TIME64_H +#define __VDSO_TIME64_H + +/* Parameters used to convert the timespec values: */ +#define MSEC_PER_SEC 1000L +#define USEC_PER_MSEC 1000L +#define NSEC_PER_USEC 1000L +#define NSEC_PER_MSEC 1000000L +#define USEC_PER_SEC 1000000L +#define NSEC_PER_SEC 1000000000L +#define FSEC_PER_SEC 1000000000000000LL + +#endif /* __VDSO_TIME64_H */ diff --git a/kernel/exit.c b/kernel/exit.c index 764960fabfa1..d70d47159640 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -103,17 +103,8 @@ static void __exit_signal(struct task_struct *tsk) #ifdef CONFIG_POSIX_TIMERS posix_cpu_timers_exit(tsk); - if (group_dead) { + if (group_dead) posix_cpu_timers_exit_group(tsk); - } else { - /* - * This can only happen if the caller is de_thread(). - * FIXME: this is the temporary hack, we should teach - * posix-cpu-timers to handle this case correctly. - */ - if (unlikely(has_group_leader_pid(tsk))) - posix_cpu_timers_exit_group(tsk); - } #endif if (group_dead) { diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 428beb69426a..7cb09c4cf21c 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -928,6 +928,15 @@ int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq) clocksource_arch_init(cs); +#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE + if (cs->vdso_clock_mode < 0 || + cs->vdso_clock_mode >= VDSO_CLOCKMODE_MAX) { + pr_warn("clocksource %s registered with invalid VDSO mode %d. Disabling VDSO support.\n", + cs->name, cs->vdso_clock_mode); + cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; + } +#endif + /* Initialize mult/shift and max_idle_ns */ __clocksource_update_freq_scale(cs, scale, freq); diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 8cce72501aea..d0a5ba37aff4 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -311,7 +311,7 @@ s64 __ktime_divns(const ktime_t kt, s64 div) div >>= 1; } tmp >>= sft; - do_div(tmp, (unsigned long) div); + do_div(tmp, (u32) div); return dclc < 0 ? -tmp : tmp; } EXPORT_SYMBOL_GPL(__ktime_divns); diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index 12858507d75a..e6ba064ce773 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -8,6 +8,7 @@ #include <linux/user_namespace.h> #include <linux/sched/signal.h> #include <linux/sched/task.h> +#include <linux/clocksource.h> #include <linux/seq_file.h> #include <linux/proc_ns.h> #include <linux/export.h> @@ -172,8 +173,8 @@ static struct timens_offset offset_from_ts(struct timespec64 off) * for vdso_data->clock_mode is a non-issue. The task is spin waiting for the * update to finish and for 'seq' to become even anyway. * - * Timens page has vdso_data->clock_mode set to VCLOCK_TIMENS which enforces - * the time namespace handling path. + * Timens page has vdso_data->clock_mode set to VDSO_CLOCKMODE_TIMENS which + * enforces the time namespace handling path. */ static void timens_setup_vdso_data(struct vdso_data *vdata, struct time_namespace *ns) @@ -183,7 +184,7 @@ static void timens_setup_vdso_data(struct vdso_data *vdata, struct timens_offset boottime = offset_from_ts(ns->offsets.boottime); vdata->seq = 1; - vdata->clock_mode = VCLOCK_TIMENS; + vdata->clock_mode = VDSO_CLOCKMODE_TIMENS; offset[CLOCK_MONOTONIC] = monotonic; offset[CLOCK_MONOTONIC_RAW] = monotonic; offset[CLOCK_MONOTONIC_COARSE] = monotonic; diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 2c48a7233b19..2fd3b3fa68bf 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -118,6 +118,16 @@ static inline int validate_clock_permissions(const clockid_t clock) return __get_task_for_clock(clock, false, false) ? 0 : -EINVAL; } +static inline enum pid_type cpu_timer_pid_type(struct k_itimer *timer) +{ + return CPUCLOCK_PERTHREAD(timer->it_clock) ? PIDTYPE_PID : PIDTYPE_TGID; +} + +static inline struct task_struct *cpu_timer_task_rcu(struct k_itimer *timer) +{ + return pid_task(timer->it.cpu.pid, cpu_timer_pid_type(timer)); +} + /* * Update expiry time from increment, and increase overrun count, * given the current clock sample. @@ -336,9 +346,7 @@ static void __thread_group_cputime(struct task_struct *tsk, u64 *samples) /* * Sample a process (thread group) clock for the given task clkid. If the * group's cputime accounting is already enabled, read the atomic - * store. Otherwise a full update is required. Task's sighand lock must be - * held to protect the task traversal on a full update. clkid is already - * validated. + * store. Otherwise a full update is required. clkid is already validated. */ static u64 cpu_clock_sample_group(const clockid_t clkid, struct task_struct *p, bool start) @@ -393,7 +401,12 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer) new_timer->kclock = &clock_posix_cpu; timerqueue_init(&new_timer->it.cpu.node); - new_timer->it.cpu.task = p; + new_timer->it.cpu.pid = get_task_pid(p, cpu_timer_pid_type(new_timer)); + /* + * get_task_for_clock() took a reference on @p. Drop it as the timer + * holds a reference on the pid of @p. + */ + put_task_struct(p); return 0; } @@ -406,13 +419,15 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer) static int posix_cpu_timer_del(struct k_itimer *timer) { struct cpu_timer *ctmr = &timer->it.cpu; - struct task_struct *p = ctmr->task; struct sighand_struct *sighand; + struct task_struct *p; unsigned long flags; int ret = 0; - if (WARN_ON_ONCE(!p)) - return -EINVAL; + rcu_read_lock(); + p = cpu_timer_task_rcu(timer); + if (!p) + goto out; /* * Protect against sighand release/switch in exit/exec and process/ @@ -434,8 +449,10 @@ static int posix_cpu_timer_del(struct k_itimer *timer) unlock_task_sighand(p, &flags); } +out: + rcu_read_unlock(); if (!ret) - put_task_struct(p); + put_pid(ctmr->pid); return ret; } @@ -484,12 +501,11 @@ void posix_cpu_timers_exit_group(struct task_struct *tsk) * Insert the timer on the appropriate list before any timers that * expire later. This must be called with the sighand lock held. */ -static void arm_timer(struct k_itimer *timer) +static void arm_timer(struct k_itimer *timer, struct task_struct *p) { int clkidx = CPUCLOCK_WHICH(timer->it_clock); struct cpu_timer *ctmr = &timer->it.cpu; u64 newexp = cpu_timer_getexpires(ctmr); - struct task_struct *p = ctmr->task; struct posix_cputimer_base *base; if (CPUCLOCK_PERTHREAD(timer->it_clock)) @@ -564,13 +580,21 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); u64 old_expires, new_expires, old_incr, val; struct cpu_timer *ctmr = &timer->it.cpu; - struct task_struct *p = ctmr->task; struct sighand_struct *sighand; + struct task_struct *p; unsigned long flags; int ret = 0; - if (WARN_ON_ONCE(!p)) - return -EINVAL; + rcu_read_lock(); + p = cpu_timer_task_rcu(timer); + if (!p) { + /* + * If p has just been reaped, we can no + * longer get any information about it at all. + */ + rcu_read_unlock(); + return -ESRCH; + } /* * Use the to_ktime conversion because that clamps the maximum @@ -587,8 +611,10 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, * If p has just been reaped, we can no * longer get any information about it at all. */ - if (unlikely(sighand == NULL)) + if (unlikely(sighand == NULL)) { + rcu_read_unlock(); return -ESRCH; + } /* * Disarm any old timer after extracting its expiry time. @@ -662,7 +688,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, */ cpu_timer_setexpires(ctmr, new_expires); if (new_expires != 0 && val < new_expires) { - arm_timer(timer); + arm_timer(timer, p); } unlock_task_sighand(p, &flags); @@ -693,6 +719,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, ret = 0; out: + rcu_read_unlock(); if (old) old->it_interval = ns_to_timespec64(old_incr); @@ -704,10 +731,12 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); struct cpu_timer *ctmr = &timer->it.cpu; u64 now, expires = cpu_timer_getexpires(ctmr); - struct task_struct *p = ctmr->task; + struct task_struct *p; - if (WARN_ON_ONCE(!p)) - return; + rcu_read_lock(); + p = cpu_timer_task_rcu(timer); + if (!p) + goto out; /* * Easy part: convert the reload time. @@ -715,36 +744,15 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp itp->it_interval = ktime_to_timespec64(timer->it_interval); if (!expires) - return; + goto out; /* * Sample the clock to take the difference with the expiry time. */ - if (CPUCLOCK_PERTHREAD(timer->it_clock)) { + if (CPUCLOCK_PERTHREAD(timer->it_clock)) now = cpu_clock_sample(clkid, p); - } else { - struct sighand_struct *sighand; - unsigned long flags; - - /* - * Protect against sighand release/switch in exit/exec and - * also make timer sampling safe if it ends up calling - * thread_group_cputime(). - */ - sighand = lock_task_sighand(p, &flags); - if (unlikely(sighand == NULL)) { - /* - * The process has been reaped. - * We can't even collect a sample any more. - * Disarm the timer, nothing else to do. - */ - cpu_timer_setexpires(ctmr, 0); - return; - } else { - now = cpu_clock_sample_group(clkid, p, false); - unlock_task_sighand(p, &flags); - } - } + else + now = cpu_clock_sample_group(clkid, p, false); if (now < expires) { itp->it_value = ns_to_timespec64(expires - now); @@ -756,6 +764,8 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp itp->it_value.tv_nsec = 1; itp->it_value.tv_sec = 0; } +out: + rcu_read_unlock(); } #define MAX_COLLECTED 20 @@ -976,56 +986,38 @@ static void check_process_timers(struct task_struct *tsk, static void posix_cpu_timer_rearm(struct k_itimer *timer) { clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); - struct cpu_timer *ctmr = &timer->it.cpu; - struct task_struct *p = ctmr->task; + struct task_struct *p; struct sighand_struct *sighand; unsigned long flags; u64 now; - if (WARN_ON_ONCE(!p)) - return; + rcu_read_lock(); + p = cpu_timer_task_rcu(timer); + if (!p) + goto out; /* * Fetch the current sample and update the timer's expiry time. */ - if (CPUCLOCK_PERTHREAD(timer->it_clock)) { + if (CPUCLOCK_PERTHREAD(timer->it_clock)) now = cpu_clock_sample(clkid, p); - bump_cpu_timer(timer, now); - if (unlikely(p->exit_state)) - return; - - /* Protect timer list r/w in arm_timer() */ - sighand = lock_task_sighand(p, &flags); - if (!sighand) - return; - } else { - /* - * Protect arm_timer() and timer sampling in case of call to - * thread_group_cputime(). - */ - sighand = lock_task_sighand(p, &flags); - if (unlikely(sighand == NULL)) { - /* - * The process has been reaped. - * We can't even collect a sample any more. - */ - cpu_timer_setexpires(ctmr, 0); - return; - } else if (unlikely(p->exit_state) && thread_group_empty(p)) { - /* If the process is dying, no need to rearm */ - goto unlock; - } + else now = cpu_clock_sample_group(clkid, p, true); - bump_cpu_timer(timer, now); - /* Leave the sighand locked for the call below. */ - } + + bump_cpu_timer(timer, now); + + /* Protect timer list r/w in arm_timer() */ + sighand = lock_task_sighand(p, &flags); + if (unlikely(sighand == NULL)) + goto out; /* * Now re-arm for the new expiry time. */ - arm_timer(timer); -unlock: + arm_timer(timer, p); unlock_task_sighand(p, &flags); +out: + rcu_read_unlock(); } /** diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index ff0eb30de346..07709ac30439 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -121,7 +121,8 @@ static struct k_itimer *__posix_timers_find(struct hlist_head *head, { struct k_itimer *timer; - hlist_for_each_entry_rcu(timer, head, t_hash) { + hlist_for_each_entry_rcu(timer, head, t_hash, + lockdep_is_held(&hash_lock)) { if ((timer->it_signal == sig) && (timer->it_id == id)) return timer; } diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c index e4332e3e2d56..fa3f800d7d76 100644 --- a/kernel/time/sched_clock.c +++ b/kernel/time/sched_clock.c @@ -208,7 +208,8 @@ sched_clock_register(u64 (*read)(void), int bits, unsigned long rate) if (sched_clock_timer.function != NULL) { /* update timeout for clock wrap */ - hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); + hrtimer_start(&sched_clock_timer, cd.wrap_kt, + HRTIMER_MODE_REL_HARD); } r = rate; @@ -254,9 +255,9 @@ void __init generic_sched_clock_init(void) * Start the timer to keep sched_clock() properly updated and * sets the initial epoch. */ - hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); sched_clock_timer.function = sched_clock_poll; - hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); + hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD); } /* @@ -293,7 +294,7 @@ void sched_clock_resume(void) struct clock_read_data *rd = &cd.read_data[0]; rd->epoch_cyc = cd.actual_read_sched_clock(); - hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); + hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD); rd->read_sched_clock = cd.actual_read_sched_clock; } diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 856280d2cbd4..9ebaab13339d 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1005,9 +1005,8 @@ static int scale64_check_overflow(u64 mult, u64 div, u64 *base) ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem))) return -EOVERFLOW; tmp *= mult; - rem *= mult; - do_div(rem, div); + rem = div64_u64(rem * mult, div); *base = tmp + rem; return 0; } diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 568564ae3597..a5221abb4594 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1829,21 +1829,23 @@ static void process_timeout(struct timer_list *t) * schedule_timeout - sleep until timeout * @timeout: timeout value in jiffies * - * Make the current task sleep until @timeout jiffies have - * elapsed. The routine will return immediately unless - * the current task state has been set (see set_current_state()). + * Make the current task sleep until @timeout jiffies have elapsed. + * The function behavior depends on the current task state + * (see also set_current_state() description): * - * You can set the task state as follows - + * %TASK_RUNNING - the scheduler is called, but the task does not sleep + * at all. That happens because sched_submit_work() does nothing for + * tasks in %TASK_RUNNING state. * * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to * pass before the routine returns unless the current task is explicitly - * woken up, (e.g. by wake_up_process())". + * woken up, (e.g. by wake_up_process()). * * %TASK_INTERRUPTIBLE - the routine may return early if a signal is * delivered to the current task or the current task is explicitly woken * up. * - * The current task state is guaranteed to be TASK_RUNNING when this + * The current task state is guaranteed to be %TASK_RUNNING when this * routine returns. * * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule @@ -1851,7 +1853,7 @@ static void process_timeout(struct timer_list *t) * value will be %MAX_SCHEDULE_TIMEOUT. * * Returns 0 when the timer has expired otherwise the remaining time in - * jiffies will be returned. In all cases the return value is guaranteed + * jiffies will be returned. In all cases the return value is guaranteed * to be non-negative. */ signed long __sched schedule_timeout(signed long timeout) diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c index 9577c89179cd..54ce6eb2ca36 100644 --- a/kernel/time/vsyscall.c +++ b/kernel/time/vsyscall.c @@ -71,13 +71,15 @@ void update_vsyscall(struct timekeeper *tk) { struct vdso_data *vdata = __arch_get_k_vdso_data(); struct vdso_timestamp *vdso_ts; + s32 clock_mode; u64 nsec; /* copy vsyscall data */ vdso_write_begin(vdata); - vdata[CS_HRES_COARSE].clock_mode = __arch_get_clock_mode(tk); - vdata[CS_RAW].clock_mode = __arch_get_clock_mode(tk); + clock_mode = tk->tkr_mono.clock->vdso_clock_mode; + vdata[CS_HRES_COARSE].clock_mode = clock_mode; + vdata[CS_RAW].clock_mode = clock_mode; /* CLOCK_REALTIME also required for time() */ vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_REALTIME]; @@ -103,10 +105,10 @@ void update_vsyscall(struct timekeeper *tk) WRITE_ONCE(vdata[CS_HRES_COARSE].hrtimer_res, hrtimer_resolution); /* - * Architectures can opt out of updating the high resolution part - * of the VDSO. + * If the current clocksource is not VDSO capable, then spare the + * update of the high reolution parts. */ - if (__arch_update_vdso_data()) + if (clock_mode != VDSO_CLOCKMODE_NONE) update_vdso_data(vdata, tk); __arch_update_vsyscall(vdata, tk); diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index f8b8ec5e63ac..a2909af4b924 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -2,30 +2,9 @@ /* * Generic userspace implementations of gettimeofday() and similar. */ -#include <linux/compiler.h> -#include <linux/math64.h> -#include <linux/time.h> -#include <linux/kernel.h> -#include <linux/hrtimer_defs.h> #include <vdso/datapage.h> #include <vdso/helpers.h> -/* - * The generic vDSO implementation requires that gettimeofday.h - * provides: - * - __arch_get_vdso_data(): to get the vdso datapage. - * - __arch_get_hw_counter(): to get the hw counter based on the - * clock_mode. - * - gettimeofday_fallback(): fallback for gettimeofday. - * - clock_gettime_fallback(): fallback for clock_gettime. - * - clock_getres_fallback(): fallback for clock_getres. - */ -#ifdef ENABLE_COMPAT_VDSO -#include <asm/vdso/compat_gettimeofday.h> -#else -#include <asm/vdso/gettimeofday.h> -#endif /* ENABLE_COMPAT_VDSO */ - #ifndef vdso_calc_delta /* * Default implementation which works for all sane clocksources. That @@ -38,6 +17,27 @@ u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) } #endif +#ifndef vdso_shift_ns +static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) +{ + return ns >> shift; +} +#endif + +#ifndef __arch_vdso_hres_capable +static inline bool __arch_vdso_hres_capable(void) +{ + return true; +} +#endif + +#ifndef vdso_clocksource_ok +static inline bool vdso_clocksource_ok(const struct vdso_data *vd) +{ + return vd->clock_mode != VDSO_CLOCKMODE_NONE; +} +#endif + #ifdef CONFIG_TIME_NS static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, struct __kernel_timespec *ts) @@ -57,14 +57,15 @@ static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, do { seq = vdso_read_begin(vd); + + if (unlikely(!vdso_clocksource_ok(vd))) + return -1; + cycles = __arch_get_hw_counter(vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; - if (unlikely((s64)cycles < 0)) - return -1; - ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); - ns >>= vd->shift; + ns = vdso_shift_ns(ns, vd->shift); sec = vdso_ts->sec; } while (unlikely(vdso_read_retry(vd, seq))); @@ -101,12 +102,16 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, u64 cycles, last, sec, ns; u32 seq; + /* Allows to compile the high resolution parts out */ + if (!__arch_vdso_hres_capable()) + return -1; + do { /* - * Open coded to handle VCLOCK_TIMENS. Time namespace + * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace * enabled tasks have a special VVAR page installed which * has vd->seq set to 1 and vd->clock_mode set to - * VCLOCK_TIMENS. For non time namespace affected tasks + * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks * this does not affect performance because if vd->seq is * odd, i.e. a concurrent update is in progress the extra * check for vd->clock_mode is just a few extra @@ -115,20 +120,20 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, */ while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { if (IS_ENABLED(CONFIG_TIME_NS) && - vd->clock_mode == VCLOCK_TIMENS) + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) return do_hres_timens(vd, clk, ts); cpu_relax(); } smp_rmb(); + if (unlikely(!vdso_clocksource_ok(vd))) + return -1; + cycles = __arch_get_hw_counter(vd->clock_mode); ns = vdso_ts->nsec; last = vd->cycle_last; - if (unlikely((s64)cycles < 0)) - return -1; - ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); - ns >>= vd->shift; + ns = vdso_shift_ns(ns, vd->shift); sec = vdso_ts->sec; } while (unlikely(vdso_read_retry(vd, seq))); @@ -187,12 +192,12 @@ static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, do { /* - * Open coded to handle VCLOCK_TIMENS. See comment in + * Open coded to handle VDSO_CLOCK_TIMENS. See comment in * do_hres(). */ while ((seq = READ_ONCE(vd->seq)) & 1) { if (IS_ENABLED(CONFIG_TIME_NS) && - vd->clock_mode == VCLOCK_TIMENS) + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) return do_coarse_timens(vd, clk, ts); cpu_relax(); } @@ -206,9 +211,9 @@ static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, } static __maybe_unused int -__cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) +__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *ts) { - const struct vdso_data *vd = __arch_get_vdso_data(); u32 msk; /* Check for negative values or invalid clocks */ @@ -233,23 +238,31 @@ __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts) } static __maybe_unused int -__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *ts) { - int ret = __cvdso_clock_gettime_common(clock, ts); + int ret = __cvdso_clock_gettime_common(vd, clock, ts); if (unlikely(ret)) return clock_gettime_fallback(clock, ts); return 0; } +static __maybe_unused int +__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts); +} + #ifdef BUILD_VDSO32 static __maybe_unused int -__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) +__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, + struct old_timespec32 *res) { struct __kernel_timespec ts; int ret; - ret = __cvdso_clock_gettime_common(clock, &ts); + ret = __cvdso_clock_gettime_common(vd, clock, &ts); if (unlikely(ret)) return clock_gettime32_fallback(clock, res); @@ -260,12 +273,18 @@ __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) return ret; } + +static __maybe_unused int +__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) +{ + return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res); +} #endif /* BUILD_VDSO32 */ static __maybe_unused int -__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +__cvdso_gettimeofday_data(const struct vdso_data *vd, + struct __kernel_old_timeval *tv, struct timezone *tz) { - const struct vdso_data *vd = __arch_get_vdso_data(); if (likely(tv != NULL)) { struct __kernel_timespec ts; @@ -279,7 +298,7 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) if (unlikely(tz != NULL)) { if (IS_ENABLED(CONFIG_TIME_NS) && - vd->clock_mode == VCLOCK_TIMENS) + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) vd = __arch_get_timens_vdso_data(); tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; @@ -289,13 +308,20 @@ __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) return 0; } +static __maybe_unused int +__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz); +} + #ifdef VDSO_HAS_TIME -static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) +static __maybe_unused __kernel_old_time_t +__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) { - const struct vdso_data *vd = __arch_get_vdso_data(); __kernel_old_time_t t; - if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) vd = __arch_get_timens_vdso_data(); t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); @@ -305,13 +331,18 @@ static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time return t; } + +static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) +{ + return __cvdso_time_data(__arch_get_vdso_data(), time); +} #endif /* VDSO_HAS_TIME */ #ifdef VDSO_HAS_CLOCK_GETRES static __maybe_unused -int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) +int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *res) { - const struct vdso_data *vd = __arch_get_vdso_data(); u32 msk; u64 ns; @@ -319,7 +350,8 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) if (unlikely((u32) clock >= MAX_CLOCKS)) return -1; - if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS) + if (IS_ENABLED(CONFIG_TIME_NS) && + vd->clock_mode == VDSO_CLOCKMODE_TIMENS) vd = __arch_get_timens_vdso_data(); /* @@ -349,23 +381,31 @@ int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res) } static __maybe_unused -int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) +int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, + struct __kernel_timespec *res) { - int ret = __cvdso_clock_getres_common(clock, res); + int ret = __cvdso_clock_getres_common(vd, clock, res); if (unlikely(ret)) return clock_getres_fallback(clock, res); return 0; } +static __maybe_unused +int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) +{ + return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res); +} + #ifdef BUILD_VDSO32 static __maybe_unused int -__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) +__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, + struct old_timespec32 *res) { struct __kernel_timespec ts; int ret; - ret = __cvdso_clock_getres_common(clock, &ts); + ret = __cvdso_clock_getres_common(vd, clock, &ts); if (unlikely(ret)) return clock_getres32_fallback(clock, res); @@ -376,5 +416,12 @@ __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) } return ret; } + +static __maybe_unused int +__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) +{ + return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(), + clock, res); +} #endif /* BUILD_VDSO32 */ #endif /* VDSO_HAS_CLOCK_GETRES */ diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 55a0a2eccbd2..acf0a69d68ce 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -2252,8 +2252,12 @@ static int check_modname_len(struct module *mod) **/ static void add_header(struct buffer *b, struct module *mod) { - buf_printf(b, "#include <linux/build-salt.h>\n"); buf_printf(b, "#include <linux/module.h>\n"); + /* + * Include build-salt.h after module.h in order to + * inherit the definitions. + */ + buf_printf(b, "#include <linux/build-salt.h>\n"); buf_printf(b, "#include <linux/vermagic.h>\n"); buf_printf(b, "#include <linux/compiler.h>\n"); buf_printf(b, "\n"); |