summaryrefslogtreecommitdiff
path: root/kernel/time/timekeeping.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/time/timekeeping.c')
-rw-r--r--kernel/time/timekeeping.c90
1 files changed, 51 insertions, 39 deletions
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 37dec7e3db43..da233cdf89b0 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -258,10 +258,9 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
tk->cycle_interval = interval;
/* Go back from cycles -> shifted ns */
- tk->xtime_interval = (u64) interval * clock->mult;
+ tk->xtime_interval = interval * clock->mult;
tk->xtime_remainder = ntpinterval - tk->xtime_interval;
- tk->raw_interval =
- ((u64) interval * clock->mult) >> clock->shift;
+ tk->raw_interval = (interval * clock->mult) >> clock->shift;
/* if changing clocks, convert xtime_nsec shift units */
if (old_clock) {
@@ -299,10 +298,10 @@ u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;
static inline u32 arch_gettimeoffset(void) { return 0; }
#endif
-static inline s64 timekeeping_delta_to_ns(struct tk_read_base *tkr,
+static inline u64 timekeeping_delta_to_ns(struct tk_read_base *tkr,
cycle_t delta)
{
- s64 nsec;
+ u64 nsec;
nsec = delta * tkr->mult + tkr->xtime_nsec;
nsec >>= tkr->shift;
@@ -311,7 +310,7 @@ static inline s64 timekeeping_delta_to_ns(struct tk_read_base *tkr,
return nsec + arch_gettimeoffset();
}
-static inline s64 timekeeping_get_ns(struct tk_read_base *tkr)
+static inline u64 timekeeping_get_ns(struct tk_read_base *tkr)
{
cycle_t delta;
@@ -319,8 +318,8 @@ static inline s64 timekeeping_get_ns(struct tk_read_base *tkr)
return timekeeping_delta_to_ns(tkr, delta);
}
-static inline s64 timekeeping_cycles_to_ns(struct tk_read_base *tkr,
- cycle_t cycles)
+static inline u64 timekeeping_cycles_to_ns(struct tk_read_base *tkr,
+ cycle_t cycles)
{
cycle_t delta;
@@ -425,6 +424,35 @@ u64 ktime_get_raw_fast_ns(void)
}
EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
+/**
+ * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
+ *
+ * To keep it NMI safe since we're accessing from tracing, we're not using a
+ * separate timekeeper with updates to monotonic clock and boot offset
+ * protected with seqlocks. This has the following minor side effects:
+ *
+ * (1) Its possible that a timestamp be taken after the boot offset is updated
+ * but before the timekeeper is updated. If this happens, the new boot offset
+ * is added to the old timekeeping making the clock appear to update slightly
+ * earlier:
+ * CPU 0 CPU 1
+ * timekeeping_inject_sleeptime64()
+ * __timekeeping_inject_sleeptime(tk, delta);
+ * timestamp();
+ * timekeeping_update(tk, TK_CLEAR_NTP...);
+ *
+ * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
+ * partially updated. Since the tk->offs_boot update is a rare event, this
+ * should be a rare occurrence which postprocessing should be able to handle.
+ */
+u64 notrace ktime_get_boot_fast_ns(void)
+{
+ struct timekeeper *tk = &tk_core.timekeeper;
+
+ return (ktime_get_mono_fast_ns() + ktime_to_ns(tk->offs_boot));
+}
+EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
+
/* Suspend-time cycles value for halted fast timekeeper. */
static cycle_t cycles_at_suspend;
@@ -623,7 +651,7 @@ static void timekeeping_forward_now(struct timekeeper *tk)
{
struct clocksource *clock = tk->tkr_mono.clock;
cycle_t cycle_now, delta;
- s64 nsec;
+ u64 nsec;
cycle_now = tk->tkr_mono.read(clock);
delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
@@ -652,7 +680,7 @@ int __getnstimeofday64(struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned long seq;
- s64 nsecs = 0;
+ u64 nsecs;
do {
seq = read_seqcount_begin(&tk_core.seq);
@@ -692,7 +720,7 @@ ktime_t ktime_get(void)
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
ktime_t base;
- s64 nsecs;
+ u64 nsecs;
WARN_ON(timekeeping_suspended);
@@ -735,7 +763,7 @@ ktime_t ktime_get_with_offset(enum tk_offsets offs)
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
ktime_t base, *offset = offsets[offs];
- s64 nsecs;
+ u64 nsecs;
WARN_ON(timekeeping_suspended);
@@ -779,7 +807,7 @@ ktime_t ktime_get_raw(void)
struct timekeeper *tk = &tk_core.timekeeper;
unsigned int seq;
ktime_t base;
- s64 nsecs;
+ u64 nsecs;
do {
seq = read_seqcount_begin(&tk_core.seq);
@@ -804,8 +832,8 @@ void ktime_get_ts64(struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
struct timespec64 tomono;
- s64 nsec;
unsigned int seq;
+ u64 nsec;
WARN_ON(timekeeping_suspended);
@@ -893,8 +921,8 @@ void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
unsigned long seq;
ktime_t base_raw;
ktime_t base_real;
- s64 nsec_raw;
- s64 nsec_real;
+ u64 nsec_raw;
+ u64 nsec_real;
cycle_t now;
WARN_ON_ONCE(timekeeping_suspended);
@@ -1052,7 +1080,7 @@ int get_device_system_crosststamp(int (*get_time_fn)
cycle_t cycles, now, interval_start;
unsigned int clock_was_set_seq = 0;
ktime_t base_real, base_raw;
- s64 nsec_real, nsec_raw;
+ u64 nsec_real, nsec_raw;
u8 cs_was_changed_seq;
unsigned long seq;
bool do_interp;
@@ -1365,7 +1393,7 @@ void getrawmonotonic64(struct timespec64 *ts)
struct timekeeper *tk = &tk_core.timekeeper;
struct timespec64 ts64;
unsigned long seq;
- s64 nsecs;
+ u64 nsecs;
do {
seq = read_seqcount_begin(&tk_core.seq);
@@ -1616,7 +1644,7 @@ void timekeeping_resume(void)
struct clocksource *clock = tk->tkr_mono.clock;
unsigned long flags;
struct timespec64 ts_new, ts_delta;
- cycle_t cycle_now, cycle_delta;
+ cycle_t cycle_now;
sleeptime_injected = false;
read_persistent_clock64(&ts_new);
@@ -1642,27 +1670,11 @@ void timekeeping_resume(void)
cycle_now = tk->tkr_mono.read(clock);
if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) &&
cycle_now > tk->tkr_mono.cycle_last) {
- u64 num, max = ULLONG_MAX;
- u32 mult = clock->mult;
- u32 shift = clock->shift;
- s64 nsec = 0;
-
- cycle_delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last,
- tk->tkr_mono.mask);
-
- /*
- * "cycle_delta * mutl" may cause 64 bits overflow, if the
- * suspended time is too long. In that case we need do the
- * 64 bits math carefully
- */
- do_div(max, mult);
- if (cycle_delta > max) {
- num = div64_u64(cycle_delta, max);
- nsec = (((u64) max * mult) >> shift) * num;
- cycle_delta -= num * max;
- }
- nsec += ((u64) cycle_delta * mult) >> shift;
+ u64 nsec, cyc_delta;
+ cyc_delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last,
+ tk->tkr_mono.mask);
+ nsec = mul_u64_u32_shr(cyc_delta, clock->mult, clock->shift);
ts_delta = ns_to_timespec64(nsec);
sleeptime_injected = true;
} else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
generated by cgit v1.2.3 (git 2.39.0) at 2024-11-30 20:35:06 +0300