diff options
23 files changed, 348 insertions, 194 deletions
diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index c4b3ee552131..2411b4c32932 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -51,8 +51,8 @@ static void __hyp_clock_work(struct work_struct *work) hyp_clock = container_of(dwork, struct hyp_trace_clock, work); - ktime_get_snapshot(&snap); - boot = ktime_to_ns(snap.boot); + ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap); + boot = ktime_to_ns(snap.systime); delta_boot = boot - hyp_clock->boot; delta_cycles = snap.cycles - hyp_clock->cycles; @@ -118,9 +118,9 @@ static void hyp_trace_clock_enable(struct hyp_trace_clock *hyp_clock, bool enabl hyp_clock->running = false; } - ktime_get_snapshot(&snap); + ktime_get_snapshot_id(CLOCK_BOOTTIME, &snap); - hyp_clock->boot = ktime_to_ns(snap.boot); + hyp_clock->boot = ktime_to_ns(snap.systime); hyp_clock->cycles = snap.cycles; hyp_clock->mult = 0; diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index 58c5fe7d7572..b11b8821c9fb 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -28,7 +28,7 @@ static void kvm_ptp_get_time(struct kvm_vcpu *vcpu, u64 *val) * system time and counter value must captured at the same * time to keep consistency and precision. */ - ktime_get_snapshot(&systime_snapshot); + ktime_get_snapshot_id(CLOCK_REALTIME, &systime_snapshot); /* * This is only valid if the current clocksource is the @@ -61,8 +61,8 @@ static void kvm_ptp_get_time(struct kvm_vcpu *vcpu, u64 *val) * in the future (about 292 years from 1970, and at that stage * nobody will give a damn about it). */ - val[0] = upper_32_bits(systime_snapshot.real); - val[1] = lower_32_bits(systime_snapshot.real); + val[0] = upper_32_bits(systime_snapshot.systime); + val[1] = lower_32_bits(systime_snapshot.systime); val[2] = upper_32_bits(cycles); val[3] = lower_32_bits(cycles); } diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index b5991d53fc0e..cb3d0ca1fa22 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -87,6 +87,27 @@ static u64 kvm_clock_get_cycles(struct clocksource *cs) return kvm_clock_read(); } +static u64 kvm_clock_get_cycles_snapshot(struct clocksource *cs, + struct clocksource_hw_snapshot *chs) +{ + struct pvclock_vcpu_time_info *src; + unsigned version; + u64 ret, tsc; + + preempt_disable_notrace(); + src = this_cpu_pvti(); + do { + version = pvclock_read_begin(src); + tsc = rdtsc_ordered(); + ret = __pvclock_read_cycles(src, tsc); + } while (pvclock_read_retry(src, version)); + preempt_enable_notrace(); + + chs->hw_cycles = tsc; + chs->hw_csid = CSID_X86_TSC; + return ret; +} + static noinstr u64 kvm_sched_clock_read(void) { return pvclock_clocksource_read_nowd(this_cpu_pvti()) - kvm_sched_clock_offset; @@ -156,13 +177,14 @@ static int kvm_cs_enable(struct clocksource *cs) } static struct clocksource kvm_clock = { - .name = "kvm-clock", - .read = kvm_clock_get_cycles, - .rating = 400, - .mask = CLOCKSOURCE_MASK(64), - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .id = CSID_X86_KVM_CLK, - .enable = kvm_cs_enable, + .name = "kvm-clock", + .read = kvm_clock_get_cycles, + .read_snapshot = kvm_clock_get_cycles_snapshot, + .rating = 400, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .id = CSID_X86_KVM_CLK, + .enable = kvm_cs_enable, }; static void kvm_register_clock(char *txt) diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c index e9f5034a1bc8..df567795d175 100644 --- a/drivers/clocksource/hyperv_timer.c +++ b/drivers/clocksource/hyperv_timer.c @@ -444,6 +444,22 @@ static u64 notrace read_hv_clock_tsc_cs(struct clocksource *arg) return read_hv_clock_tsc(); } +static u64 notrace read_hv_clock_tsc_cs_snapshot(struct clocksource *arg, + struct clocksource_hw_snapshot *chs) +{ + u64 time; + + if (hv_read_tsc_page_tsc(tsc_page, &chs->hw_cycles, &time)) { + chs->hw_csid = CSID_X86_TSC; + } else { + chs->hw_cycles = 0; + chs->hw_csid = CSID_GENERIC; + time = read_hv_clock_msr(); + } + + return time; +} + static u64 noinstr read_hv_sched_clock_tsc(void) { return (read_hv_clock_tsc() - hv_sched_clock_offset) * @@ -492,18 +508,19 @@ static int hv_cs_enable(struct clocksource *cs) #endif static struct clocksource hyperv_cs_tsc = { - .name = "hyperv_clocksource_tsc_page", - .rating = 500, - .read = read_hv_clock_tsc_cs, - .mask = CLOCKSOURCE_MASK(64), - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .suspend= suspend_hv_clock_tsc, - .resume = resume_hv_clock_tsc, + .name = "hyperv_clocksource_tsc_page", + .rating = 500, + .read = read_hv_clock_tsc_cs, + .read_snapshot = read_hv_clock_tsc_cs_snapshot, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .suspend = suspend_hv_clock_tsc, + .resume = resume_hv_clock_tsc, #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK - .enable = hv_cs_enable, - .vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK, + .enable = hv_cs_enable, + .vdso_clock_mode = VDSO_CLOCKMODE_HVCLOCK, #else - .vdso_clock_mode = VDSO_CLOCKMODE_NONE, + .vdso_clock_mode = VDSO_CLOCKMODE_NONE, #endif }; diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index c72c2bfdcffb..2697073dbf90 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -2310,10 +2310,10 @@ int sja1105_static_config_reload(struct sja1105_private *priv, goto out; } - t1 = timespec64_to_ns(&ptp_sts_before.pre_ts); - t2 = timespec64_to_ns(&ptp_sts_before.post_ts); - t3 = timespec64_to_ns(&ptp_sts_after.pre_ts); - t4 = timespec64_to_ns(&ptp_sts_after.post_ts); + t1 = ktime_to_ns(ptp_sts_before.pre_sts.systime); + t2 = ktime_to_ns(ptp_sts_before.post_sts.systime); + t3 = ktime_to_ns(ptp_sts_after.pre_sts.systime); + t4 = ktime_to_ns(ptp_sts_after.post_sts.systime); /* Mid point, corresponds to pre-reset PTPCLKVAL */ t12 = t1 + (t2 - t1) / 2; /* Mid point, corresponds to post-reset PTPCLKVAL, aka 0 */ diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c index 36df742c326c..f9e4ec6f7ebb 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp.c @@ -2065,11 +2065,13 @@ static const struct ice_crosststamp_cfg ice_crosststamp_cfg_e830 = { /** * struct ice_crosststamp_ctx - Device cross timestamp context * @snapshot: snapshot of system clocks for historic interpolation + * @snapshot_clock_id: System clock ID for @snapshot * @pf: pointer to the PF private structure * @cfg: pointer to hardware configuration for cross timestamp */ struct ice_crosststamp_ctx { struct system_time_snapshot snapshot; + clockid_t snapshot_clock_id; struct ice_pf *pf; const struct ice_crosststamp_cfg *cfg; }; @@ -2115,7 +2117,7 @@ static int ice_capture_crosststamp(ktime_t *device, } /* Snapshot system time for historic interpolation */ - ktime_get_snapshot(&ctx->snapshot); + ktime_get_snapshot_id(ctx->snapshot_clock_id, &ctx->snapshot); /* Program cmd to master timer */ ice_ptp_src_cmd(hw, ICE_PTP_READ_TIME); @@ -2176,6 +2178,7 @@ static int ice_ptp_getcrosststamp(struct ptp_clock_info *info, { struct ice_pf *pf = ptp_info_to_pf(info); struct ice_crosststamp_ctx ctx = { + .snapshot_clock_id = cts->clock_id, .pf = pf, }; diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index 17236813965d..46d625b15f44 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -326,6 +326,7 @@ struct igc_adapter { struct timespec64 prev_ptp_time; /* Pre-reset PTP clock */ ktime_t ptp_reset_start; /* Reset time in clock mono */ struct system_time_snapshot snapshot; + clockid_t snapshot_clock_id; struct mutex ptm_lock; /* Only allow one PTM transaction at a time */ char fw_version[32]; diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c index 3d6b2264164a..b40aba9ab685 100644 --- a/drivers/net/ethernet/intel/igc/igc_ptp.c +++ b/drivers/net/ethernet/intel/igc/igc_ptp.c @@ -1049,7 +1049,7 @@ static int igc_phc_get_syncdevicetime(ktime_t *device, */ do { /* Get a snapshot of system clocks to use as historic value. */ - ktime_get_snapshot(&adapter->snapshot); + ktime_get_snapshot_id(adapter->snapshot_clock_id, &adapter->snapshot); igc_ptm_trigger(hw); @@ -1103,6 +1103,8 @@ static int igc_ptp_getcrosststamp(struct ptp_clock_info *ptp, /* This blocks until any in progress PTM transactions complete */ mutex_lock(&adapter->ptm_lock); + adapter->snapshot_clock_id = cts->clock_id; + ret = get_device_system_crosststamp(igc_phc_get_syncdevicetime, adapter, &adapter->snapshot, cts); mutex_unlock(&adapter->ptm_lock); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c index d785f1b4f2e1..5df786133e4b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c @@ -340,7 +340,7 @@ static int mlx5_ptp_getcrosststamp(struct ptp_clock_info *ptp, goto unlock; } - ktime_get_snapshot(&history_begin); + ktime_get_snapshot_id(cts->clock_id, &history_begin); err = get_device_system_crosststamp(mlx5_mtctr_syncdevicetime, mdev, &history_begin, cts); @@ -366,7 +366,7 @@ static int mlx5_ptp_getcrosscycles(struct ptp_clock_info *ptp, goto unlock; } - ktime_get_snapshot(&history_begin); + ktime_get_snapshot_id(cts->clock_id, &history_begin); err = get_device_system_crosststamp(mlx5_mtctr_syncdevicecyclestime, mdev, &history_begin, cts); diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c index c65f4b56a327..f829156d42b3 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c @@ -250,7 +250,8 @@ iwl_mld_phc_get_crosstimestamp(struct ptp_clock_info *ptp, /* System (wall) time */ ktime_t sys_time; - memset(xtstamp, 0, sizeof(struct system_device_crosststamp)); + if (xtstamp->clock_id != CLOCK_REALTIME) + return -ENOTSUPP; ret = iwl_mld_get_crosstimestamp_fw(mld, &gp2, &sys_time); if (ret) { @@ -270,7 +271,7 @@ iwl_mld_phc_get_crosstimestamp(struct ptp_clock_info *ptp, /* System monotonic raw time is not used */ xtstamp->device = ns_to_ktime(gp2_ns); - xtstamp->sys_realtime = sys_time; + xtstamp->sys_systime = sys_time; return ret; } diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c index f7b620136c85..bcd6f7cead2a 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c @@ -160,13 +160,14 @@ iwl_mvm_phc_get_crosstimestamp(struct ptp_clock_info *ptp, /* System (wall) time */ ktime_t sys_time; - memset(xtstamp, 0, sizeof(struct system_device_crosststamp)); - if (!mvm->ptp_data.ptp_clock) { IWL_ERR(mvm, "No PHC clock registered\n"); return -ENODEV; } + if (xtstamp->clock_id != CLOCK_REALTIME) + return -ENOTSUPP; + mutex_lock(&mvm->mutex); if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_SYNCED_TIME)) { ret = iwl_mvm_get_crosstimestamp_fw(mvm, &gp2, &sys_time); @@ -184,7 +185,7 @@ iwl_mvm_phc_get_crosstimestamp(struct ptp_clock_info *ptp, /* System monotonic raw time is not used */ xtstamp->device = (ktime_t)gp2_ns; - xtstamp->sys_realtime = sys_time; + xtstamp->sys_systime = sys_time; out: mutex_unlock(&mvm->mutex); diff --git a/drivers/pps/generators/pps_gen-dummy.c b/drivers/pps/generators/pps_gen-dummy.c index 547fa7fe29f4..a4395543c4bb 100644 --- a/drivers/pps/generators/pps_gen-dummy.c +++ b/drivers/pps/generators/pps_gen-dummy.c @@ -39,11 +39,7 @@ static void pps_gen_ktimer_event(struct timer_list *unused) static int pps_gen_dummy_get_time(struct pps_gen_device *pps_gen, struct timespec64 *time) { - struct system_time_snapshot snap; - - ktime_get_snapshot(&snap); - *time = ktime_to_timespec64(snap.real); - + ktime_get_real_ts64(time); return 0; } diff --git a/drivers/pps/generators/pps_gen_tio.c b/drivers/pps/generators/pps_gen_tio.c index de00a85bfafa..9483d126ada0 100644 --- a/drivers/pps/generators/pps_gen_tio.c +++ b/drivers/pps/generators/pps_gen_tio.c @@ -189,11 +189,7 @@ static int pps_tio_gen_enable(struct pps_gen_device *pps_gen, bool enable) static int pps_tio_get_time(struct pps_gen_device *pps_gen, struct timespec64 *time) { - struct system_time_snapshot snap; - - ktime_get_snapshot(&snap); - *time = ktime_to_timespec64(snap.real); - + ktime_get_real_ts64(time); return 0; } diff --git a/drivers/ptp/ptp_chardev.c b/drivers/ptp/ptp_chardev.c index c61cf9edac48..dc23cd708cfe 100644 --- a/drivers/ptp/ptp_chardev.c +++ b/drivers/ptp/ptp_chardev.c @@ -317,8 +317,8 @@ typedef int (*ptp_crosststamp_fn)(struct ptp_clock_info *, static long ptp_sys_offset_precise(struct ptp_clock *ptp, void __user *arg, ptp_crosststamp_fn crosststamp_fn) { + struct system_device_crosststamp xtstamp = { .clock_id = CLOCK_REALTIME }; struct ptp_sys_offset_precise precise_offset; - struct system_device_crosststamp xtstamp; struct timespec64 ts; int err; @@ -333,7 +333,7 @@ static long ptp_sys_offset_precise(struct ptp_clock *ptp, void __user *arg, ts = ktime_to_timespec64(xtstamp.device); precise_offset.device.sec = ts.tv_sec; precise_offset.device.nsec = ts.tv_nsec; - ts = ktime_to_timespec64(xtstamp.sys_realtime); + ts = ktime_to_timespec64(xtstamp.sys_systime); precise_offset.sys_realtime.sec = ts.tv_sec; precise_offset.sys_realtime.nsec = ts.tv_nsec; ts = ktime_to_timespec64(xtstamp.sys_monoraw); @@ -386,15 +386,19 @@ static long ptp_sys_offset_extended(struct ptp_clock *ptp, void __user *arg, return err; /* Filter out disabled or unavailable clocks */ - if (sts.pre_ts.tv_sec < 0 || sts.post_ts.tv_sec < 0) + if (!sts.pre_sts.valid || !sts.post_sts.valid) return -EINVAL; - extoff->ts[i][0].sec = sts.pre_ts.tv_sec; - extoff->ts[i][0].nsec = sts.pre_ts.tv_nsec; extoff->ts[i][1].sec = ts.tv_sec; extoff->ts[i][1].nsec = ts.tv_nsec; - extoff->ts[i][2].sec = sts.post_ts.tv_sec; - extoff->ts[i][2].nsec = sts.post_ts.tv_nsec; + + ts = ktime_to_timespec64(sts.pre_sts.systime); + extoff->ts[i][0].sec = ts.tv_sec; + extoff->ts[i][0].nsec = ts.tv_nsec; + + ts = ktime_to_timespec64(sts.post_sts.systime); + extoff->ts[i][2].sec = ts.tv_sec; + extoff->ts[i][2].nsec = ts.tv_nsec; } return copy_to_user(arg, extoff, sizeof(*extoff)) ? -EFAULT : 0; diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c index 735385539b9f..8231c2d7785e 100644 --- a/drivers/ptp/ptp_ocp.c +++ b/drivers/ptp/ptp_ocp.c @@ -1491,11 +1491,8 @@ __ptp_ocp_gettime_locked(struct ptp_ocp *bp, struct timespec64 *ts, } ptp_read_system_postts(sts); - if (sts && bp->ts_window_adjust) { - s64 ns = timespec64_to_ns(&sts->post_ts); - - sts->post_ts = ns_to_timespec64(ns - bp->ts_window_adjust); - } + if (sts && bp->ts_window_adjust) + sts->post_sts.systime -= bp->ts_window_adjust; time_ns = ioread32(&bp->reg->time_ns); time_sec = ioread32(&bp->reg->time_sec); @@ -4600,8 +4597,8 @@ ptp_ocp_summary_show(struct seq_file *s, void *data) struct timespec64 sys_ts; s64 pre_ns, post_ns, ns; - pre_ns = timespec64_to_ns(&sts.pre_ts); - post_ns = timespec64_to_ns(&sts.post_ts); + pre_ns = ktime_to_ns(sts.pre_sts.systime); + post_ns = ktime_to_ns(sts.post_sts.systime); ns = (pre_ns + post_ns) / 2; ns += (s64)bp->utc_tai_offset * NSEC_PER_SEC; sys_ts = ns_to_timespec64(ns); diff --git a/drivers/ptp/ptp_vmclock.c b/drivers/ptp/ptp_vmclock.c index 8b630eb916b5..eebdcd5ebc08 100644 --- a/drivers/ptp/ptp_vmclock.c +++ b/drivers/ptp/ptp_vmclock.c @@ -101,7 +101,6 @@ static int vmclock_get_crosststamp(struct vmclock_state *st, struct timespec64 *tspec) { ktime_t deadline = ktime_add(ktime_get(), VMCLOCK_MAX_WAIT); - struct system_time_snapshot systime_snapshot; uint64_t cycle, delta, seq, frac_sec; #ifdef CONFIG_X86 @@ -132,17 +131,19 @@ static int vmclock_get_crosststamp(struct vmclock_state *st, * will be derived from the *same* counter value. * * If the system isn't using the same counter, then the value - * from ktime_get_snapshot() will still be used as pre_ts, and - * ptp_read_system_postts() is called to populate postts after - * calling get_cycles(). - * - * The conversion to timespec64 happens further down, outside - * the seq_count loop. + * from ptp_read_system_prets() will still be used as pre_ts, + * and ptp_read_system_postts() is called to populate postts + * after calling get_cycles(). */ if (sts) { - ktime_get_snapshot(&systime_snapshot); - if (systime_snapshot.cs_id == st->cs_id) { - cycle = systime_snapshot.cycles; + ptp_read_system_prets(sts); + if (sts->pre_sts.cs_id == st->cs_id) { + cycle = sts->pre_sts.cycles; + sts->post_sts = sts->pre_sts; + } else if (sts->pre_sts.hw_csid == st->cs_id && + sts->pre_sts.hw_cycles) { + cycle = sts->pre_sts.hw_cycles; + sts->post_sts = sts->pre_sts; } else { cycle = get_cycles(); ptp_read_system_postts(sts); @@ -180,12 +181,6 @@ static int vmclock_get_crosststamp(struct vmclock_state *st, system_counter->cs_id = st->cs_id; } - if (sts) { - sts->pre_ts = ktime_to_timespec64(systime_snapshot.real); - if (systime_snapshot.cs_id == st->cs_id) - sts->post_ts = sts->pre_ts; - } - return 0; } @@ -272,7 +267,7 @@ static int ptp_vmclock_getcrosststamp(struct ptp_clock_info *ptp, if (ret == -ENODEV) { struct system_time_snapshot systime_snapshot; - ktime_get_snapshot(&systime_snapshot); + ktime_get_snapshot_id(CLOCK_REALTIME, &systime_snapshot); if (systime_snapshot.cs_id == CSID_X86_TSC || systime_snapshot.cs_id == CSID_X86_KVM_CLK) { diff --git a/drivers/virtio/virtio_rtc_ptp.c b/drivers/virtio/virtio_rtc_ptp.c index f84599950cd4..ff8d834493dc 100644 --- a/drivers/virtio/virtio_rtc_ptp.c +++ b/drivers/virtio/virtio_rtc_ptp.c @@ -139,7 +139,7 @@ static int viortc_ptp_getcrosststamp(struct ptp_clock_info *ptp, if (ret) return ret; - ktime_get_snapshot(&history_begin); + ktime_get_snapshot_id(xtstamp->clock_id, &history_begin); if (history_begin.cs_id != cs_id) return -EOPNOTSUPP; diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index b12a6d19aa60..283d7297aa79 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -32,6 +32,21 @@ struct module; #include <vdso/clocksource.h> /** + * struct clocksource_hw_snapshot - Snapshot for the underlying hardware counter of derived + * clocksources like kvmclock or Hyper-V scaled TSC + * @hw_cycles: The hardware counter value + * @hw_csid: Clocksource ID of the hardware counter + * + * Such clocksources must implement the read_snapshot() callback and fill in the + * hardware counter value, the clocksource ID of the hardware counter and derive + * the actual clocksource cycles from @hw_cycles to provide an atomic snapshot + */ +struct clocksource_hw_snapshot { + u64 hw_cycles; + enum clocksource_ids hw_csid; +}; + +/** * struct clocksource - hardware abstraction for a free running counter * Provides mostly state-free accessors to the underlying hardware. * This is the structure used for system time. @@ -72,6 +87,14 @@ struct module; * @flags: Flags describing special properties * @base: Hardware abstraction for clock on which a clocksource * is based + * @read_snapshot: Extended @read() function for clocksources such as + * kvmclock or the Hyper-V scaled TSC where the actual + * clocksource value for timekeeping is calculated from an + * underlying hardware counter. Returns the timekeeping + * relevant cycle value and stores the raw value of the + * underlying counter from which it was calculated + * including the clocksource ID of that counter in the + * clocksource hardware snapshot. * @enable: Optional function to enable the clocksource * @disable: Optional function to disable the clocksource * @suspend: Optional suspend function for the clocksource @@ -113,6 +136,7 @@ struct clocksource { unsigned long flags; struct clocksource_base *base; + u64 (*read_snapshot)(struct clocksource *cs, struct clocksource_hw_snapshot *chs); int (*enable)(struct clocksource *cs); void (*disable)(struct clocksource *cs); void (*suspend)(struct clocksource *cs); diff --git a/include/linux/pps_kernel.h b/include/linux/pps_kernel.h index aab0aebb529e..9f088c9023b1 100644 --- a/include/linux/pps_kernel.h +++ b/include/linux/pps_kernel.h @@ -99,12 +99,14 @@ static inline void timespec_to_pps_ktime(struct pps_ktime *kt, static inline void pps_get_ts(struct pps_event_time *ts) { +#ifdef CONFIG_NTP_PPS struct system_time_snapshot snap; - ktime_get_snapshot(&snap); - ts->ts_real = ktime_to_timespec64(snap.real); -#ifdef CONFIG_NTP_PPS - ts->ts_raw = ktime_to_timespec64(snap.raw); + ktime_get_snapshot_id(CLOCK_REALTIME, &snap); + ts->ts_real = ktime_to_timespec64(snap.systime); + ts->ts_raw = ktime_to_timespec64(snap.monoraw); +#else + ktime_get_real_ts64(&ts->ts_real); #endif } diff --git a/include/linux/ptp_clock_kernel.h b/include/linux/ptp_clock_kernel.h index 884364596dd3..36a27a910595 100644 --- a/include/linux/ptp_clock_kernel.h +++ b/include/linux/ptp_clock_kernel.h @@ -12,6 +12,7 @@ #include <linux/pps_kernel.h> #include <linux/ptp_clock.h> #include <linux/timecounter.h> +#include <linux/timekeeping.h> #include <linux/skbuff.h> #define PTP_CLOCK_NAME_LEN 32 @@ -45,13 +46,13 @@ struct system_device_crosststamp; /** * struct ptp_system_timestamp - system time corresponding to a PHC timestamp - * @pre_ts: system timestamp before capturing PHC - * @post_ts: system timestamp after capturing PHC - * @clockid: clock-base used for capturing the system timestamps + * @pre_sts: system time snapshot before capturing PHC + * @post_sts: system time snapshot after capturing PHC + * @clockid: clock-base used for capturing the system timestamps */ struct ptp_system_timestamp { - struct timespec64 pre_ts; - struct timespec64 post_ts; + struct system_time_snapshot pre_sts; + struct system_time_snapshot post_sts; clockid_t clockid; }; @@ -510,13 +511,13 @@ static inline ktime_t ptp_convert_timestamp(const ktime_t *hwtstamp, static inline void ptp_read_system_prets(struct ptp_system_timestamp *sts) { if (sts) - ktime_get_clock_ts64(sts->clockid, &sts->pre_ts); + ktime_get_snapshot_id(sts->clockid, &sts->pre_sts); } static inline void ptp_read_system_postts(struct ptp_system_timestamp *sts) { if (sts) - ktime_get_clock_ts64(sts->clockid, &sts->post_ts); + ktime_get_snapshot_id(sts->clockid, &sts->post_sts); } #endif diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index aee2c1a46e47..984a866d293b 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -276,37 +276,30 @@ static inline bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt) { ret #endif /** - * struct system_time_snapshot - simultaneous raw/real time capture with - * counter value - * @cycles: Clocksource counter value to produce the system times - * @real: Realtime system time - * @boot: Boot time - * @raw: Monotonic raw system time - * @cs_id: Clocksource ID + * struct system_time_snapshot - Simultaneous time capture of CLOCK_MONOTONIC_RAW, + * a selected CLOCK_* and the clocksource counter value + * @cycles: Clocksource counter value to produce the system times + * @hw_cycles: For derived clocksources, the hardware counter value from + * which @cycles was derived + * @systime: The system time of the selected CLOCK ID + * @monoraw: Monotonic raw system time + * @cs_id: Clocksource ID + * @hw_csid: Clocksource ID of the underlying hardware counter for derived + * clocksources which implement the read_snapshot() callback. * @clock_was_set_seq: The sequence number of clock-was-set events * @cs_was_changed_seq: The sequence number of clocksource change events + * @valid: True if the snapshot is valid */ struct system_time_snapshot { u64 cycles; - ktime_t real; - ktime_t boot; - ktime_t raw; + u64 hw_cycles; + ktime_t systime; + ktime_t monoraw; enum clocksource_ids cs_id; + enum clocksource_ids hw_csid; unsigned int clock_was_set_seq; u8 cs_was_changed_seq; -}; - -/** - * struct system_device_crosststamp - system/device cross-timestamp - * (synchronized capture) - * @device: Device time - * @sys_realtime: Realtime simultaneous with device time - * @sys_monoraw: Monotonic raw simultaneous with device time - */ -struct system_device_crosststamp { - ktime_t device; - ktime_t sys_realtime; - ktime_t sys_monoraw; + u8 valid; }; /** @@ -325,6 +318,23 @@ struct system_counterval_t { bool use_nsecs; }; +/** + * struct system_device_crosststamp - system/device cross-timestamp + * (synchronized capture) + * @clock_id: System time Clock ID to capture + * @device: Device time + * @sys_counter: Clocksource counter value simultaneous with device time + * @sys_systime: System time for @clock_id + * @sys_monoraw: Monotonic raw simultaneous with device time + */ +struct system_device_crosststamp { + clockid_t clock_id; + ktime_t device; + struct system_counterval_t sys_counter; + ktime_t sys_systime; + ktime_t sys_monoraw; +}; + extern bool ktime_real_to_base_clock(ktime_t treal, enum clocksource_ids base_id, u64 *cycles); extern bool timekeeping_clocksource_has_base(enum clocksource_ids id); @@ -341,9 +351,10 @@ extern int get_device_system_crosststamp( struct system_device_crosststamp *xtstamp); /* - * Simultaneously snapshot realtime and monotonic raw clocks + * Simultaneously snapshot a given clock with MONOTONIC_RAW and the underlying + * clocksource counter value. */ -extern void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot); +extern void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot); /* * Persistent clock related interfaces diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index c493a4010305..0d5b67f609bb 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -67,6 +67,7 @@ static inline bool tk_is_aux(const struct timekeeper *tk) { return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST; } +static inline struct tk_data *aux_get_tk_data(clockid_t id); #else static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts) { @@ -77,6 +78,10 @@ static inline bool tk_is_aux(const struct timekeeper *tk) { return false; } +static inline struct tk_data *aux_get_tk_data(clockid_t id) +{ + return NULL; +} #endif static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs) @@ -315,6 +320,7 @@ static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr) return clock->read(clock); } + static inline void clocksource_disable_inline_read(void) { } static inline void clocksource_enable_inline_read(void) { } #endif @@ -1182,44 +1188,107 @@ noinstr time64_t __ktime_get_real_seconds(void) return tk->xtime_sec; } -/** - * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter - * @systime_snapshot: pointer to struct receiving the system time snapshot - */ -void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot) +static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr, + struct clocksource_hw_snapshot *chs) { - struct timekeeper *tk = &tk_core.timekeeper; + struct clocksource *clock = READ_ONCE(tkr->clock); + + if (unlikely(clock->read_snapshot)) + return clock->read_snapshot(clock, chs); + + return clock->read(clock); +} + + +/** + * ktime_get_snapshot_id - Simultaneously snapshot a given clock ID with + * CLOCK_MONOTONIC_RAW and the underlying + * clocksource counter value. + * @clock_id: The clock ID to snapshot + * @systime_snapshot: Pointer to struct receiving the system time snapshot + */ +void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot) +{ + ktime_t base_raw, base_sys, offs_sys, *offs, offs_zero = 0; + u64 nsec_raw, nsec_sys, now; + struct timekeeper *tk; + struct tk_data *tkd; unsigned int seq; - ktime_t base_raw; - ktime_t base_real; - ktime_t base_boot; - u64 nsec_raw; - u64 nsec_real; - u64 now; - WARN_ON_ONCE(timekeeping_suspended); + /* Invalidate the snapshot for all failure cases */ + systime_snapshot->valid = false; + + if (WARN_ON_ONCE(timekeeping_suspended)) + return; + + switch (clock_id) { + case CLOCK_REALTIME: + tkd = &tk_core; + offs = &tk_core.timekeeper.offs_real; + break; + /* Map RAW to MONOTONIC so the loop below is trivial */ + case CLOCK_MONOTONIC_RAW: + case CLOCK_MONOTONIC: + tkd = &tk_core; + offs = &offs_zero; + break; + case CLOCK_BOOTTIME: + tkd = &tk_core; + offs = &tk_core.timekeeper.offs_boot; + break; + case CLOCK_AUX ... CLOCK_AUX_LAST: + tkd = aux_get_tk_data(clock_id); + if (!tkd) + return; + offs = &tkd->timekeeper.offs_aux; + break; + default: + WARN_ON_ONCE(1); + return; + } + + tk = &tkd->timekeeper; do { - seq = read_seqcount_begin(&tk_core.seq); - now = tk_clock_read(&tk->tkr_mono); + struct clocksource_hw_snapshot chs = { }; + + seq = read_seqcount_begin(&tkd->seq); + + /* Aux clocks can be invalid */ + if (!tk->clock_valid) + return; + + now = tk_clock_read_snapshot(&tk->tkr_mono, &chs); systime_snapshot->cs_id = tk->tkr_mono.clock->id; + + systime_snapshot->hw_cycles = chs.hw_cycles; + systime_snapshot->hw_csid = chs.hw_csid; + systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq; systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq; - base_real = ktime_add(tk->tkr_mono.base, - tk_core.timekeeper.offs_real); - base_boot = ktime_add(tk->tkr_mono.base, - tk_core.timekeeper.offs_boot); + + base_sys = tk->tkr_mono.base; + offs_sys = *offs; base_raw = tk->tkr_raw.base; - nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now); - nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now); - } while (read_seqcount_retry(&tk_core.seq, seq)); + + nsec_sys = timekeeping_cycles_to_ns(&tk->tkr_mono, now); + nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now); + } while (read_seqcount_retry(&tkd->seq, seq)); systime_snapshot->cycles = now; - systime_snapshot->real = ktime_add_ns(base_real, nsec_real); - systime_snapshot->boot = ktime_add_ns(base_boot, nsec_real); - systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw); + systime_snapshot->systime = ktime_add_ns(base_sys, offs_sys + nsec_sys); + systime_snapshot->monoraw = ktime_add_ns(base_raw, nsec_raw); + + /* + * Special case for PTP. Just transfer the raw time into sys, + * so the call sites can consistently use snap::systime. + */ + if (clock_id == CLOCK_MONOTONIC_RAW) + systime_snapshot->systime = systime_snapshot->monoraw; + /* Tell the consumer that this snapshot is valid */ + systime_snapshot->valid = true; } -EXPORT_SYMBOL_GPL(ktime_get_snapshot); +EXPORT_SYMBOL_GPL(ktime_get_snapshot_id); /* Scale base by mult/div checking for overflow */ static int scale64_check_overflow(u64 mult, u64 div, u64 *base) @@ -1262,7 +1331,7 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history, struct system_device_crosststamp *ts) { struct timekeeper *tk = &tk_core.timekeeper; - u64 corr_raw, corr_real; + u64 corr_raw, corr_sys; bool interp_forward; int ret; @@ -1279,8 +1348,7 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history, * Scale the monotonic raw time delta by: * partial_history_cycles / total_history_cycles */ - corr_raw = (u64)ktime_to_ns( - ktime_sub(ts->sys_monoraw, history->raw)); + corr_raw = (u64)ktime_to_ns(ktime_sub(ts->sys_monoraw, history->monoraw)); ret = scale64_check_overflow(partial_history_cycles, total_history_cycles, &corr_raw); if (ret) @@ -1288,30 +1356,29 @@ static int adjust_historical_crosststamp(struct system_time_snapshot *history, /* * If there is a discontinuity in the history, scale monotonic raw - * correction by: - * mult(real)/mult(raw) yielding the realtime correction - * Otherwise, calculate the realtime correction similar to monotonic - * raw calculation + * correction by: + * mult(sys)/mult(raw) yielding the system time correction + * + * Otherwise, calculate the system time correction similar to monotonic + * raw calculation */ if (discontinuity) { - corr_real = mul_u64_u32_div - (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult); + corr_sys = mul_u64_u32_div(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult); } else { - corr_real = (u64)ktime_to_ns( - ktime_sub(ts->sys_realtime, history->real)); - ret = scale64_check_overflow(partial_history_cycles, - total_history_cycles, &corr_real); + corr_sys = (u64)ktime_to_ns(ktime_sub(ts->sys_systime, history->systime)); + ret = scale64_check_overflow(partial_history_cycles, total_history_cycles, + &corr_sys); if (ret) return ret; } - /* Fixup monotonic raw and real time time values */ + /* Fixup monotonic raw and system time time values */ if (interp_forward) { - ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw); - ts->sys_realtime = ktime_add_ns(history->real, corr_real); + ts->sys_monoraw = ktime_add_ns(history->monoraw, corr_raw); + ts->sys_systime = ktime_add_ns(history->systime, corr_sys); } else { ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw); - ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real); + ts->sys_systime = ktime_sub_ns(ts->sys_systime, corr_sys); } return 0; @@ -1368,6 +1435,8 @@ static bool convert_base_to_cs(struct system_counterval_t *scv) return false; scv->cycles += base->offset; + /* Set the clocksource ID as scv::cycles is now clocksource based */ + scv->cs_id = cs->id; return true; } @@ -1435,11 +1504,11 @@ EXPORT_SYMBOL_GPL(ktime_real_to_base_clock); /** * get_device_system_crosststamp - Synchronously capture system/device timestamp - * @get_time_fn: Callback to get simultaneous device time and - * system counter from the device driver + * @get_time_fn: Callback to get simultaneous device time and system counter + * from the device driver * @ctx: Context passed to get_time_fn() - * @history_begin: Historical reference point used to interpolate system - * time when counter provided by the driver is before the current interval + * @history_begin: Historical reference point used to interpolate system time when + * the counter value provided by the driver is before the current interval * @xtstamp: Receives simultaneously captured system and device time * * Reads a timestamp from a device and correlates it to system time @@ -1452,36 +1521,54 @@ int get_device_system_crosststamp(int (*get_time_fn) struct system_time_snapshot *history_begin, struct system_device_crosststamp *xtstamp) { - struct system_counterval_t system_counterval = {}; - struct timekeeper *tk = &tk_core.timekeeper; - u64 cycles, now, interval_start; - unsigned int clock_was_set_seq = 0; - ktime_t base_real, base_raw; - u64 nsec_real, nsec_raw; + u64 syscnt_cycles, cycles, now, interval_start; + unsigned int seq, clock_was_set_seq = 0; + ktime_t base_sys, base_raw, *offs; + u64 nsec_sys, nsec_raw; u8 cs_was_changed_seq; - unsigned int seq; bool do_interp; + struct timekeeper *tk; + struct tk_data *tkd; int ret; + switch (xtstamp->clock_id) { + case CLOCK_REALTIME: + tkd = &tk_core; + offs = &tk_core.timekeeper.offs_real; + break; + case CLOCK_AUX ... CLOCK_AUX_LAST: + tkd = aux_get_tk_data(xtstamp->clock_id); + if (!tkd) + return -ENODEV; + offs = &tkd->timekeeper.offs_aux; + break; + default: + WARN_ON_ONCE(1); + return -ENODEV; + } + + tk = &tkd->timekeeper; + do { - seq = read_seqcount_begin(&tk_core.seq); + seq = read_seqcount_begin(&tkd->seq); /* * Try to synchronously capture device time and a system * counter value calling back into the device driver */ - ret = get_time_fn(&xtstamp->device, &system_counterval, ctx); + ret = get_time_fn(&xtstamp->device, &xtstamp->sys_counter, ctx); if (ret) return ret; /* * Verify that the clocksource ID associated with the captured * system counter value is the same as for the currently - * installed timekeeper clocksource + * installed timekeeper clocksource and convert to it. */ - if (system_counterval.cs_id == CSID_GENERIC || - !convert_base_to_cs(&system_counterval)) + if (xtstamp->sys_counter.cs_id == CSID_GENERIC || + !convert_base_to_cs(&xtstamp->sys_counter)) return -ENODEV; - cycles = system_counterval.cycles; + + cycles = syscnt_cycles = xtstamp->sys_counter.cycles; /* * Check whether the system counter value provided by the @@ -1498,15 +1585,14 @@ int get_device_system_crosststamp(int (*get_time_fn) do_interp = false; } - base_real = ktime_add(tk->tkr_mono.base, - tk_core.timekeeper.offs_real); + base_sys = ktime_add(tk->tkr_mono.base, *offs); base_raw = tk->tkr_raw.base; - nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles); + nsec_sys = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles); nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, cycles); - } while (read_seqcount_retry(&tk_core.seq, seq)); + } while (read_seqcount_retry(&tkd->seq, seq)); - xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real); + xtstamp->sys_systime = ktime_add_ns(base_sys, nsec_sys); xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw); /* @@ -1523,24 +1609,19 @@ int get_device_system_crosststamp(int (*get_time_fn) * clocksource change */ if (!history_begin || - !timestamp_in_interval(history_begin->cycles, - cycles, system_counterval.cycles) || + !timestamp_in_interval(history_begin->cycles, cycles, syscnt_cycles) || history_begin->cs_was_changed_seq != cs_was_changed_seq) return -EINVAL; - partial_history_cycles = cycles - system_counterval.cycles; + + partial_history_cycles = cycles - syscnt_cycles; total_history_cycles = cycles - history_begin->cycles; - discontinuity = - history_begin->clock_was_set_seq != clock_was_set_seq; + discontinuity = history_begin->clock_was_set_seq != clock_was_set_seq; - ret = adjust_historical_crosststamp(history_begin, - partial_history_cycles, - total_history_cycles, - discontinuity, xtstamp); - if (ret) - return ret; + ret = adjust_historical_crosststamp(history_begin, partial_history_cycles, + total_history_cycles, discontinuity, xtstamp); } - return 0; + return ret; } EXPORT_SYMBOL_GPL(get_device_system_crosststamp); diff --git a/sound/hda/common/controller.c b/sound/hda/common/controller.c index a847546753db..afec5c5546ec 100644 --- a/sound/hda/common/controller.c +++ b/sound/hda/common/controller.c @@ -491,9 +491,9 @@ static int azx_get_time_info(struct snd_pcm_substream *substream, struct snd_pcm_audio_tstamp_config *audio_tstamp_config, struct snd_pcm_audio_tstamp_report *audio_tstamp_report) { + struct system_device_crosststamp xtstamp = { .clock_id = CLOCK_REALTIME }; struct azx_dev *azx_dev = get_azx_dev(substream); struct snd_pcm_runtime *runtime = substream->runtime; - struct system_device_crosststamp xtstamp; int ret; u64 nsec; @@ -527,7 +527,7 @@ static int azx_get_time_info(struct snd_pcm_substream *substream, break; default: - *system_ts = ktime_to_timespec64(xtstamp.sys_realtime); + *system_ts = ktime_to_timespec64(xtstamp.sys_systime); break; } |