From ca2768bbf5c48d8c048877dfbceafcebc3f06fa6 Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Tue, 23 Jan 2024 17:46:56 +0100 Subject: hrtimers: Update formatting of documentation Documentation of functions lacks the annotations which are used by kernel-doc and *.rst to make appearance in rendered documents more user-friendly. Use those annotations to improve user-friendliness. While at it prevent duplication of comments and use a reference instead. Signed-off-by: Anna-Maria Behnsen Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20240123164702.55612-3-anna-maria@linutronix.de --- kernel/time/hrtimer.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'kernel/time/hrtimer.c') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 760793998cdd..4c8dd633ab4a 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1021,21 +1021,23 @@ void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) } /** - * hrtimer_forward - forward the timer expiry + * hrtimer_forward() - forward the timer expiry * @timer: hrtimer to forward * @now: forward past this time * @interval: the interval to forward * * Forward the timer expiry so it will expire in the future. - * Returns the number of overruns. * - * Can be safely called from the callback function of @timer. If - * called from other contexts @timer must neither be enqueued nor - * running the callback and the caller needs to take care of - * serialization. + * .. note:: + * This only updates the timer expiry value and does not requeue the timer. * - * Note: This only updates the timer expiry value and does not requeue - * the timer. + * There is also a variant of the function hrtimer_forward_now(). + * + * Context: Can be safely called from the callback function of @timer. If called + * from other contexts @timer must neither be enqueued nor running the + * callback and the caller needs to take care of serialization. + * + * Return: The number of overruns are returned. */ u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) { -- cgit v1.2.3-70-g09d2 From 56c2cb10120894be40c40a9bf0ce798da14c50f6 Mon Sep 17 00:00:00 2001 From: Costa Shulyupin Date: Thu, 22 Feb 2024 22:08:56 +0200 Subject: hrtimer: Select housekeeping CPU during migration During CPU-down hotplug, hrtimers may migrate to isolated CPUs, compromising CPU isolation. Address this issue by masking valid CPUs for hrtimers using housekeeping_cpumask(HK_TYPE_TIMER). Suggested-by: Waiman Long Signed-off-by: Costa Shulyupin Signed-off-by: Thomas Gleixner Reviewed-by: Waiman Long Link: https://lore.kernel.org/r/20240222200856.569036-1-costa.shul@redhat.com --- kernel/time/hrtimer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel/time/hrtimer.c') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 5a98b35b0576..1fd106af747d 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -2225,8 +2226,8 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, int hrtimers_cpu_dying(unsigned int dying_cpu) { + int i, ncpu = cpumask_any_and(cpu_active_mask, housekeeping_cpumask(HK_TYPE_TIMER)); struct hrtimer_cpu_base *old_base, *new_base; - int i, ncpu = cpumask_first(cpu_active_mask); tick_cancel_sched_timer(dying_cpu); -- cgit v1.2.3-70-g09d2 From ffb7e01c4e654d5c8bf2ce2a4830b826fa1f149e Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Sun, 25 Feb 2024 23:54:53 +0100 Subject: tick/nohz: Remove duplicate between tick_nohz_switch_to_nohz() and tick_setup_sched_timer() The ts->sched_timer initialization work of tick_nohz_switch_to_nohz() is almost the same as that of tick_setup_sched_timer(), so adjust the latter to get it reused by tick_nohz_switch_to_nohz(). This also makes the low resolution mode sched_timer benefit from the tick skew boot option. Signed-off-by: Peng Liu Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Reviewed-by: Thomas Gleixner Link: https://lore.kernel.org/r/20240225225508.11587-2-frederic@kernel.org --- kernel/time/hrtimer.c | 2 +- kernel/time/tick-sched.c | 39 ++++++++++++++++++--------------------- kernel/time/tick-sched.h | 2 +- 3 files changed, 20 insertions(+), 23 deletions(-) (limited to 'kernel/time/hrtimer.c') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 1fd106af747d..95f1f351dcd9 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -747,7 +747,7 @@ static void hrtimer_switch_to_hres(void) base->hres_active = 1; hrtimer_resolution = HIGH_RES_NSEC; - tick_setup_sched_timer(); + tick_setup_sched_timer(NOHZ_MODE_HIGHRES); /* "Retrigger" the interrupt to get things going */ retrigger_next_event(NULL); } diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 417bb7f880ca..d4901654148d 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -1471,9 +1471,6 @@ static inline void tick_nohz_activate(struct tick_sched *ts, int mode) */ static void tick_nohz_switch_to_nohz(void) { - struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); - ktime_t next; - if (!tick_nohz_enabled) return; @@ -1482,16 +1479,9 @@ static void tick_nohz_switch_to_nohz(void) /* * Recycle the hrtimer in 'ts', so we can share the - * hrtimer_forward_now() function with the highres code. + * highres code. */ - hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); - /* Get the next period */ - next = tick_init_jiffy_update(); - - hrtimer_set_expires(&ts->sched_timer, next); - hrtimer_forward_now(&ts->sched_timer, TICK_NSEC); - tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); - tick_nohz_activate(ts, NOHZ_MODE_LOWRES); + tick_setup_sched_timer(NOHZ_MODE_LOWRES); } static inline void tick_nohz_irq_enter(void) @@ -1570,7 +1560,11 @@ static enum hrtimer_restart tick_nohz_highres_handler(struct hrtimer *timer) return HRTIMER_RESTART; } +#else +#define tick_nohz_highres_handler NULL +#endif /* CONFIG_HIGH_RES_TIMERS */ +#if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS static int sched_skew_tick; static int __init skew_tick(char *str) @@ -1583,15 +1577,17 @@ early_param("skew_tick", skew_tick); /** * tick_setup_sched_timer - setup the tick emulation timer + * @mode: tick_nohz_mode to setup for */ -void tick_setup_sched_timer(void) +void tick_setup_sched_timer(int mode) { struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); - ktime_t now = ktime_get(); /* Emulate tick processing via per-CPU hrtimers: */ hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); - ts->sched_timer.function = tick_nohz_highres_handler; + + if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && mode == NOHZ_MODE_HIGHRES) + ts->sched_timer.function = tick_nohz_highres_handler; /* Get the next period (per-CPU) */ hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); @@ -1604,13 +1600,14 @@ void tick_setup_sched_timer(void) hrtimer_add_expires_ns(&ts->sched_timer, offset); } - hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); - hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD); - tick_nohz_activate(ts, NOHZ_MODE_HIGHRES); + hrtimer_forward_now(&ts->sched_timer, TICK_NSEC); + if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && mode == NOHZ_MODE_HIGHRES) + hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD); + else + tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); + tick_nohz_activate(ts, mode); } -#endif /* HIGH_RES_TIMERS */ -#if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS void tick_cancel_sched_timer(int cpu) { struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); @@ -1632,7 +1629,7 @@ void tick_cancel_sched_timer(int cpu) ts->idle_calls = idle_calls; ts->idle_sleeps = idle_sleeps; } -#endif +#endif /* CONFIG_NO_HZ_COMMON || CONFIG_HIGH_RES_TIMERS */ /* * Async notification about clocksource changes diff --git a/kernel/time/tick-sched.h b/kernel/time/tick-sched.h index 5ed5a9d41d5a..35808bbb8a47 100644 --- a/kernel/time/tick-sched.h +++ b/kernel/time/tick-sched.h @@ -102,7 +102,7 @@ struct tick_sched { extern struct tick_sched *tick_get_tick_sched(int cpu); -extern void tick_setup_sched_timer(void); +extern void tick_setup_sched_timer(int mode); #if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS extern void tick_cancel_sched_timer(int cpu); #else -- cgit v1.2.3-70-g09d2 From f04e51220ad5cf35540f67f3ca15c8617c1f0bef Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Sun, 25 Feb 2024 23:55:00 +0100 Subject: tick: Move tick cancellation up to CPUHP_AP_TICK_DYING The tick hrtimer is cancelled right before hrtimers are migrated. This is done from the hrtimer subsystem even though it shouldn't know about its actual users. Move instead the tick hrtimer cancellation to the relevant CPU hotplug state that aims at centralizing high level tick shutdown operations so that the related flow is easy to follow. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Reviewed-by: Thomas Gleixner Link: https://lore.kernel.org/r/20240225225508.11587-9-frederic@kernel.org --- kernel/time/hrtimer.c | 2 -- kernel/time/tick-common.c | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel/time/hrtimer.c') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 95f1f351dcd9..3e95474199ac 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2229,8 +2229,6 @@ int hrtimers_cpu_dying(unsigned int dying_cpu) int i, ncpu = cpumask_any_and(cpu_active_mask, housekeeping_cpumask(HK_TYPE_TIMER)); struct hrtimer_cpu_base *old_base, *new_base; - tick_cancel_sched_timer(dying_cpu); - old_base = this_cpu_ptr(&hrtimer_bases); new_base = &per_cpu(hrtimer_bases, ncpu); diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index a89ef450fda7..b4af8c743b73 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c @@ -410,6 +410,8 @@ int tick_cpu_dying(unsigned int dying_cpu) if (tick_do_timer_cpu == dying_cpu) tick_do_timer_cpu = cpumask_first(cpu_online_mask); + tick_cancel_sched_timer(dying_cpu); + return 0; } -- cgit v1.2.3-70-g09d2 From 7988e5ae2be70b110db9d4b8ec163bd42e67d3be Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Sun, 25 Feb 2024 23:55:05 +0100 Subject: tick: Split nohz and highres features from nohz_mode The nohz mode field tells about low resolution nohz mode or high resolution nohz mode but it doesn't tell about high resolution non-nohz mode. In order to retrieve the latter state, tick_cancel_sched_timer() must fiddle with struct hrtimer's internals to guess if the tick has been initialized in high resolution. Move instead the nohz mode field information into the tick flags and provide two new bits: one to know if the tick is in nohz mode and another one to know if the tick is in high resolution. The combination of those two flags provides all the needed informations to determine which of the three tick modes is running. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Reviewed-by: Thomas Gleixner Link: https://lore.kernel.org/r/20240225225508.11587-14-frederic@kernel.org --- kernel/time/hrtimer.c | 2 +- kernel/time/tick-sched.c | 32 +++++++++++++++++--------------- kernel/time/tick-sched.h | 13 +++++-------- kernel/time/timer_list.c | 5 +++-- 4 files changed, 26 insertions(+), 26 deletions(-) (limited to 'kernel/time/hrtimer.c') diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 3e95474199ac..70625dff62ce 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -747,7 +747,7 @@ static void hrtimer_switch_to_hres(void) base->hres_active = 1; hrtimer_resolution = HIGH_RES_NSEC; - tick_setup_sched_timer(NOHZ_MODE_HIGHRES); + tick_setup_sched_timer(true); /* "Retrigger" the interrupt to get things going */ retrigger_next_event(NULL); } diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 4aa7ce04a72c..dcb9f0394182 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -831,7 +831,7 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) /* Forward the time to expire in the future */ hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); - if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { + if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) { hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD); } else { @@ -1040,14 +1040,14 @@ static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu) * the tick timer. */ if (unlikely(expires == KTIME_MAX)) { - if (ts->nohz_mode == NOHZ_MODE_HIGHRES) + if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) hrtimer_cancel(&ts->sched_timer); else tick_program_event(KTIME_MAX, 1); return; } - if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { + if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) { hrtimer_start(&ts->sched_timer, expires, HRTIMER_MODE_ABS_PINNED_HARD); } else { @@ -1108,7 +1108,7 @@ static void tick_nohz_full_update_tick(struct tick_sched *ts) if (!tick_nohz_full_cpu(smp_processor_id())) return; - if (ts->nohz_mode == NOHZ_MODE_INACTIVE) + if (!tick_sched_flag_test(ts, TS_FLAG_NOHZ)) return; __tick_nohz_full_update_tick(ts, ktime_get()); @@ -1168,7 +1168,7 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) return false; } - if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) + if (unlikely(!tick_sched_flag_test(ts, TS_FLAG_NOHZ))) return false; if (need_resched()) @@ -1487,11 +1487,11 @@ static void tick_nohz_lowres_handler(struct clock_event_device *dev) tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); } -static inline void tick_nohz_activate(struct tick_sched *ts, int mode) +static inline void tick_nohz_activate(struct tick_sched *ts) { if (!tick_nohz_enabled) return; - ts->nohz_mode = mode; + tick_sched_flag_set(ts, TS_FLAG_NOHZ); /* One update is enough */ if (!test_and_set_bit(0, &tick_nohz_active)) timers_update_nohz(); @@ -1512,7 +1512,7 @@ static void tick_nohz_switch_to_nohz(void) * Recycle the hrtimer in 'ts', so we can share the * highres code. */ - tick_setup_sched_timer(NOHZ_MODE_LOWRES); + tick_setup_sched_timer(false); } static inline void tick_nohz_irq_enter(void) @@ -1540,7 +1540,7 @@ static inline void tick_nohz_irq_enter(void) static inline void tick_nohz_switch_to_nohz(void) { } static inline void tick_nohz_irq_enter(void) { } -static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { } +static inline void tick_nohz_activate(struct tick_sched *ts) { } #endif /* CONFIG_NO_HZ_COMMON */ @@ -1567,15 +1567,17 @@ early_param("skew_tick", skew_tick); * tick_setup_sched_timer - setup the tick emulation timer * @mode: tick_nohz_mode to setup for */ -void tick_setup_sched_timer(int mode) +void tick_setup_sched_timer(bool hrtimer) { struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); /* Emulate tick processing via per-CPU hrtimers: */ hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); - if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && mode == NOHZ_MODE_HIGHRES) + if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && hrtimer) { + tick_sched_flag_set(ts, TS_FLAG_HIGHRES); ts->sched_timer.function = tick_nohz_handler; + } /* Get the next period (per-CPU) */ hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); @@ -1589,11 +1591,11 @@ void tick_setup_sched_timer(int mode) } hrtimer_forward_now(&ts->sched_timer, TICK_NSEC); - if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && mode == NOHZ_MODE_HIGHRES) + if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && hrtimer) hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD); else tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); - tick_nohz_activate(ts, mode); + tick_nohz_activate(ts); } void tick_cancel_sched_timer(int cpu) @@ -1602,7 +1604,7 @@ void tick_cancel_sched_timer(int cpu) ktime_t idle_sleeptime, iowait_sleeptime; unsigned long idle_calls, idle_sleeps; - if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && ts->sched_timer.base) + if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) hrtimer_cancel(&ts->sched_timer); idle_sleeptime = ts->idle_sleeptime; @@ -1652,7 +1654,7 @@ int tick_check_oneshot_change(int allow_nohz) if (!test_and_clear_bit(0, &ts->check_clocks)) return 0; - if (ts->nohz_mode != NOHZ_MODE_INACTIVE) + if (tick_sched_flag_test(ts, TS_FLAG_NOHZ)) return 0; if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available()) diff --git a/kernel/time/tick-sched.h b/kernel/time/tick-sched.h index 07a4c0144c47..bbe72a078985 100644 --- a/kernel/time/tick-sched.h +++ b/kernel/time/tick-sched.h @@ -14,12 +14,6 @@ struct tick_device { enum tick_device_mode mode; }; -enum tick_nohz_mode { - NOHZ_MODE_INACTIVE, - NOHZ_MODE_LOWRES, - NOHZ_MODE_HIGHRES, -}; - /* The CPU is in the tick idle mode */ #define TS_FLAG_INIDLE BIT(0) /* The idle tick has been stopped */ @@ -31,6 +25,10 @@ enum tick_nohz_mode { #define TS_FLAG_IDLE_ACTIVE BIT(2) /* CPU was the last one doing do_timer before going idle */ #define TS_FLAG_DO_TIMER_LAST BIT(3) +/* NO_HZ is enabled */ +#define TS_FLAG_NOHZ BIT(4) +/* High resolution tick mode */ +#define TS_FLAG_HIGHRES BIT(5) /** * struct tick_sched - sched tick emulation and no idle tick control/stats @@ -84,7 +82,6 @@ struct tick_sched { ktime_t idle_entrytime; /* Tick stop */ - enum tick_nohz_mode nohz_mode; unsigned long last_jiffies; u64 timer_expires_base; u64 timer_expires; @@ -107,7 +104,7 @@ struct tick_sched { extern struct tick_sched *tick_get_tick_sched(int cpu); -extern void tick_setup_sched_timer(int mode); +extern void tick_setup_sched_timer(bool hrtimer); #if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS extern void tick_cancel_sched_timer(int cpu); #else diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index 38f81d836fc5..1c311c46da50 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c @@ -152,7 +152,8 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now) { struct tick_sched *ts = tick_get_tick_sched(cpu); - P(nohz_mode); + P_flag(nohz, TS_FLAG_NOHZ); + P_flag(highres, TS_FLAG_HIGHRES); P_ns(last_tick); P_flag(tick_stopped, TS_FLAG_STOPPED); P(idle_jiffies); @@ -259,7 +260,7 @@ static void timer_list_show_tickdevices_header(struct seq_file *m) static inline void timer_list_header(struct seq_file *m, u64 now) { - SEQ_printf(m, "Timer List Version: v0.9\n"); + SEQ_printf(m, "Timer List Version: v0.10\n"); SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES); SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now); SEQ_printf(m, "\n"); -- cgit v1.2.3-70-g09d2