From 87d8b9eb7eb6669aad6435a51e9862362141ba76 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Thu, 18 Jul 2013 16:21:14 -0700 Subject: clocksource: Extract max nsec calculation into separate function We need to calculate the same number in the clocksource code and the sched_clock code, so extract this code into its own function. We also drop the min_t and just use min() because the two types are the same. Signed-off-by: Stephen Boyd Signed-off-by: John Stultz --- include/linux/clocksource.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include') diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index dbbf8aa7731b..67301a405712 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -292,6 +292,8 @@ extern void clocksource_resume(void); extern struct clocksource * __init __weak clocksource_default_clock(void); extern void clocksource_mark_unstable(struct clocksource *cs); +extern u64 +clocks_calc_max_nsecs(u32 mult, u32 shift, u32 maxadj, u64 mask); extern void clocks_calc_mult_shift(u32 *mult, u32 *shift, u32 from, u32 to, u32 minsec); -- cgit v1.2.3-70-g09d2 From e7e3ff1bfe9c42ee31172e9afdc0383a9e595e29 Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Thu, 18 Jul 2013 16:21:17 -0700 Subject: sched_clock: Add support for >32 bit sched_clock The ARM architected system counter has at least 56 usable bits. Add support for counters with more than 32 bits to the generic sched_clock implementation so we can increase the time between wakeups due to dealing with wrap-around on these devices while benefiting from the irqtime accounting and suspend/resume handling that the generic sched_clock code already has. On my system using 56 bits over 32 bits changes the wraparound time from a few minutes to an hour. For faster running counters (GHz range) this is even more important because we may not be able to execute the timer in time to deal with the wraparound if only 32 bits are used. We choose a maxsec value of 3600 seconds because we assume no system will go idle for more than an hour. In the future we may need to increase this value. Note: All users should switch over to the 64-bit read function so we can remove setup_sched_clock() in favor of sched_clock_register(). Cc: Russell King Signed-off-by: Stephen Boyd Signed-off-by: John Stultz --- include/linux/sched_clock.h | 2 ++ kernel/time/sched_clock.c | 46 +++++++++++++++++++++++++++++++-------------- 2 files changed, 34 insertions(+), 14 deletions(-) (limited to 'include') diff --git a/include/linux/sched_clock.h b/include/linux/sched_clock.h index fa7922c80a41..eca7abeb86fc 100644 --- a/include/linux/sched_clock.h +++ b/include/linux/sched_clock.h @@ -15,6 +15,8 @@ static inline void sched_clock_postinit(void) { } #endif extern void setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate); +extern void sched_clock_register(u64 (*read)(void), int bits, + unsigned long rate); extern unsigned long long (*sched_clock_func)(void); diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c index c018ffc59937..f388baeaf2b6 100644 --- a/kernel/time/sched_clock.c +++ b/kernel/time/sched_clock.c @@ -16,11 +16,12 @@ #include #include #include +#include struct clock_data { ktime_t wrap_kt; u64 epoch_ns; - u32 epoch_cyc; + u64 epoch_cyc; seqcount_t seq; unsigned long rate; u32 mult; @@ -37,14 +38,25 @@ static struct clock_data cd = { .mult = NSEC_PER_SEC / HZ, }; -static u32 __read_mostly sched_clock_mask = 0xffffffff; +static u64 __read_mostly sched_clock_mask; -static u32 notrace jiffy_sched_clock_read(void) +static u64 notrace jiffy_sched_clock_read(void) { - return (u32)(jiffies - INITIAL_JIFFIES); + /* + * We don't need to use get_jiffies_64 on 32-bit arches here + * because we register with BITS_PER_LONG + */ + return (u64)(jiffies - INITIAL_JIFFIES); +} + +static u32 __read_mostly (*read_sched_clock_32)(void); + +static u64 notrace read_sched_clock_32_wrapper(void) +{ + return read_sched_clock_32(); } -static u32 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read; +static u64 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read; static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) { @@ -54,8 +66,8 @@ static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) static unsigned long long notrace sched_clock_32(void) { u64 epoch_ns; - u32 epoch_cyc; - u32 cyc; + u64 epoch_cyc; + u64 cyc; unsigned long seq; if (cd.suspended) @@ -78,7 +90,7 @@ static unsigned long long notrace sched_clock_32(void) static void notrace update_sched_clock(void) { unsigned long flags; - u32 cyc; + u64 cyc; u64 ns; cyc = read_sched_clock(); @@ -101,7 +113,8 @@ static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt) return HRTIMER_RESTART; } -void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate) +void __init sched_clock_register(u64 (*read)(void), int bits, + unsigned long rate) { unsigned long r; u64 res, wrap; @@ -110,14 +123,13 @@ void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate) if (cd.rate > rate) return; - BUG_ON(bits > 32); WARN_ON(!irqs_disabled()); read_sched_clock = read; - sched_clock_mask = (1 << bits) - 1; + sched_clock_mask = CLOCKSOURCE_MASK(bits); cd.rate = rate; /* calculate the mult/shift to convert counter ticks to ns. */ - clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 0); + clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 3600); r = rate; if (r >= 4000000) { @@ -130,7 +142,7 @@ void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate) r_unit = ' '; /* calculate how many ns until we wrap */ - wrap = cyc_to_ns((1ULL << bits) - 1, cd.mult, cd.shift); + wrap = clocks_calc_max_nsecs(cd.mult, cd.shift, 0, sched_clock_mask); cd.wrap_kt = ns_to_ktime(wrap - (wrap >> 3)); /* calculate the ns resolution of this counter */ @@ -152,6 +164,12 @@ void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate) pr_debug("Registered %pF as sched_clock source\n", read); } +void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate) +{ + read_sched_clock_32 = read; + sched_clock_register(read_sched_clock_32_wrapper, bits, rate); +} + unsigned long long __read_mostly (*sched_clock_func)(void) = sched_clock_32; unsigned long long notrace sched_clock(void) @@ -166,7 +184,7 @@ void __init sched_clock_postinit(void) * make it the final one one. */ if (read_sched_clock == jiffy_sched_clock_read) - setup_sched_clock(jiffy_sched_clock_read, 32, HZ); + sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); update_sched_clock(); -- cgit v1.2.3-70-g09d2 From 28061758dc83df445a05af347b5ce55ccd968c03 Mon Sep 17 00:00:00 2001 From: Sudeep KarkadaNagesha Date: Tue, 13 Aug 2013 13:43:26 +0100 Subject: ARM/ARM64: arch_timer: add macros for bits in control register Add macros to describe the bitfields in the ARM architected timer control register to make code easy to understand. Reviewed-by: Lorenzo Pieralisi Reviewed-by: Will Deacon Acked-by: Catalin Marinas Acked-by: Olof Johansson Signed-off-by: Sudeep KarkadaNagesha --- arch/arm/include/asm/arch_timer.h | 9 +++++++-- arch/arm64/include/asm/arch_timer.h | 12 ++++++++---- include/clocksource/arm_arch_timer.h | 8 ++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/arch/arm/include/asm/arch_timer.h b/arch/arm/include/asm/arch_timer.h index 5665134bfa3e..d57e04de8e8c 100644 --- a/arch/arm/include/asm/arch_timer.h +++ b/arch/arm/include/asm/arch_timer.h @@ -93,8 +93,13 @@ static inline void arch_counter_set_user_access(void) asm volatile("mrc p15, 0, %0, c14, c1, 0" : "=r" (cntkctl)); - /* disable user access to everything */ - cntkctl &= ~((3 << 8) | (7 << 0)); + /* Disable user access to both physical/virtual counters/timers */ + /* Also disable virtual event stream */ + cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN + | ARCH_TIMER_USR_VT_ACCESS_EN + | ARCH_TIMER_VIRT_EVT_EN + | ARCH_TIMER_USR_VCT_ACCESS_EN + | ARCH_TIMER_USR_PCT_ACCESS_EN); asm volatile("mcr p15, 0, %0, c14, c1, 0" : : "r" (cntkctl)); } diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index c9f1d2816c2b..2b9722f42729 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -96,12 +96,16 @@ static inline void arch_counter_set_user_access(void) { u32 cntkctl; - /* Disable user access to the timers and the physical counter. */ asm volatile("mrs %0, cntkctl_el1" : "=r" (cntkctl)); - cntkctl &= ~((3 << 8) | (1 << 0)); - /* Enable user access to the virtual counter and frequency. */ - cntkctl |= (1 << 1); + /* Disable user access to the timers and the physical counter */ + cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN + | ARCH_TIMER_USR_VT_ACCESS_EN + | ARCH_TIMER_USR_PCT_ACCESS_EN); + + /* Enable user access to the virtual counter */ + cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN; + asm volatile("msr cntkctl_el1, %0" : : "r" (cntkctl)); } diff --git a/include/clocksource/arm_arch_timer.h b/include/clocksource/arm_arch_timer.h index 93b7f96f9c59..8707dae4cee2 100644 --- a/include/clocksource/arm_arch_timer.h +++ b/include/clocksource/arm_arch_timer.h @@ -33,6 +33,14 @@ enum arch_timer_reg { #define ARCH_TIMER_MEM_PHYS_ACCESS 2 #define ARCH_TIMER_MEM_VIRT_ACCESS 3 +#define ARCH_TIMER_USR_PCT_ACCESS_EN (1 << 0) /* physical counter */ +#define ARCH_TIMER_USR_VCT_ACCESS_EN (1 << 1) /* virtual counter */ +#define ARCH_TIMER_VIRT_EVT_EN (1 << 2) +#define ARCH_TIMER_EVT_TRIGGER_SHIFT (4) +#define ARCH_TIMER_EVT_TRIGGER_MASK (0xF << ARCH_TIMER_EVT_TRIGGER_SHIFT) +#define ARCH_TIMER_USR_VT_ACCESS_EN (1 << 8) /* virtual timer registers */ +#define ARCH_TIMER_USR_PT_ACCESS_EN (1 << 9) /* physical timer registers */ + #ifdef CONFIG_ARM_ARCH_TIMER extern u32 arch_timer_get_rate(void); -- cgit v1.2.3-70-g09d2 From 037f637767a82907efedda78d3ff405c34020075 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 23 Aug 2013 15:32:29 +0100 Subject: drivers: clocksource: add support for ARM architected timer event stream The ARM architected timer can generate events (used for waking up CPUs executing the wfe instruction) at a frequency represented as a power-of-2 divisor of the clock rate. An event stream might be used: - To implement wfe-based timeouts for userspace locking implementations. - To impose a timeout on a wfe for safeguarding against any programming error in case an expected event is not generated. This patch computes the event stream frequency aiming for a period of 100us between events. It uses ARM/ARM64 specific backends to configure and enable the event stream. Cc: Lorenzo Pieralisi Reviewed-by: Catalin Marinas Acked-by: Olof Johansson Signed-off-by: Will Deacon [sudeep: moving ARM/ARM64 changes into separate patches and adding Kconfig option] Signed-off-by: Sudeep KarkadaNagesha --- drivers/clocksource/Kconfig | 15 +++++++++++++++ drivers/clocksource/arm_arch_timer.c | 15 +++++++++++++++ include/clocksource/arm_arch_timer.h | 2 ++ 3 files changed, 32 insertions(+) (limited to 'include') diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 41c69469ce20..559d80335446 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -74,6 +74,21 @@ config ARM_ARCH_TIMER bool select CLKSRC_OF if OF +config ARM_ARCH_TIMER_EVTSTREAM + bool "Support for ARM architected timer event stream generation" + default y if ARM_ARCH_TIMER + help + This option enables support for event stream generation based on + the ARM architected timer. It is used for waking up CPUs executing + the wfe instruction at a frequency represented as a power-of-2 + divisor of the clock rate. + The main use of the event stream is wfe-based timeouts of userspace + locking implementations. It might also be useful for imposing timeout + on wfe to safeguard against any programming errors in case an expected + event is not generated. + This must be disabled for hardware validation purposes to detect any + hardware anomalies of missing events. + config ARM_GLOBAL_TIMER bool select CLKSRC_OF if OF diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index fbd9ccd5e114..105f8ffa66a8 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -294,6 +294,19 @@ static void __arch_timer_setup(unsigned type, clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff); } +static void arch_timer_configure_evtstream(void) +{ + int evt_stream_div, pos; + + /* Find the closest power of two to the divisor */ + evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ; + pos = fls(evt_stream_div); + if (pos > 1 && !(evt_stream_div & (1 << (pos - 2)))) + pos--; + /* enable event stream */ + arch_timer_evtstrm_enable(min(pos, 15)); +} + static int arch_timer_setup(struct clock_event_device *clk) { __arch_timer_setup(ARCH_CP15_TIMER, clk); @@ -307,6 +320,8 @@ static int arch_timer_setup(struct clock_event_device *clk) } arch_counter_set_user_access(); + if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM)) + arch_timer_configure_evtstream(); return 0; } diff --git a/include/clocksource/arm_arch_timer.h b/include/clocksource/arm_arch_timer.h index 8707dae4cee2..6d26b40cbf5d 100644 --- a/include/clocksource/arm_arch_timer.h +++ b/include/clocksource/arm_arch_timer.h @@ -41,6 +41,8 @@ enum arch_timer_reg { #define ARCH_TIMER_USR_VT_ACCESS_EN (1 << 8) /* virtual timer registers */ #define ARCH_TIMER_USR_PT_ACCESS_EN (1 << 9) /* physical timer registers */ +#define ARCH_TIMER_EVT_STREAM_FREQ 10000 /* 100us */ + #ifdef CONFIG_ARM_ARCH_TIMER extern u32 arch_timer_get_rate(void); -- cgit v1.2.3-70-g09d2 From 3713c0cfd06fa49729a12929a7ee8b7ad48f3c02 Mon Sep 17 00:00:00 2001 From: Soren Brinkmann Date: Wed, 18 Sep 2013 11:48:35 -0700 Subject: clockchips: Add FEAT_PERCPU clockevent flag Add the flag CLOCK_EVT_FEAT_PERCPU which is supposed to be set for per cpu clockevent devices. Signed-off-by: Soren Brinkmann Signed-off-by: Daniel Lezcano Acked-by: Michal Simek --- include/linux/clockchips.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/linux/clockchips.h b/include/linux/clockchips.h index 0857922e8ad0..493aa021c7a9 100644 --- a/include/linux/clockchips.h +++ b/include/linux/clockchips.h @@ -60,6 +60,7 @@ enum clock_event_mode { * Core shall set the interrupt affinity dynamically in broadcast mode */ #define CLOCK_EVT_FEAT_DYNIRQ 0x000020 +#define CLOCK_EVT_FEAT_PERCPU 0x000040 /** * struct clock_event_device - clock event device descriptor -- cgit v1.2.3-70-g09d2 From b4042ceaabbd913bc5b397ddd1e396eeb312d72f Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Thu, 18 Jul 2013 16:21:19 -0700 Subject: sched_clock: Remove sched_clock_func() hook Nobody is using sched_clock_func() anymore now that sched_clock supports up to 64 bits. Remove the hook so that new code only uses sched_clock_register(). Signed-off-by: Stephen Boyd Signed-off-by: John Stultz --- include/linux/sched_clock.h | 2 -- kernel/time/sched_clock.c | 9 +-------- 2 files changed, 1 insertion(+), 10 deletions(-) (limited to 'include') diff --git a/include/linux/sched_clock.h b/include/linux/sched_clock.h index eca7abeb86fc..cddf0c2940b6 100644 --- a/include/linux/sched_clock.h +++ b/include/linux/sched_clock.h @@ -18,6 +18,4 @@ extern void setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate); extern void sched_clock_register(u64 (*read)(void), int bits, unsigned long rate); -extern unsigned long long (*sched_clock_func)(void); - #endif diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c index f388baeaf2b6..68b799375981 100644 --- a/kernel/time/sched_clock.c +++ b/kernel/time/sched_clock.c @@ -63,7 +63,7 @@ static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) return (cyc * mult) >> shift; } -static unsigned long long notrace sched_clock_32(void) +unsigned long long notrace sched_clock(void) { u64 epoch_ns; u64 epoch_cyc; @@ -170,13 +170,6 @@ void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate) sched_clock_register(read_sched_clock_32_wrapper, bits, rate); } -unsigned long long __read_mostly (*sched_clock_func)(void) = sched_clock_32; - -unsigned long long notrace sched_clock(void) -{ - return sched_clock_func(); -} - void __init sched_clock_postinit(void) { /* -- cgit v1.2.3-70-g09d2