diff options
| -rw-r--r-- | include/linux/kernel.h | 4 | ||||
| -rw-r--r-- | include/linux/sched.h | 3 | ||||
| -rw-r--r-- | kernel/sched/core.c | 28 |
3 files changed, 21 insertions, 14 deletions
diff --git a/include/linux/kernel.h b/include/linux/kernel.h index f95ee786e4ef..e8696e4a45aa 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -111,7 +111,7 @@ static __always_inline void might_resched(void) #endif /* CONFIG_PREEMPT_* */ #ifdef CONFIG_DEBUG_ATOMIC_SLEEP -extern void __might_resched(const char *file, int line, int preempt_offset); +extern void __might_resched(const char *file, int line, unsigned int offsets); extern void __might_sleep(const char *file, int line); extern void __cant_sleep(const char *file, int line, int preempt_offset); extern void __cant_migrate(const char *file, int line); @@ -169,7 +169,7 @@ extern void __cant_migrate(const char *file, int line); # define non_block_end() WARN_ON(current->non_block_count-- == 0) #else static inline void __might_resched(const char *file, int line, - int preempt_offset) { } + unsigned int offsets) { } static inline void __might_sleep(const char *file, int line) { } # define might_sleep() do { might_resched(); } while (0) # define cant_sleep() do { } while (0) diff --git a/include/linux/sched.h b/include/linux/sched.h index 7a989f2487f8..b448c7460577 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2046,6 +2046,9 @@ extern int __cond_resched_lock(spinlock_t *lock); extern int __cond_resched_rwlock_read(rwlock_t *lock); extern int __cond_resched_rwlock_write(rwlock_t *lock); +#define MIGHT_RESCHED_RCU_SHIFT 8 +#define MIGHT_RESCHED_PREEMPT_MASK ((1U << MIGHT_RESCHED_RCU_SHIFT) - 1) + #define cond_resched_lock(lock) ({ \ __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \ __cond_resched_lock(lock); \ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0a27cb8f72a9..8d3fa0768e5b 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9468,12 +9468,6 @@ void __init sched_init(void) } #ifdef CONFIG_DEBUG_ATOMIC_SLEEP -static inline int preempt_count_equals(int preempt_offset) -{ - int nested = preempt_count() + rcu_preempt_depth(); - - return (nested == preempt_offset); -} void __might_sleep(const char *file, int line) { @@ -9505,7 +9499,16 @@ static void print_preempt_disable_ip(int preempt_offset, unsigned long ip) print_ip_sym(KERN_ERR, ip); } -void __might_resched(const char *file, int line, int preempt_offset) +static inline bool resched_offsets_ok(unsigned int offsets) +{ + unsigned int nested = preempt_count(); + + nested += rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT; + + return nested == offsets; +} + +void __might_resched(const char *file, int line, unsigned int offsets) { /* Ratelimiting timestamp: */ static unsigned long prev_jiffy; @@ -9515,7 +9518,7 @@ void __might_resched(const char *file, int line, int preempt_offset) /* WARN_ON_ONCE() by default, no rate limit required: */ rcu_sleep_check(); - if ((preempt_count_equals(preempt_offset) && !irqs_disabled() && + if ((resched_offsets_ok(offsets) && !irqs_disabled() && !is_idle_task(current) && !current->non_block_count) || system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || oops_in_progress) @@ -9534,11 +9537,11 @@ void __might_resched(const char *file, int line, int preempt_offset) in_atomic(), irqs_disabled(), current->non_block_count, current->pid, current->comm); pr_err("preempt_count: %x, expected: %x\n", preempt_count(), - preempt_offset); + offsets & MIGHT_RESCHED_PREEMPT_MASK); if (IS_ENABLED(CONFIG_PREEMPT_RCU)) { - pr_err("RCU nest depth: %d, expected: 0\n", - rcu_preempt_depth()); + pr_err("RCU nest depth: %d, expected: %u\n", + rcu_preempt_depth(), offsets >> MIGHT_RESCHED_RCU_SHIFT); } if (task_stack_end_corrupted(current)) @@ -9548,7 +9551,8 @@ void __might_resched(const char *file, int line, int preempt_offset) if (irqs_disabled()) print_irqtrace_events(current); - print_preempt_disable_ip(preempt_offset, preempt_disable_ip); + print_preempt_disable_ip(offsets & MIGHT_RESCHED_PREEMPT_MASK, + preempt_disable_ip); dump_stack(); add_taint(TAINT_WARN, LOCKDEP_STILL_OK); |