rcu: Conditionally compile RCU's eqs warnings
This commit applies some warning-omission micro-optimizations to RCU's various extended-quiescent-state functions, which are on the kernel/user hotpath for CONFIG_NO_HZ_FULL=y. Reported-by: Rik van Riel <riel@redhat.com> Reported by: Mike Galbraith <umgwanakikbuti@gmail.com> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
parent
e72aeafc66
commit
1ce46ee597
@ -620,7 +620,8 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
|
||||
struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
|
||||
|
||||
trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
|
||||
if (!user && !is_idle_task(current)) {
|
||||
if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
|
||||
!user && !is_idle_task(current)) {
|
||||
struct task_struct *idle __maybe_unused =
|
||||
idle_task(smp_processor_id());
|
||||
|
||||
@ -639,7 +640,8 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
|
||||
smp_mb__before_atomic(); /* See above. */
|
||||
atomic_inc(&rdtp->dynticks);
|
||||
smp_mb__after_atomic(); /* Force ordering with next sojourn. */
|
||||
WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
|
||||
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
|
||||
atomic_read(&rdtp->dynticks) & 0x1);
|
||||
rcu_dynticks_task_enter();
|
||||
|
||||
/*
|
||||
@ -665,7 +667,8 @@ static void rcu_eqs_enter(bool user)
|
||||
|
||||
rdtp = this_cpu_ptr(&rcu_dynticks);
|
||||
oldval = rdtp->dynticks_nesting;
|
||||
WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0);
|
||||
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
|
||||
(oldval & DYNTICK_TASK_NEST_MASK) == 0);
|
||||
if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) {
|
||||
rdtp->dynticks_nesting = 0;
|
||||
rcu_eqs_enter_common(oldval, user);
|
||||
@ -738,7 +741,8 @@ void rcu_irq_exit(void)
|
||||
rdtp = this_cpu_ptr(&rcu_dynticks);
|
||||
oldval = rdtp->dynticks_nesting;
|
||||
rdtp->dynticks_nesting--;
|
||||
WARN_ON_ONCE(rdtp->dynticks_nesting < 0);
|
||||
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
|
||||
rdtp->dynticks_nesting < 0);
|
||||
if (rdtp->dynticks_nesting)
|
||||
trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting);
|
||||
else
|
||||
@ -763,10 +767,12 @@ static void rcu_eqs_exit_common(long long oldval, int user)
|
||||
atomic_inc(&rdtp->dynticks);
|
||||
/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
|
||||
smp_mb__after_atomic(); /* See above. */
|
||||
WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
|
||||
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
|
||||
!(atomic_read(&rdtp->dynticks) & 0x1));
|
||||
rcu_cleanup_after_idle();
|
||||
trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
|
||||
if (!user && !is_idle_task(current)) {
|
||||
if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
|
||||
!user && !is_idle_task(current)) {
|
||||
struct task_struct *idle __maybe_unused =
|
||||
idle_task(smp_processor_id());
|
||||
|
||||
@ -790,7 +796,7 @@ static void rcu_eqs_exit(bool user)
|
||||
|
||||
rdtp = this_cpu_ptr(&rcu_dynticks);
|
||||
oldval = rdtp->dynticks_nesting;
|
||||
WARN_ON_ONCE(oldval < 0);
|
||||
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
|
||||
if (oldval & DYNTICK_TASK_NEST_MASK) {
|
||||
rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE;
|
||||
} else {
|
||||
@ -863,7 +869,8 @@ void rcu_irq_enter(void)
|
||||
rdtp = this_cpu_ptr(&rcu_dynticks);
|
||||
oldval = rdtp->dynticks_nesting;
|
||||
rdtp->dynticks_nesting++;
|
||||
WARN_ON_ONCE(rdtp->dynticks_nesting == 0);
|
||||
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
|
||||
rdtp->dynticks_nesting == 0);
|
||||
if (oldval)
|
||||
trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting);
|
||||
else
|
||||
|
@ -1373,6 +1373,17 @@ config RCU_TRACE
|
||||
Say Y here if you want to enable RCU tracing
|
||||
Say N if you are unsure.
|
||||
|
||||
config RCU_EQS_DEBUG
|
||||
bool "Use this when adding any sort of NO_HZ support to your arch"
|
||||
depends on DEBUG_KERNEL
|
||||
help
|
||||
This option provides consistency checks in RCU's handling of
|
||||
NO_HZ. These checks have proven quite helpful in detecting
|
||||
bugs in arch-specific NO_HZ code.
|
||||
|
||||
Say N here if you need ultimate kernel/user switch latencies
|
||||
Say Y if you are unsure
|
||||
|
||||
endmenu # "RCU Debugging"
|
||||
|
||||
config DEBUG_BLOCK_EXT_DEVT
|
||||
|
Loading…
Reference in New Issue
Block a user