b8f8c3cf0a
Jack Ren and Eric Miao tracked down the following long standing
problem in the NOHZ code:
scheduler switch to idle task
enable interrupts
Window starts here
----> interrupt happens (does not set NEED_RESCHED)
irq_exit() stops the tick
----> interrupt happens (does set NEED_RESCHED)
return from schedule()
cpu_idle(): preempt_disable();
Window ends here
The interrupts can happen at any point inside the race window. The
first interrupt stops the tick, the second one causes the scheduler to
rerun and switch away from idle again and we end up with the tick
disabled.
The fact that it needs two interrupts where the first one does not set
NEED_RESCHED and the second one does made the bug obscure and extremly
hard to reproduce and analyse. Kudos to Jack and Eric.
Solution: Limit the NOHZ functionality to the idle loop to make sure
that we can not run into such a situation ever again.
cpu_idle()
{
preempt_disable();
while(1) {
tick_nohz_stop_sched_tick(1); <- tell NOHZ code that we
are in the idle loop
while (!need_resched())
halt();
tick_nohz_restart_sched_tick(); <- disables NOHZ mode
preempt_enable_no_resched();
schedule();
preempt_disable();
}
}
In hindsight we should have done this forever, but ...
/me grabs a large brown paperbag.
Debugged-by: Jack Ren <jack.ren@marvell.com>,
Debugged-by: eric miao <eric.y.miao@gmail.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
130 lines
3.9 KiB
C
130 lines
3.9 KiB
C
/* linux/include/linux/tick.h
|
|
*
|
|
* This file contains the structure definitions for tick related functions
|
|
*
|
|
*/
|
|
#ifndef _LINUX_TICK_H
|
|
#define _LINUX_TICK_H
|
|
|
|
#include <linux/clockchips.h>
|
|
|
|
#ifdef CONFIG_GENERIC_CLOCKEVENTS
|
|
|
|
enum tick_device_mode {
|
|
TICKDEV_MODE_PERIODIC,
|
|
TICKDEV_MODE_ONESHOT,
|
|
};
|
|
|
|
struct tick_device {
|
|
struct clock_event_device *evtdev;
|
|
enum tick_device_mode mode;
|
|
};
|
|
|
|
enum tick_nohz_mode {
|
|
NOHZ_MODE_INACTIVE,
|
|
NOHZ_MODE_LOWRES,
|
|
NOHZ_MODE_HIGHRES,
|
|
};
|
|
|
|
/**
|
|
* struct tick_sched - sched tick emulation and no idle tick control/stats
|
|
* @sched_timer: hrtimer to schedule the periodic tick in high
|
|
* resolution mode
|
|
* @idle_tick: Store the last idle tick expiry time when the tick
|
|
* timer is modified for idle sleeps. This is necessary
|
|
* to resume the tick timer operation in the timeline
|
|
* when the CPU returns from idle
|
|
* @tick_stopped: Indicator that the idle tick has been stopped
|
|
* @idle_jiffies: jiffies at the entry to idle for idle time accounting
|
|
* @idle_calls: Total number of idle calls
|
|
* @idle_sleeps: Number of idle calls, where the sched tick was stopped
|
|
* @idle_entrytime: Time when the idle call was entered
|
|
* @idle_waketime: Time when the idle was interrupted
|
|
* @idle_exittime: Time when the idle state was left
|
|
* @idle_sleeptime: Sum of the time slept in idle with sched tick stopped
|
|
* @sleep_length: Duration of the current idle sleep
|
|
*/
|
|
struct tick_sched {
|
|
struct hrtimer sched_timer;
|
|
unsigned long check_clocks;
|
|
enum tick_nohz_mode nohz_mode;
|
|
ktime_t idle_tick;
|
|
int inidle;
|
|
int tick_stopped;
|
|
unsigned long idle_jiffies;
|
|
unsigned long idle_calls;
|
|
unsigned long idle_sleeps;
|
|
int idle_active;
|
|
ktime_t idle_entrytime;
|
|
ktime_t idle_waketime;
|
|
ktime_t idle_exittime;
|
|
ktime_t idle_sleeptime;
|
|
ktime_t idle_lastupdate;
|
|
ktime_t sleep_length;
|
|
unsigned long last_jiffies;
|
|
unsigned long next_jiffies;
|
|
ktime_t idle_expires;
|
|
};
|
|
|
|
extern void __init tick_init(void);
|
|
extern int tick_is_oneshot_available(void);
|
|
extern struct tick_device *tick_get_device(int cpu);
|
|
|
|
# ifdef CONFIG_HIGH_RES_TIMERS
|
|
extern int tick_init_highres(void);
|
|
extern int tick_program_event(ktime_t expires, int force);
|
|
extern void tick_setup_sched_timer(void);
|
|
extern void tick_cancel_sched_timer(int cpu);
|
|
# else
|
|
static inline void tick_cancel_sched_timer(int cpu) { }
|
|
# endif /* HIGHRES */
|
|
|
|
# ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
|
|
extern struct tick_device *tick_get_broadcast_device(void);
|
|
extern cpumask_t *tick_get_broadcast_mask(void);
|
|
|
|
# ifdef CONFIG_TICK_ONESHOT
|
|
extern cpumask_t *tick_get_broadcast_oneshot_mask(void);
|
|
# endif
|
|
|
|
# endif /* BROADCAST */
|
|
|
|
# ifdef CONFIG_TICK_ONESHOT
|
|
extern void tick_clock_notify(void);
|
|
extern int tick_check_oneshot_change(int allow_nohz);
|
|
extern struct tick_sched *tick_get_tick_sched(int cpu);
|
|
# else
|
|
static inline void tick_clock_notify(void) { }
|
|
static inline int tick_check_oneshot_change(int allow_nohz) { return 0; }
|
|
# endif
|
|
|
|
#else /* CONFIG_GENERIC_CLOCKEVENTS */
|
|
static inline void tick_init(void) { }
|
|
static inline void tick_cancel_sched_timer(int cpu) { }
|
|
static inline void tick_clock_notify(void) { }
|
|
static inline int tick_check_oneshot_change(int allow_nohz) { return 0; }
|
|
#endif /* !CONFIG_GENERIC_CLOCKEVENTS */
|
|
|
|
# ifdef CONFIG_NO_HZ
|
|
extern void tick_nohz_stop_sched_tick(int inidle);
|
|
extern void tick_nohz_restart_sched_tick(void);
|
|
extern void tick_nohz_update_jiffies(void);
|
|
extern ktime_t tick_nohz_get_sleep_length(void);
|
|
extern void tick_nohz_stop_idle(int cpu);
|
|
extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
|
|
# else
|
|
static inline void tick_nohz_stop_sched_tick(int inidle) { }
|
|
static inline void tick_nohz_restart_sched_tick(void) { }
|
|
static inline void tick_nohz_update_jiffies(void) { }
|
|
static inline ktime_t tick_nohz_get_sleep_length(void)
|
|
{
|
|
ktime_t len = { .tv64 = NSEC_PER_SEC/HZ };
|
|
|
|
return len;
|
|
}
|
|
static inline void tick_nohz_stop_idle(int cpu) { }
|
|
static inline u64 get_cpu_idle_time_us(int cpu, u64 *unused) { return 0; }
|
|
# endif /* !NO_HZ */
|
|
|
|
#endif
|