Merge branch 'tip/perf/core' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-2.6-trace into perf/core
Conflicts: include/linux/perf_event.h Merge reason: pick up the latest jump-label enhancements, they are cooked ready. Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
commit
32673822e4
@ -20,16 +20,18 @@
|
||||
#define WORD_INSN ".word"
|
||||
#endif
|
||||
|
||||
#define JUMP_LABEL(key, label) \
|
||||
do { \
|
||||
asm goto("1:\tnop\n\t" \
|
||||
"nop\n\t" \
|
||||
".pushsection __jump_table, \"a\"\n\t" \
|
||||
WORD_INSN " 1b, %l[" #label "], %0\n\t" \
|
||||
".popsection\n\t" \
|
||||
: : "i" (key) : : label); \
|
||||
} while (0)
|
||||
|
||||
static __always_inline bool arch_static_branch(struct jump_label_key *key)
|
||||
{
|
||||
asm goto("1:\tnop\n\t"
|
||||
"nop\n\t"
|
||||
".pushsection __jump_table, \"aw\"\n\t"
|
||||
WORD_INSN " 1b, %l[l_yes], %0\n\t"
|
||||
".popsection\n\t"
|
||||
: : "i" (key) : : l_yes);
|
||||
return false;
|
||||
l_yes:
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
|
||||
|
@ -88,6 +88,7 @@ config S390
|
||||
select HAVE_KERNEL_XZ
|
||||
select HAVE_GET_USER_PAGES_FAST
|
||||
select HAVE_ARCH_MUTEX_CPU_RELAX
|
||||
select HAVE_ARCH_JUMP_LABEL if !MARCH_G5
|
||||
select ARCH_INLINE_SPIN_TRYLOCK
|
||||
select ARCH_INLINE_SPIN_TRYLOCK_BH
|
||||
select ARCH_INLINE_SPIN_LOCK
|
||||
|
37
arch/s390/include/asm/jump_label.h
Normal file
37
arch/s390/include/asm/jump_label.h
Normal file
@ -0,0 +1,37 @@
|
||||
#ifndef _ASM_S390_JUMP_LABEL_H
|
||||
#define _ASM_S390_JUMP_LABEL_H
|
||||
|
||||
#include <linux/types.h>
|
||||
|
||||
#define JUMP_LABEL_NOP_SIZE 6
|
||||
|
||||
#ifdef CONFIG_64BIT
|
||||
#define ASM_PTR ".quad"
|
||||
#define ASM_ALIGN ".balign 8"
|
||||
#else
|
||||
#define ASM_PTR ".long"
|
||||
#define ASM_ALIGN ".balign 4"
|
||||
#endif
|
||||
|
||||
static __always_inline bool arch_static_branch(struct jump_label_key *key)
|
||||
{
|
||||
asm goto("0: brcl 0,0\n"
|
||||
".pushsection __jump_table, \"aw\"\n"
|
||||
ASM_ALIGN "\n"
|
||||
ASM_PTR " 0b, %l[label], %0\n"
|
||||
".popsection\n"
|
||||
: : "X" (key) : : label);
|
||||
return false;
|
||||
label:
|
||||
return true;
|
||||
}
|
||||
|
||||
typedef unsigned long jump_label_t;
|
||||
|
||||
struct jump_entry {
|
||||
jump_label_t code;
|
||||
jump_label_t target;
|
||||
jump_label_t key;
|
||||
};
|
||||
|
||||
#endif
|
@ -23,7 +23,7 @@ CFLAGS_sysinfo.o += -Iinclude/math-emu -Iarch/s390/math-emu -w
|
||||
obj-y := bitmap.o traps.o time.o process.o base.o early.o setup.o \
|
||||
processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o \
|
||||
s390_ext.o debug.o irq.o ipl.o dis.o diag.o mem_detect.o \
|
||||
vdso.o vtime.o sysinfo.o nmi.o sclp.o
|
||||
vdso.o vtime.o sysinfo.o nmi.o sclp.o jump_label.o
|
||||
|
||||
obj-y += $(if $(CONFIG_64BIT),entry64.o,entry.o)
|
||||
obj-y += $(if $(CONFIG_64BIT),reipl64.o,reipl.o)
|
||||
|
59
arch/s390/kernel/jump_label.c
Normal file
59
arch/s390/kernel/jump_label.c
Normal file
@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Jump label s390 support
|
||||
*
|
||||
* Copyright IBM Corp. 2011
|
||||
* Author(s): Jan Glauber <jang@linux.vnet.ibm.com>
|
||||
*/
|
||||
#include <linux/module.h>
|
||||
#include <linux/uaccess.h>
|
||||
#include <linux/stop_machine.h>
|
||||
#include <linux/jump_label.h>
|
||||
#include <asm/ipl.h>
|
||||
|
||||
#ifdef HAVE_JUMP_LABEL
|
||||
|
||||
struct insn {
|
||||
u16 opcode;
|
||||
s32 offset;
|
||||
} __packed;
|
||||
|
||||
struct insn_args {
|
||||
unsigned long *target;
|
||||
struct insn *insn;
|
||||
ssize_t size;
|
||||
};
|
||||
|
||||
static int __arch_jump_label_transform(void *data)
|
||||
{
|
||||
struct insn_args *args = data;
|
||||
int rc;
|
||||
|
||||
rc = probe_kernel_write(args->target, args->insn, args->size);
|
||||
WARN_ON_ONCE(rc < 0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void arch_jump_label_transform(struct jump_entry *entry,
|
||||
enum jump_label_type type)
|
||||
{
|
||||
struct insn_args args;
|
||||
struct insn insn;
|
||||
|
||||
if (type == JUMP_LABEL_ENABLE) {
|
||||
/* brcl 15,offset */
|
||||
insn.opcode = 0xc0f4;
|
||||
insn.offset = (entry->target - entry->code) >> 1;
|
||||
} else {
|
||||
/* brcl 0,0 */
|
||||
insn.opcode = 0xc004;
|
||||
insn.offset = 0;
|
||||
}
|
||||
|
||||
args.target = (void *) entry->code;
|
||||
args.insn = &insn;
|
||||
args.size = JUMP_LABEL_NOP_SIZE;
|
||||
|
||||
stop_machine(__arch_jump_label_transform, &args, NULL);
|
||||
}
|
||||
|
||||
#endif
|
@ -7,17 +7,20 @@
|
||||
|
||||
#define JUMP_LABEL_NOP_SIZE 4
|
||||
|
||||
#define JUMP_LABEL(key, label) \
|
||||
do { \
|
||||
asm goto("1:\n\t" \
|
||||
"nop\n\t" \
|
||||
"nop\n\t" \
|
||||
".pushsection __jump_table, \"a\"\n\t"\
|
||||
".align 4\n\t" \
|
||||
".word 1b, %l[" #label "], %c0\n\t" \
|
||||
".popsection \n\t" \
|
||||
: : "i" (key) : : label);\
|
||||
} while (0)
|
||||
static __always_inline bool arch_static_branch(struct jump_label_key *key)
|
||||
{
|
||||
asm goto("1:\n\t"
|
||||
"nop\n\t"
|
||||
"nop\n\t"
|
||||
".pushsection __jump_table, \"aw\"\n\t"
|
||||
".align 4\n\t"
|
||||
".word 1b, %l[l_yes], %c0\n\t"
|
||||
".popsection \n\t"
|
||||
: : "i" (key) : : l_yes);
|
||||
return false;
|
||||
l_yes:
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
|
||||
|
@ -4,7 +4,6 @@
|
||||
#include <linux/types.h>
|
||||
#include <linux/stddef.h>
|
||||
#include <linux/stringify.h>
|
||||
#include <linux/jump_label.h>
|
||||
#include <asm/asm.h>
|
||||
|
||||
/*
|
||||
@ -191,7 +190,7 @@ extern void *text_poke(void *addr, const void *opcode, size_t len);
|
||||
extern void *text_poke_smp(void *addr, const void *opcode, size_t len);
|
||||
extern void text_poke_smp_batch(struct text_poke_param *params, int n);
|
||||
|
||||
#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
|
||||
#if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL)
|
||||
#define IDEAL_NOP_SIZE_5 5
|
||||
extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
|
||||
extern void arch_init_ideal_nop5(void);
|
||||
|
@ -5,20 +5,25 @@
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <asm/nops.h>
|
||||
#include <asm/asm.h>
|
||||
|
||||
#define JUMP_LABEL_NOP_SIZE 5
|
||||
|
||||
# define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
|
||||
#define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
|
||||
|
||||
# define JUMP_LABEL(key, label) \
|
||||
do { \
|
||||
asm goto("1:" \
|
||||
JUMP_LABEL_INITIAL_NOP \
|
||||
".pushsection __jump_table, \"aw\" \n\t"\
|
||||
_ASM_PTR "1b, %l[" #label "], %c0 \n\t" \
|
||||
".popsection \n\t" \
|
||||
: : "i" (key) : : label); \
|
||||
} while (0)
|
||||
static __always_inline bool arch_static_branch(struct jump_label_key *key)
|
||||
{
|
||||
asm goto("1:"
|
||||
JUMP_LABEL_INITIAL_NOP
|
||||
".pushsection __jump_table, \"aw\" \n\t"
|
||||
_ASM_ALIGN "\n\t"
|
||||
_ASM_PTR "1b, %l[l_yes], %c0 \n\t"
|
||||
".popsection \n\t"
|
||||
: : "i" (key) : : l_yes);
|
||||
return false;
|
||||
l_yes:
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
|
||||
|
@ -679,7 +679,7 @@ void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n)
|
||||
__stop_machine(stop_machine_text_poke, (void *)&tpp, NULL);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
|
||||
#if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL)
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
unsigned char ideal_nop5[5] = { 0x66, 0x66, 0x66, 0x66, 0x90 };
|
||||
|
@ -24,6 +24,7 @@
|
||||
#include <linux/bug.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/gfp.h>
|
||||
#include <linux/jump_label.h>
|
||||
|
||||
#include <asm/system.h>
|
||||
#include <asm/page.h>
|
||||
|
@ -170,6 +170,10 @@
|
||||
STRUCT_ALIGN(); \
|
||||
*(__tracepoints) \
|
||||
/* implement dynamic printk debug */ \
|
||||
. = ALIGN(8); \
|
||||
VMLINUX_SYMBOL(__start___jump_table) = .; \
|
||||
*(__jump_table) \
|
||||
VMLINUX_SYMBOL(__stop___jump_table) = .; \
|
||||
. = ALIGN(8); \
|
||||
VMLINUX_SYMBOL(__start___verbose) = .; \
|
||||
*(__verbose) \
|
||||
@ -228,8 +232,6 @@
|
||||
\
|
||||
BUG_TABLE \
|
||||
\
|
||||
JUMP_TABLE \
|
||||
\
|
||||
/* PCI quirks */ \
|
||||
.pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \
|
||||
VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \
|
||||
@ -589,14 +591,6 @@
|
||||
#define BUG_TABLE
|
||||
#endif
|
||||
|
||||
#define JUMP_TABLE \
|
||||
. = ALIGN(8); \
|
||||
__jump_table : AT(ADDR(__jump_table) - LOAD_OFFSET) { \
|
||||
VMLINUX_SYMBOL(__start___jump_table) = .; \
|
||||
*(__jump_table) \
|
||||
VMLINUX_SYMBOL(__stop___jump_table) = .; \
|
||||
}
|
||||
|
||||
#ifdef CONFIG_PM_TRACE
|
||||
#define TRACEDATA \
|
||||
. = ALIGN(4); \
|
||||
|
@ -1,8 +1,6 @@
|
||||
#ifndef _DYNAMIC_DEBUG_H
|
||||
#define _DYNAMIC_DEBUG_H
|
||||
|
||||
#include <linux/jump_label.h>
|
||||
|
||||
/* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which
|
||||
* bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They
|
||||
* use independent hash functions, to reduce the chance of false positives.
|
||||
|
@ -1,20 +1,43 @@
|
||||
#ifndef _LINUX_JUMP_LABEL_H
|
||||
#define _LINUX_JUMP_LABEL_H
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <linux/compiler.h>
|
||||
|
||||
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
|
||||
|
||||
struct jump_label_key {
|
||||
atomic_t enabled;
|
||||
struct jump_entry *entries;
|
||||
#ifdef CONFIG_MODULES
|
||||
struct jump_label_mod *next;
|
||||
#endif
|
||||
};
|
||||
|
||||
# include <asm/jump_label.h>
|
||||
# define HAVE_JUMP_LABEL
|
||||
#endif
|
||||
|
||||
enum jump_label_type {
|
||||
JUMP_LABEL_DISABLE = 0,
|
||||
JUMP_LABEL_ENABLE,
|
||||
JUMP_LABEL_DISABLE
|
||||
};
|
||||
|
||||
struct module;
|
||||
|
||||
#ifdef HAVE_JUMP_LABEL
|
||||
|
||||
#ifdef CONFIG_MODULES
|
||||
#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL}
|
||||
#else
|
||||
#define JUMP_LABEL_INIT {{ 0 }, NULL}
|
||||
#endif
|
||||
|
||||
static __always_inline bool static_branch(struct jump_label_key *key)
|
||||
{
|
||||
return arch_static_branch(key);
|
||||
}
|
||||
|
||||
extern struct jump_entry __start___jump_table[];
|
||||
extern struct jump_entry __stop___jump_table[];
|
||||
|
||||
@ -23,37 +46,37 @@ extern void jump_label_unlock(void);
|
||||
extern void arch_jump_label_transform(struct jump_entry *entry,
|
||||
enum jump_label_type type);
|
||||
extern void arch_jump_label_text_poke_early(jump_label_t addr);
|
||||
extern void jump_label_update(unsigned long key, enum jump_label_type type);
|
||||
extern void jump_label_apply_nops(struct module *mod);
|
||||
extern int jump_label_text_reserved(void *start, void *end);
|
||||
|
||||
#define jump_label_enable(key) \
|
||||
jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
|
||||
|
||||
#define jump_label_disable(key) \
|
||||
jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
|
||||
extern void jump_label_inc(struct jump_label_key *key);
|
||||
extern void jump_label_dec(struct jump_label_key *key);
|
||||
extern bool jump_label_enabled(struct jump_label_key *key);
|
||||
extern void jump_label_apply_nops(struct module *mod);
|
||||
|
||||
#else
|
||||
|
||||
#define JUMP_LABEL(key, label) \
|
||||
do { \
|
||||
if (unlikely(*key)) \
|
||||
goto label; \
|
||||
} while (0)
|
||||
#include <asm/atomic.h>
|
||||
|
||||
#define jump_label_enable(cond_var) \
|
||||
do { \
|
||||
*(cond_var) = 1; \
|
||||
} while (0)
|
||||
#define JUMP_LABEL_INIT {ATOMIC_INIT(0)}
|
||||
|
||||
#define jump_label_disable(cond_var) \
|
||||
do { \
|
||||
*(cond_var) = 0; \
|
||||
} while (0)
|
||||
struct jump_label_key {
|
||||
atomic_t enabled;
|
||||
};
|
||||
|
||||
static inline int jump_label_apply_nops(struct module *mod)
|
||||
static __always_inline bool static_branch(struct jump_label_key *key)
|
||||
{
|
||||
return 0;
|
||||
if (unlikely(atomic_read(&key->enabled)))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline void jump_label_inc(struct jump_label_key *key)
|
||||
{
|
||||
atomic_inc(&key->enabled);
|
||||
}
|
||||
|
||||
static inline void jump_label_dec(struct jump_label_key *key)
|
||||
{
|
||||
atomic_dec(&key->enabled);
|
||||
}
|
||||
|
||||
static inline int jump_label_text_reserved(void *start, void *end)
|
||||
@ -64,16 +87,16 @@ static inline int jump_label_text_reserved(void *start, void *end)
|
||||
static inline void jump_label_lock(void) {}
|
||||
static inline void jump_label_unlock(void) {}
|
||||
|
||||
#endif
|
||||
static inline bool jump_label_enabled(struct jump_label_key *key)
|
||||
{
|
||||
return !!atomic_read(&key->enabled);
|
||||
}
|
||||
|
||||
#define COND_STMT(key, stmt) \
|
||||
do { \
|
||||
__label__ jl_enabled; \
|
||||
JUMP_LABEL(key, jl_enabled); \
|
||||
if (0) { \
|
||||
jl_enabled: \
|
||||
stmt; \
|
||||
} \
|
||||
} while (0)
|
||||
static inline int jump_label_apply_nops(struct module *mod)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
@ -1,44 +0,0 @@
|
||||
#ifndef _LINUX_JUMP_LABEL_REF_H
|
||||
#define _LINUX_JUMP_LABEL_REF_H
|
||||
|
||||
#include <linux/jump_label.h>
|
||||
#include <asm/atomic.h>
|
||||
|
||||
#ifdef HAVE_JUMP_LABEL
|
||||
|
||||
static inline void jump_label_inc(atomic_t *key)
|
||||
{
|
||||
if (atomic_add_return(1, key) == 1)
|
||||
jump_label_enable(key);
|
||||
}
|
||||
|
||||
static inline void jump_label_dec(atomic_t *key)
|
||||
{
|
||||
if (atomic_dec_and_test(key))
|
||||
jump_label_disable(key);
|
||||
}
|
||||
|
||||
#else /* !HAVE_JUMP_LABEL */
|
||||
|
||||
static inline void jump_label_inc(atomic_t *key)
|
||||
{
|
||||
atomic_inc(key);
|
||||
}
|
||||
|
||||
static inline void jump_label_dec(atomic_t *key)
|
||||
{
|
||||
atomic_dec(key);
|
||||
}
|
||||
|
||||
#undef JUMP_LABEL
|
||||
#define JUMP_LABEL(key, label) \
|
||||
do { \
|
||||
if (unlikely(__builtin_choose_expr( \
|
||||
__builtin_types_compatible_p(typeof(key), atomic_t *), \
|
||||
atomic_read((atomic_t *)(key)), *(key)))) \
|
||||
goto label; \
|
||||
} while (0)
|
||||
|
||||
#endif /* HAVE_JUMP_LABEL */
|
||||
|
||||
#endif /* _LINUX_JUMP_LABEL_REF_H */
|
@ -505,7 +505,7 @@ struct perf_guest_info_callbacks {
|
||||
#include <linux/ftrace.h>
|
||||
#include <linux/cpu.h>
|
||||
#include <linux/irq_work.h>
|
||||
#include <linux/jump_label_ref.h>
|
||||
#include <linux/jump_label.h>
|
||||
#include <asm/atomic.h>
|
||||
#include <asm/local.h>
|
||||
|
||||
@ -1034,7 +1034,7 @@ static inline int is_software_event(struct perf_event *event)
|
||||
return event->pmu->task_ctx_nr == perf_sw_context;
|
||||
}
|
||||
|
||||
extern atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
|
||||
extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
|
||||
|
||||
extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64);
|
||||
|
||||
@ -1063,22 +1063,21 @@ perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
|
||||
{
|
||||
struct pt_regs hot_regs;
|
||||
|
||||
JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);
|
||||
return;
|
||||
|
||||
have_event:
|
||||
if (!regs) {
|
||||
perf_fetch_caller_regs(&hot_regs);
|
||||
regs = &hot_regs;
|
||||
if (static_branch(&perf_swevent_enabled[event_id])) {
|
||||
if (!regs) {
|
||||
perf_fetch_caller_regs(&hot_regs);
|
||||
regs = &hot_regs;
|
||||
}
|
||||
__perf_sw_event(event_id, nr, nmi, regs, addr);
|
||||
}
|
||||
__perf_sw_event(event_id, nr, nmi, regs, addr);
|
||||
}
|
||||
|
||||
extern atomic_t perf_sched_events;
|
||||
extern struct jump_label_key perf_sched_events;
|
||||
|
||||
static inline void perf_event_task_sched_in(struct task_struct *task)
|
||||
{
|
||||
COND_STMT(&perf_sched_events, __perf_event_task_sched_in(task));
|
||||
if (static_branch(&perf_sched_events))
|
||||
__perf_event_task_sched_in(task);
|
||||
}
|
||||
|
||||
static inline
|
||||
|
@ -29,7 +29,7 @@ struct tracepoint_func {
|
||||
|
||||
struct tracepoint {
|
||||
const char *name; /* Tracepoint name */
|
||||
int state; /* State. */
|
||||
struct jump_label_key key;
|
||||
void (*regfunc)(void);
|
||||
void (*unregfunc)(void);
|
||||
struct tracepoint_func __rcu *funcs;
|
||||
@ -146,9 +146,7 @@ void tracepoint_update_probe_range(struct tracepoint * const *begin,
|
||||
extern struct tracepoint __tracepoint_##name; \
|
||||
static inline void trace_##name(proto) \
|
||||
{ \
|
||||
JUMP_LABEL(&__tracepoint_##name.state, do_trace); \
|
||||
return; \
|
||||
do_trace: \
|
||||
if (static_branch(&__tracepoint_##name.key)) \
|
||||
__DO_TRACE(&__tracepoint_##name, \
|
||||
TP_PROTO(data_proto), \
|
||||
TP_ARGS(data_args), \
|
||||
@ -176,14 +174,14 @@ do_trace: \
|
||||
* structures, so we create an array of pointers that will be used for iteration
|
||||
* on the tracepoints.
|
||||
*/
|
||||
#define DEFINE_TRACE_FN(name, reg, unreg) \
|
||||
static const char __tpstrtab_##name[] \
|
||||
__attribute__((section("__tracepoints_strings"))) = #name; \
|
||||
struct tracepoint __tracepoint_##name \
|
||||
__attribute__((section("__tracepoints"))) = \
|
||||
{ __tpstrtab_##name, 0, reg, unreg, NULL }; \
|
||||
static struct tracepoint * const __tracepoint_ptr_##name __used \
|
||||
__attribute__((section("__tracepoints_ptrs"))) = \
|
||||
#define DEFINE_TRACE_FN(name, reg, unreg) \
|
||||
static const char __tpstrtab_##name[] \
|
||||
__attribute__((section("__tracepoints_strings"))) = #name; \
|
||||
struct tracepoint __tracepoint_##name \
|
||||
__attribute__((section("__tracepoints"))) = \
|
||||
{ __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\
|
||||
static struct tracepoint * const __tracepoint_ptr_##name __used \
|
||||
__attribute__((section("__tracepoints_ptrs"))) = \
|
||||
&__tracepoint_##name;
|
||||
|
||||
#define DEFINE_TRACE(name) \
|
||||
|
@ -2,43 +2,23 @@
|
||||
* jump label support
|
||||
*
|
||||
* Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
|
||||
* Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com>
|
||||
*
|
||||
*/
|
||||
#include <linux/jump_label.h>
|
||||
#include <linux/memory.h>
|
||||
#include <linux/uaccess.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/list.h>
|
||||
#include <linux/jhash.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/sort.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/jump_label.h>
|
||||
|
||||
#ifdef HAVE_JUMP_LABEL
|
||||
|
||||
#define JUMP_LABEL_HASH_BITS 6
|
||||
#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
|
||||
static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
|
||||
|
||||
/* mutex to protect coming/going of the the jump_label table */
|
||||
static DEFINE_MUTEX(jump_label_mutex);
|
||||
|
||||
struct jump_label_entry {
|
||||
struct hlist_node hlist;
|
||||
struct jump_entry *table;
|
||||
int nr_entries;
|
||||
/* hang modules off here */
|
||||
struct hlist_head modules;
|
||||
unsigned long key;
|
||||
};
|
||||
|
||||
struct jump_label_module_entry {
|
||||
struct hlist_node hlist;
|
||||
struct jump_entry *table;
|
||||
int nr_entries;
|
||||
struct module *mod;
|
||||
};
|
||||
|
||||
void jump_label_lock(void)
|
||||
{
|
||||
mutex_lock(&jump_label_mutex);
|
||||
@ -49,6 +29,11 @@ void jump_label_unlock(void)
|
||||
mutex_unlock(&jump_label_mutex);
|
||||
}
|
||||
|
||||
bool jump_label_enabled(struct jump_label_key *key)
|
||||
{
|
||||
return !!atomic_read(&key->enabled);
|
||||
}
|
||||
|
||||
static int jump_label_cmp(const void *a, const void *b)
|
||||
{
|
||||
const struct jump_entry *jea = a;
|
||||
@ -64,7 +49,7 @@ static int jump_label_cmp(const void *a, const void *b)
|
||||
}
|
||||
|
||||
static void
|
||||
sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
|
||||
jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
|
||||
{
|
||||
unsigned long size;
|
||||
|
||||
@ -73,118 +58,25 @@ sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
|
||||
sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
|
||||
}
|
||||
|
||||
static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
|
||||
static void jump_label_update(struct jump_label_key *key, int enable);
|
||||
|
||||
void jump_label_inc(struct jump_label_key *key)
|
||||
{
|
||||
struct hlist_head *head;
|
||||
struct hlist_node *node;
|
||||
struct jump_label_entry *e;
|
||||
u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
|
||||
|
||||
head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
|
||||
hlist_for_each_entry(e, node, head, hlist) {
|
||||
if (key == e->key)
|
||||
return e;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static struct jump_label_entry *
|
||||
add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
|
||||
{
|
||||
struct hlist_head *head;
|
||||
struct jump_label_entry *e;
|
||||
u32 hash;
|
||||
|
||||
e = get_jump_label_entry(key);
|
||||
if (e)
|
||||
return ERR_PTR(-EEXIST);
|
||||
|
||||
e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
|
||||
if (!e)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
hash = jhash((void *)&key, sizeof(jump_label_t), 0);
|
||||
head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
|
||||
e->key = key;
|
||||
e->table = table;
|
||||
e->nr_entries = nr_entries;
|
||||
INIT_HLIST_HEAD(&(e->modules));
|
||||
hlist_add_head(&e->hlist, head);
|
||||
return e;
|
||||
}
|
||||
|
||||
static int
|
||||
build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
|
||||
{
|
||||
struct jump_entry *iter, *iter_begin;
|
||||
struct jump_label_entry *entry;
|
||||
int count;
|
||||
|
||||
sort_jump_label_entries(start, stop);
|
||||
iter = start;
|
||||
while (iter < stop) {
|
||||
entry = get_jump_label_entry(iter->key);
|
||||
if (!entry) {
|
||||
iter_begin = iter;
|
||||
count = 0;
|
||||
while ((iter < stop) &&
|
||||
(iter->key == iter_begin->key)) {
|
||||
iter++;
|
||||
count++;
|
||||
}
|
||||
entry = add_jump_label_entry(iter_begin->key,
|
||||
count, iter_begin);
|
||||
if (IS_ERR(entry))
|
||||
return PTR_ERR(entry);
|
||||
} else {
|
||||
WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/***
|
||||
* jump_label_update - update jump label text
|
||||
* @key - key value associated with a a jump label
|
||||
* @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
|
||||
*
|
||||
* Will enable/disable the jump for jump label @key, depending on the
|
||||
* value of @type.
|
||||
*
|
||||
*/
|
||||
|
||||
void jump_label_update(unsigned long key, enum jump_label_type type)
|
||||
{
|
||||
struct jump_entry *iter;
|
||||
struct jump_label_entry *entry;
|
||||
struct hlist_node *module_node;
|
||||
struct jump_label_module_entry *e_module;
|
||||
int count;
|
||||
if (atomic_inc_not_zero(&key->enabled))
|
||||
return;
|
||||
|
||||
jump_label_lock();
|
||||
entry = get_jump_label_entry((jump_label_t)key);
|
||||
if (entry) {
|
||||
count = entry->nr_entries;
|
||||
iter = entry->table;
|
||||
while (count--) {
|
||||
if (kernel_text_address(iter->code))
|
||||
arch_jump_label_transform(iter, type);
|
||||
iter++;
|
||||
}
|
||||
/* eanble/disable jump labels in modules */
|
||||
hlist_for_each_entry(e_module, module_node, &(entry->modules),
|
||||
hlist) {
|
||||
count = e_module->nr_entries;
|
||||
iter = e_module->table;
|
||||
while (count--) {
|
||||
if (iter->key &&
|
||||
kernel_text_address(iter->code))
|
||||
arch_jump_label_transform(iter, type);
|
||||
iter++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (atomic_add_return(1, &key->enabled) == 1)
|
||||
jump_label_update(key, JUMP_LABEL_ENABLE);
|
||||
jump_label_unlock();
|
||||
}
|
||||
|
||||
void jump_label_dec(struct jump_label_key *key)
|
||||
{
|
||||
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex))
|
||||
return;
|
||||
|
||||
jump_label_update(key, JUMP_LABEL_DISABLE);
|
||||
jump_label_unlock();
|
||||
}
|
||||
|
||||
@ -197,41 +89,254 @@ static int addr_conflict(struct jump_entry *entry, void *start, void *end)
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_MODULES
|
||||
|
||||
static int module_conflict(void *start, void *end)
|
||||
static int __jump_label_text_reserved(struct jump_entry *iter_start,
|
||||
struct jump_entry *iter_stop, void *start, void *end)
|
||||
{
|
||||
struct hlist_head *head;
|
||||
struct hlist_node *node, *node_next, *module_node, *module_node_next;
|
||||
struct jump_label_entry *e;
|
||||
struct jump_label_module_entry *e_module;
|
||||
struct jump_entry *iter;
|
||||
int i, count;
|
||||
int conflict = 0;
|
||||
|
||||
for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
|
||||
head = &jump_label_table[i];
|
||||
hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
|
||||
hlist_for_each_entry_safe(e_module, module_node,
|
||||
module_node_next,
|
||||
&(e->modules), hlist) {
|
||||
count = e_module->nr_entries;
|
||||
iter = e_module->table;
|
||||
while (count--) {
|
||||
if (addr_conflict(iter, start, end)) {
|
||||
conflict = 1;
|
||||
goto out;
|
||||
}
|
||||
iter++;
|
||||
}
|
||||
}
|
||||
}
|
||||
iter = iter_start;
|
||||
while (iter < iter_stop) {
|
||||
if (addr_conflict(iter, start, end))
|
||||
return 1;
|
||||
iter++;
|
||||
}
|
||||
out:
|
||||
return conflict;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __jump_label_update(struct jump_label_key *key,
|
||||
struct jump_entry *entry, int enable)
|
||||
{
|
||||
for (; entry->key == (jump_label_t)(unsigned long)key; entry++) {
|
||||
/*
|
||||
* entry->code set to 0 invalidates module init text sections
|
||||
* kernel_text_address() verifies we are not in core kernel
|
||||
* init code, see jump_label_invalidate_module_init().
|
||||
*/
|
||||
if (entry->code && kernel_text_address(entry->code))
|
||||
arch_jump_label_transform(entry, enable);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Not all archs need this.
|
||||
*/
|
||||
void __weak arch_jump_label_text_poke_early(jump_label_t addr)
|
||||
{
|
||||
}
|
||||
|
||||
static __init int jump_label_init(void)
|
||||
{
|
||||
struct jump_entry *iter_start = __start___jump_table;
|
||||
struct jump_entry *iter_stop = __stop___jump_table;
|
||||
struct jump_label_key *key = NULL;
|
||||
struct jump_entry *iter;
|
||||
|
||||
jump_label_lock();
|
||||
jump_label_sort_entries(iter_start, iter_stop);
|
||||
|
||||
for (iter = iter_start; iter < iter_stop; iter++) {
|
||||
arch_jump_label_text_poke_early(iter->code);
|
||||
if (iter->key == (jump_label_t)(unsigned long)key)
|
||||
continue;
|
||||
|
||||
key = (struct jump_label_key *)(unsigned long)iter->key;
|
||||
atomic_set(&key->enabled, 0);
|
||||
key->entries = iter;
|
||||
#ifdef CONFIG_MODULES
|
||||
key->next = NULL;
|
||||
#endif
|
||||
}
|
||||
jump_label_unlock();
|
||||
|
||||
return 0;
|
||||
}
|
||||
early_initcall(jump_label_init);
|
||||
|
||||
#ifdef CONFIG_MODULES
|
||||
|
||||
struct jump_label_mod {
|
||||
struct jump_label_mod *next;
|
||||
struct jump_entry *entries;
|
||||
struct module *mod;
|
||||
};
|
||||
|
||||
static int __jump_label_mod_text_reserved(void *start, void *end)
|
||||
{
|
||||
struct module *mod;
|
||||
|
||||
mod = __module_text_address((unsigned long)start);
|
||||
if (!mod)
|
||||
return 0;
|
||||
|
||||
WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
|
||||
|
||||
return __jump_label_text_reserved(mod->jump_entries,
|
||||
mod->jump_entries + mod->num_jump_entries,
|
||||
start, end);
|
||||
}
|
||||
|
||||
static void __jump_label_mod_update(struct jump_label_key *key, int enable)
|
||||
{
|
||||
struct jump_label_mod *mod = key->next;
|
||||
|
||||
while (mod) {
|
||||
__jump_label_update(key, mod->entries, enable);
|
||||
mod = mod->next;
|
||||
}
|
||||
}
|
||||
|
||||
/***
|
||||
* apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
|
||||
* @mod: module to patch
|
||||
*
|
||||
* Allow for run-time selection of the optimal nops. Before the module
|
||||
* loads patch these with arch_get_jump_label_nop(), which is specified by
|
||||
* the arch specific jump label code.
|
||||
*/
|
||||
void jump_label_apply_nops(struct module *mod)
|
||||
{
|
||||
struct jump_entry *iter_start = mod->jump_entries;
|
||||
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
|
||||
struct jump_entry *iter;
|
||||
|
||||
/* if the module doesn't have jump label entries, just return */
|
||||
if (iter_start == iter_stop)
|
||||
return;
|
||||
|
||||
for (iter = iter_start; iter < iter_stop; iter++)
|
||||
arch_jump_label_text_poke_early(iter->code);
|
||||
}
|
||||
|
||||
static int jump_label_add_module(struct module *mod)
|
||||
{
|
||||
struct jump_entry *iter_start = mod->jump_entries;
|
||||
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
|
||||
struct jump_entry *iter;
|
||||
struct jump_label_key *key = NULL;
|
||||
struct jump_label_mod *jlm;
|
||||
|
||||
/* if the module doesn't have jump label entries, just return */
|
||||
if (iter_start == iter_stop)
|
||||
return 0;
|
||||
|
||||
jump_label_sort_entries(iter_start, iter_stop);
|
||||
|
||||
for (iter = iter_start; iter < iter_stop; iter++) {
|
||||
if (iter->key == (jump_label_t)(unsigned long)key)
|
||||
continue;
|
||||
|
||||
key = (struct jump_label_key *)(unsigned long)iter->key;
|
||||
|
||||
if (__module_address(iter->key) == mod) {
|
||||
atomic_set(&key->enabled, 0);
|
||||
key->entries = iter;
|
||||
key->next = NULL;
|
||||
continue;
|
||||
}
|
||||
|
||||
jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL);
|
||||
if (!jlm)
|
||||
return -ENOMEM;
|
||||
|
||||
jlm->mod = mod;
|
||||
jlm->entries = iter;
|
||||
jlm->next = key->next;
|
||||
key->next = jlm;
|
||||
|
||||
if (jump_label_enabled(key))
|
||||
__jump_label_update(key, iter, JUMP_LABEL_ENABLE);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void jump_label_del_module(struct module *mod)
|
||||
{
|
||||
struct jump_entry *iter_start = mod->jump_entries;
|
||||
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
|
||||
struct jump_entry *iter;
|
||||
struct jump_label_key *key = NULL;
|
||||
struct jump_label_mod *jlm, **prev;
|
||||
|
||||
for (iter = iter_start; iter < iter_stop; iter++) {
|
||||
if (iter->key == (jump_label_t)(unsigned long)key)
|
||||
continue;
|
||||
|
||||
key = (struct jump_label_key *)(unsigned long)iter->key;
|
||||
|
||||
if (__module_address(iter->key) == mod)
|
||||
continue;
|
||||
|
||||
prev = &key->next;
|
||||
jlm = key->next;
|
||||
|
||||
while (jlm && jlm->mod != mod) {
|
||||
prev = &jlm->next;
|
||||
jlm = jlm->next;
|
||||
}
|
||||
|
||||
if (jlm) {
|
||||
*prev = jlm->next;
|
||||
kfree(jlm);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void jump_label_invalidate_module_init(struct module *mod)
|
||||
{
|
||||
struct jump_entry *iter_start = mod->jump_entries;
|
||||
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
|
||||
struct jump_entry *iter;
|
||||
|
||||
for (iter = iter_start; iter < iter_stop; iter++) {
|
||||
if (within_module_init(iter->code, mod))
|
||||
iter->code = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
jump_label_module_notify(struct notifier_block *self, unsigned long val,
|
||||
void *data)
|
||||
{
|
||||
struct module *mod = data;
|
||||
int ret = 0;
|
||||
|
||||
switch (val) {
|
||||
case MODULE_STATE_COMING:
|
||||
jump_label_lock();
|
||||
ret = jump_label_add_module(mod);
|
||||
if (ret)
|
||||
jump_label_del_module(mod);
|
||||
jump_label_unlock();
|
||||
break;
|
||||
case MODULE_STATE_GOING:
|
||||
jump_label_lock();
|
||||
jump_label_del_module(mod);
|
||||
jump_label_unlock();
|
||||
break;
|
||||
case MODULE_STATE_LIVE:
|
||||
jump_label_lock();
|
||||
jump_label_invalidate_module_init(mod);
|
||||
jump_label_unlock();
|
||||
break;
|
||||
}
|
||||
|
||||
return notifier_from_errno(ret);
|
||||
}
|
||||
|
||||
struct notifier_block jump_label_module_nb = {
|
||||
.notifier_call = jump_label_module_notify,
|
||||
.priority = 1, /* higher than tracepoints */
|
||||
};
|
||||
|
||||
static __init int jump_label_init_module(void)
|
||||
{
|
||||
return register_module_notifier(&jump_label_module_nb);
|
||||
}
|
||||
early_initcall(jump_label_init_module);
|
||||
|
||||
#endif /* CONFIG_MODULES */
|
||||
|
||||
/***
|
||||
* jump_label_text_reserved - check if addr range is reserved
|
||||
@ -248,237 +353,29 @@ static int module_conflict(void *start, void *end)
|
||||
*/
|
||||
int jump_label_text_reserved(void *start, void *end)
|
||||
{
|
||||
struct jump_entry *iter;
|
||||
struct jump_entry *iter_start = __start___jump_table;
|
||||
struct jump_entry *iter_stop = __start___jump_table;
|
||||
int conflict = 0;
|
||||
int ret = __jump_label_text_reserved(__start___jump_table,
|
||||
__stop___jump_table, start, end);
|
||||
|
||||
iter = iter_start;
|
||||
while (iter < iter_stop) {
|
||||
if (addr_conflict(iter, start, end)) {
|
||||
conflict = 1;
|
||||
goto out;
|
||||
}
|
||||
iter++;
|
||||
}
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
/* now check modules */
|
||||
#ifdef CONFIG_MODULES
|
||||
conflict = module_conflict(start, end);
|
||||
ret = __jump_label_mod_text_reserved(start, end);
|
||||
#endif
|
||||
out:
|
||||
return conflict;
|
||||
}
|
||||
|
||||
/*
|
||||
* Not all archs need this.
|
||||
*/
|
||||
void __weak arch_jump_label_text_poke_early(jump_label_t addr)
|
||||
{
|
||||
}
|
||||
|
||||
static __init int init_jump_label(void)
|
||||
{
|
||||
int ret;
|
||||
struct jump_entry *iter_start = __start___jump_table;
|
||||
struct jump_entry *iter_stop = __stop___jump_table;
|
||||
struct jump_entry *iter;
|
||||
|
||||
jump_label_lock();
|
||||
ret = build_jump_label_hashtable(__start___jump_table,
|
||||
__stop___jump_table);
|
||||
iter = iter_start;
|
||||
while (iter < iter_stop) {
|
||||
arch_jump_label_text_poke_early(iter->code);
|
||||
iter++;
|
||||
}
|
||||
jump_label_unlock();
|
||||
return ret;
|
||||
}
|
||||
early_initcall(init_jump_label);
|
||||
|
||||
#ifdef CONFIG_MODULES
|
||||
|
||||
static struct jump_label_module_entry *
|
||||
add_jump_label_module_entry(struct jump_label_entry *entry,
|
||||
struct jump_entry *iter_begin,
|
||||
int count, struct module *mod)
|
||||
{
|
||||
struct jump_label_module_entry *e;
|
||||
|
||||
e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
|
||||
if (!e)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
e->mod = mod;
|
||||
e->nr_entries = count;
|
||||
e->table = iter_begin;
|
||||
hlist_add_head(&e->hlist, &entry->modules);
|
||||
return e;
|
||||
}
|
||||
|
||||
static int add_jump_label_module(struct module *mod)
|
||||
{
|
||||
struct jump_entry *iter, *iter_begin;
|
||||
struct jump_label_entry *entry;
|
||||
struct jump_label_module_entry *module_entry;
|
||||
int count;
|
||||
|
||||
/* if the module doesn't have jump label entries, just return */
|
||||
if (!mod->num_jump_entries)
|
||||
return 0;
|
||||
|
||||
sort_jump_label_entries(mod->jump_entries,
|
||||
mod->jump_entries + mod->num_jump_entries);
|
||||
iter = mod->jump_entries;
|
||||
while (iter < mod->jump_entries + mod->num_jump_entries) {
|
||||
entry = get_jump_label_entry(iter->key);
|
||||
iter_begin = iter;
|
||||
count = 0;
|
||||
while ((iter < mod->jump_entries + mod->num_jump_entries) &&
|
||||
(iter->key == iter_begin->key)) {
|
||||
iter++;
|
||||
count++;
|
||||
}
|
||||
if (!entry) {
|
||||
entry = add_jump_label_entry(iter_begin->key, 0, NULL);
|
||||
if (IS_ERR(entry))
|
||||
return PTR_ERR(entry);
|
||||
}
|
||||
module_entry = add_jump_label_module_entry(entry, iter_begin,
|
||||
count, mod);
|
||||
if (IS_ERR(module_entry))
|
||||
return PTR_ERR(module_entry);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void remove_jump_label_module(struct module *mod)
|
||||
{
|
||||
struct hlist_head *head;
|
||||
struct hlist_node *node, *node_next, *module_node, *module_node_next;
|
||||
struct jump_label_entry *e;
|
||||
struct jump_label_module_entry *e_module;
|
||||
int i;
|
||||
|
||||
/* if the module doesn't have jump label entries, just return */
|
||||
if (!mod->num_jump_entries)
|
||||
return;
|
||||
|
||||
for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
|
||||
head = &jump_label_table[i];
|
||||
hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
|
||||
hlist_for_each_entry_safe(e_module, module_node,
|
||||
module_node_next,
|
||||
&(e->modules), hlist) {
|
||||
if (e_module->mod == mod) {
|
||||
hlist_del(&e_module->hlist);
|
||||
kfree(e_module);
|
||||
}
|
||||
}
|
||||
if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
|
||||
hlist_del(&e->hlist);
|
||||
kfree(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void remove_jump_label_module_init(struct module *mod)
|
||||
{
|
||||
struct hlist_head *head;
|
||||
struct hlist_node *node, *node_next, *module_node, *module_node_next;
|
||||
struct jump_label_entry *e;
|
||||
struct jump_label_module_entry *e_module;
|
||||
struct jump_entry *iter;
|
||||
int i, count;
|
||||
|
||||
/* if the module doesn't have jump label entries, just return */
|
||||
if (!mod->num_jump_entries)
|
||||
return;
|
||||
|
||||
for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
|
||||
head = &jump_label_table[i];
|
||||
hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
|
||||
hlist_for_each_entry_safe(e_module, module_node,
|
||||
module_node_next,
|
||||
&(e->modules), hlist) {
|
||||
if (e_module->mod != mod)
|
||||
continue;
|
||||
count = e_module->nr_entries;
|
||||
iter = e_module->table;
|
||||
while (count--) {
|
||||
if (within_module_init(iter->code, mod))
|
||||
iter->key = 0;
|
||||
iter++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
jump_label_module_notify(struct notifier_block *self, unsigned long val,
|
||||
void *data)
|
||||
{
|
||||
struct module *mod = data;
|
||||
int ret = 0;
|
||||
|
||||
switch (val) {
|
||||
case MODULE_STATE_COMING:
|
||||
jump_label_lock();
|
||||
ret = add_jump_label_module(mod);
|
||||
if (ret)
|
||||
remove_jump_label_module(mod);
|
||||
jump_label_unlock();
|
||||
break;
|
||||
case MODULE_STATE_GOING:
|
||||
jump_label_lock();
|
||||
remove_jump_label_module(mod);
|
||||
jump_label_unlock();
|
||||
break;
|
||||
case MODULE_STATE_LIVE:
|
||||
jump_label_lock();
|
||||
remove_jump_label_module_init(mod);
|
||||
jump_label_unlock();
|
||||
break;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/***
|
||||
* apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
|
||||
* @mod: module to patch
|
||||
*
|
||||
* Allow for run-time selection of the optimal nops. Before the module
|
||||
* loads patch these with arch_get_jump_label_nop(), which is specified by
|
||||
* the arch specific jump label code.
|
||||
*/
|
||||
void jump_label_apply_nops(struct module *mod)
|
||||
static void jump_label_update(struct jump_label_key *key, int enable)
|
||||
{
|
||||
struct jump_entry *iter;
|
||||
struct jump_entry *entry = key->entries;
|
||||
|
||||
/* if the module doesn't have jump label entries, just return */
|
||||
if (!mod->num_jump_entries)
|
||||
return;
|
||||
/* if there are no users, entry can be NULL */
|
||||
if (entry)
|
||||
__jump_label_update(key, entry, enable);
|
||||
|
||||
iter = mod->jump_entries;
|
||||
while (iter < mod->jump_entries + mod->num_jump_entries) {
|
||||
arch_jump_label_text_poke_early(iter->code);
|
||||
iter++;
|
||||
}
|
||||
#ifdef CONFIG_MODULES
|
||||
__jump_label_mod_update(key, enable);
|
||||
#endif
|
||||
}
|
||||
|
||||
struct notifier_block jump_label_module_nb = {
|
||||
.notifier_call = jump_label_module_notify,
|
||||
.priority = 0,
|
||||
};
|
||||
|
||||
static __init int init_jump_label_module(void)
|
||||
{
|
||||
return register_module_notifier(&jump_label_module_nb);
|
||||
}
|
||||
early_initcall(init_jump_label_module);
|
||||
|
||||
#endif /* CONFIG_MODULES */
|
||||
|
||||
#endif
|
||||
|
@ -125,7 +125,7 @@ enum event_type_t {
|
||||
* perf_sched_events : >0 events exist
|
||||
* perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
|
||||
*/
|
||||
atomic_t perf_sched_events __read_mostly;
|
||||
struct jump_label_key perf_sched_events __read_mostly;
|
||||
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
|
||||
|
||||
static atomic_t nr_mmap_events __read_mostly;
|
||||
@ -5429,7 +5429,7 @@ static int swevent_hlist_get(struct perf_event *event)
|
||||
return err;
|
||||
}
|
||||
|
||||
atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
|
||||
struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
|
||||
|
||||
static void sw_perf_event_destroy(struct perf_event *event)
|
||||
{
|
||||
|
@ -2013,9 +2013,10 @@ enum print_line_t print_trace_line(struct trace_iterator *iter)
|
||||
{
|
||||
enum print_line_t ret;
|
||||
|
||||
if (iter->lost_events)
|
||||
trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
|
||||
iter->cpu, iter->lost_events);
|
||||
if (iter->lost_events &&
|
||||
!trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
|
||||
iter->cpu, iter->lost_events))
|
||||
return TRACE_TYPE_PARTIAL_LINE;
|
||||
|
||||
if (iter->trace && iter->trace->print_line) {
|
||||
ret = iter->trace->print_line(iter);
|
||||
@ -3229,6 +3230,14 @@ tracing_read_pipe(struct file *filp, char __user *ubuf,
|
||||
|
||||
if (iter->seq.len >= cnt)
|
||||
break;
|
||||
|
||||
/*
|
||||
* Setting the full flag means we reached the trace_seq buffer
|
||||
* size and we should leave by partial output condition above.
|
||||
* One of the trace_seq_* functions is not used properly.
|
||||
*/
|
||||
WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
|
||||
iter->ent->type);
|
||||
}
|
||||
trace_access_unlock(iter->cpu_file);
|
||||
trace_event_read_unlock();
|
||||
|
@ -830,6 +830,9 @@ EXPORT_SYMBOL_GPL(unregister_ftrace_event);
|
||||
enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags,
|
||||
struct trace_event *event)
|
||||
{
|
||||
if (!trace_seq_printf(&iter->seq, "type: %d\n", iter->ent->type))
|
||||
return TRACE_TYPE_PARTIAL_LINE;
|
||||
|
||||
return TRACE_TYPE_HANDLED;
|
||||
}
|
||||
|
||||
|
@ -32,7 +32,7 @@ static DEFINE_MUTEX(btrace_mutex);
|
||||
|
||||
struct trace_bprintk_fmt {
|
||||
struct list_head list;
|
||||
char fmt[0];
|
||||
const char *fmt;
|
||||
};
|
||||
|
||||
static inline struct trace_bprintk_fmt *lookup_format(const char *fmt)
|
||||
@ -49,6 +49,7 @@ static
|
||||
void hold_module_trace_bprintk_format(const char **start, const char **end)
|
||||
{
|
||||
const char **iter;
|
||||
char *fmt;
|
||||
|
||||
mutex_lock(&btrace_mutex);
|
||||
for (iter = start; iter < end; iter++) {
|
||||
@ -58,14 +59,18 @@ void hold_module_trace_bprintk_format(const char **start, const char **end)
|
||||
continue;
|
||||
}
|
||||
|
||||
tb_fmt = kmalloc(offsetof(struct trace_bprintk_fmt, fmt)
|
||||
+ strlen(*iter) + 1, GFP_KERNEL);
|
||||
if (tb_fmt) {
|
||||
tb_fmt = kmalloc(sizeof(*tb_fmt), GFP_KERNEL);
|
||||
if (tb_fmt)
|
||||
fmt = kmalloc(strlen(*iter) + 1, GFP_KERNEL);
|
||||
if (tb_fmt && fmt) {
|
||||
list_add_tail(&tb_fmt->list, &trace_bprintk_fmt_list);
|
||||
strcpy(tb_fmt->fmt, *iter);
|
||||
strcpy(fmt, *iter);
|
||||
tb_fmt->fmt = fmt;
|
||||
*iter = tb_fmt->fmt;
|
||||
} else
|
||||
} else {
|
||||
kfree(tb_fmt);
|
||||
*iter = NULL;
|
||||
}
|
||||
}
|
||||
mutex_unlock(&btrace_mutex);
|
||||
}
|
||||
@ -84,6 +89,76 @@ static int module_trace_bprintk_format_notify(struct notifier_block *self,
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* The debugfs/tracing/printk_formats file maps the addresses with
|
||||
* the ASCII formats that are used in the bprintk events in the
|
||||
* buffer. For userspace tools to be able to decode the events from
|
||||
* the buffer, they need to be able to map the address with the format.
|
||||
*
|
||||
* The addresses of the bprintk formats are in their own section
|
||||
* __trace_printk_fmt. But for modules we copy them into a link list.
|
||||
* The code to print the formats and their addresses passes around the
|
||||
* address of the fmt string. If the fmt address passed into the seq
|
||||
* functions is within the kernel core __trace_printk_fmt section, then
|
||||
* it simply uses the next pointer in the list.
|
||||
*
|
||||
* When the fmt pointer is outside the kernel core __trace_printk_fmt
|
||||
* section, then we need to read the link list pointers. The trick is
|
||||
* we pass the address of the string to the seq function just like
|
||||
* we do for the kernel core formats. To get back the structure that
|
||||
* holds the format, we simply use containerof() and then go to the
|
||||
* next format in the list.
|
||||
*/
|
||||
static const char **
|
||||
find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos)
|
||||
{
|
||||
struct trace_bprintk_fmt *mod_fmt;
|
||||
|
||||
if (list_empty(&trace_bprintk_fmt_list))
|
||||
return NULL;
|
||||
|
||||
/*
|
||||
* v will point to the address of the fmt record from t_next
|
||||
* v will be NULL from t_start.
|
||||
* If this is the first pointer or called from start
|
||||
* then we need to walk the list.
|
||||
*/
|
||||
if (!v || start_index == *pos) {
|
||||
struct trace_bprintk_fmt *p;
|
||||
|
||||
/* search the module list */
|
||||
list_for_each_entry(p, &trace_bprintk_fmt_list, list) {
|
||||
if (start_index == *pos)
|
||||
return &p->fmt;
|
||||
start_index++;
|
||||
}
|
||||
/* pos > index */
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* v points to the address of the fmt field in the mod list
|
||||
* structure that holds the module print format.
|
||||
*/
|
||||
mod_fmt = container_of(v, typeof(*mod_fmt), fmt);
|
||||
if (mod_fmt->list.next == &trace_bprintk_fmt_list)
|
||||
return NULL;
|
||||
|
||||
mod_fmt = container_of(mod_fmt->list.next, typeof(*mod_fmt), list);
|
||||
|
||||
return &mod_fmt->fmt;
|
||||
}
|
||||
|
||||
static void format_mod_start(void)
|
||||
{
|
||||
mutex_lock(&btrace_mutex);
|
||||
}
|
||||
|
||||
static void format_mod_stop(void)
|
||||
{
|
||||
mutex_unlock(&btrace_mutex);
|
||||
}
|
||||
|
||||
#else /* !CONFIG_MODULES */
|
||||
__init static int
|
||||
module_trace_bprintk_format_notify(struct notifier_block *self,
|
||||
@ -91,6 +166,13 @@ module_trace_bprintk_format_notify(struct notifier_block *self,
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
static inline const char **
|
||||
find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
static inline void format_mod_start(void) { }
|
||||
static inline void format_mod_stop(void) { }
|
||||
#endif /* CONFIG_MODULES */
|
||||
|
||||
|
||||
@ -153,20 +235,33 @@ int __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap)
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(__ftrace_vprintk);
|
||||
|
||||
static const char **find_next(void *v, loff_t *pos)
|
||||
{
|
||||
const char **fmt = v;
|
||||
int start_index;
|
||||
|
||||
if (!fmt)
|
||||
fmt = __start___trace_bprintk_fmt + *pos;
|
||||
|
||||
start_index = __stop___trace_bprintk_fmt - __start___trace_bprintk_fmt;
|
||||
|
||||
if (*pos < start_index)
|
||||
return fmt;
|
||||
|
||||
return find_next_mod_format(start_index, v, fmt, pos);
|
||||
}
|
||||
|
||||
static void *
|
||||
t_start(struct seq_file *m, loff_t *pos)
|
||||
{
|
||||
const char **fmt = __start___trace_bprintk_fmt + *pos;
|
||||
|
||||
if ((unsigned long)fmt >= (unsigned long)__stop___trace_bprintk_fmt)
|
||||
return NULL;
|
||||
return fmt;
|
||||
format_mod_start();
|
||||
return find_next(NULL, pos);
|
||||
}
|
||||
|
||||
static void *t_next(struct seq_file *m, void * v, loff_t *pos)
|
||||
{
|
||||
(*pos)++;
|
||||
return t_start(m, pos);
|
||||
return find_next(v, pos);
|
||||
}
|
||||
|
||||
static int t_show(struct seq_file *m, void *v)
|
||||
@ -205,6 +300,7 @@ static int t_show(struct seq_file *m, void *v)
|
||||
|
||||
static void t_stop(struct seq_file *m, void *p)
|
||||
{
|
||||
format_mod_stop();
|
||||
}
|
||||
|
||||
static const struct seq_operations show_format_seq_ops = {
|
||||
|
@ -251,9 +251,9 @@ static void set_tracepoint(struct tracepoint_entry **entry,
|
||||
{
|
||||
WARN_ON(strcmp((*entry)->name, elem->name) != 0);
|
||||
|
||||
if (elem->regfunc && !elem->state && active)
|
||||
if (elem->regfunc && !jump_label_enabled(&elem->key) && active)
|
||||
elem->regfunc();
|
||||
else if (elem->unregfunc && elem->state && !active)
|
||||
else if (elem->unregfunc && jump_label_enabled(&elem->key) && !active)
|
||||
elem->unregfunc();
|
||||
|
||||
/*
|
||||
@ -264,13 +264,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
|
||||
* is used.
|
||||
*/
|
||||
rcu_assign_pointer(elem->funcs, (*entry)->funcs);
|
||||
if (!elem->state && active) {
|
||||
jump_label_enable(&elem->state);
|
||||
elem->state = active;
|
||||
} else if (elem->state && !active) {
|
||||
jump_label_disable(&elem->state);
|
||||
elem->state = active;
|
||||
}
|
||||
if (active && !jump_label_enabled(&elem->key))
|
||||
jump_label_inc(&elem->key);
|
||||
else if (!active && jump_label_enabled(&elem->key))
|
||||
jump_label_dec(&elem->key);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -281,13 +278,11 @@ static void set_tracepoint(struct tracepoint_entry **entry,
|
||||
*/
|
||||
static void disable_tracepoint(struct tracepoint *elem)
|
||||
{
|
||||
if (elem->unregfunc && elem->state)
|
||||
if (elem->unregfunc && jump_label_enabled(&elem->key))
|
||||
elem->unregfunc();
|
||||
|
||||
if (elem->state) {
|
||||
jump_label_disable(&elem->state);
|
||||
elem->state = 0;
|
||||
}
|
||||
if (jump_label_enabled(&elem->key))
|
||||
jump_label_dec(&elem->key);
|
||||
rcu_assign_pointer(elem->funcs, NULL);
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user