AcoSail/kernel_optimize_test
8
0
Fork 0
forked from luck/tmp_suning_uos_patched
Code Pull Requests Activity

mm/kmemleak: turn kmemleak_lock and object->lock to raw_spinlock_t

Browse Source 
kmemleak_lock as a rwlock on RT can possibly be acquired in atomic
context which does work.

Since the kmemleak operation is performed in atomic context make it a
raw_spinlock_t so it can also be acquired on RT.  This is used for
debugging and is not enabled by default in a production like environment
(where performance/latency matters) so it makes sense to make it a
raw_spinlock_t instead trying to get rid of the atomic context.  Turn
also the kmemleak_object->lock into raw_spinlock_t which is acquired
(nested) while the kmemleak_lock is held.

The time spent in "echo scan > kmemleak" slightly improved on 64core box
with this patch applied after boot.

[bigeasy@linutronix.de: redo the description, update comments. Merge the individual bits:  He Zhe did the kmemleak_lock, Liu Haitao the ->lock and Yongxin Liu forwarded Liu's patch.]
Link: http://lkml.kernel.org/r/20191219170834.4tah3prf2gdothz4@linutronix.de
Link: https://lkml.kernel.org/r/20181218150744.GB20197@arrakis.emea.arm.com
Link: https://lkml.kernel.org/r/1542877459-144382-1-git-send-email-zhe.he@windriver.com
Link: https://lkml.kernel.org/r/20190927082230.34152-1-yongxin.liu@windriver.com
Signed-off-by: He Zhe <zhe.he@windriver.com>
Signed-off-by: Liu Haitao <haitao.liu@windriver.com>
Signed-off-by: Yongxin Liu <yongxin.liu@windriver.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
He Zhe 2020-01-30 22:12:00 -08:00 committed by Linus Torvalds
parent 90e9f6a66c
commit 8c96f1bc6f
1 changed files with 56 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
mm/kmemleak.c
View File

@ -13,7 +13,7 @@
* *
* The following locks and mutexes are used by kmemleak: * The following locks and mutexes are used by kmemleak:
* *
* - kmemleak_lock (rwlock): protects the object_list modifications and * - kmemleak_lock (raw_spinlock_t): protects the object_list modifications and
* accesses to the object_tree_root. The object_list is the main list * accesses to the object_tree_root. The object_list is the main list
* holding the metadata (struct kmemleak_object) for the allocated memory * holding the metadata (struct kmemleak_object) for the allocated memory
* blocks. The object_tree_root is a red black tree used to look-up * blocks. The object_tree_root is a red black tree used to look-up
@ -22,13 +22,13 @@
* object_tree_root in the create_object() function called from the * object_tree_root in the create_object() function called from the
* kmemleak_alloc() callback and removed in delete_object() called from the * kmemleak_alloc() callback and removed in delete_object() called from the
* kmemleak_free() callback * kmemleak_free() callback
* - kmemleak_object.lock (spinlock): protects a kmemleak_object. Accesses to * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
* the metadata (e.g. count) are protected by this lock. Note that some * Accesses to the metadata (e.g. count) are protected by this lock. Note
* members of this structure may be protected by other means (atomic or * that some members of this structure may be protected by other means
* kmemleak_lock). This lock is also held when scanning the corresponding * (atomic or kmemleak_lock). This lock is also held when scanning the
* memory block to avoid the kernel freeing it via the kmemleak_free() * corresponding memory block to avoid the kernel freeing it via the
* callback. This is less heavyweight than holding a global lock like * kmemleak_free() callback. This is less heavyweight than holding a global
* kmemleak_lock during scanning * lock like kmemleak_lock during scanning.
* - scan_mutex (mutex): ensures that only one thread may scan the memory for * - scan_mutex (mutex): ensures that only one thread may scan the memory for
* unreferenced objects at a time. The gray_list contains the objects which * unreferenced objects at a time. The gray_list contains the objects which
* are already referenced or marked as false positives and need to be * are already referenced or marked as false positives and need to be
@ -135,7 +135,7 @@ struct kmemleak_scan_area {
* (use_count) and freed using the RCU mechanism. * (use_count) and freed using the RCU mechanism.
*/ */
struct kmemleak_object { struct kmemleak_object {
spinlock_t lock; raw_spinlock_t lock;
unsigned int flags; /* object status flags */ unsigned int flags; /* object status flags */
struct list_head object_list; struct list_head object_list;
struct list_head gray_list; struct list_head gray_list;
@ -191,8 +191,8 @@ static int mem_pool_free_count = ARRAY_SIZE(mem_pool);
static LIST_HEAD(mem_pool_free_list); static LIST_HEAD(mem_pool_free_list);
/* search tree for object boundaries */ /* search tree for object boundaries */
static struct rb_root object_tree_root = RB_ROOT; static struct rb_root object_tree_root = RB_ROOT;
/* rw_lock protecting the access to object_list and object_tree_root */ /* protecting the access to object_list and object_tree_root */
static DEFINE_RWLOCK(kmemleak_lock); static DEFINE_RAW_SPINLOCK(kmemleak_lock);
/* allocation caches for kmemleak internal data */ /* allocation caches for kmemleak internal data */
static struct kmem_cache *object_cache; static struct kmem_cache *object_cache;
@ -426,7 +426,7 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
} }
/* slab allocation failed, try the memory pool */ /* slab allocation failed, try the memory pool */
write_lock_irqsave(&kmemleak_lock, flags); raw_spin_lock_irqsave(&kmemleak_lock, flags);
object = list_first_entry_or_null(&mem_pool_free_list, object = list_first_entry_or_null(&mem_pool_free_list,
typeof(*object), object_list); typeof(*object), object_list);
if (object) if (object)
@ -435,7 +435,7 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
object = &mem_pool[--mem_pool_free_count]; object = &mem_pool[--mem_pool_free_count];
else else
pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n"); pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");
write_unlock_irqrestore(&kmemleak_lock, flags); raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
return object; return object;
} }
@ -453,9 +453,9 @@ static void mem_pool_free(struct kmemleak_object *object)
} }
/* add the object to the memory pool free list */ /* add the object to the memory pool free list */
write_lock_irqsave(&kmemleak_lock, flags); raw_spin_lock_irqsave(&kmemleak_lock, flags);
list_add(&object->object_list, &mem_pool_free_list); list_add(&object->object_list, &mem_pool_free_list);
write_unlock_irqrestore(&kmemleak_lock, flags); raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
} }
/* /*
@ -514,9 +514,9 @@ static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
struct kmemleak_object *object; struct kmemleak_object *object;
rcu_read_lock(); rcu_read_lock();
read_lock_irqsave(&kmemleak_lock, flags); raw_spin_lock_irqsave(&kmemleak_lock, flags);
object = lookup_object(ptr, alias); object = lookup_object(ptr, alias);
read_unlock_irqrestore(&kmemleak_lock, flags); raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
/* check whether the object is still available */ /* check whether the object is still available */
if (object && !get_object(object)) if (object && !get_object(object))
@ -546,11 +546,11 @@ static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int ali
unsigned long flags; unsigned long flags;
struct kmemleak_object *object; struct kmemleak_object *object;
write_lock_irqsave(&kmemleak_lock, flags); raw_spin_lock_irqsave(&kmemleak_lock, flags);
object = lookup_object(ptr, alias); object = lookup_object(ptr, alias);
if (object) if (object)
__remove_object(object); __remove_object(object);
write_unlock_irqrestore(&kmemleak_lock, flags); raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
return object; return object;
} }
@ -585,7 +585,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
INIT_LIST_HEAD(&object->object_list); INIT_LIST_HEAD(&object->object_list);
INIT_LIST_HEAD(&object->gray_list); INIT_LIST_HEAD(&object->gray_list);
INIT_HLIST_HEAD(&object->area_list); INIT_HLIST_HEAD(&object->area_list);
spin_lock_init(&object->lock); raw_spin_lock_init(&object->lock);
atomic_set(&object->use_count, 1); atomic_set(&object->use_count, 1);
object->flags = OBJECT_ALLOCATED; object->flags = OBJECT_ALLOCATED;
object->pointer = ptr; object->pointer = ptr;
@ -617,7 +617,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
/* kernel backtrace */ /* kernel backtrace */
object->trace_len = __save_stack_trace(object->trace); object->trace_len = __save_stack_trace(object->trace);
write_lock_irqsave(&kmemleak_lock, flags); raw_spin_lock_irqsave(&kmemleak_lock, flags);
untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr); untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
min_addr = min(min_addr, untagged_ptr); min_addr = min(min_addr, untagged_ptr);
@ -649,7 +649,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
list_add_tail_rcu(&object->object_list, &object_list); list_add_tail_rcu(&object->object_list, &object_list);
out: out:
write_unlock_irqrestore(&kmemleak_lock, flags); raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
return object; return object;
} }
@ -667,9 +667,9 @@ static void __delete_object(struct kmemleak_object *object)
* Locking here also ensures that the corresponding memory block * Locking here also ensures that the corresponding memory block
* cannot be freed when it is being scanned. * cannot be freed when it is being scanned.
*/ */
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
object->flags &= ~OBJECT_ALLOCATED; object->flags &= ~OBJECT_ALLOCATED;
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
put_object(object); put_object(object);
} }
@ -739,9 +739,9 @@ static void paint_it(struct kmemleak_object *object, int color)
{ {
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
__paint_it(object, color); __paint_it(object, color);
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
} }
static void paint_ptr(unsigned long ptr, int color) static void paint_ptr(unsigned long ptr, int color)
@ -798,7 +798,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
if (scan_area_cache) if (scan_area_cache)
area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp)); area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
if (!area) { if (!area) {
pr_warn_once("Cannot allocate a scan area, scanning the full object\n"); pr_warn_once("Cannot allocate a scan area, scanning the full object\n");
/* mark the object for full scan to avoid false positives */ /* mark the object for full scan to avoid false positives */
@ -820,7 +820,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
hlist_add_head(&area->node, &object->area_list); hlist_add_head(&area->node, &object->area_list);
out_unlock: out_unlock:
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
put_object(object); put_object(object);
} }
@ -842,9 +842,9 @@ static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
return; return;
} }
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
object->excess_ref = excess_ref; object->excess_ref = excess_ref;
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
put_object(object); put_object(object);
} }
@ -864,9 +864,9 @@ static void object_no_scan(unsigned long ptr)
return; return;
} }
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
object->flags |= OBJECT_NO_SCAN; object->flags |= OBJECT_NO_SCAN;
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
put_object(object); put_object(object);
} }
@ -1026,9 +1026,9 @@ void __ref kmemleak_update_trace(const void *ptr)
return; return;
} }
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
object->trace_len = __save_stack_trace(object->trace); object->trace_len = __save_stack_trace(object->trace);
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
put_object(object); put_object(object);
} }
@ -1233,7 +1233,7 @@ static void scan_block(void *_start, void *_end,
unsigned long flags; unsigned long flags;
unsigned long untagged_ptr; unsigned long untagged_ptr;
read_lock_irqsave(&kmemleak_lock, flags); raw_spin_lock_irqsave(&kmemleak_lock, flags);
for (ptr = start; ptr < end; ptr++) { for (ptr = start; ptr < end; ptr++) {
struct kmemleak_object *object; struct kmemleak_object *object;
unsigned long pointer; unsigned long pointer;
@ -1268,7 +1268,7 @@ static void scan_block(void *_start, void *_end,
* previously acquired in scan_object(). These locks are * previously acquired in scan_object(). These locks are
* enclosed by scan_mutex. * enclosed by scan_mutex.
*/ */
spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
/* only pass surplus references (object already gray) */ /* only pass surplus references (object already gray) */
if (color_gray(object)) { if (color_gray(object)) {
excess_ref = object->excess_ref; excess_ref = object->excess_ref;
@ -1277,7 +1277,7 @@ static void scan_block(void *_start, void *_end,
excess_ref = 0; excess_ref = 0;
update_refs(object); update_refs(object);
} }
spin_unlock(&object->lock); raw_spin_unlock(&object->lock);
if (excess_ref) { if (excess_ref) {
object = lookup_object(excess_ref, 0); object = lookup_object(excess_ref, 0);
@ -1286,12 +1286,12 @@ static void scan_block(void *_start, void *_end,
if (object == scanned) if (object == scanned)
/* circular reference, ignore */ /* circular reference, ignore */
continue; continue;
spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
update_refs(object); update_refs(object);
spin_unlock(&object->lock); raw_spin_unlock(&object->lock);
} }
} }
read_unlock_irqrestore(&kmemleak_lock, flags); raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
} }
/* /*
@ -1324,7 +1324,7 @@ static void scan_object(struct kmemleak_object *object)
* Once the object->lock is acquired, the corresponding memory block * Once the object->lock is acquired, the corresponding memory block
* cannot be freed (the same lock is acquired in delete_object). * cannot be freed (the same lock is acquired in delete_object).
*/ */
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
if (object->flags & OBJECT_NO_SCAN) if (object->flags & OBJECT_NO_SCAN)
goto out; goto out;
if (!(object->flags & OBJECT_ALLOCATED)) if (!(object->flags & OBJECT_ALLOCATED))
@ -1344,9 +1344,9 @@ static void scan_object(struct kmemleak_object *object)
if (start >= end) if (start >= end)
break; break;
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
cond_resched(); cond_resched();
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
} while (object->flags & OBJECT_ALLOCATED); } while (object->flags & OBJECT_ALLOCATED);
} else } else
hlist_for_each_entry(area, &object->area_list, node) hlist_for_each_entry(area, &object->area_list, node)
@ -1354,7 +1354,7 @@ static void scan_object(struct kmemleak_object *object)
(void *)(area->start + area->size), (void *)(area->start + area->size),
object); object);
out: out:
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
} }
/* /*
@ -1407,7 +1407,7 @@ static void kmemleak_scan(void)
/* prepare the kmemleak_object's */ /* prepare the kmemleak_object's */
rcu_read_lock(); rcu_read_lock();
list_for_each_entry_rcu(object, &object_list, object_list) { list_for_each_entry_rcu(object, &object_list, object_list) {
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
#ifdef DEBUG #ifdef DEBUG
/* /*
* With a few exceptions there should be a maximum of * With a few exceptions there should be a maximum of
@ -1424,7 +1424,7 @@ static void kmemleak_scan(void)
if (color_gray(object) && get_object(object)) if (color_gray(object) && get_object(object))
list_add_tail(&object->gray_list, &gray_list); list_add_tail(&object->gray_list, &gray_list);
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
} }
rcu_read_unlock(); rcu_read_unlock();
@ -1492,14 +1492,14 @@ static void kmemleak_scan(void)
*/ */
rcu_read_lock(); rcu_read_lock();
list_for_each_entry_rcu(object, &object_list, object_list) { list_for_each_entry_rcu(object, &object_list, object_list) {
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
if (color_white(object) && (object->flags & OBJECT_ALLOCATED) if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
&& update_checksum(object) && get_object(object)) { && update_checksum(object) && get_object(object)) {
/* color it gray temporarily */ /* color it gray temporarily */
object->count = object->min_count; object->count = object->min_count;
list_add_tail(&object->gray_list, &gray_list); list_add_tail(&object->gray_list, &gray_list);
} }
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
} }
rcu_read_unlock(); rcu_read_unlock();
@ -1519,7 +1519,7 @@ static void kmemleak_scan(void)
*/ */
rcu_read_lock(); rcu_read_lock();
list_for_each_entry_rcu(object, &object_list, object_list) { list_for_each_entry_rcu(object, &object_list, object_list) {
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
if (unreferenced_object(object) && if (unreferenced_object(object) &&
!(object->flags & OBJECT_REPORTED)) { !(object->flags & OBJECT_REPORTED)) {
object->flags |= OBJECT_REPORTED; object->flags |= OBJECT_REPORTED;
@ -1529,7 +1529,7 @@ static void kmemleak_scan(void)
new_leaks++; new_leaks++;
} }
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
} }
rcu_read_unlock(); rcu_read_unlock();
@ -1681,10 +1681,10 @@ static int kmemleak_seq_show(struct seq_file *seq, void *v)
struct kmemleak_object *object = v; struct kmemleak_object *object = v;
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object)) if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
print_unreferenced(seq, object); print_unreferenced(seq, object);
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
return 0; return 0;
} }
@ -1714,9 +1714,9 @@ static int dump_str_object_info(const char *str)
return -EINVAL; return -EINVAL;
} }
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
dump_object_info(object); dump_object_info(object);
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
put_object(object); put_object(object);
return 0; return 0;
@ -1735,11 +1735,11 @@ static void kmemleak_clear(void)
rcu_read_lock(); rcu_read_lock();
list_for_each_entry_rcu(object, &object_list, object_list) { list_for_each_entry_rcu(object, &object_list, object_list) {
spin_lock_irqsave(&object->lock, flags); raw_spin_lock_irqsave(&object->lock, flags);
if ((object->flags & OBJECT_REPORTED) && if ((object->flags & OBJECT_REPORTED) &&
unreferenced_object(object)) unreferenced_object(object))
__paint_it(object, KMEMLEAK_GREY); __paint_it(object, KMEMLEAK_GREY);
spin_unlock_irqrestore(&object->lock, flags); raw_spin_unlock_irqrestore(&object->lock, flags);
} }
rcu_read_unlock(); rcu_read_unlock();