Lines Matching +full:scan +full:- +full:count
1 // SPDX-License-Identifier: GPL-2.0-only
9 * Documentation/dev-tools/kmemleak.rst.
12 * ----------------
16 * - kmemleak_lock (raw_spinlock_t): protects the object_list as well as
21 * red black trees used to look-up metadata based on a pointer to the
26 * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
27 * Accesses to the metadata (e.g. count) are protected by this lock. Note
33 * - scan_mutex (mutex): ensures that only one thread may scan the memory for
37 * scan_mutex is held. At the end of a scan, the gray_list is always empty.
46 * scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
48 * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
53 * 0, this count can no longer be incremented and put_object() schedules the
111 #define SECS_FIRST_SCAN 60 /* delay before the first scan */
125 #define KMEMLEAK_BLACK -1
130 * object->lock. Insertions or deletions from object_list, gray_list or
132 * the notes on locking above). These objects are reference-counted
142 /* object usage count; object freed when use_count == 0 */
152 int count; member
167 /* flag set to not scan the object */
169 /* flag set to fully scan the object when scan_area allocation failed */
173 /* flag set for per-CPU pointers */
193 /* the list of gray-colored objects (see color_gray comment below) */
225 /* minimum and maximum address that may be valid per-CPU pointers */
290 * with the object->lock held.
295 const u8 *ptr = (const u8 *)object->pointer; in hex_dump_object()
298 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in hex_dump_object()
301 if (object->flags & OBJECT_PERCPU) in hex_dump_object()
302 ptr = (const u8 *)this_cpu_ptr((void __percpu *)object->pointer); in hex_dump_object()
305 len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE); in hex_dump_object()
307 if (object->flags & OBJECT_PERCPU) in hex_dump_object()
319 * Object colors, encoded with count and min_count:
320 * - white - orphan object, not enough references to it (count < min_count)
321 * - gray - not orphan, not marked as false positive (min_count == 0) or
322 * sufficient references to it (count >= min_count)
323 * - black - ignore, it doesn't contain references (e.g. text section)
324 * (min_count == -1). No function defined for this color.
328 return object->count != KMEMLEAK_BLACK && in color_white()
329 object->count < object->min_count; in color_white()
334 return object->min_count != KMEMLEAK_BLACK && in color_gray()
335 object->count >= object->min_count; in color_gray()
345 return (color_white(object) && object->flags & OBJECT_ALLOCATED) && in unreferenced_object()
346 time_before_eq(object->jiffies + jiffies_min_age, in unreferenced_object()
352 if (object->flags & OBJECT_PHYS) in __object_type_str()
354 if (object->flags & OBJECT_PERCPU) in __object_type_str()
361 * print_unreferenced function must be called with the object->lock held.
370 nr_entries = stack_depot_fetch(object->trace_handle, &entries); in print_unreferenced()
373 object->pointer, object->size); in print_unreferenced()
375 object->comm, object->pid, object->jiffies); in print_unreferenced()
377 warn_or_seq_printf(seq, " backtrace (crc %x):\n", object->checksum); in print_unreferenced()
388 * the object->lock held.
393 __object_type_str(object), object->pointer, object->size); in dump_object_info()
395 object->comm, object->pid, object->jiffies); in dump_object_info()
396 pr_notice(" min_count = %d\n", object->min_count); in dump_object_info()
397 pr_notice(" count = %d\n", object->count); in dump_object_info()
398 pr_notice(" flags = 0x%x\n", object->flags); in dump_object_info()
399 pr_notice(" checksum = %u\n", object->checksum); in dump_object_info()
401 if (object->trace_handle) in dump_object_info()
402 stack_depot_print(object->trace_handle); in dump_object_info()
415 * Look-up a memory block metadata (kmemleak_object) in the object search
423 struct rb_node *rb = object_tree(objflags)->rb_node; in __lookup_object()
431 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in __lookup_object()
434 rb = object->rb_node.rb_left; in __lookup_object()
435 else if (untagged_objp + object->size <= untagged_ptr) in __lookup_object()
436 rb = object->rb_node.rb_right; in __lookup_object()
449 /* Look-up a kmemleak object which allocated with virtual address. */
463 return atomic_inc_not_zero(&object->use_count); in get_object()
487 list_del(&object->object_list); in mem_pool_alloc()
489 object = &mem_pool[--mem_pool_free_count]; in mem_pool_alloc()
511 list_add(&object->object_list, &mem_pool_free_list); in mem_pool_free()
529 hlist_for_each_entry_safe(area, tmp, &object->area_list, node) { in free_object_rcu()
530 hlist_del(&area->node); in free_object_rcu()
537 * Decrement the object use_count. Once the count is 0, free the object using
538 * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
540 * recursive call to the kernel allocator. Lock-less RCU object_list traversal
545 if (!atomic_dec_and_test(&object->use_count)) in put_object()
549 WARN_ON(object->flags & OBJECT_ALLOCATED); in put_object()
557 call_rcu(&object->rcu, free_object_rcu); in put_object()
559 free_object_rcu(&object->rcu); in put_object()
596 rb_erase(&object->rb_node, object_tree(object->flags)); in __remove_object()
597 if (!(object->del_state & DELSTATE_NO_DELETE)) in __remove_object()
598 list_del_rcu(&object->object_list); in __remove_object()
599 object->del_state |= DELSTATE_REMOVED; in __remove_object()
663 INIT_LIST_HEAD(&object->object_list); in __alloc_object()
664 INIT_LIST_HEAD(&object->gray_list); in __alloc_object()
665 INIT_HLIST_HEAD(&object->area_list); in __alloc_object()
666 raw_spin_lock_init(&object->lock); in __alloc_object()
667 atomic_set(&object->use_count, 1); in __alloc_object()
668 object->excess_ref = 0; in __alloc_object()
669 object->count = 0; /* white color initially */ in __alloc_object()
670 object->checksum = 0; in __alloc_object()
671 object->del_state = 0; in __alloc_object()
675 object->pid = 0; in __alloc_object()
676 strscpy(object->comm, "hardirq"); in __alloc_object()
678 object->pid = 0; in __alloc_object()
679 strscpy(object->comm, "softirq"); in __alloc_object()
681 object->pid = current->pid; in __alloc_object()
685 * dependency issues with current->alloc_lock. In the worst in __alloc_object()
688 strscpy(object->comm, current->comm); in __alloc_object()
692 object->trace_handle = set_track_prepare(); in __alloc_object()
706 object->flags = OBJECT_ALLOCATED | objflags; in __link_object()
707 object->pointer = ptr; in __link_object()
708 object->size = kfence_ksize((void *)ptr) ?: size; in __link_object()
709 object->min_count = min_count; in __link_object()
710 object->jiffies = jiffies; in __link_object()
715 * address. And update min_percpu_addr max_percpu_addr for per-CPU in __link_object()
725 link = &object_tree(objflags)->rb_node; in __link_object()
730 untagged_objp = (unsigned long)kasan_reset_tag((void *)parent->pointer); in __link_object()
732 link = &parent->rb_node.rb_left; in __link_object()
733 else if (untagged_objp + parent->size <= untagged_ptr) in __link_object()
734 link = &parent->rb_node.rb_right; in __link_object()
739 * No need for parent->lock here since "parent" cannot in __link_object()
743 return -EEXIST; in __link_object()
746 rb_link_node(&object->rb_node, rb_parent, link); in __link_object()
747 rb_insert_color(&object->rb_node, object_tree(objflags)); in __link_object()
748 list_add_tail_rcu(&object->object_list, &object_list); in __link_object()
789 /* Create kmemleak object corresponding to a per-CPU allocation. */
803 WARN_ON(!(object->flags & OBJECT_ALLOCATED)); in __delete_object()
804 WARN_ON(atomic_read(&object->use_count) < 1); in __delete_object()
810 raw_spin_lock_irqsave(&object->lock, flags); in __delete_object()
811 object->flags &= ~OBJECT_ALLOCATED; in __delete_object()
812 raw_spin_unlock_irqrestore(&object->lock, flags); in __delete_object()
869 start = object->pointer; in delete_object_part()
870 end = object->pointer + object->size; in delete_object_part()
872 !__link_object(object_l, start, ptr - start, in delete_object_part()
873 object->min_count, objflags)) in delete_object_part()
876 !__link_object(object_r, ptr + size, end - ptr - size, in delete_object_part()
877 object->min_count, objflags)) in delete_object_part()
894 object->min_count = color; in __paint_it()
896 object->flags |= OBJECT_NO_SCAN; in __paint_it()
903 raw_spin_lock_irqsave(&object->lock, flags); in paint_it()
905 raw_spin_unlock_irqrestore(&object->lock, flags); in paint_it()
925 * Mark an object permanently as gray-colored so that it can no longer be
934 * Mark the object as black-colored so that it is ignored from scans and
944 * be reported as a leak during the next scan until its checksum is updated.
958 raw_spin_lock_irqsave(&object->lock, flags); in reset_checksum()
959 object->checksum = 0; in reset_checksum()
960 raw_spin_unlock_irqrestore(&object->lock, flags); in reset_checksum()
966 * kmemleak will only scan these ranges rather than the whole memory block.
978 kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n", in add_scan_area()
984 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in add_scan_area()
990 raw_spin_lock_irqsave(&object->lock, flags); in add_scan_area()
992 pr_warn_once("Cannot allocate a scan area, scanning the full object\n"); in add_scan_area()
993 /* mark the object for full scan to avoid false positives */ in add_scan_area()
994 object->flags |= OBJECT_FULL_SCAN; in add_scan_area()
998 size = untagged_objp + object->size - untagged_ptr; in add_scan_area()
999 } else if (untagged_ptr + size > untagged_objp + object->size) { in add_scan_area()
1000 kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr); in add_scan_area()
1006 INIT_HLIST_NODE(&area->node); in add_scan_area()
1007 area->start = ptr; in add_scan_area()
1008 area->size = size; in add_scan_area()
1010 hlist_add_head(&area->node, &object->area_list); in add_scan_area()
1012 raw_spin_unlock_irqrestore(&object->lock, flags); in add_scan_area()
1034 raw_spin_lock_irqsave(&object->lock, flags); in object_set_excess_ref()
1035 object->excess_ref = excess_ref; in object_set_excess_ref()
1036 raw_spin_unlock_irqrestore(&object->lock, flags); in object_set_excess_ref()
1056 raw_spin_lock_irqsave(&object->lock, flags); in object_no_scan()
1057 object->flags |= OBJECT_NO_SCAN; in object_no_scan()
1058 raw_spin_unlock_irqrestore(&object->lock, flags); in object_no_scan()
1063 * kmemleak_alloc - register a newly allocated object
1069 * the object is never reported as a leak. If @min_count is -1,
1087 * kmemleak_alloc_percpu - register a newly allocated __percpu object
1106 * kmemleak_vmalloc - register a newly vmalloc'ed object
1123 create_object((unsigned long)area->addr, size, 2, gfp); in kmemleak_vmalloc()
1125 (unsigned long)area->addr); in kmemleak_vmalloc()
1131 * kmemleak_free - unregister a previously registered object
1147 * kmemleak_free_part - partially unregister a previously registered object
1165 * kmemleak_free_percpu - unregister a previously registered __percpu object
1181 * kmemleak_update_trace - update object allocation stack trace
1208 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_update_trace()
1209 object->trace_handle = trace_handle; in kmemleak_update_trace()
1210 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_update_trace()
1217 * kmemleak_not_leak - mark an allocated object as false positive
1233 * kmemleak_transient_leak - mark an allocated object as transient false positive
1238 * is part of a singly linked list and the ->next reference to it is changed.
1250 * kmemleak_ignore - ignore an allocated object
1268 * kmemleak_scan_area - limit the range to be scanned in an allocated object
1270 * represents the start of the scan area
1271 * @size: size of the scan area
1275 * contain references to other objects. Kmemleak will only scan these areas
1288 * kmemleak_no_scan - do not scan an allocated object
1291 * This function notifies kmemleak not to scan the given memory block. Useful
1293 * references to other objects. Kmemleak will not scan such objects reducing
1306 * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1326 * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1342 * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1360 u32 old_csum = object->checksum; in update_checksum()
1362 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in update_checksum()
1367 if (object->flags & OBJECT_PERCPU) { in update_checksum()
1370 object->checksum = 0; in update_checksum()
1372 void *ptr = per_cpu_ptr((void __percpu *)object->pointer, cpu); in update_checksum()
1374 object->checksum ^= crc32(0, kasan_reset_tag((void *)ptr), object->size); in update_checksum()
1377 object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size); in update_checksum()
1382 return object->checksum != old_csum; in update_checksum()
1386 * Update an object's references. object->lock must be held by the caller.
1391 /* non-orphan, ignored or new */ in update_refs()
1396 * Increase the object's reference count (number of pointers to the in update_refs()
1397 * memory block). If this count reaches the required minimum, the in update_refs()
1401 object->count++; in update_refs()
1405 list_add_tail(&object->gray_list, &gray_list); in update_refs()
1427 * object->use_count cannot be dropped to 0 while the object in pointer_update_refs()
1439 * Avoid the lockdep recursive warning on object->lock being in pointer_update_refs()
1443 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in pointer_update_refs()
1446 excess_ref = object->excess_ref; in pointer_update_refs()
1452 raw_spin_unlock(&object->lock); in pointer_update_refs()
1461 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in pointer_update_refs()
1463 raw_spin_unlock(&object->lock); in pointer_update_refs()
1480 if (current->mm) in scan_should_stop()
1489 * Scan a memory block (exclusive range) for valid pointers and add those
1497 unsigned long *end = _end - (BYTES_PER_POINTER - 1); in scan_block()
1518 * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency.
1535 * Scan a memory block corresponding to a kmemleak_object. A condition is
1536 * that object->use_count >= 1.
1544 * Once the object->lock is acquired, the corresponding memory block in scan_object()
1547 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1548 if (object->flags & OBJECT_NO_SCAN) in scan_object()
1550 if (!(object->flags & OBJECT_ALLOCATED)) in scan_object()
1554 if (object->flags & OBJECT_PERCPU) { in scan_object()
1558 void *start = per_cpu_ptr((void __percpu *)object->pointer, cpu); in scan_object()
1559 void *end = start + object->size; in scan_object()
1563 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1565 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1566 if (!(object->flags & OBJECT_ALLOCATED)) in scan_object()
1569 } else if (hlist_empty(&object->area_list) || in scan_object()
1570 object->flags & OBJECT_FULL_SCAN) { in scan_object()
1571 void *start = object->flags & OBJECT_PHYS ? in scan_object()
1572 __va((phys_addr_t)object->pointer) : in scan_object()
1573 (void *)object->pointer; in scan_object()
1574 void *end = start + object->size; in scan_object()
1585 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1587 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1588 } while (object->flags & OBJECT_ALLOCATED); in scan_object()
1590 hlist_for_each_entry(area, &object->area_list, node) in scan_object()
1591 scan_block((void *)area->start, in scan_object()
1592 (void *)(area->start + area->size), in scan_object()
1596 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1600 * Scan the objects already referenced (gray objects). More objects will be
1613 while (&object->gray_list != &gray_list) { in scan_gray_list()
1620 tmp = list_entry(object->gray_list.next, typeof(*object), in scan_gray_list()
1624 list_del(&object->gray_list); in scan_gray_list()
1643 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1645 object->del_state |= DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1653 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1654 list_del_rcu(&object->object_list); in kmemleak_cond_resched()
1655 object->del_state &= ~DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1662 * Scan data sections and all the referenced memory blocks allocated via the
1678 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1684 if (atomic_read(&object->use_count) > 1) { in kmemleak_scan()
1685 pr_debug("object->use_count = %d\n", in kmemleak_scan()
1686 atomic_read(&object->use_count)); in kmemleak_scan()
1692 if ((object->flags & OBJECT_PHYS) && in kmemleak_scan()
1693 !(object->flags & OBJECT_NO_SCAN)) { in kmemleak_scan()
1694 unsigned long phys = object->pointer; in kmemleak_scan()
1697 PHYS_PFN(phys + object->size) > max_low_pfn) in kmemleak_scan()
1701 /* reset the reference count (whiten the object) */ in kmemleak_scan()
1702 object->count = 0; in kmemleak_scan()
1704 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1706 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1714 /* per-cpu sections scanning */ in kmemleak_scan()
1725 unsigned long start_pfn = zone->zone_start_pfn; in kmemleak_scan()
1738 /* only scan pages belonging to this zone */ in kmemleak_scan()
1741 /* only scan if page is in use */ in kmemleak_scan()
1767 * Scan the objects already referenced from the sections scanned in kmemleak_scan()
1774 * scan and color them gray until the next scan. in kmemleak_scan()
1784 * the next scan. in kmemleak_scan()
1788 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1789 if (color_white(object) && (object->flags & OBJECT_ALLOCATED) in kmemleak_scan()
1792 object->count = object->min_count; in kmemleak_scan()
1793 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1795 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1800 * Re-scan the gray list for modified unreferenced objects. in kmemleak_scan()
1821 * the next scan. in kmemleak_scan()
1825 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1827 !(object->flags & OBJECT_REPORTED)) { in kmemleak_scan()
1828 object->flags |= OBJECT_REPORTED; in kmemleak_scan()
1835 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1850 * at the end of a memory scan are reported but only the first time.
1860 * Wait before the first scan to allow the system to fully initialize. in kmemleak_scan_thread()
1876 /* wait before the next scan */ in kmemleak_scan_thread()
1896 pr_warn("Failed to create the scan thread\n"); in start_scan_thread()
1929 if (n-- > 0) in kmemleak_seq_start()
1987 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_seq_show()
1988 if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object)) in kmemleak_seq_show()
1990 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_seq_show()
2015 raw_spin_lock_irqsave(&object->lock, flags); in __dump_str_object_info()
2017 raw_spin_unlock_irqrestore(&object->lock, flags); in __dump_str_object_info()
2030 return -EINVAL; in dump_str_object_info()
2038 return -EINVAL; in dump_str_object_info()
2056 raw_spin_lock_irq(&object->lock); in kmemleak_clear()
2057 if ((object->flags & OBJECT_REPORTED) && in kmemleak_clear()
2060 raw_spin_unlock_irq(&object->lock); in kmemleak_clear()
2070 * File write operation to configure kmemleak at run-time. The following
2072 * off - disable kmemleak (irreversible)
2073 * stack=on - enable the task stacks scanning
2074 * stack=off - disable the tasks stacks scanning
2075 * scan=on - start the automatic memory scanning thread
2076 * scan=off - stop the automatic memory scanning thread
2077 * scan=... - set the automatic memory scanning period in seconds (0 to
2079 * scan - trigger a memory scan
2080 * clear - mark all current reported unreferenced kmemleak objects as
2083 * dump=... - dump information about the object found at the given address
2092 buf_size = min(size, (sizeof(buf) - 1)); in kmemleak_write()
2094 return -EFAULT; in kmemleak_write()
2110 ret = -EPERM; in kmemleak_write()
2120 else if (strncmp(buf, "scan=on", 7) == 0) in kmemleak_write()
2122 else if (strncmp(buf, "scan=off", 8) == 0) in kmemleak_write()
2124 else if (strncmp(buf, "scan=", 5) == 0) { in kmemleak_write()
2141 } else if (strncmp(buf, "scan", 4) == 0) in kmemleak_write()
2146 ret = -EINVAL; in kmemleak_write()
2183 * no previous scan thread (otherwise, kmemleak may still have some useful
2193 * longer track object freeing. Ordering of the scan thread stopping and in kmemleak_do_cleanup()
2231 * Allow boot-time kmemleak disabling (enabled by default).
2236 return -EINVAL; in kmemleak_boot_config()
2244 return -EINVAL; in kmemleak_boot_config()
2271 create_object((unsigned long)_sdata, _edata - _sdata, in kmemleak_init()
2273 create_object((unsigned long)__bss_start, __bss_stop - __bss_start, in kmemleak_init()
2278 __end_ro_after_init - __start_ro_after_init, in kmemleak_init()
2296 * two clean-up threads but serialized by scan_mutex. in kmemleak_late_init()
2299 return -ENOMEM; in kmemleak_late_init()