Lines Matching full:hlock
227 static inline struct lock_class *hlock_class(struct held_lock *hlock) in hlock_class() argument
229 unsigned int class_idx = hlock->class_idx; in hlock_class()
231 /* Don't re-read hlock->class_idx, can't use READ_ONCE() on bitfield */ in hlock_class()
243 * At this point, if the passed hlock->class_idx is still garbage, in hlock_class()
345 static void lock_release_holdtime(struct held_lock *hlock) in lock_release_holdtime() argument
353 holdtime = lockstat_clock() - hlock->holdtime_stamp; in lock_release_holdtime()
355 stats = get_lock_stats(hlock_class(hlock)); in lock_release_holdtime()
356 if (hlock->read) in lock_release_holdtime()
362 static inline void lock_release_holdtime(struct held_lock *hlock) in lock_release_holdtime() argument
429 static inline u16 hlock_id(struct held_lock *hlock) in hlock_id() argument
433 return (hlock->class_idx | (hlock->read << MAX_LOCKDEP_KEYS_BITS)); in hlock_id()
717 static void __print_lock_name(struct held_lock *hlock, struct lock_class *class) in __print_lock_name() argument
732 if (hlock && class->print_fn) in __print_lock_name()
733 class->print_fn(hlock->instance); in __print_lock_name()
737 static void print_lock_name(struct held_lock *hlock, struct lock_class *class) in print_lock_name() argument
744 __print_lock_name(hlock, class); in print_lock_name()
762 static void print_lock(struct held_lock *hlock) in print_lock() argument
766 * extra careful, the hlock might have been released and cleared. in print_lock()
769 * to print the lock unless the hlock class_idx does not point to a in print_lock()
774 struct lock_class *lock = hlock_class(hlock); in print_lock()
781 printk(KERN_CONT "%px", hlock->instance); in print_lock()
782 print_lock_name(hlock, lock); in print_lock()
783 printk(KERN_CONT ", at: %pS\n", (void *)hlock->acquire_ip); in print_lock()
1713 * Initialize a lock_list entry @lock based on a lock acquisition @hlock as the
1716 * ->only_xr of the initial lock node is set to @hlock->read == 2, to make sure
1717 * that <prev> -> @hlock and @hlock -> <whatever __bfs() found> is not -(*R)->
1721 struct held_lock *hlock) in bfs_init_root() argument
1723 __bfs_init_root(lock, hlock_class(hlock)); in bfs_init_root()
1724 lock->only_xr = (hlock->read == 2); in bfs_init_root()
1730 * ->only_xr of the initial lock node is set to @hlock->read != 0, to make sure
1731 * that <next> -> @hlock and @hlock -> <whatever backwards BFS found> is not
1735 struct held_lock *hlock) in bfs_init_rootb() argument
1737 __bfs_init_root(lock, hlock_class(hlock)); in bfs_init_rootb()
1738 lock->only_xr = (hlock->read != 0); in bfs_init_rootb()
2042 struct held_lock *hlock = (struct held_lock *)data; in hlock_conflict() local
2044 return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */ in hlock_conflict()
2045 (hlock->read == 0 || /* B -> A is -(E*)-> */ in hlock_conflict()
2954 struct held_lock *hlock = (struct held_lock *)data; in hlock_equal() local
2956 return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */ in hlock_equal()
2957 (hlock->read == 2 || /* A -> B is -(*R)-> */ in hlock_equal()
3306 struct held_lock *hlock; in check_prevs_add() local
3325 hlock = curr->held_locks + depth - 1; in check_prevs_add()
3327 if (hlock->check) { in check_prevs_add()
3328 int ret = check_prev_add(curr, hlock, next, distance, &trace); in check_prevs_add()
3338 if (!hlock->trylock) in check_prevs_add()
3506 * hlock entries in the primordial chain block is almost used up. That
3615 struct held_lock *hlock) in get_first_held_lock() argument
3622 if (hlock_curr->irq_context != hlock->irq_context) in get_first_held_lock()
3647 struct held_lock *hlock; in print_chain_keys_held_locks() local
3655 hlock = curr->held_locks + i; in print_chain_keys_held_locks()
3656 chain_key = print_chain_key_iteration(hlock_id(hlock), chain_key); in print_chain_keys_held_locks()
3658 print_lock(hlock); in print_chain_keys_held_locks()
3715 struct held_lock *hlock, in check_no_collision() argument
3721 i = get_first_held_lock(curr, hlock); in check_no_collision()
3724 print_collision(curr, hlock, chain); in check_no_collision()
3732 print_collision(curr, hlock, chain); in check_no_collision()
3775 struct held_lock *hlock, in add_chain_cache() argument
3802 chain->irq_context = hlock->irq_context; in add_chain_cache()
3803 i = get_first_held_lock(curr, hlock); in add_chain_cache()
3828 chain_hlocks[chain->base + j] = hlock_id(hlock); in add_chain_cache()
3861 struct held_lock *hlock, in lookup_chain_cache_add() argument
3864 struct lock_class *class = hlock_class(hlock); in lookup_chain_cache_add()
3869 if (!check_no_collision(curr, hlock, chain)) in lookup_chain_cache_add()
3899 if (!add_chain_cache(curr, hlock, chain_key)) in lookup_chain_cache_add()
3906 struct held_lock *hlock, in validate_chain() argument
3919 if (!hlock->trylock && hlock->check && in validate_chain()
3920 lookup_chain_cache_add(curr, hlock, chain_key)) { in validate_chain()
3939 int ret = check_deadlock(curr, hlock); in validate_chain()
3952 if (!check_prevs_add(curr, hlock)) in validate_chain()
3967 struct held_lock *hlock, in validate_chain() argument
3983 struct held_lock *hlock, *prev_hlock = NULL; in check_chain_key() local
3988 hlock = curr->held_locks + i; in check_chain_key()
3989 if (chain_key != hlock->prev_chain_key) { in check_chain_key()
3998 (unsigned long long)hlock->prev_chain_key); in check_chain_key()
4003 * hlock->class_idx can't go beyond MAX_LOCKDEP_KEYS, but is in check_chain_key()
4006 if (DEBUG_LOCKS_WARN_ON(!test_bit(hlock->class_idx, lock_classes_in_use))) in check_chain_key()
4010 hlock->irq_context)) in check_chain_key()
4012 chain_key = iterate_chain_key(chain_key, hlock_id(hlock)); in check_chain_key()
4013 prev_hlock = hlock; in check_chain_key()
4354 struct held_lock *hlock; in mark_held_locks() local
4359 hlock = curr->held_locks + i; in mark_held_locks()
4361 if (hlock->read) in mark_held_locks()
4366 if (!hlock->check) in mark_held_locks()
4369 if (!mark_lock(curr, hlock, hlock_bit)) in mark_held_locks()
4662 mark_usage(struct task_struct *curr, struct held_lock *hlock, int check) in mark_usage() argument
4671 if (!hlock->trylock) { in mark_usage()
4672 if (hlock->read) { in mark_usage()
4674 if (!mark_lock(curr, hlock, in mark_usage()
4678 if (!mark_lock(curr, hlock, in mark_usage()
4683 if (!mark_lock(curr, hlock, LOCK_USED_IN_HARDIRQ)) in mark_usage()
4686 if (!mark_lock(curr, hlock, LOCK_USED_IN_SOFTIRQ)) in mark_usage()
4696 if (!hlock->hardirqs_off && !hlock->sync) { in mark_usage()
4697 if (hlock->read) { in mark_usage()
4698 if (!mark_lock(curr, hlock, in mark_usage()
4702 if (!mark_lock(curr, hlock, in mark_usage()
4706 if (!mark_lock(curr, hlock, in mark_usage()
4710 if (!mark_lock(curr, hlock, in mark_usage()
4718 if (!mark_lock(curr, hlock, LOCK_USED)) in mark_usage()
4731 struct held_lock *hlock) in separate_irq_context() argument
4747 if (prev_hlock->irq_context != hlock->irq_context) in separate_irq_context()
4846 struct held_lock *hlock) in print_lock_invalid_wait_context() argument
4864 print_lock(hlock); in print_lock_invalid_wait_context()
4954 mark_usage(struct task_struct *curr, struct held_lock *hlock, int check) in mark_usage() argument
4965 struct held_lock *hlock) in separate_irq_context() argument
5081 struct held_lock *hlock) in print_lock_nested_lock_not_held() argument
5097 print_lock(hlock); in print_lock_nested_lock_not_held()
5100 pr_warn("%s\n", hlock->nest_lock->name); in print_lock_nested_lock_not_held()
5131 struct held_lock *hlock; in __lock_acquire() local
5195 hlock = curr->held_locks + depth - 1; in __lock_acquire()
5196 if (hlock->class_idx == class_idx && nest_lock) { in __lock_acquire()
5200 if (!hlock->references) in __lock_acquire()
5201 hlock->references++; in __lock_acquire()
5203 hlock->references += references; in __lock_acquire()
5206 if (DEBUG_LOCKS_WARN_ON(hlock->references < references)) in __lock_acquire()
5213 hlock = curr->held_locks + depth; in __lock_acquire()
5220 hlock->class_idx = class_idx; in __lock_acquire()
5221 hlock->acquire_ip = ip; in __lock_acquire()
5222 hlock->instance = lock; in __lock_acquire()
5223 hlock->nest_lock = nest_lock; in __lock_acquire()
5224 hlock->irq_context = task_irq_context(curr); in __lock_acquire()
5225 hlock->trylock = trylock; in __lock_acquire()
5226 hlock->read = read; in __lock_acquire()
5227 hlock->check = check; in __lock_acquire()
5228 hlock->sync = !!sync; in __lock_acquire()
5229 hlock->hardirqs_off = !!hardirqs_off; in __lock_acquire()
5230 hlock->references = references; in __lock_acquire()
5232 hlock->waittime_stamp = 0; in __lock_acquire()
5233 hlock->holdtime_stamp = lockstat_clock(); in __lock_acquire()
5235 hlock->pin_count = pin_count; in __lock_acquire()
5236 hlock->seq_count = seq; in __lock_acquire()
5238 if (check_wait_context(curr, hlock)) in __lock_acquire()
5242 if (!mark_usage(curr, hlock, check)) in __lock_acquire()
5271 hlock->prev_chain_key = chain_key; in __lock_acquire()
5272 if (separate_irq_context(curr, hlock)) { in __lock_acquire()
5276 chain_key = iterate_chain_key(chain_key, hlock_id(hlock)); in __lock_acquire()
5279 print_lock_nested_lock_not_held(curr, hlock); in __lock_acquire()
5284 WARN_ON_ONCE(depth && !hlock_class(hlock - 1)->key); in __lock_acquire()
5285 WARN_ON_ONCE(!hlock_class(hlock)->key); in __lock_acquire()
5288 if (!validate_chain(curr, hlock, chain_head, chain_key)) in __lock_acquire()
5292 if (hlock->sync) in __lock_acquire()
5354 static noinstr int match_held_lock(const struct held_lock *hlock, in match_held_lock() argument
5357 if (hlock->instance == lock) in match_held_lock()
5360 if (hlock->references) { in match_held_lock()
5380 if (DEBUG_LOCKS_WARN_ON(!hlock->nest_lock)) in match_held_lock()
5383 if (hlock->class_idx == class - lock_classes) in match_held_lock()
5395 struct held_lock *ret, *hlock, *prev_hlock; in find_held_lock() local
5399 hlock = curr->held_locks + i; in find_held_lock()
5400 ret = hlock; in find_held_lock()
5401 if (match_held_lock(hlock, lock)) in find_held_lock()
5405 for (i--, prev_hlock = hlock--; in find_held_lock()
5407 i--, prev_hlock = hlock--) { in find_held_lock()
5411 if (prev_hlock->irq_context != hlock->irq_context) { in find_held_lock()
5415 if (match_held_lock(hlock, lock)) { in find_held_lock()
5416 ret = hlock; in find_held_lock()
5429 struct held_lock *hlock; in reacquire_held_locks() local
5435 for (hlock = curr->held_locks + idx; idx < depth; idx++, hlock++) { in reacquire_held_locks()
5436 switch (__lock_acquire(hlock->instance, in reacquire_held_locks()
5437 hlock_class(hlock)->subclass, in reacquire_held_locks()
5438 hlock->trylock, in reacquire_held_locks()
5439 hlock->read, hlock->check, in reacquire_held_locks()
5440 hlock->hardirqs_off, in reacquire_held_locks()
5441 hlock->nest_lock, hlock->acquire_ip, in reacquire_held_locks()
5442 hlock->references, hlock->pin_count, 0, hlock->seq_count)) { in reacquire_held_locks()
5465 struct held_lock *hlock; in __lock_set_class() local
5480 hlock = find_held_lock(curr, lock, depth, &i); in __lock_set_class()
5481 if (!hlock) { in __lock_set_class()
5493 hlock->class_idx = class - lock_classes; in __lock_set_class()
5496 curr->curr_chain_key = hlock->prev_chain_key; in __lock_set_class()
5514 struct held_lock *hlock; in __lock_downgrade() local
5528 hlock = find_held_lock(curr, lock, depth, &i); in __lock_downgrade()
5529 if (!hlock) { in __lock_downgrade()
5535 curr->curr_chain_key = hlock->prev_chain_key; in __lock_downgrade()
5537 WARN(hlock->read, "downgrading a read lock"); in __lock_downgrade()
5538 hlock->read = 1; in __lock_downgrade()
5539 hlock->acquire_ip = ip; in __lock_downgrade()
5568 struct held_lock *hlock; in __lock_release() local
5588 hlock = find_held_lock(curr, lock, depth, &i); in __lock_release()
5589 if (!hlock) { in __lock_release()
5594 if (hlock->instance == lock) in __lock_release()
5595 lock_release_holdtime(hlock); in __lock_release()
5597 WARN(hlock->pin_count, "releasing a pinned lock\n"); in __lock_release()
5599 if (hlock->references) { in __lock_release()
5600 hlock->references--; in __lock_release()
5601 if (hlock->references) { in __lock_release()
5612 * We have the right lock to unlock, 'hlock' points to it. in __lock_release()
5618 curr->curr_chain_key = hlock->prev_chain_key; in __lock_release()
5652 struct held_lock *hlock = curr->held_locks + i; in __lock_is_held() local
5654 if (match_held_lock(hlock, lock)) { in __lock_is_held()
5655 if (read == -1 || !!hlock->read == read) in __lock_is_held()
5675 struct held_lock *hlock = curr->held_locks + i; in __lock_pin_lock() local
5677 if (match_held_lock(hlock, lock)) { in __lock_pin_lock()
5684 hlock->pin_count += cookie.val; in __lock_pin_lock()
5702 struct held_lock *hlock = curr->held_locks + i; in __lock_repin_lock() local
5704 if (match_held_lock(hlock, lock)) { in __lock_repin_lock()
5705 hlock->pin_count += cookie.val; in __lock_repin_lock()
5722 struct held_lock *hlock = curr->held_locks + i; in __lock_unpin_lock() local
5724 if (match_held_lock(hlock, lock)) { in __lock_unpin_lock()
5727 if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n")) in __lock_unpin_lock()
5730 pin_count = hlock->pin_count - cookie.val; in __lock_unpin_lock()
5735 hlock->pin_count = pin_count; in __lock_unpin_lock()
5752 struct held_lock *hlock = curr->held_locks + i; in __lock_sequence() local
5754 if (match_held_lock(hlock, lock)) in __lock_sequence()
5755 return hlock->seq_count; in __lock_sequence()
5844 static void verify_lock_unused(struct lockdep_map *lock, struct held_lock *hlock, int subclass) in verify_lock_unused() argument
5858 if (!hlock->read) in verify_lock_unused()
5864 hlock->class_idx = class - lock_classes; in verify_lock_unused()
5866 print_usage_bug(current, hlock, LOCK_USED, LOCK_USAGE_STATES); in verify_lock_unused()
5921 struct held_lock hlock; in lock_acquire() local
5923 hlock.acquire_ip = ip; in lock_acquire()
5924 hlock.instance = lock; in lock_acquire()
5925 hlock.nest_lock = nest_lock; in lock_acquire()
5926 hlock.irq_context = 2; // XXX in lock_acquire()
5927 hlock.trylock = trylock; in lock_acquire()
5928 hlock.read = read; in lock_acquire()
5929 hlock.check = check; in lock_acquire()
5930 hlock.hardirqs_off = true; in lock_acquire()
5931 hlock.references = 0; in lock_acquire()
5933 verify_lock_unused(lock, &hlock, subclass); in lock_acquire()
6135 struct held_lock *hlock; in __lock_contended() local
6151 hlock = find_held_lock(curr, lock, depth, &i); in __lock_contended()
6152 if (!hlock) { in __lock_contended()
6157 if (hlock->instance != lock) in __lock_contended()
6160 hlock->waittime_stamp = lockstat_clock(); in __lock_contended()
6162 contention_point = lock_point(hlock_class(hlock)->contention_point, ip); in __lock_contended()
6163 contending_point = lock_point(hlock_class(hlock)->contending_point, in __lock_contended()
6166 stats = get_lock_stats(hlock_class(hlock)); in __lock_contended()
6172 stats->bounces[bounce_contended + !!hlock->read]++; in __lock_contended()
6179 struct held_lock *hlock; in __lock_acquired() local
6196 hlock = find_held_lock(curr, lock, depth, &i); in __lock_acquired()
6197 if (!hlock) { in __lock_acquired()
6202 if (hlock->instance != lock) in __lock_acquired()
6206 if (hlock->waittime_stamp) { in __lock_acquired()
6208 waittime = now - hlock->waittime_stamp; in __lock_acquired()
6209 hlock->holdtime_stamp = now; in __lock_acquired()
6212 stats = get_lock_stats(hlock_class(hlock)); in __lock_acquired()
6214 if (hlock->read) in __lock_acquired()
6220 stats->bounces[bounce_acquired + !!hlock->read]++; in __lock_acquired()
6769 const void *mem_to, struct held_lock *hlock) in print_freed_lock_bug() argument
6785 print_lock(hlock); in print_freed_lock_bug()
6809 struct held_lock *hlock; in debug_check_no_locks_freed() local
6818 hlock = curr->held_locks + i; in debug_check_no_locks_freed()
6820 if (not_in_range(mem_from, mem_len, hlock->instance, in debug_check_no_locks_freed()
6821 sizeof(*hlock->instance))) in debug_check_no_locks_freed()
6824 print_freed_lock_bug(curr, mem_from, mem_from + mem_len, hlock); in debug_check_no_locks_freed()