300 void lock_stats(struct lock_class *class, struct lock_class_stats *stats)  in lock_stats()  argument
307 			&per_cpu(cpu_lock_stats, cpu)[class - lock_classes];  in lock_stats()
326 void clear_lock_stats(struct lock_class *class)  in clear_lock_stats()  argument
332 			&per_cpu(cpu_lock_stats, cpu)[class - lock_classes];  in clear_lock_stats()
336 	memset(class->contention_point, 0, sizeof(class->contention_point));  in clear_lock_stats()
337 	memset(class->contending_point, 0, sizeof(class->contending_point));  in clear_lock_stats()
340 static struct lock_class_stats *get_lock_stats(struct lock_class *class)  in get_lock_stats()  argument
342 	return &this_cpu_ptr(cpu_lock_stats)[class - lock_classes];  in get_lock_stats()
498 static int class_filter(struct lock_class *class)  in class_filter()  argument
502 	if (class->name_version == 1 &&  in class_filter()
503 			!strcmp(class->name, "lockname"))  in class_filter()
505 	if (class->name_version == 1 &&  in class_filter()
506 			!strcmp(class->name, "&struct->lockfield"))  in class_filter()
514 static int verbose(struct lock_class *class)  in verbose()  argument
517 	return class_filter(class);  in verbose()
678 static char get_usage_char(struct lock_class *class, enum lock_usage_bit bit)  in get_usage_char()  argument
694 	if (class->usage_mask & lock_flag(bit + LOCK_USAGE_DIR_MASK)) {  in get_usage_char()
696 		if (class->usage_mask & lock_flag(bit))  in get_usage_char()
698 	} else if (class->usage_mask & lock_flag(bit))  in get_usage_char()
704 void get_usage_chars(struct lock_class *class, char usage[LOCK_USAGE_CHARS])  in get_usage_chars()  argument
709 	usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE);	\  in get_usage_chars()
710 	usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE##_READ);  in get_usage_chars()
717 static void __print_lock_name(struct held_lock *hlock, struct lock_class *class)  in __print_lock_name()  argument
722 	name = class->name;  in __print_lock_name()
724 		name = __get_key_name(class->key, str);  in __print_lock_name()
728 		if (class->name_version > 1)  in __print_lock_name()
729 			printk(KERN_CONT "#%d", class->name_version);  in __print_lock_name()
730 		if (class->subclass)  in __print_lock_name()
731 			printk(KERN_CONT "/%d", class->subclass);  in __print_lock_name()
732 		if (hlock && class->print_fn)  in __print_lock_name()
733 			class->print_fn(hlock->instance);  in __print_lock_name()
737 static void print_lock_name(struct held_lock *hlock, struct lock_class *class)  in print_lock_name()  argument
741 	get_usage_chars(class, usage);  in print_lock_name()
744 	__print_lock_name(hlock, class);  in print_lock_name()
746 			class->wait_type_outer ?: class->wait_type_inner,  in print_lock_name()
747 			class->wait_type_inner);  in print_lock_name()
815 static int very_verbose(struct lock_class *class)  in very_verbose()  argument
818 	return class_filter(class);  in very_verbose()
869 	struct lock_class *class;  in count_matching_names()  local
875 	list_for_each_entry(class, &all_lock_classes, lock_entry) {  in count_matching_names()
876 		if (new_class->key - new_class->subclass == class->key)  in count_matching_names()
877 			return class->name_version;  in count_matching_names()
878 		if (class->name && !strcmp(class->name, new_class->name))  in count_matching_names()
879 			count = max(count, class->name_version);  in count_matching_names()
891 	struct lock_class *class;  in look_up_lock_class()  local
933 	hlist_for_each_entry_rcu_notrace(class, hash_head, hash_entry) {  in look_up_lock_class()
934 		if (class->key == key) {  in look_up_lock_class()
939 			WARN_ONCE(class->name != lock->name &&  in look_up_lock_class()
942 				  lock->name, lock->key, class->name);  in look_up_lock_class()
943 			return class;  in look_up_lock_class()
1013 	struct lock_class *class;  in in_any_class_list()  local
1017 		class = &lock_classes[i];  in in_any_class_list()
1018 		if (in_list(e, &class->locks_after) ||  in in_any_class_list()
1019 		    in_list(e, &class->locks_before))  in in_any_class_list()
1036 			       e->class && e->class->name ? e->class->name :  in class_lock_list_valid()
1071 static bool in_any_zapped_class_list(struct lock_class *class)  in in_any_zapped_class_list()  argument
1077 		if (in_list(&class->lock_entry, &pf->zapped))  in in_any_zapped_class_list()
1086 	struct lock_class *class;  in __check_data_structures()  local
1094 		class = &lock_classes[i];  in __check_data_structures()
1095 		if (!in_list(&class->lock_entry, &all_lock_classes) &&  in __check_data_structures()
1096 		    !in_list(&class->lock_entry, &free_lock_classes) &&  in __check_data_structures()
1097 		    !in_any_zapped_class_list(class)) {  in __check_data_structures()
1099 			       class, class->name ? : "(?)");  in __check_data_structures()
1106 		class = &lock_classes[i];  in __check_data_structures()
1107 		if (!class_lock_list_valid(class, &class->locks_before))  in __check_data_structures()
1109 		if (!class_lock_list_valid(class, &class->locks_after))  in __check_data_structures()
1131 			       e->class->name ? : "(?)",  in __check_data_structures()
1146 			       e->class && e->class->name ? e->class->name :  in __check_data_structures()
1289 	struct lock_class *class;  in register_lock_class()  local
1294 	class = look_up_lock_class(lock, subclass);  in register_lock_class()
1295 	if (likely(class))  in register_lock_class()
1315 	hlist_for_each_entry_rcu(class, hash_head, hash_entry) {  in register_lock_class()
1316 		if (class->key == key)  in register_lock_class()
1323 	class = list_first_entry_or_null(&free_lock_classes, typeof(*class),  in register_lock_class()
1325 	if (!class) {  in register_lock_class()
1337 	__set_bit(class - lock_classes, lock_classes_in_use);  in register_lock_class()
1339 	class->key = key;  in register_lock_class()
1340 	class->name = lock->name;  in register_lock_class()
1341 	class->subclass = subclass;  in register_lock_class()
1342 	WARN_ON_ONCE(!list_empty(&class->locks_before));  in register_lock_class()
1343 	WARN_ON_ONCE(!list_empty(&class->locks_after));  in register_lock_class()
1344 	class->name_version = count_matching_names(class);  in register_lock_class()
1345 	class->wait_type_inner = lock->wait_type_inner;  in register_lock_class()
1346 	class->wait_type_outer = lock->wait_type_outer;  in register_lock_class()
1347 	class->lock_type = lock->lock_type;  in register_lock_class()
1352 	hlist_add_head_rcu(&class->hash_entry, hash_head);  in register_lock_class()
1357 	list_move_tail(&class->lock_entry, &all_lock_classes);  in register_lock_class()
1358 	idx = class - lock_classes;  in register_lock_class()
1362 	if (verbose(class)) {  in register_lock_class()
1366 		printk("\nnew class %px: %s", class->key, class->name);  in register_lock_class()
1367 		if (class->name_version > 1)  in register_lock_class()
1368 			printk(KERN_CONT "#%d", class->name_version);  in register_lock_class()
1382 		lock->class_cache[0] = class;  in register_lock_class()
1384 		lock->class_cache[subclass] = class;  in register_lock_class()
1390 	if (DEBUG_LOCKS_WARN_ON(class->subclass != subclass))  in register_lock_class()
1393 	return class;  in register_lock_class()
1438 	entry->class = this;  in add_lock_to_list()
1530 	lock->class->dep_gen_id = lockdep_dependency_gen_id;  in mark_lock_accessed()
1541 	return lock->class->dep_gen_id == lockdep_dependency_gen_id;  in lock_accessed()
1570 	void *lock_class = lock->class;  in get_dep_list()
1661 				   struct lock_class *class)  in __bfs_init_root()  argument
1663 	lock->class = class;  in __bfs_init_root()
1753 		if (!lock->class)  in __bfs()
1881 	print_lock_name(NULL, target->class);  in print_circular_bug_entry()
1893 	struct lock_class *parent = prt->class;  in print_circular_lock_scenario()
2000 	return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */  in hlock_conflict()
2077 unsigned long lockdep_count_forward_deps(struct lock_class *class)  in lockdep_count_forward_deps()  argument
2082 	__bfs_init_root(&this, class);  in lockdep_count_forward_deps()
2103 unsigned long lockdep_count_backward_deps(struct lock_class *class)  in lockdep_count_backward_deps()  argument
2108 	__bfs_init_root(&this, class);  in lockdep_count_backward_deps()
2227 		*(unsigned long *)mask |= entry->class->usage_mask;  in usage_accumulate()
2229 		*(unsigned long *)mask |= (entry->class->usage_mask & LOCKF_IRQ);  in usage_accumulate()
2246 		return !!(entry->class->usage_mask & *(unsigned long *)mask);  in usage_match()
2248 		return !!((entry->class->usage_mask & LOCKF_IRQ) & *(unsigned long *)mask);  in usage_match()
2253 	if (entry->class->lock_type == LD_LOCK_NORMAL)  in usage_skip()
2282 	if (entry->class->lock_type == LD_LOCK_PERCPU &&  in usage_skip()
2283 	    DEBUG_LOCKS_WARN_ON(entry->class->wait_type_inner < LD_WAIT_CONFIG))  in usage_skip()
2331 static void print_lock_class_header(struct lock_class *class, int depth)  in print_lock_class_header()  argument
2336 	print_lock_name(NULL, class);  in print_lock_class_header()
2338 	printk(KERN_CONT " ops: %lu", debug_class_ops_read(class));  in print_lock_class_header()
2343 		if (class->usage_mask & (1 << bit)) {  in print_lock_class_header()
2348 			print_lock_trace(class->usage_traces[bit], len);  in print_lock_class_header()
2354 		depth, "", class->key, class->key);  in print_lock_class_header()
2420 		print_lock_class_header(entry->class, depth);  in print_shortest_lock_dependencies()
2467 		print_lock_class_header(entry->class, depth);  in print_shortest_lock_dependencies_backwards()
2496 	struct lock_class *safe_class = safe_entry->class;  in print_irq_lock_scenario()
2497 	struct lock_class *unsafe_class = unsafe_entry->class;  in print_irq_lock_scenario()
2587 	print_lock_name(NULL, backwards_entry->class);  in print_bad_irq_dependency()
2590 	print_lock_trace(backwards_entry->class->usage_traces[bit1], 1);  in print_bad_irq_dependency()
2593 	print_lock_name(NULL, forwards_entry->class);  in print_bad_irq_dependency()
2597 	print_lock_trace(forwards_entry->class->usage_traces[bit2], 1);  in print_bad_irq_dependency()
2837 	backward_mask = original_mask(target_entry1->class->usage_mask & LOCKF_ENABLED_IRQ_ALL);  in check_irq_usage()
2851 	ret = find_exclusive_match(target_entry->class->usage_mask,  in check_irq_usage()
2852 				   target_entry1->class->usage_mask,  in check_irq_usage()
2912 	return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */  in hlock_equal()
3013 	struct lock_class *class = hlock_class(prev);  in print_deadlock_bug()  local
3031 	if (class->cmp_fn) {  in print_deadlock_bug()
3033 			class->cmp_fn(prev->instance, next->instance));  in print_deadlock_bug()
3059 	struct lock_class *class;  in check_deadlock()  local
3080 		class = hlock_class(prev);  in check_deadlock()
3082 		if (class->cmp_fn &&  in check_deadlock()
3083 		    class->cmp_fn(prev->instance, next->instance) < 0)  in check_deadlock()
3148 		struct lock_class *class = hlock_class(prev);  in check_prev_add()  local
3150 		if (class->cmp_fn &&  in check_prev_add()
3151 		    class->cmp_fn(prev->instance, next->instance) < 0)  in check_prev_add()
3181 		if (entry->class == hlock_class(next)) {  in check_prev_add()
3203 				if (entry->class == hlock_class(prev)) {  in check_prev_add()
3820 	struct lock_class *class = hlock_class(hlock);  in lookup_chain_cache_add()  local
3828 		if (very_verbose(class)) {  in lookup_chain_cache_add()
3832 					class->key, class->name);  in lookup_chain_cache_add()
3838 	if (very_verbose(class)) {  in lookup_chain_cache_add()
3840 			(unsigned long long)chain_key, class->key, class->name);  in lookup_chain_cache_add()
3991 	struct lock_class *class = hlock_class(lock);  in print_usage_bug_scenario()  local
3997 	__print_lock_name(lock, class);  in print_usage_bug_scenario()
4001 	__print_lock_name(lock, class);  in print_usage_bug_scenario()
4093 	print_lock_name(NULL, other->class);  in print_irq_inversion_bug()
4111 			middle ? middle->class : root->class, other->class);  in print_irq_inversion_bug()
4114 			middle ? middle->class : other->class, root->class);  in print_irq_inversion_bug()
4154 	if (target_entry->class->usage_mask & lock_flag(bit)) {  in check_usage_forwards()
4189 	if (target_entry->class->usage_mask & lock_flag(bit)) {  in check_usage_backwards()
4223 static int HARDIRQ_verbose(struct lock_class *class)  in HARDIRQ_verbose()  argument
4226 	return class_filter(class);  in HARDIRQ_verbose()
4231 static int SOFTIRQ_verbose(struct lock_class *class)  in SOFTIRQ_verbose()  argument
4234 	return class_filter(class);  in SOFTIRQ_verbose()
4239 static int (*state_verbose_f[])(struct lock_class *class) = {
4247 				struct lock_class *class)  in state_verbose()  argument
4249 	return state_verbose_f[bit >> LOCK_USAGE_DIR_MASK](class);  in state_verbose()
4879 		struct lock_class *class = hlock_class(prev);  in check_wait_context()  local
4880 		u8 prev_inner = class->wait_type_inner;  in check_wait_context()
4896 			if (unlikely(class->lock_type == LD_LOCK_WAIT_OVERRIDE))  in check_wait_context()
5009 	struct lock_class *class = lock->class_cache[0];  in lockdep_set_lock_cmp_fn()  local
5015 	if (!class)  in lockdep_set_lock_cmp_fn()
5016 		class = register_lock_class(lock, 0, 0);  in lockdep_set_lock_cmp_fn()
5018 	if (class) {  in lockdep_set_lock_cmp_fn()
5019 		WARN_ON(class->cmp_fn	&& class->cmp_fn != cmp_fn);  in lockdep_set_lock_cmp_fn()
5020 		WARN_ON(class->print_fn && class->print_fn != print_fn);  in lockdep_set_lock_cmp_fn()
5022 		class->cmp_fn	= cmp_fn;  in lockdep_set_lock_cmp_fn()
5023 		class->print_fn = print_fn;  in lockdep_set_lock_cmp_fn()
5083 	struct lock_class *class = NULL;  in __lock_acquire()  local
5107 		class = lock->class_cache[subclass];  in __lock_acquire()
5111 	if (unlikely(!class)) {  in __lock_acquire()
5112 		class = register_lock_class(lock, subclass, 0);  in __lock_acquire()
5113 		if (!class)  in __lock_acquire()
5117 	debug_class_ops_inc(class);  in __lock_acquire()
5119 	if (very_verbose(class)) {  in __lock_acquire()
5121 		printk("\nacquire class [%px] %s", class->key, class->name);  in __lock_acquire()
5122 		if (class->name_version > 1)  in __lock_acquire()
5123 			printk(KERN_CONT "#%d", class->name_version);  in __lock_acquire()
5141 	class_idx = class - lock_classes;  in __lock_acquire()
5168 	if (DEBUG_LOCKS_WARN_ON(!class))  in __lock_acquire()
5310 		const struct lock_class *class = lock->class_cache[0];  in match_held_lock()  local
5312 		if (!class)  in match_held_lock()
5313 			class = look_up_lock_class(lock, 0);  in match_held_lock()
5321 		if (!class)  in match_held_lock()
5332 		if (hlock->class_idx == class - lock_classes)  in match_held_lock()
5415 	struct lock_class *class;  in __lock_set_class()  local
5439 	class = register_lock_class(lock, subclass, 0);  in __lock_set_class()
5440 	hlock->class_idx = class - lock_classes;  in __lock_set_class()
5773 	struct lock_class *class = look_up_lock_class(lock, subclass);  in verify_lock_unused()  local
5777 	if (!class)  in verify_lock_unused()
5787 	if (!(class->usage_mask & mask))  in verify_lock_unused()
5790 	hlock->class_idx = class - lock_classes;  in verify_lock_unused()
6192 					 struct lock_class *class)  in remove_class_from_lock_chain()  argument
6198 		if (chain_hlock_class_idx(chain_hlocks[i]) != class - lock_classes)  in remove_class_from_lock_chain()
6227 					  struct lock_class *class)  in remove_class_from_lock_chains()  argument
6236 			remove_class_from_lock_chain(pf, chain, class);  in remove_class_from_lock_chains()
6244 static void zap_class(struct pending_free *pf, struct lock_class *class)  in zap_class()  argument
6249 	WARN_ON_ONCE(!class->key);  in zap_class()
6257 		if (entry->class != class && entry->links_to != class)  in zap_class()
6263 	if (list_empty(&class->locks_after) &&  in zap_class()
6264 	    list_empty(&class->locks_before)) {  in zap_class()
6265 		list_move_tail(&class->lock_entry, &pf->zapped);  in zap_class()
6266 		hlist_del_rcu(&class->hash_entry);  in zap_class()
6267 		WRITE_ONCE(class->key, NULL);  in zap_class()
6268 		WRITE_ONCE(class->name, NULL);  in zap_class()
6270 		if (class->usage_mask == 0)  in zap_class()
6273 		__clear_bit(class - lock_classes, lock_classes_in_use);  in zap_class()
6274 		if (class - lock_classes == max_lock_class_idx)  in zap_class()
6278 			  class->name);  in zap_class()
6281 	remove_class_from_lock_chains(pf, class);  in zap_class()
6285 static void reinit_class(struct lock_class *class)  in reinit_class()  argument
6287 	WARN_ON_ONCE(!class->lock_entry.next);  in reinit_class()
6288 	WARN_ON_ONCE(!list_empty(&class->locks_after));  in reinit_class()
6289 	WARN_ON_ONCE(!list_empty(&class->locks_before));  in reinit_class()
6290 	memset_startat(class, 0, key);  in reinit_class()
6291 	WARN_ON_ONCE(!class->lock_entry.next);  in reinit_class()
6292 	WARN_ON_ONCE(!list_empty(&class->locks_after));  in reinit_class()
6293 	WARN_ON_ONCE(!list_empty(&class->locks_before));  in reinit_class()
6341 	struct lock_class *class;  in __free_zapped_classes()  local
6345 	list_for_each_entry(class, &pf->zapped, lock_entry)  in __free_zapped_classes()
6346 		reinit_class(class);  in __free_zapped_classes()
6396 	struct lock_class *class;  in __lockdep_free_key_range()  local
6403 		hlist_for_each_entry_rcu(class, head, hash_entry) {  in __lockdep_free_key_range()
6404 			if (!within(class->key, start, size) &&  in __lockdep_free_key_range()
6405 			    !within(class->name, start, size))  in __lockdep_free_key_range()
6407 			zap_class(pf, class);  in __lockdep_free_key_range()
6480 	struct lock_class *class;  in lock_class_cache_is_registered()  local
6486 		hlist_for_each_entry_rcu(class, head, hash_entry) {  in lock_class_cache_is_registered()
6488 				if (lock->class_cache[j] == class)  in lock_class_cache_is_registered()
6499 	struct lock_class *class;  in __lockdep_reset_lock()  local
6509 		class = look_up_lock_class(lock, j);  in __lockdep_reset_lock()
6510 		if (class)  in __lockdep_reset_lock()
6511 			zap_class(pf, class);  in __lockdep_reset_lock()