40 		       const struct bucket_table *tbl,  in head_hashfn()  argument
43 	return rht_head_hashfn(ht, tbl, he, ht->p);  in head_hashfn()
55 int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash)  in lockdep_rht_bucket_is_held()  argument
59 	if (unlikely(tbl->nest))  in lockdep_rht_bucket_is_held()
61 	return bit_spin_is_locked(0, (unsigned long *)&tbl->buckets[hash]);  in lockdep_rht_bucket_is_held()
69 	const struct bucket_table *tbl)  in nested_table_top()  argument
74 	return (void *)rcu_dereference_protected(tbl->buckets[0], 1);  in nested_table_top()
96 static void nested_bucket_table_free(const struct bucket_table *tbl)  in nested_bucket_table_free()  argument
98 	unsigned int size = tbl->size >> tbl->nest;  in nested_bucket_table_free()
99 	unsigned int len = 1 << tbl->nest;  in nested_bucket_table_free()
103 	ntbl = nested_table_top(tbl);  in nested_bucket_table_free()
111 static void bucket_table_free(const struct bucket_table *tbl)  in bucket_table_free()  argument
113 	if (tbl->nest)  in bucket_table_free()
114 		nested_bucket_table_free(tbl);  in bucket_table_free()
116 	kvfree(tbl);  in bucket_table_free()
119 static void bucket_table_free_atomic(const struct bucket_table *tbl)  in bucket_table_free_atomic()  argument
121 	if (tbl->nest)  in bucket_table_free_atomic()
122 		nested_bucket_table_free(tbl);  in bucket_table_free_atomic()
124 	kvfree_atomic(tbl);  in bucket_table_free_atomic()
163 	struct bucket_table *tbl;  in nested_bucket_table_alloc()  local
169 	size = sizeof(*tbl) + sizeof(tbl->buckets[0]);  in nested_bucket_table_alloc()
171 	tbl = alloc_hooks_tag(ht->alloc_tag,  in nested_bucket_table_alloc()
173 	if (!tbl)  in nested_bucket_table_alloc()
176 	if (!nested_table_alloc(ht, (union nested_table __rcu **)tbl->buckets,  in nested_bucket_table_alloc()
178 		kfree(tbl);  in nested_bucket_table_alloc()
182 	tbl->nest = (ilog2(nbuckets) - 1) % shift + 1;  in nested_bucket_table_alloc()
184 	return tbl;  in nested_bucket_table_alloc()
191 	struct bucket_table *tbl = NULL;  in bucket_table_alloc()  local
196 	tbl = alloc_hooks_tag(ht->alloc_tag,  in bucket_table_alloc()
197 			kvmalloc_node_align_noprof(struct_size(tbl, buckets, nbuckets),  in bucket_table_alloc()
202 	if (tbl == NULL && !gfpflags_allow_blocking(gfp)) {  in bucket_table_alloc()
203 		tbl = nested_bucket_table_alloc(ht, nbuckets, gfp);  in bucket_table_alloc()
207 	if (tbl == NULL)  in bucket_table_alloc()
210 	lockdep_init_map(&tbl->dep_map, "rhashtable_bucket", &__key, 0);  in bucket_table_alloc()
212 	tbl->size = size;  in bucket_table_alloc()
214 	rcu_head_init(&tbl->rcu);  in bucket_table_alloc()
215 	INIT_LIST_HEAD(&tbl->walkers);  in bucket_table_alloc()
217 	tbl->hash_rnd = get_random_u32();  in bucket_table_alloc()
220 		INIT_RHT_NULLS_HEAD(tbl->buckets[i]);  in bucket_table_alloc()
222 	return tbl;  in bucket_table_alloc()
226 						  struct bucket_table *tbl)  in rhashtable_last_table()  argument
231 		new_tbl = tbl;  in rhashtable_last_table()
232 		tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in rhashtable_last_table()
233 	} while (tbl);  in rhashtable_last_table()
242 	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);  in rhashtable_rehash_one()
293 	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);  in rhashtable_rehash_chain()
331 	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);  in rhashtable_rehash_table()
349 	rcu_assign_pointer(ht->tbl, new_tbl);  in rhashtable_rehash_table()
353 		walker->tbl = NULL;  in rhashtable_rehash_table()
408 	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);  in rhashtable_shrink()
429 	struct bucket_table *tbl;  in rht_deferred_worker()  local
435 	tbl = rht_dereference(ht->tbl, ht);  in rht_deferred_worker()
436 	tbl = rhashtable_last_table(ht, tbl);  in rht_deferred_worker()
438 	if (rht_grow_above_75(ht, tbl))  in rht_deferred_worker()
439 		err = rhashtable_rehash_alloc(ht, tbl, tbl->size * 2);  in rht_deferred_worker()
440 	else if (ht->p.automatic_shrinking && rht_shrink_below_30(ht, tbl))  in rht_deferred_worker()
442 	else if (tbl->nest)  in rht_deferred_worker()
443 		err = rhashtable_rehash_alloc(ht, tbl, tbl->size);  in rht_deferred_worker()
482 				    struct bucket_table *tbl)  in rhashtable_insert_rehash()  argument
489 	old_tbl = rht_dereference_rcu(ht->tbl, ht);  in rhashtable_insert_rehash()
491 	size = tbl->size;  in rhashtable_insert_rehash()
495 	if (rht_grow_above_75(ht, tbl))  in rhashtable_insert_rehash()
498 	else if (old_tbl != tbl)  in rhashtable_insert_rehash()
507 	err = rhashtable_rehash_attach(ht, tbl, new_tbl);  in rhashtable_insert_rehash()
519 	if (likely(rcu_access_pointer(tbl->future_tbl)))  in rhashtable_insert_rehash()
531 				   struct bucket_table *tbl, unsigned int hash,  in rhashtable_lookup_one()  argument
543 	rht_for_each_from(head, rht_ptr(bkt, tbl, hash), tbl, hash) {  in rhashtable_lookup_one()
563 		head = rht_dereference_bucket(head->next, tbl, hash);  in rhashtable_lookup_one()
582 	struct bucket_table *tbl, unsigned int hash, struct rhash_head *obj,  in rhashtable_insert_one()  argument
594 	new_tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in rhashtable_insert_one()
601 	if (unlikely(rht_grow_above_max(ht, tbl)))  in rhashtable_insert_one()
604 	if (unlikely(rht_grow_above_100(ht, tbl)) &&  in rhashtable_insert_one()
608 	head = rht_ptr(bkt, tbl, hash);  in rhashtable_insert_one()
630 	struct bucket_table *tbl;  in rhashtable_try_insert()  local
636 	new_tbl = rcu_dereference(ht->tbl);  in rhashtable_try_insert()
639 		tbl = new_tbl;  in rhashtable_try_insert()
640 		hash = rht_head_hashfn(ht, tbl, obj, ht->p);  in rhashtable_try_insert()
641 		if (rcu_access_pointer(tbl->future_tbl))  in rhashtable_try_insert()
643 			bkt = rht_bucket_var(tbl, hash);  in rhashtable_try_insert()
645 			bkt = rht_bucket_insert(ht, tbl, hash);  in rhashtable_try_insert()
647 			new_tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in rhashtable_try_insert()
652 			flags = rht_lock(tbl, bkt);  in rhashtable_try_insert()
653 			data = rhashtable_lookup_one(ht, bkt, tbl,  in rhashtable_try_insert()
655 			new_tbl = rhashtable_insert_one(ht, bkt, tbl,  in rhashtable_try_insert()
663 			rht_unlock(tbl, bkt, flags);  in rhashtable_try_insert()
665 			if (inserted && rht_grow_above_75(ht, tbl))  in rhashtable_try_insert()
671 		data = ERR_PTR(rhashtable_insert_rehash(ht, tbl) ?:  in rhashtable_try_insert()
699 	struct rhashtable *ht, struct bucket_table *tbl,  in __rhashtable_next_in_table()  argument
709 		b = rht_key_hashfn(ht, tbl, prev_key, params);  in __rhashtable_next_in_table()
710 		rht_for_each_rcu(he, tbl, b) {  in __rhashtable_next_in_table()
727 	for (; b < tbl->size; b++)  in __rhashtable_next_in_table()
728 		rht_for_each_rcu(he, tbl, b)  in __rhashtable_next_in_table()
773 	struct bucket_table *tbl;  in rhashtable_next_key()  local
779 	tbl = rht_dereference_rcu(ht->tbl, ht);  in rhashtable_next_key()
781 		he = __rhashtable_next_in_table(ht, tbl, prev_key);  in rhashtable_next_key()
788 		tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in rhashtable_next_key()
789 	} while (tbl);  in rhashtable_next_key()
824 	iter->walker.tbl =  in rhashtable_walk_enter()
825 		rcu_dereference_protected(ht->tbl, lockdep_is_held(&ht->lock));  in rhashtable_walk_enter()
826 	list_add(&iter->walker.list, &iter->walker.tbl->walkers);  in rhashtable_walk_enter()
840 	if (iter->walker.tbl)  in rhashtable_walk_exit()
873 	if (iter->walker.tbl)  in rhashtable_walk_start_check()
879 	if (!iter->walker.tbl) {  in rhashtable_walk_start_check()
880 		iter->walker.tbl = rht_dereference_rcu(ht->tbl, ht);  in rhashtable_walk_start_check()
894 		rht_for_each_rcu(p, iter->walker.tbl, iter->slot) {  in rhashtable_walk_start_check()
909 		rht_for_each_rcu(p, iter->walker.tbl, iter->slot) {  in rhashtable_walk_start_check()
940 	struct bucket_table *tbl = iter->walker.tbl;  in __rhashtable_walk_find_next()  local
946 	if (!tbl)  in __rhashtable_walk_find_next()
949 	for (; iter->slot < tbl->size; iter->slot++) {  in __rhashtable_walk_find_next()
952 		rht_for_each_rcu(p, tbl, iter->slot) {  in __rhashtable_walk_find_next()
986 	iter->walker.tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in __rhashtable_walk_find_next()
987 	if (iter->walker.tbl) {  in __rhashtable_walk_find_next()
1084 	struct bucket_table *tbl = iter->walker.tbl;  in rhashtable_walk_stop()  local
1086 	if (!tbl)  in rhashtable_walk_stop()
1092 	if (rcu_head_after_call_rcu(&tbl->rcu, bucket_table_free_rcu))  in rhashtable_walk_stop()
1094 		iter->walker.tbl = NULL;  in rhashtable_walk_stop()
1096 		list_add(&iter->walker.list, &tbl->walkers);  in rhashtable_walk_stop()
1169 	struct bucket_table *tbl;  in __rhashtable_init_noprof()  local
1214 	tbl = bucket_table_alloc(ht, size, GFP_KERNEL);  in __rhashtable_init_noprof()
1215 	if (unlikely(tbl == NULL)) {  in __rhashtable_init_noprof()
1217 		tbl = bucket_table_alloc(ht, size, GFP_KERNEL | __GFP_NOFAIL);  in __rhashtable_init_noprof()
1222 	RCU_INIT_POINTER(ht->tbl, tbl);  in __rhashtable_init_noprof()
1295 	struct bucket_table *tbl, *next_tbl;  in rhashtable_free_and_destroy()  local
1309 	tbl = rcu_dereference_raw(ht->tbl);  in rhashtable_free_and_destroy()
1312 		for (i = 0; i < tbl->size; i++) {  in rhashtable_free_and_destroy()
1316 			for (pos = rht_ptr_exclusive(rht_bucket(tbl, i)),  in rhashtable_free_and_destroy()
1327 	next_tbl = rcu_dereference_raw(tbl->future_tbl);  in rhashtable_free_and_destroy()
1328 	bucket_table_free(tbl);  in rhashtable_free_and_destroy()
1330 		tbl = next_tbl;  in rhashtable_free_and_destroy()
1343 	const struct bucket_table *tbl, unsigned int hash)  in __rht_bucket_nested()  argument
1346 	unsigned int index = hash & ((1 << tbl->nest) - 1);  in __rht_bucket_nested()
1347 	unsigned int size = tbl->size >> tbl->nest;  in __rht_bucket_nested()
1351 	ntbl = nested_table_top(tbl);  in __rht_bucket_nested()
1352 	ntbl = rht_dereference_bucket_rcu(ntbl[index].table, tbl, hash);  in __rht_bucket_nested()
1353 	subhash >>= tbl->nest;  in __rht_bucket_nested()
1358 						  tbl, hash);  in __rht_bucket_nested()
1372 	const struct bucket_table *tbl, unsigned int hash)  in rht_bucket_nested()  argument
1378 	return __rht_bucket_nested(tbl, hash) ?: &rhnull;  in rht_bucket_nested()
1383 	struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)  in rht_bucket_nested_insert()  argument
1386 	unsigned int index = hash & ((1 << tbl->nest) - 1);  in rht_bucket_nested_insert()
1387 	unsigned int size = tbl->size >> tbl->nest;  in rht_bucket_nested_insert()
1390 	ntbl = nested_table_top(tbl);  in rht_bucket_nested_insert()
1391 	hash >>= tbl->nest;  in rht_bucket_nested_insert()