439 	struct node_barn *barn;  member
458 	return n->barn;  in get_barn()
3090 static struct slab_sheaf *barn_get_empty_sheaf(struct node_barn *barn,  in barn_get_empty_sheaf()  argument
3096 	if (!data_race(barn->nr_empty))  in barn_get_empty_sheaf()
3100 		spin_lock_irqsave(&barn->lock, flags);  in barn_get_empty_sheaf()
3101 	else if (!spin_trylock_irqsave(&barn->lock, flags))  in barn_get_empty_sheaf()
3104 	if (likely(barn->nr_empty)) {  in barn_get_empty_sheaf()
3105 		empty = list_first_entry(&barn->sheaves_empty,  in barn_get_empty_sheaf()
3108 		barn->nr_empty--;  in barn_get_empty_sheaf()
3111 	spin_unlock_irqrestore(&barn->lock, flags);  in barn_get_empty_sheaf()
3122 static void barn_put_empty_sheaf(struct node_barn *barn, struct slab_sheaf *sheaf)  in barn_put_empty_sheaf()  argument
3126 	spin_lock_irqsave(&barn->lock, flags);  in barn_put_empty_sheaf()
3128 	list_add(&sheaf->barn_list, &barn->sheaves_empty);  in barn_put_empty_sheaf()
3129 	barn->nr_empty++;  in barn_put_empty_sheaf()
3131 	spin_unlock_irqrestore(&barn->lock, flags);  in barn_put_empty_sheaf()
3134 static void barn_put_full_sheaf(struct node_barn *barn, struct slab_sheaf *sheaf)  in barn_put_full_sheaf()  argument
3138 	spin_lock_irqsave(&barn->lock, flags);  in barn_put_full_sheaf()
3140 	list_add(&sheaf->barn_list, &barn->sheaves_full);  in barn_put_full_sheaf()
3141 	barn->nr_full++;  in barn_put_full_sheaf()
3143 	spin_unlock_irqrestore(&barn->lock, flags);  in barn_put_full_sheaf()
3146 static struct slab_sheaf *barn_get_full_or_empty_sheaf(struct node_barn *barn)  in barn_get_full_or_empty_sheaf()  argument
3151 	if (!data_race(barn->nr_full) && !data_race(barn->nr_empty))  in barn_get_full_or_empty_sheaf()
3154 	spin_lock_irqsave(&barn->lock, flags);  in barn_get_full_or_empty_sheaf()
3156 	if (barn->nr_full) {  in barn_get_full_or_empty_sheaf()
3157 		sheaf = list_first_entry(&barn->sheaves_full, struct slab_sheaf,  in barn_get_full_or_empty_sheaf()
3160 		barn->nr_full--;  in barn_get_full_or_empty_sheaf()
3161 	} else if (barn->nr_empty) {  in barn_get_full_or_empty_sheaf()
3162 		sheaf = list_first_entry(&barn->sheaves_empty,  in barn_get_full_or_empty_sheaf()
3165 		barn->nr_empty--;  in barn_get_full_or_empty_sheaf()
3168 	spin_unlock_irqrestore(&barn->lock, flags);  in barn_get_full_or_empty_sheaf()
3179 barn_replace_empty_sheaf(struct node_barn *barn, struct slab_sheaf *empty,  in barn_replace_empty_sheaf()  argument
3185 	if (!data_race(barn->nr_full))  in barn_replace_empty_sheaf()
3189 		spin_lock_irqsave(&barn->lock, flags);  in barn_replace_empty_sheaf()
3190 	else if (!spin_trylock_irqsave(&barn->lock, flags))  in barn_replace_empty_sheaf()
3193 	if (likely(barn->nr_full)) {  in barn_replace_empty_sheaf()
3194 		full = list_first_entry(&barn->sheaves_full, struct slab_sheaf,  in barn_replace_empty_sheaf()
3197 		list_add(&empty->barn_list, &barn->sheaves_empty);  in barn_replace_empty_sheaf()
3198 		barn->nr_full--;  in barn_replace_empty_sheaf()
3199 		barn->nr_empty++;  in barn_replace_empty_sheaf()
3202 	spin_unlock_irqrestore(&barn->lock, flags);  in barn_replace_empty_sheaf()
3212 barn_replace_full_sheaf(struct node_barn *barn, struct slab_sheaf *full,  in barn_replace_full_sheaf()  argument
3219 	if (data_race(barn->nr_full) >= MAX_FULL_SHEAVES)  in barn_replace_full_sheaf()
3221 	if (!data_race(barn->nr_empty))  in barn_replace_full_sheaf()
3225 		spin_lock_irqsave(&barn->lock, flags);  in barn_replace_full_sheaf()
3226 	else if (!spin_trylock_irqsave(&barn->lock, flags))  in barn_replace_full_sheaf()
3229 	if (likely(barn->nr_empty)) {  in barn_replace_full_sheaf()
3230 		empty = list_first_entry(&barn->sheaves_empty, struct slab_sheaf,  in barn_replace_full_sheaf()
3233 		list_add(&full->barn_list, &barn->sheaves_full);  in barn_replace_full_sheaf()
3234 		barn->nr_empty--;  in barn_replace_full_sheaf()
3235 		barn->nr_full++;  in barn_replace_full_sheaf()
3240 	spin_unlock_irqrestore(&barn->lock, flags);  in barn_replace_full_sheaf()
3245 static void barn_init(struct node_barn *barn)  in barn_init()  argument
3247 	spin_lock_init(&barn->lock);  in barn_init()
3248 	INIT_LIST_HEAD(&barn->sheaves_full);  in barn_init()
3249 	INIT_LIST_HEAD(&barn->sheaves_empty);  in barn_init()
3250 	barn->nr_full = 0;  in barn_init()
3251 	barn->nr_empty = 0;  in barn_init()
3254 static void barn_shrink(struct kmem_cache *s, struct node_barn *barn)  in barn_shrink()  argument
3261 	spin_lock_irqsave(&barn->lock, flags);  in barn_shrink()
3263 	list_splice_init(&barn->sheaves_full, &full_list);  in barn_shrink()
3264 	barn->nr_full = 0;  in barn_shrink()
3265 	list_splice_init(&barn->sheaves_empty, &empty_list);  in barn_shrink()
3266 	barn->nr_empty = 0;  in barn_shrink()
3268 	spin_unlock_irqrestore(&barn->lock, flags);  in barn_shrink()
4569 	struct node_barn *barn;  in __pcs_replace_empty_main()  local
4585 	barn = get_barn(s);  in __pcs_replace_empty_main()
4586 	if (!barn) {  in __pcs_replace_empty_main()
4593 	full = barn_replace_empty_sheaf(barn, pcs->main, allow_spin);  in __pcs_replace_empty_main()
4608 			empty = barn_get_empty_sheaf(barn, true);  in __pcs_replace_empty_main()
4651 			barn_put_empty_sheaf(barn, pcs->main);  in __pcs_replace_empty_main()
4662 		barn_put_empty_sheaf(barn, pcs->spare);  in __pcs_replace_empty_main()
4668 	barn_put_full_sheaf(barn, full);  in __pcs_replace_empty_main()
4767 		struct node_barn *barn;  in alloc_from_pcs_bulk()  local
4779 		barn = get_barn(s);  in alloc_from_pcs_bulk()
4780 		if (!barn) {  in alloc_from_pcs_bulk()
4785 		full = barn_replace_empty_sheaf(barn, pcs->main,  in alloc_from_pcs_bulk()
4968 	struct node_barn *barn;  in kmem_cache_prefill_sheaf()  local
5006 		barn = get_barn(s);  in kmem_cache_prefill_sheaf()
5009 		if (barn)  in kmem_cache_prefill_sheaf()
5010 			sheaf = barn_get_full_or_empty_sheaf(barn);  in kmem_cache_prefill_sheaf()
5052 	struct node_barn *barn;  in kmem_cache_return_sheaf()  local
5063 	barn = get_barn(s);  in kmem_cache_return_sheaf()
5082 	if (!barn || data_race(barn->nr_full) >= MAX_FULL_SHEAVES ||  in kmem_cache_return_sheaf()
5089 	barn_put_full_sheaf(barn, sheaf);  in kmem_cache_return_sheaf()
5599 		struct node_barn *barn)  in __pcs_install_empty_sheaf()  argument
5615 		barn_put_empty_sheaf(barn, empty);  in __pcs_install_empty_sheaf()
5622 		barn_put_empty_sheaf(barn, empty);  in __pcs_install_empty_sheaf()
5630 	barn_put_full_sheaf(barn, pcs->main);  in __pcs_install_empty_sheaf()
5648 	struct node_barn *barn;  in __pcs_replace_full_main()  local
5660 	barn = get_barn(s);  in __pcs_replace_full_main()
5661 	if (!barn) {  in __pcs_replace_full_main()
5669 		empty = barn_get_empty_sheaf(barn, allow_spin);  in __pcs_replace_full_main()
5683 	empty = barn_replace_full_sheaf(barn, pcs->main, allow_spin);  in __pcs_replace_full_main()
5752 		barn_put_empty_sheaf(barn, empty);  in __pcs_replace_full_main()
5757 	__pcs_install_empty_sheaf(s, pcs, empty, barn);  in __pcs_replace_full_main()
5796 	struct node_barn *barn = NULL;  in rcu_free_sheaf()  local
5821 	barn = n->barn;  in rcu_free_sheaf()
5833 	if (data_race(barn->nr_full) < MAX_FULL_SHEAVES) {  in rcu_free_sheaf()
5835 		barn_put_full_sheaf(barn, sheaf);  in rcu_free_sheaf()
5844 	if (barn && data_race(barn->nr_empty) < MAX_EMPTY_SHEAVES) {  in rcu_free_sheaf()
5845 		barn_put_empty_sheaf(barn, sheaf);  in rcu_free_sheaf()
5883 		struct node_barn *barn;  in __kfree_rcu_sheaf()  local
5897 		barn = get_barn(s);  in __kfree_rcu_sheaf()
5898 		if (!barn) {  in __kfree_rcu_sheaf()
5903 		empty = barn_get_empty_sheaf(barn, true);  in __kfree_rcu_sheaf()
5918 			barn_put_empty_sheaf(barn, empty);  in __kfree_rcu_sheaf()
5925 			barn_put_empty_sheaf(barn, empty);  in __kfree_rcu_sheaf()
5977 	struct node_barn *barn;  in free_to_pcs_bulk()  local
6018 	barn = get_barn(s);  in free_to_pcs_bulk()
6019 	if (!barn)  in free_to_pcs_bulk()
6023 		empty = barn_get_empty_sheaf(barn, true);  in free_to_pcs_bulk()
6037 	empty = barn_replace_full_sheaf(barn, pcs->main, true);  in free_to_pcs_bulk()
7433 init_kmem_cache_node(struct kmem_cache_node *n, struct node_barn *barn)  in init_kmem_cache_node()  argument
7443 	n->barn = barn;  in init_kmem_cache_node()
7444 	if (barn)  in init_kmem_cache_node()
7445 		barn_init(barn);  in init_kmem_cache_node()
7557 		if (n->barn) {  in free_kmem_cache_nodes()
7558 			WARN_ON(n->barn->nr_full);  in free_kmem_cache_nodes()
7559 			WARN_ON(n->barn->nr_empty);  in free_kmem_cache_nodes()
7560 			kfree(n->barn);  in free_kmem_cache_nodes()
7561 			n->barn = NULL;  in free_kmem_cache_nodes()
7585 		struct node_barn *barn = NULL;  in init_kmem_cache_nodes()  local
7593 			barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node);  in init_kmem_cache_nodes()
7595 			if (!barn)  in init_kmem_cache_nodes()
7602 			kfree(barn);  in init_kmem_cache_nodes()
7606 		init_kmem_cache_node(n, barn);  in init_kmem_cache_nodes()
7900 		if (n->barn)  in __kmem_cache_shutdown()
7901 			barn_shrink(s, n->barn);  in __kmem_cache_shutdown()
8116 		if (n->barn)  in __kmem_cache_do_shrink()
8117 			barn_shrink(s, n->barn);  in __kmem_cache_do_shrink()
8198 		struct node_barn *barn = NULL;  in slab_mem_going_online_callback()  local
8208 			barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, nid);  in slab_mem_going_online_callback()
8210 			if (!barn) {  in slab_mem_going_online_callback()
8223 			kfree(barn);  in slab_mem_going_online_callback()
8228 		init_kmem_cache_node(n, barn);  in slab_mem_going_online_callback()
8316 		struct node_barn *barn;  in bootstrap_cache_sheaves()  local
8318 		barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node);  in bootstrap_cache_sheaves()
8320 		if (!barn) {  in bootstrap_cache_sheaves()
8325 		barn_init(barn);  in bootstrap_cache_sheaves()
8326 		get_node(s, node)->barn = barn;  in bootstrap_cache_sheaves()