1 // SPDX-License-Identifier: GPL-2.0
29 	unit = info->unit;  in shrinker_unit_free()
30 	nr = DIV_ROUND_UP(info->map_nr_max, SHRINKER_UNIT_BITS);  in shrinker_unit_free()
45 	int nr = DIV_ROUND_UP(new->map_nr_max, SHRINKER_UNIT_BITS);  in shrinker_unit_alloc()
46 	int start = old ? DIV_ROUND_UP(old->map_nr_max, SHRINKER_UNIT_BITS) : 0;  in shrinker_unit_alloc()
53 			return -ENOMEM;  in shrinker_unit_alloc()
56 		new->unit[i] = unit;  in shrinker_unit_alloc()
69 		pn = memcg->nodeinfo[nid];  in free_shrinker_info()
70 		info = rcu_dereference_protected(pn->shrinker_info, true);  in free_shrinker_info()
73 		rcu_assign_pointer(pn->shrinker_info, NULL);  in free_shrinker_info()
89 		info->map_nr_max = shrinker_nr_max;  in alloc_shrinker_info()
94 		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);  in alloc_shrinker_info()
103 	return -ENOMEM;  in alloc_shrinker_info()
109 	return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info,  in shrinker_info_protected()
121 		pn = memcg->nodeinfo[nid];  in expand_one_shrinker_info()
128 		if (new_nr_max <= old->map_nr_max)  in expand_one_shrinker_info()
133 			return -ENOMEM;  in expand_one_shrinker_info()
135 		new->map_nr_max = new_nr_max;  in expand_one_shrinker_info()
137 		memcpy(new->unit, old->unit, old_size);  in expand_one_shrinker_info()
140 			return -ENOMEM;  in expand_one_shrinker_info()
143 		rcu_assign_pointer(pn->shrinker_info, new);  in expand_one_shrinker_info()
203 		info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);  in set_shrinker_bit()
204 		unit = info->unit[shrinker_id_to_index(shrinker_id)];  in set_shrinker_bit()
205 		if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) {  in set_shrinker_bit()
208 			set_bit(shrinker_id_to_offset(shrinker_id), unit->map);  in set_shrinker_bit()
218 	int id, ret = -ENOMEM;  in shrinker_memcg_alloc()
221 		return -ENOSYS;  in shrinker_memcg_alloc()
234 	shrinker->id = id;  in shrinker_memcg_alloc()
243 	int id = shrinker->id;  in shrinker_memcg_remove()
260 	info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);  in xchg_nr_deferred_memcg()
261 	unit = info->unit[shrinker_id_to_index(shrinker->id)];  in xchg_nr_deferred_memcg()
262 	nr_deferred = atomic_long_xchg(&unit->nr_deferred[shrinker_id_to_offset(shrinker->id)], 0);  in xchg_nr_deferred_memcg()
276 	info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);  in add_nr_deferred_memcg()
277 	unit = info->unit[shrinker_id_to_index(shrinker->id)];  in add_nr_deferred_memcg()
279 		atomic_long_add_return(nr, &unit->nr_deferred[shrinker_id_to_offset(shrinker->id)]);  in add_nr_deferred_memcg()
302 		for (index = 0; index < shrinker_id_to_index(child_info->map_nr_max); index++) {  in reparent_shrinker_deferred()
303 			child_unit = child_info->unit[index];  in reparent_shrinker_deferred()
304 			parent_unit = parent_info->unit[index];  in reparent_shrinker_deferred()
306 				nr = atomic_long_read(&child_unit->nr_deferred[offset]);  in reparent_shrinker_deferred()
307 				atomic_long_add(nr, &parent_unit->nr_deferred[offset]);  in reparent_shrinker_deferred()
316 	return -ENOSYS;  in shrinker_memcg_alloc()
339 	int nid = sc->nid;  in xchg_nr_deferred()
341 	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))  in xchg_nr_deferred()
344 	if (sc->memcg &&  in xchg_nr_deferred()
345 	    (shrinker->flags & SHRINKER_MEMCG_AWARE))  in xchg_nr_deferred()
347 					      sc->memcg);  in xchg_nr_deferred()
349 	return atomic_long_xchg(&shrinker->nr_deferred[nid], 0);  in xchg_nr_deferred()
356 	int nid = sc->nid;  in add_nr_deferred()
358 	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))  in add_nr_deferred()
361 	if (sc->memcg &&  in add_nr_deferred()
362 	    (shrinker->flags & SHRINKER_MEMCG_AWARE))  in add_nr_deferred()
364 					     sc->memcg);  in add_nr_deferred()
366 	return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]);  in add_nr_deferred()
380 	long batch_size = shrinker->batch ? shrinker->batch  in do_shrink_slab()
384 	freeable = shrinker->count_objects(shrinker, shrinkctl);  in do_shrink_slab()
395 	if (shrinker->seeks) {  in do_shrink_slab()
398 		do_div(delta, shrinker->seeks);  in do_shrink_slab()
420 	 * up failing allocations although there are plenty of reclaimable  in do_shrink_slab()
435 		shrinkctl->nr_to_scan = nr_to_scan;  in do_shrink_slab()
436 		shrinkctl->nr_scanned = nr_to_scan;  in do_shrink_slab()
437 		ret = shrinker->scan_objects(shrinker, shrinkctl);  in do_shrink_slab()
442 		count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);  in do_shrink_slab()
443 		total_scan -= shrinkctl->nr_scanned;  in do_shrink_slab()
444 		scanned += shrinkctl->nr_scanned;  in do_shrink_slab()
455 	next_deferred = max_t(long, (nr + delta - scanned), 0);  in do_shrink_slab()
464 	trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan);  in do_shrink_slab()
492 	 *  step 1: use rcu_read_lock() to guarantee existence of the  in shrink_slab_memcg()
494 	 *  step 2: after getting shrinker_info_unit we can safely release the  in shrink_slab_memcg()
496 	 *  step 3: traverse the bitmap and calculate shrinker_id  in shrink_slab_memcg()
497 	 *  step 4: use rcu_read_lock() to guarantee existence of the shrinker.  in shrink_slab_memcg()
498 	 *  step 5: use shrinker_id to find the shrinker, then use  in shrink_slab_memcg()
502 	 *  step 6: do shrinker_put() paired with step 5 to put the refcount,  in shrink_slab_memcg()
503 	 *          if the refcount reaches 0, then wake up the waiter in  in shrink_slab_memcg()
509 	 *  step 7: we have already exited the read-side of rcu critical section  in shrink_slab_memcg()
511 	 *          released in expand_one_shrinker_info(), so go back to step 1  in shrink_slab_memcg()
516 	info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);  in shrink_slab_memcg()
520 	if (index < shrinker_id_to_index(info->map_nr_max)) {  in shrink_slab_memcg()
523 		unit = info->unit[index];  in shrink_slab_memcg()
527 		for_each_set_bit(offset, unit->map, SHRINKER_UNIT_BITS) {  in shrink_slab_memcg()
539 				clear_bit(offset, unit->map);  in shrink_slab_memcg()
545 			/* Call non-slab shrinkers even though kmem is disabled */  in shrink_slab_memcg()
547 			    !(shrinker->flags & SHRINKER_NONSLAB))  in shrink_slab_memcg()
552 				clear_bit(offset, unit->map);  in shrink_slab_memcg()
595  * shrink_slab - shrink slab caches
609  * @priority is sc->priority, we take the number of objects and >> by priority
638 	 *  step 1: use rcu_read_lock() to guarantee existence of the shrinker  in shrink_slab()
640 	 *  step 2: use shrinker_try_get() to try get the refcount, if successful,  in shrink_slab()
644 	 *  step 3: *MUST* to reacquire the RCU lock before calling shrinker_put(),  in shrink_slab()
647 	 *  step 4: do shrinker_put() paired with step 2 to put the refcount,  in shrink_slab()
648 	 *          if the refcount reaches 0, then wake up the waiter in  in shrink_slab()
695 	shrinker->flags = flags | SHRINKER_ALLOCATED;  in shrinker_alloc()
696 	shrinker->seeks = DEFAULT_SEEKS;  in shrinker_alloc()
700 		if (err == -ENOSYS) {  in shrinker_alloc()
701 			/* Memcg is not supported, fallback to non-memcg-aware shrinker. */  in shrinker_alloc()
702 			shrinker->flags &= ~SHRINKER_MEMCG_AWARE;  in shrinker_alloc()
716 	 *  - non-memcg-aware shrinkers  in shrinker_alloc()
717 	 *  - !CONFIG_MEMCG  in shrinker_alloc()
718 	 *  - memcg is disabled by kernel command line  in shrinker_alloc()
720 	size = sizeof(*shrinker->nr_deferred);  in shrinker_alloc()
724 	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);  in shrinker_alloc()
725 	if (!shrinker->nr_deferred)  in shrinker_alloc()
740 	if (unlikely(!(shrinker->flags & SHRINKER_ALLOCATED))) {  in shrinker_register()
746 	list_add_tail_rcu(&shrinker->list, &shrinker_list);  in shrinker_register()
747 	shrinker->flags |= SHRINKER_REGISTERED;  in shrinker_register()
751 	init_completion(&shrinker->done);  in shrinker_register()
753 	 * Now the shrinker is fully set up, take the first reference to it to  in shrinker_register()
757 	refcount_set(&shrinker->refcount, 1);  in shrinker_register()
765 	kfree(shrinker->nr_deferred);  in shrinker_free_rcu_cb()
777 	if (shrinker->flags & SHRINKER_REGISTERED) {  in shrinker_free()
786 		wait_for_completion(&shrinker->done);  in shrinker_free()
790 	if (shrinker->flags & SHRINKER_REGISTERED) {  in shrinker_free()
795 		list_del_rcu(&shrinker->list);  in shrinker_free()
797 		shrinker->flags &= ~SHRINKER_REGISTERED;  in shrinker_free()
802 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in shrinker_free()
809 	call_rcu(&shrinker->rcu, shrinker_free_rcu_cb);  in shrinker_free()