Lines Matching full:pool
113 static void dmemcg_pool_get(struct dmem_cgroup_pool_state *pool) in dmemcg_pool_get() argument
115 refcount_inc(&pool->ref); in dmemcg_pool_get()
118 static bool dmemcg_pool_tryget(struct dmem_cgroup_pool_state *pool) in dmemcg_pool_tryget() argument
120 return refcount_inc_not_zero(&pool->ref); in dmemcg_pool_tryget()
123 static void dmemcg_pool_put(struct dmem_cgroup_pool_state *pool) in dmemcg_pool_put() argument
125 if (!refcount_dec_and_test(&pool->ref)) in dmemcg_pool_put()
128 call_rcu(&pool->rcu, dmemcg_pool_free_rcu); in dmemcg_pool_put()
133 struct dmem_cgroup_pool_state *pool = container_of(rcu, typeof(*pool), rcu); in dmemcg_pool_free_rcu() local
135 if (pool->parent) in dmemcg_pool_free_rcu()
136 dmemcg_pool_put(pool->parent); in dmemcg_pool_free_rcu()
137 kref_put(&pool->region->ref, dmemcg_free_region); in dmemcg_pool_free_rcu()
138 kfree(pool); in dmemcg_pool_free_rcu()
141 static void free_cg_pool(struct dmem_cgroup_pool_state *pool) in free_cg_pool() argument
143 list_del(&pool->region_node); in free_cg_pool()
144 dmemcg_pool_put(pool); in free_cg_pool()
148 set_resource_min(struct dmem_cgroup_pool_state *pool, u64 val) in set_resource_min() argument
150 page_counter_set_min(&pool->cnt, val); in set_resource_min()
154 set_resource_low(struct dmem_cgroup_pool_state *pool, u64 val) in set_resource_low() argument
156 page_counter_set_low(&pool->cnt, val); in set_resource_low()
160 set_resource_max(struct dmem_cgroup_pool_state *pool, u64 val) in set_resource_max() argument
162 page_counter_set_max(&pool->cnt, val); in set_resource_max()
165 static u64 get_resource_low(struct dmem_cgroup_pool_state *pool) in get_resource_low() argument
167 return pool ? READ_ONCE(pool->cnt.low) : 0; in get_resource_low()
170 static u64 get_resource_min(struct dmem_cgroup_pool_state *pool) in get_resource_min() argument
172 return pool ? READ_ONCE(pool->cnt.min) : 0; in get_resource_min()
175 static u64 get_resource_max(struct dmem_cgroup_pool_state *pool) in get_resource_max() argument
177 return pool ? READ_ONCE(pool->cnt.max) : PAGE_COUNTER_MAX; in get_resource_max()
180 static u64 get_resource_current(struct dmem_cgroup_pool_state *pool) in get_resource_current() argument
182 return pool ? page_counter_read(&pool->cnt) : 0; in get_resource_current()
195 struct dmem_cgroup_pool_state *pool; in dmemcs_offline() local
198 list_for_each_entry_rcu(pool, &dmemcs->pools, css_node) in dmemcs_offline()
199 reset_all_resource_limits(pool); in dmemcs_offline()
206 struct dmem_cgroup_pool_state *pool, *next; in dmemcs_free() local
209 list_for_each_entry_safe(pool, next, &dmemcs->pools, css_node) { in dmemcs_free()
211 *The pool is dead and all references are 0, in dmemcs_free()
214 list_del(&pool->css_node); in dmemcs_free()
215 free_cg_pool(pool); in dmemcs_free()
236 struct dmem_cgroup_pool_state *pool; in find_cg_pool_locked() local
238 list_for_each_entry_rcu(pool, &dmemcs->pools, css_node, spin_is_locked(&dmemcg_lock)) in find_cg_pool_locked()
239 if (pool->region == region) in find_cg_pool_locked()
240 return pool; in find_cg_pool_locked()
245 static struct dmem_cgroup_pool_state *pool_parent(struct dmem_cgroup_pool_state *pool) in pool_parent() argument
247 if (!pool->cnt.parent) in pool_parent()
250 return container_of(pool->cnt.parent, typeof(*pool), cnt); in pool_parent()
260 struct dmem_cgroup_pool_state *pool, *found_pool; in dmem_cgroup_calculate_protection() local
270 list_for_each_entry_rcu(pool, &dmemcg_iter->pools, css_node) { in dmem_cgroup_calculate_protection()
271 if (pool->region == limit_pool->region) { in dmem_cgroup_calculate_protection()
272 found_pool = pool; in dmem_cgroup_calculate_protection()
290 * @limit_pool: The pool for which we hit limits
291 * @test_pool: The pool for which to test
306 struct dmem_cgroup_pool_state *pool = test_pool; in dmem_cgroup_state_evict_valuable() local
310 /* Can always evict from current pool, despite limits */ in dmem_cgroup_state_evict_valuable()
318 for (pool = test_pool; pool && limit_pool != pool; pool = pool_parent(pool)) in dmem_cgroup_state_evict_valuable()
321 if (!pool) in dmem_cgroup_state_evict_valuable()
359 struct dmem_cgroup_pool_state *pool, *ppool = NULL; in alloc_pool_single() local
362 pool = kzalloc_obj(*pool, GFP_NOWAIT); in alloc_pool_single()
363 if (!pool) in alloc_pool_single()
366 pool = *allocpool; in alloc_pool_single()
370 pool->region = region; in alloc_pool_single()
371 pool->cs = dmemcs; in alloc_pool_single()
376 page_counter_init(&pool->cnt, in alloc_pool_single()
378 reset_all_resource_limits(pool); in alloc_pool_single()
379 refcount_set(&pool->ref, 1); in alloc_pool_single()
381 if (ppool && !pool->parent) { in alloc_pool_single()
382 pool->parent = ppool; in alloc_pool_single()
386 list_add_tail_rcu(&pool->css_node, &dmemcs->pools); in alloc_pool_single()
387 list_add_tail(&pool->region_node, ®ion->pools); in alloc_pool_single()
390 pool->inited = true; in alloc_pool_single()
392 pool->inited = ppool ? ppool->inited : false; in alloc_pool_single()
393 return pool; in alloc_pool_single()
400 struct dmem_cgroup_pool_state *pool, *ppool, *retpool; in get_cg_pool_locked() local
404 * Recursively create pool, we may not initialize yet on in get_cg_pool_locked()
408 pool = find_cg_pool_locked(p, region); in get_cg_pool_locked()
409 if (!pool) in get_cg_pool_locked()
410 pool = alloc_pool_single(p, region, allocpool); in get_cg_pool_locked()
412 if (IS_ERR(pool)) in get_cg_pool_locked()
413 return pool; in get_cg_pool_locked()
415 if (p == dmemcs && pool->inited) in get_cg_pool_locked()
416 return pool; in get_cg_pool_locked()
418 if (pool->inited) in get_cg_pool_locked()
422 retpool = pool = find_cg_pool_locked(dmemcs, region); in get_cg_pool_locked()
424 if (pool->inited) in get_cg_pool_locked()
431 pool->cnt.parent = &ppool->cnt; in get_cg_pool_locked()
432 if (ppool && !pool->parent) { in get_cg_pool_locked()
433 pool->parent = ppool; in get_cg_pool_locked()
436 pool->inited = true; in get_cg_pool_locked()
438 pool = ppool; in get_cg_pool_locked()
447 struct dmem_cgroup_pool_state *pool, *next; in dmemcg_free_rcu() local
449 list_for_each_entry_safe(pool, next, ®ion->pools, region_node) in dmemcg_free_rcu()
450 free_cg_pool(pool); in dmemcg_free_rcu()
470 struct dmem_cgroup_pool_state *pool, *next; in dmem_cgroup_unregister_region() local
480 list_for_each_entry_safe(pool, next, ®ion->pools, region_node) { in dmem_cgroup_unregister_region()
481 list_del_rcu(&pool->css_node); in dmem_cgroup_unregister_region()
482 list_del(&pool->region_node); in dmem_cgroup_unregister_region()
483 dmemcg_pool_put(pool); in dmem_cgroup_unregister_region()
557 * @pool: &dmem_cgroup_pool_state
559 * Called to drop a reference to the limiting pool returned by
562 void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool) in dmem_cgroup_pool_state_put() argument
564 if (pool) { in dmem_cgroup_pool_state_put()
565 css_put(&pool->cs->css); in dmem_cgroup_pool_state_put()
566 dmemcg_pool_put(pool); in dmem_cgroup_pool_state_put()
574 struct dmem_cgroup_pool_state *pool, *allocpool = NULL; in get_cg_pool_unlocked() local
578 pool = find_cg_pool_locked(cg, region); in get_cg_pool_unlocked()
579 if (pool && !READ_ONCE(pool->inited)) in get_cg_pool_unlocked()
580 pool = NULL; in get_cg_pool_unlocked()
581 if (pool && !dmemcg_pool_tryget(pool)) in get_cg_pool_unlocked()
582 pool = NULL; in get_cg_pool_unlocked()
585 while (!pool) { in get_cg_pool_unlocked()
588 pool = get_cg_pool_locked(cg, region, &allocpool); in get_cg_pool_unlocked()
590 pool = ERR_PTR(-ENODEV); in get_cg_pool_unlocked()
591 if (!IS_ERR(pool)) in get_cg_pool_unlocked()
592 dmemcg_pool_get(pool); in get_cg_pool_unlocked()
595 if (pool == ERR_PTR(-ENOMEM)) { in get_cg_pool_unlocked()
596 pool = NULL; in get_cg_pool_unlocked()
602 pool = NULL; in get_cg_pool_unlocked()
609 return pool; in get_cg_pool_unlocked()
613 * dmem_cgroup_uncharge() - Uncharge a pool.
614 * @pool: Pool to uncharge.
618 * Must be called with the returned pool as argument,
621 void dmem_cgroup_uncharge(struct dmem_cgroup_pool_state *pool, u64 size) in dmem_cgroup_uncharge() argument
623 if (!pool) in dmem_cgroup_uncharge()
626 page_counter_uncharge(&pool->cnt, size); in dmem_cgroup_uncharge()
627 css_put(&pool->cs->css); in dmem_cgroup_uncharge()
628 dmemcg_pool_put(pool); in dmem_cgroup_uncharge()
636 * @ret_pool: On succesfull allocation, the pool that is charged.
637 * @ret_limit_pool: On a failed allocation, the limiting pool.
645 * will be set to the pool for which the limit is hit. This can be used for
656 struct dmem_cgroup_pool_state *pool; in dmem_cgroup_try_charge() local
670 pool = get_cg_pool_unlocked(cg, region); in dmem_cgroup_try_charge()
671 if (IS_ERR(pool)) { in dmem_cgroup_try_charge()
672 ret = PTR_ERR(pool); in dmem_cgroup_try_charge()
676 if (!page_counter_try_charge(&pool->cnt, size, &fail)) { in dmem_cgroup_try_charge()
682 dmemcg_pool_put(pool); in dmem_cgroup_try_charge()
688 *ret_pool = pool; in dmem_cgroup_try_charge()
735 struct dmem_cgroup_pool_state *pool = NULL; in dmemcg_limit_write() local
769 pool = get_cg_pool_unlocked(dmemcs, region); in dmemcg_limit_write()
770 if (IS_ERR(pool)) { in dmemcg_limit_write()
771 err = PTR_ERR(pool); in dmemcg_limit_write()
776 apply(pool, new_limit); in dmemcg_limit_write()
777 dmemcg_pool_put(pool); in dmemcg_limit_write()
795 struct dmem_cgroup_pool_state *pool = find_cg_pool_locked(dmemcs, region); in dmemcg_limit_show() local
800 val = fn(pool); in dmemcg_limit_show()