Lines Matching defs:pool
3 * kernel/workqueue.c - generic async execution with shared worker pool
19 * executed in process context. The worker pool is shared and
66 * A bound pool is either associated or disassociated with its CPU.
73 * be executing on any CPU. The pool behaves as an unbound one.
80 * BH pool is per-CPU and always DISASSOCIATED.
82 POOL_BH = 1 << 0, /* is a BH pool */
109 UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
159 * L: pool->lock protected. Access with pool->lock held.
161 * LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for
164 * K: Only modified by worker while holding pool->lock. Can be safely read by
165 * self, while holding pool->lock or from IRQ context if %current is the
196 raw_spinlock_t lock; /* the pool lock */
199 int id; /* I: pool ID */
203 bool cpu_stall; /* WD: stalled cpu bound pool */
208 * but w/ pool->lock held. The readers grab pool->lock and are
240 * Destruction of pool is RCU protected to allow dereferences
264 * The per-pool workqueue. While queued, bits below WORK_PWQ_SHIFT
270 struct worker_pool *pool; /* I: the associated pool */
283 * pwq->inactive_works instead of pool->worklist and marked with
290 * pool->worklist or worker->scheduled. Those work itmes are only struct
300 struct work_struct mayday_cursor; /* L: cursor on pool->worklist */
341 raw_spinlock_t lock; /* nests inside pool locks */
508 /* PL: hash of all unbound pools keyed by pool->attrs */
519 * process context while holding a pool lock. Bounce to a dedicated kthread
552 static void show_one_worker_pool(struct worker_pool *pool);
562 #define for_each_bh_worker_pool(pool, cpu) \
563 for ((pool) = &per_cpu(bh_worker_pools, cpu)[0]; \
564 (pool) < &per_cpu(bh_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
565 (pool)++)
567 #define for_each_cpu_worker_pool(pool, cpu) \
568 for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
569 (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
570 (pool)++)
574 * @pool: iteration cursor
578 * locked. If the pool needs to be used beyond the locking in effect, the
579 * caller is responsible for guaranteeing that the pool stays online.
584 #define for_each_pool(pool, pi) \
585 idr_for_each_entry(&worker_pool_idr, pool, pi) \
592 * @pool: worker_pool to iterate workers of
599 #define for_each_pool_worker(worker, pool) \
600 list_for_each_entry((worker), &(pool)->workers, node) \
718 * worker_pool_assign_id - allocate ID and assign it to @pool
719 * @pool: the pool pointer of interest
724 static int worker_pool_assign_id(struct worker_pool *pool)
730 ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
733 pool->id = ret;
762 * default pwq is always mapped to the pool with the current effective cpumask.
766 return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask;
785 static unsigned long pool_offq_flags(struct worker_pool *pool)
787 return (pool->flags & POOL_BH) ? WORK_OFFQ_BH : 0;
793 * is cleared and the high bits contain OFFQ flags and pool ID.
796 * can be used to set the pwq, pool or clear work->data. These functions should
799 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
890 * All fields of the returned pool are accessible as long as the above
891 * mentioned locking is in effect. If the returned pool needs to be used
893 * returned pool is and stays online.
905 return work_struct_pwq(data)->pool;
939 * they're being called with pool->lock held.
950 static bool need_more_worker(struct worker_pool *pool)
952 return !list_empty(&pool->worklist) && !pool->nr_running;
956 static bool may_start_working(struct worker_pool *pool)
958 return pool->nr_idle;
962 static bool keep_working(struct worker_pool *pool)
964 return !list_empty(&pool->worklist) && (pool->nr_running <= 1);
968 static bool need_to_create_worker(struct worker_pool *pool)
970 return need_more_worker(pool) && !may_start_working(pool);
974 static bool too_many_workers(struct worker_pool *pool)
976 bool managing = pool->flags & POOL_MANAGER_ACTIVE;
977 int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
978 int nr_busy = pool->nr_workers - nr_idle;
992 struct worker_pool *pool = worker->pool;
994 lockdep_assert_held(&pool->lock);
999 pool->nr_running--;
1014 struct worker_pool *pool = worker->pool;
1017 lockdep_assert_held(&pool->lock);
1028 pool->nr_running++;
1031 /* Return the first idle worker. Called with pool->lock held. */
1032 static struct worker *first_idle_worker(struct worker_pool *pool)
1034 if (unlikely(list_empty(&pool->idle_list)))
1037 return list_first_entry(&pool->idle_list, struct worker, entry);
1048 * raw_spin_lock_irq(pool->lock).
1052 struct worker_pool *pool = worker->pool;
1061 pool->nr_idle++;
1065 list_add(&worker->entry, &pool->idle_list);
1067 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1068 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1071 WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running);
1081 * raw_spin_lock_irq(pool->lock).
1085 struct worker_pool *pool = worker->pool;
1090 pool->nr_idle--;
1096 * @pool: pool of interest
1099 * Find a worker which is executing @work on @pool by searching
1100 * @pool->busy_hash which is keyed by the address of @work. For a worker
1121 * raw_spin_lock_irq(pool->lock).
1127 static struct worker *find_worker_executing_work(struct worker_pool *pool,
1132 hash_for_each_possible(pool->busy_hash, worker, hentry,
1159 * raw_spin_lock_irq(pool->lock).
1192 * executed by another worker in the same pool, it'll be punted there.
1204 struct worker_pool *pool = worker->pool;
1207 lockdep_assert_held(&pool->lock);
1221 * __queue_work() ensures that @work doesn't jump to a different pool
1222 * while still running in the previous pool. Here, we should ensure that
1224 * pool. Check whether anyone is already processing the work. If so,
1227 collision = find_worker_executing_work(pool, work);
1237 static struct irq_work *bh_pool_irq_work(struct worker_pool *pool)
1239 int high = pool->attrs->nice == HIGHPRI_NICE_LEVEL ? 1 : 0;
1241 return &per_cpu(bh_pool_irq_works, pool->cpu)[high];
1244 static void kick_bh_pool(struct worker_pool *pool)
1248 if (unlikely(pool->cpu != smp_processor_id() &&
1249 !(pool->flags & POOL_BH_DRAINING))) {
1250 irq_work_queue_on(bh_pool_irq_work(pool), pool->cpu);
1254 if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
1262 * @pool: pool to kick
1264 * @pool may have pending work items. Wake up worker if necessary. Returns
1267 static bool kick_pool(struct worker_pool *pool)
1269 struct worker *worker = first_idle_worker(pool);
1272 lockdep_assert_held(&pool->lock);
1274 if (!need_more_worker(pool) || !worker)
1277 if (pool->flags & POOL_BH) {
1278 kick_bh_pool(pool);
1298 * If @pool has non-strict affinity, @worker might have ended up outside
1301 if (!pool->attrs->affn_strict &&
1302 !cpumask_test_cpu(p->wake_cpu, pool->attrs->__pod_cpumask)) {
1303 struct work_struct *work = list_first_entry(&pool->worklist,
1305 int wake_cpu = cpumask_any_and_distribute(pool->attrs->__pod_cpumask,
1429 * and leave with an unexpected pool->nr_running == 1 on the newly unbound
1430 * pool. Protect against such race.
1434 worker->pool->nr_running++;
1456 struct worker_pool *pool;
1466 pool = worker->pool;
1473 raw_spin_lock_irq(&pool->lock);
1481 raw_spin_unlock_irq(&pool->lock);
1485 pool->nr_running--;
1486 if (kick_pool(pool))
1489 raw_spin_unlock_irq(&pool->lock);
1503 struct worker_pool *pool = worker->pool;
1520 * @pool->nr_running until it wakes up. As wq_worker_sleeping() also
1530 raw_spin_lock(&pool->lock);
1536 if (kick_pool(pool))
1539 raw_spin_unlock(&pool->lock);
1655 * @pwq has positive refcnt and be holding the matching pool->lock.
1659 lockdep_assert_held(&pwq->pool->lock);
1669 * destruction. The caller should be holding the matching pool->lock.
1673 lockdep_assert_held(&pwq->pool->lock);
1677 * @pwq can't be released under pool->lock, bounce to a dedicated
1684 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1696 raw_spin_lock_irq(&pwq->pool->lock);
1698 raw_spin_unlock_irq(&pwq->pool->lock);
1714 if (list_empty(&pwq->pool->worklist))
1715 pwq->pool->last_progress_ts = jiffies;
1716 move_linked_works(work, &pwq->pool->worklist, NULL);
1744 struct worker_pool *pool = pwq->pool;
1745 struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node);
1748 lockdep_assert_held(&pool->lock);
1864 raw_spin_lock_irq(&pwq->pool->lock);
1879 kick_pool(pwq->pool);
1882 raw_spin_unlock_irq(&pwq->pool->lock);
1910 * If @pwq is for a different pool than @locked_pool, we need to lock
1911 * @pwq->pool->lock. Let's trylock first. If unsuccessful, do the unlock
1913 * nested inside pool locks.
1915 if (pwq->pool != locked_pool) {
1917 locked_pool = pwq->pool;
1953 /* if activating a foreign pool, make sure it's running */
1954 if (pwq->pool != caller_pool)
1955 kick_pool(pwq->pool);
1971 * For unbound workqueues, this function may temporarily drop @pwq->pool->lock.
1975 struct worker_pool *pool = pwq->pool;
1976 struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);
1978 lockdep_assert_held(&pool->lock);
2013 node_activate_pending_pwq(nna, pool);
2025 * For unbound workqueues, this function may temporarily drop @pwq->pool->lock
2030 * raw_spin_lock_irq(pool->lock).
2093 struct worker_pool *pool;
2120 pool = get_work_pool(work);
2121 if (!pool)
2124 raw_spin_lock(&pool->lock);
2128 * to pwq on queueing and to pool on dequeueing are done under
2129 * pwq->pool->lock. This in turn guarantees that, if work->data
2130 * points to pwq which is associated with a locked pool, the work
2131 * item is currently queued on that pool.
2134 if (pwq && pwq->pool == pool) {
2154 move_linked_works(work, &pwq->pool->worklist, NULL);
2159 * work->data points to pwq iff queued. Let's point to pool. As
2162 set_work_pool_and_keep_pending(work, pool->id,
2163 pool_offq_flags(pool));
2168 raw_spin_unlock(&pool->lock);
2172 raw_spin_unlock(&pool->lock);
2208 * insert_work - insert a work into a pool
2218 * raw_spin_lock_irq(pool->lock).
2279 struct worker_pool *last_pool, *pool;
2312 pool = pwq->pool;
2315 * If @work was previously on a different pool, it might still be
2317 * pool to guarantee non-reentrancy.
2324 if (last_pool && last_pool != pool && !(wq->flags & __WQ_ORDERED)) {
2333 pool = pwq->pool;
2334 WARN_ON_ONCE(pool != last_pool);
2338 raw_spin_lock(&pool->lock);
2341 raw_spin_lock(&pool->lock);
2353 raw_spin_unlock(&pool->lock);
2377 if (list_empty(&pool->worklist))
2378 pool->last_progress_ts = jiffies;
2381 insert_work(pwq, work, &pool->worklist, work_flags);
2382 kick_pool(pool);
2389 raw_spin_unlock(&pool->lock);
2697 static cpumask_t *pool_allowed_cpus(struct worker_pool *pool)
2699 if (pool->cpu < 0 && pool->attrs->affn_strict)
2700 return pool->attrs->__pod_cpumask;
2702 return pool->attrs->cpumask;
2706 * worker_attach_to_pool() - attach a worker to a pool
2708 * @pool: the target pool
2710 * Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
2711 * cpu-binding of @worker are kept coordinated with the pool across
2715 struct worker_pool *pool)
2724 if (pool->flags & POOL_DISASSOCIATED) {
2727 WARN_ON_ONCE(pool->flags & POOL_BH);
2728 kthread_set_per_cpu(worker->task, pool->cpu);
2732 set_cpus_allowed_ptr(worker->task, pool_allowed_cpus(pool));
2734 list_add_tail(&worker->node, &pool->workers);
2735 worker->pool = pool;
2761 * worker_detach_from_pool() - detach a worker from its pool
2762 * @worker: worker which is attached to its pool
2765 * caller worker shouldn't access to the pool after detached except it has
2766 * other reference to the pool.
2770 struct worker_pool *pool = worker->pool;
2773 WARN_ON_ONCE(pool->flags & POOL_BH);
2777 worker->pool = NULL;
2780 /* clear leftover flags without pool->lock after it is detached */
2785 struct worker_pool *pool)
2791 if (pool) {
2792 if (pool->cpu >= 0)
2794 pool->cpu, worker->id,
2795 pool->attrs->nice < 0 ? "H" : "");
2798 pool->id, worker->id);
2806 * @pool: pool the new worker will belong to
2808 * Create and start a new worker which is attached to @pool.
2816 static struct worker *create_worker(struct worker_pool *pool)
2822 id = ida_alloc(&pool->worker_ida, GFP_KERNEL);
2829 worker = alloc_worker(pool->node);
2837 if (!(pool->flags & POOL_BH)) {
2840 format_worker_id(id_buf, sizeof(id_buf), worker, pool);
2842 pool->node, "%s", id_buf);
2854 set_user_nice(worker->task, pool->attrs->nice);
2855 kthread_bind_mask(worker->task, pool_allowed_cpus(pool));
2858 /* successful, attach the worker to the pool */
2859 worker_attach_to_pool(worker, pool);
2862 raw_spin_lock_irq(&pool->lock);
2864 worker->pool->nr_workers++;
2869 * check if not woken up soon. As kick_pool() is noop if @pool is empty,
2875 raw_spin_unlock_irq(&pool->lock);
2880 ida_free(&pool->worker_ida, id);
2907 * @list: transfer worker away from its pool->idle_list and into list
2909 * Tag @worker for destruction and adjust @pool stats accordingly. The worker
2913 * raw_spin_lock_irq(pool->lock).
2917 struct worker_pool *pool = worker->pool;
2919 lockdep_assert_held(&pool->lock);
2928 pool->nr_workers--;
2929 pool->nr_idle--;
2941 * @t: The pool's idle_timer that just expired
2945 * pool is at the too_many_workers() tipping point would cause too much timer
2951 struct worker_pool *pool = timer_container_of(pool, t, idle_timer);
2954 if (work_pending(&pool->idle_cull_work))
2957 raw_spin_lock_irq(&pool->lock);
2959 if (too_many_workers(pool)) {
2964 worker = list_last_entry(&pool->idle_list, struct worker, entry);
2969 mod_timer(&pool->idle_timer, expires);
2971 raw_spin_unlock_irq(&pool->lock);
2974 queue_work(system_dfl_wq, &pool->idle_cull_work);
2979 * @work: the pool's work for handling these idle workers
2981 * This goes through a pool's idle workers and gets rid of those that have been
2990 struct worker_pool *pool = container_of(work, struct worker_pool, idle_cull_work);
3000 raw_spin_lock_irq(&pool->lock);
3002 while (too_many_workers(pool)) {
3006 worker = list_last_entry(&pool->idle_list, struct worker, entry);
3010 mod_timer(&pool->idle_timer, expires);
3017 raw_spin_unlock_irq(&pool->lock);
3049 struct worker_pool *pool = timer_container_of(pool, t, mayday_timer);
3052 raw_spin_lock_irq(&pool->lock);
3055 if (need_to_create_worker(pool)) {
3062 list_for_each_entry(work, &pool->worklist, entry)
3067 raw_spin_unlock_irq(&pool->lock);
3069 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
3074 * @pool: pool to create a new worker for
3076 * Create a new worker for @pool if necessary. @pool is guaranteed to
3079 * sent to all rescuers with works scheduled on @pool to resolve
3086 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
3090 static void maybe_create_worker(struct worker_pool *pool)
3091 __releases(&pool->lock)
3092 __acquires(&pool->lock)
3095 raw_spin_unlock_irq(&pool->lock);
3098 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
3101 if (create_worker(pool) || !need_to_create_worker(pool))
3106 if (!need_to_create_worker(pool))
3110 timer_delete_sync(&pool->mayday_timer);
3111 raw_spin_lock_irq(&pool->lock);
3114 * created as @pool->lock was dropped and the new worker might have
3117 if (need_to_create_worker(pool))
3122 static void worker_lock_callback(struct worker_pool *pool)
3124 spin_lock(&pool->cb_lock);
3127 static void worker_unlock_callback(struct worker_pool *pool)
3129 spin_unlock(&pool->cb_lock);
3132 static void workqueue_callback_cancel_wait_running(struct worker_pool *pool)
3134 spin_lock(&pool->cb_lock);
3135 spin_unlock(&pool->cb_lock);
3140 static void worker_lock_callback(struct worker_pool *pool) { }
3141 static void worker_unlock_callback(struct worker_pool *pool) { }
3142 static void workqueue_callback_cancel_wait_running(struct worker_pool *pool) { }
3147 * manage_workers - manage worker pool
3150 * Assume the manager role and manage the worker pool @worker belongs
3152 * pool. The exclusion is handled automatically by this function.
3159 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
3163 * %false if the pool doesn't need management and the caller can safely
3170 struct worker_pool *pool = worker->pool;
3172 if (pool->flags & POOL_MANAGER_ACTIVE)
3175 pool->flags |= POOL_MANAGER_ACTIVE;
3176 pool->manager = worker;
3178 maybe_create_worker(pool);
3180 pool->manager = NULL;
3181 pool->flags &= ~POOL_MANAGER_ACTIVE;
3198 * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
3201 __releases(&pool->lock)
3202 __acquires(&pool->lock)
3205 struct worker_pool *pool = worker->pool;
3208 bool bh_draining = pool->flags & POOL_BH_DRAINING;
3222 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
3223 raw_smp_processor_id() != pool->cpu);
3227 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
3255 * Kick @pool if necessary. It's always noop for per-cpu worker pools
3260 kick_pool(pool);
3263 * Record the last pool and clear PENDING which should be the last
3264 * update to @work. Also, do this inside @pool->lock so that
3268 set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool));
3271 raw_spin_unlock_irq(&pool->lock);
3337 raw_spin_lock_irq(&pool->lock);
3371 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
3382 worker->pool->last_progress_ts = jiffies;
3414 struct worker_pool *pool = worker->pool;
3419 raw_spin_lock_irq(&pool->lock);
3423 raw_spin_unlock_irq(&pool->lock);
3426 * The worker is dead and PF_WQ_WORKER is cleared, worker->pool
3429 worker->pool = NULL;
3430 ida_free(&pool->worker_ida, worker->id);
3437 if (!need_more_worker(pool))
3441 if (unlikely(!may_start_working(pool)) && manage_workers(worker))
3462 list_first_entry(&pool->worklist,
3467 } while (keep_working(pool));
3472 * pool->lock is held and there's no work to process and no need to
3474 * pool->lock or from local cpu, so setting the current state
3475 * before releasing pool->lock is enough to prevent losing any
3480 raw_spin_unlock_irq(&pool->lock);
3487 struct worker_pool *pool = pwq->pool;
3496 if (!need_to_create_worker(pool)) {
3498 * The pool has idle workers and doesn't need the rescuer, so it
3502 * In PERCPU pool with concurrency enabled, having idle workers
3508 * the pool may run out of idle workers, and it will have to
3514 * relapse, using (pool->flags & POOL_MANAGER_ACTIVE), though
3517 if (!(pool->flags & POOL_MANAGER_ACTIVE) ||
3524 work = list_first_entry(&pool->worklist, struct work_struct, entry);
3529 list_for_each_entry_safe_from(work, n, &pool->worklist, entry) {
3548 * Regular work processing on a pool may block trying to create a new
3554 * When such condition is possible, the pool summons rescuers of all
3555 * workqueues which have works queued on the pool and let them process
3594 struct worker_pool *pool = pwq->pool;
3602 worker_attach_to_pool(rescuer, pool);
3604 raw_spin_lock_irq(&pool->lock);
3617 pwq->nr_active && need_to_create_worker(pool)) {
3630 * Leave this pool. Notify regular workers; otherwise, we end up
3633 kick_pool(pool);
3635 raw_spin_unlock_irq(&pool->lock);
3640 * Put the reference grabbed by send_mayday(). @pool might
3664 struct worker_pool *pool = worker->pool;
3668 worker_lock_callback(pool);
3669 raw_spin_lock_irq(&pool->lock);
3676 if (!need_more_worker(pool))
3684 list_first_entry(&pool->worklist,
3689 } while (keep_working(pool) &&
3695 kick_pool(pool);
3696 raw_spin_unlock_irq(&pool->lock);
3697 worker_unlock_callback(pool);
3713 struct worker_pool *pool =
3715 if (need_more_worker(pool))
3716 bh_worker(list_first_entry(&pool->workers, struct worker, node));
3721 struct worker_pool *pool;
3729 struct worker_pool *pool = dead_work->pool;
3733 * @pool's CPU is dead and we want to execute its still pending work
3735 * its CPU is dead, @pool can't be kicked and, as work execution path
3737 * @pool with %POOL_BH_DRAINING for the special treatments.
3739 raw_spin_lock_irq(&pool->lock);
3740 pool->flags |= POOL_BH_DRAINING;
3741 raw_spin_unlock_irq(&pool->lock);
3743 bh_worker(list_first_entry(&pool->workers, struct worker, node));
3745 raw_spin_lock_irq(&pool->lock);
3746 pool->flags &= ~POOL_BH_DRAINING;
3747 repeat = need_more_worker(pool);
3748 raw_spin_unlock_irq(&pool->lock);
3756 if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
3771 * items which shouldn't be requeued on the same pool. Shouldn't take long.
3778 struct worker_pool *pool = &per_cpu(bh_worker_pools, cpu)[i];
3781 if (!need_more_worker(pool))
3785 dead_work.pool = pool;
3788 if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
3868 * raw_spin_lock_irq(pool->lock).
3880 * debugobject calls are safe here even with pool->lock locked
3968 * Most of the time, pwqs within the same pool will be linked
3970 * of pwq iters, the pool is the same, only doing lock/unlock
3971 * if the pool has changed. This can largely reduce expensive
3975 if (current_pool != pwq->pool) {
3978 current_pool = pwq->pool;
4230 raw_spin_lock_irq(&pwq->pool->lock);
4232 raw_spin_unlock_irq(&pwq->pool->lock);
4256 struct worker_pool *pool;
4261 pool = get_work_pool(work);
4262 if (!pool) {
4267 raw_spin_lock_irq(&pool->lock);
4271 if (unlikely(pwq->pool != pool))
4274 worker = find_worker_executing_work(pool, work);
4284 raw_spin_unlock_irq(&pool->lock);
4303 raw_spin_unlock_irq(&pool->lock);
4340 struct worker_pool *pool;
4343 pool = get_work_pool(work);
4344 if (pool)
4345 workqueue_callback_cancel_wait_running(pool);
4770 * fields as copying is used for both pool and wq attrs. Instead,
4865 * @pool: worker_pool to initialize
4867 * Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
4870 * inside @pool proper are initialized and put_unbound_pool() can be called
4871 * on @pool safely to release it.
4873 static int init_worker_pool(struct worker_pool *pool)
4875 raw_spin_lock_init(&pool->lock);
4876 pool->id = -1;
4877 pool->cpu = -1;
4878 pool->node = NUMA_NO_NODE;
4879 pool->flags |= POOL_DISASSOCIATED;
4880 pool->last_progress_ts = jiffies;
4881 INIT_LIST_HEAD(&pool->worklist);
4882 INIT_LIST_HEAD(&pool->idle_list);
4883 hash_init(pool->busy_hash);
4885 timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
4886 INIT_WORK(&pool->idle_cull_work, idle_cull_fn);
4888 timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);
4890 INIT_LIST_HEAD(&pool->workers);
4892 ida_init(&pool->worker_ida);
4893 INIT_HLIST_NODE(&pool->hash_node);
4894 pool->refcnt = 1;
4896 spin_lock_init(&pool->cb_lock);
4900 pool->attrs = alloc_workqueue_attrs();
4901 if (!pool->attrs)
4904 wqattrs_clear_for_pool(pool->attrs);
5022 struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
5024 ida_destroy(&pool->worker_ida);
5025 free_workqueue_attrs(pool->attrs);
5026 kfree(pool);
5031 * @pool: worker_pool to put
5033 * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
5040 static void put_unbound_pool(struct worker_pool *pool)
5047 if (--pool->refcnt)
5051 if (WARN_ON(!(pool->cpu < 0)) ||
5052 WARN_ON(!list_empty(&pool->worklist)))
5056 if (pool->id >= 0)
5057 idr_remove(&worker_pool_idr, pool->id);
5058 hash_del(&pool->hash_node);
5062 * @pool's workers from blocking on attach_mutex. We're the last
5063 * manager and @pool gets freed with the flag set.
5067 * pwq->refcnt == pool->refcnt == 0
5068 * which implies no work queued to the pool, which implies no worker can
5071 * drops pool->lock
5075 !(pool->flags & POOL_MANAGER_ACTIVE),
5079 raw_spin_lock_irq(&pool->lock);
5080 if (!(pool->flags & POOL_MANAGER_ACTIVE)) {
5081 pool->flags |= POOL_MANAGER_ACTIVE;
5084 raw_spin_unlock_irq(&pool->lock);
5088 while ((worker = first_idle_worker(pool)))
5090 WARN_ON(pool->nr_workers || pool->nr_idle);
5091 raw_spin_unlock_irq(&pool->lock);
5100 timer_delete_sync(&pool->idle_timer);
5101 cancel_work_sync(&pool->idle_cull_work);
5102 timer_delete_sync(&pool->mayday_timer);
5105 call_rcu(&pool->rcu, rcu_free_pool);
5126 struct worker_pool *pool;
5131 /* do we already have a matching pool? */
5132 hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
5133 if (wqattrs_equal(pool->attrs, attrs)) {
5134 pool->refcnt++;
5135 return pool;
5148 pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, node);
5149 if (!pool || init_worker_pool(pool) < 0)
5152 pool->node = node;
5153 copy_workqueue_attrs(pool->attrs, attrs);
5154 wqattrs_clear_for_pool(pool->attrs);
5156 if (worker_pool_assign_id(pool) < 0)
5160 if (wq_online && !create_worker(pool))
5164 hash_add(unbound_pool_hash, &pool->hash_node, hash);
5166 return pool;
5168 if (pool)
5169 put_unbound_pool(pool);
5182 struct worker_pool *pool = pwq->pool;
5205 put_unbound_pool(pool);
5211 wq_node_nr_active(pwq->wq, pwq->pool->node);
5230 /* initialize newly allocated @pwq which is associated with @wq and @pool */
5232 struct worker_pool *pool)
5238 pwq->pool = pool;
5251 * The cursor work should only be in the pwq->pool->worklist, and
5280 /* obtain a pool matching @attr and create a pwq associating the pool and @wq */
5284 struct worker_pool *pool;
5289 pool = get_unbound_pool(attrs);
5290 if (!pool)
5293 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
5295 put_unbound_pool(pool);
5299 init_pwq(pwq, wq, pool);
5564 if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
5583 raw_spin_lock_irq(&pwq->pool->lock);
5585 raw_spin_unlock_irq(&pwq->pool->lock);
5613 struct worker_pool *pool;
5615 pool = &(per_cpu_ptr(pools, cpu)[highpri]);
5619 pool->node);
5623 init_pwq(*pwq_p, wq, pool);
5779 raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
5782 kick_pool(pwq->pool);
5784 raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
6045 raw_spin_lock_irq(&pwq->pool->lock);
6050 raw_spin_unlock_irq(&pwq->pool->lock);
6056 raw_spin_unlock_irq(&pwq->pool->lock);
6229 struct worker_pool *pool;
6237 pool = get_work_pool(work);
6238 if (pool) {
6239 raw_spin_lock_irqsave(&pool->lock, irq_flags);
6240 if (find_worker_executing_work(pool, work))
6242 raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
6322 static void pr_cont_pool_info(struct worker_pool *pool)
6324 pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
6325 if (pool->node != NUMA_NO_NODE)
6326 pr_cont(" node=%d", pool->node);
6327 pr_cont(" flags=0x%x", pool->flags);
6328 if (pool->flags & POOL_BH)
6330 pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
6332 pr_cont(" nice=%d", pool->attrs->nice);
6337 struct worker_pool *pool = worker->pool;
6339 if (pool->flags & POOL_BH)
6341 pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
6394 struct worker_pool *pool = pwq->pool;
6400 pr_info(" pwq %d:", pool->id);
6401 pr_cont_pool_info(pool);
6407 hash_for_each(pool->busy_hash, bkt, worker, hentry) {
6417 hash_for_each(pool->busy_hash, bkt, worker, hentry) {
6434 list_for_each_entry(work, &pool->worklist, entry) {
6444 list_for_each_entry(work, &pool->worklist, entry) {
6490 raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
6501 raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
6513 * show_one_worker_pool - dump state of specified worker pool
6514 * @pool: worker pool whose state will be printed
6516 static void show_one_worker_pool(struct worker_pool *pool)
6523 raw_spin_lock_irqsave(&pool->lock, irq_flags);
6524 if (pool->nr_workers == pool->nr_idle)
6528 if (!list_empty(&pool->worklist))
6529 hung = jiffies_to_msecs(jiffies - pool->last_progress_ts) / 1000;
6537 pr_info("pool %d:", pool->id);
6538 pr_cont_pool_info(pool);
6539 pr_cont(" hung=%lus workers=%d", hung, pool->nr_workers);
6540 if (pool->manager)
6542 task_pid_nr(pool->manager->task));
6543 list_for_each_entry(worker, &pool->idle_list, entry) {
6551 raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
6569 struct worker_pool *pool;
6579 for_each_pool(pool, pi)
6580 show_one_worker_pool(pool);
6611 /* stabilize PF_WQ_WORKER and worker pool association */
6616 struct worker_pool *pool = worker->pool;
6619 off = format_worker_id(buf, size, worker, pool);
6621 if (pool) {
6622 raw_spin_lock_irq(&pool->lock);
6636 raw_spin_unlock_irq(&pool->lock);
6652 * pool which make migrating pending and scheduled works very
6664 struct worker_pool *pool;
6667 for_each_cpu_worker_pool(pool, cpu) {
6669 raw_spin_lock_irq(&pool->lock);
6679 for_each_pool_worker(worker, pool)
6682 pool->flags |= POOL_DISASSOCIATED;
6688 * long as the worklist is not empty. This pool now behaves as
6689 * an unbound (in terms of concurrency management) pool which
6690 * are served by workers tied to the pool.
6692 pool->nr_running = 0;
6699 kick_pool(pool);
6701 raw_spin_unlock_irq(&pool->lock);
6703 for_each_pool_worker(worker, pool)
6711 * rebind_workers - rebind all workers of a pool to the associated CPU
6712 * @pool: pool of interest
6714 * @pool->cpu is coming online. Rebind all workers to the CPU.
6716 static void rebind_workers(struct worker_pool *pool)
6729 for_each_pool_worker(worker, pool) {
6730 kthread_set_per_cpu(worker->task, pool->cpu);
6732 pool_allowed_cpus(pool)) < 0);
6735 raw_spin_lock_irq(&pool->lock);
6737 pool->flags &= ~POOL_DISASSOCIATED;
6739 for_each_pool_worker(worker, pool) {
6763 raw_spin_unlock_irq(&pool->lock);
6768 * @pool: unbound pool of interest
6771 * An unbound pool may end up with a cpumask which doesn't have any online
6772 * CPUs. When a worker of such pool get scheduled, the scheduler resets
6773 * its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
6776 static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
6783 /* is @cpu allowed for @pool? */
6784 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
6787 cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
6790 for_each_pool_worker(worker, pool)
6796 struct worker_pool *pool;
6798 for_each_cpu_worker_pool(pool, cpu) {
6799 if (pool->nr_workers)
6801 if (!create_worker(pool))
6809 struct worker_pool *pool;
6817 for_each_pool(pool, pi) {
6819 if (pool->flags & POOL_BH)
6823 if (pool->cpu == cpu)
6824 rebind_workers(pool);
6825 else if (pool->cpu < 0)
6826 restore_unbound_workers_cpumask(pool, cpu);
6933 * pool->worklist.
6936 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
7006 * frozen works are transferred to their respective pool worklists.
7009 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
7064 struct worker_pool *pool;
7071 if (wq->rescuer && !wq->rescuer->pool)
7076 for_each_cpu_worker_pool(pool, cpu) {
7077 if (!(pool->flags & POOL_DISASSOCIATED))
7079 for_each_pool_worker(worker, pool)
7654 * wait_event_idle() with PF_WQ_WORKER cleared) can stall the pool just as
7657 static void show_cpu_pool_busy_workers(struct worker_pool *pool)
7663 raw_spin_lock_irqsave(&pool->lock, irq_flags);
7665 hash_for_each(pool->busy_hash, bkt, worker, hentry) {
7673 pr_info("pool %d:\n", pool->id);
7679 raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
7684 struct worker_pool *pool;
7691 for_each_pool(pool, pi) {
7692 if (pool->cpu_stall)
7693 show_cpu_pool_busy_workers(pool);
7737 struct worker_pool *pool;
7744 for_each_pool(pool, pi) {
7747 pool->cpu_stall = false;
7748 if (list_empty(&pool->worklist))
7757 /* get the latest of pool and touched timestamps */
7758 if (pool->cpu >= 0)
7759 touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
7762 pool_ts = READ_ONCE(pool->last_progress_ts);
7775 * under pool->lock. The lock makes sure that the check reads
7776 * an updated pool->last_progress_ts when this CPU saw
7777 * an already updated pool->worklist above. It seems better
7782 scoped_guard(raw_spinlock_irqsave, &pool->lock) {
7783 pool_ts = pool->last_progress_ts;
7795 if (pool->cpu >= 0 && !(pool->flags & POOL_BH)) {
7796 pool->cpu_stall = true;
7799 pr_emerg("BUG: workqueue lockup - pool");
7800 pr_cont_pool_info(pool);
7905 static void __init init_cpu_worker_pool(struct worker_pool *pool, int cpu, int nice)
7907 BUG_ON(init_worker_pool(pool));
7908 pool->cpu = cpu;
7909 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
7910 cpumask_copy(pool->attrs->__pod_cpumask, cpumask_of(cpu));
7911 pool->attrs->nice = nice;
7912 pool->attrs->affn_strict = true;
7913 pool->node = cpu_to_node(cpu);
7915 /* alloc pool ID */
7917 BUG_ON(worker_pool_assign_id(pool));
7982 struct worker_pool *pool;
7985 for_each_bh_worker_pool(pool, cpu) {
7986 init_cpu_worker_pool(pool, cpu, std_nice[i]);
7987 pool->flags |= POOL_BH;
7988 init_irq_work(bh_pool_irq_work(pool), irq_work_fns[i]);
7993 for_each_cpu_worker_pool(pool, cpu)
7994 init_cpu_worker_pool(pool, cpu, std_nice[i++]);
8090 struct worker_pool *pool;
8102 for_each_bh_worker_pool(pool, cpu)
8103 pool->node = cpu_to_node(cpu);
8104 for_each_cpu_worker_pool(pool, cpu)
8105 pool->node = cpu_to_node(cpu);
8117 * Create the initial workers. A BH pool has one pseudo worker that
8123 for_each_bh_worker_pool(pool, cpu)
8124 BUG_ON(!create_worker(pool));
8127 for_each_cpu_worker_pool(pool, cpu) {
8128 pool->flags &= ~POOL_DISASSOCIATED;
8129 BUG_ON(!create_worker(pool));
8133 hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
8134 BUG_ON(!create_worker(pool));
8407 * worker pool. Explicitly call unbound_wq_update_pwq() on all workqueue