Lines Matching defs:pwq
266 * point to the pwq; thus, pwqs need to be aligned at two's power of the
282 * When pwq->nr_active >= max_active, new work item is queued to
283 * pwq->inactive_works instead of pool->worklist and marked with
287 * nr_active and all work items in pwq->inactive_works are marked with
289 * in pwq->inactive_works. Some of them are ready to run in
293 * WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
305 * Release of unbound pwq is punted to a kthread_worker. See put_pwq()
307 * RCU protected so that the first pwq can be determined without
331 * The following struct is used to enforce per-node max_active. When a pwq wants
334 * ->max, the pwq is queued on ->pending_pwqs. As in-flight work items finish
518 * I: kthread_worker to release pwq's. pwq release needs to be bounced to a
551 static void show_pwq(struct pool_workqueue *pwq);
606 * @pwq: iteration cursor
610 * If the pwq needs to be used beyond the locking in effect, the caller is
611 * responsible for guaranteeing that the pwq stays online.
616 #define for_each_pwq(pwq, wq) \
617 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
762 * default pwq is always mapped to the pool with the current effective cpumask.
792 * contain the pointer to the queued pwq. Once execution starts, the flag
796 * can be used to set the pwq, pool or clear work->data. These functions should
799 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
801 * queued anywhere after initialization until it is sync canceled. pwq is
810 static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
813 set_work_data(work, (unsigned long)pwq | WORK_STRUCT_PENDING |
1502 struct pool_workqueue *pwq = worker->current_pwq;
1505 if (!pwq)
1508 pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;
1534 pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;
1537 pwq->stats[PWQ_STAT_CM_WAKEUP]++;
1652 * @pwq: pool_workqueue to get
1654 * Obtain an extra reference on @pwq. The caller should guarantee that
1655 * @pwq has positive refcnt and be holding the matching pool->lock.
1657 static void get_pwq(struct pool_workqueue *pwq)
1659 lockdep_assert_held(&pwq->pool->lock);
1660 WARN_ON_ONCE(pwq->refcnt <= 0);
1661 pwq->refcnt++;
1666 * @pwq: pool_workqueue to put
1668 * Drop a reference of @pwq. If its refcnt reaches zero, schedule its
1671 static void put_pwq(struct pool_workqueue *pwq)
1673 lockdep_assert_held(&pwq->pool->lock);
1674 if (likely(--pwq->refcnt))
1677 * @pwq can't be released under pool->lock, bounce to a dedicated
1680 kthread_queue_work(pwq_release_worker, &pwq->release_work);
1685 * @pwq: pool_workqueue to put (can be %NULL)
1687 * put_pwq() with locking. This function also allows %NULL @pwq.
1689 static void put_pwq_unlocked(struct pool_workqueue *pwq)
1691 if (pwq) {
1696 raw_spin_lock_irq(&pwq->pool->lock);
1697 put_pwq(pwq);
1698 raw_spin_unlock_irq(&pwq->pool->lock);
1702 static bool pwq_is_empty(struct pool_workqueue *pwq)
1704 return !pwq->nr_active && list_empty(&pwq->inactive_works);
1707 static void __pwq_activate_work(struct pool_workqueue *pwq,
1714 if (list_empty(&pwq->pool->worklist))
1715 pwq->pool->last_progress_ts = jiffies;
1716 move_linked_works(work, &pwq->pool->worklist, NULL);
1734 * pwq_tryinc_nr_active - Try to increment nr_active for a pwq
1735 * @pwq: pool_workqueue of interest
1738 * Try to increment nr_active for @pwq. Returns %true if an nr_active count is
1741 static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
1743 struct workqueue_struct *wq = pwq->wq;
1744 struct worker_pool *pool = pwq->pool;
1751 /* BH or per-cpu workqueue, pwq->nr_active is sufficient */
1752 obtained = pwq->nr_active < READ_ONCE(wq->max_active);
1756 if (unlikely(pwq->plugged))
1760 * Unbound workqueue uses per-node shared nr_active $nna. If @pwq is
1768 if (!list_empty(&pwq->pending_node) && likely(!fill))
1784 if (list_empty(&pwq->pending_node))
1785 list_add_tail(&pwq->pending_node, &nna->pending_pwqs);
1794 * If @fill, @pwq might have already been pending. Being spuriously
1798 list_del_init(&pwq->pending_node);
1804 pwq->nr_active++;
1809 * pwq_activate_first_inactive - Activate the first inactive work item on a pwq
1810 * @pwq: pool_workqueue of interest
1813 * Activate the first inactive work item of @pwq if available and allowed by
1819 static bool pwq_activate_first_inactive(struct pool_workqueue *pwq, bool fill)
1822 list_first_entry_or_null(&pwq->inactive_works,
1825 if (work && pwq_tryinc_nr_active(pwq, fill)) {
1826 __pwq_activate_work(pwq, work);
1835 * @wq: workqueue_struct where its oldest pwq is to be unplugged
1838 * oldest pwq is unplugged, the others are plugged to suspend execution to
1850 * When the oldest pwq is drained and removed, this function should be called
1852 * pwq's are linked into wq->pwqs with the oldest first, so the first one in
1857 struct pool_workqueue *pwq;
1862 pwq = list_first_entry_or_null(&wq->pwqs, struct pool_workqueue,
1864 raw_spin_lock_irq(&pwq->pool->lock);
1865 if (pwq->plugged) {
1866 pwq->plugged = false;
1867 if (pwq_activate_first_inactive(pwq, true)) {
1871 * pwq to nna->pending_pwqs if the count can't be
1872 * obtained. We need to restore both for the pwq being
1875 * inactive, puts the pwq on pending_pwqs.
1877 pwq_activate_first_inactive(pwq, false);
1879 kick_pool(pwq->pool);
1882 raw_spin_unlock_irq(&pwq->pool->lock);
1886 * node_activate_pending_pwq - Activate a pending pwq on a wq_node_nr_active
1887 * @nna: wq_node_nr_active to activate a pending pwq for
1890 * Activate a pwq in @nna->pending_pwqs. Called with @caller_pool locked.
1897 struct pool_workqueue *pwq;
1904 pwq = list_first_entry_or_null(&nna->pending_pwqs,
1906 if (!pwq)
1910 * If @pwq is for a different pool than @locked_pool, we need to lock
1911 * @pwq->pool->lock. Let's trylock first. If unsuccessful, do the unlock
1915 if (pwq->pool != locked_pool) {
1917 locked_pool = pwq->pool;
1927 * $pwq may not have any inactive work items due to e.g. cancellations.
1930 work = list_first_entry_or_null(&pwq->inactive_works,
1933 list_del_init(&pwq->pending_node);
1939 * $pwq still has inactive work items, rotate it to the end of the
1945 pwq->nr_active++;
1946 __pwq_activate_work(pwq, work);
1948 if (list_empty(&pwq->inactive_works))
1949 list_del_init(&pwq->pending_node);
1951 list_move_tail(&pwq->pending_node, &nna->pending_pwqs);
1954 if (pwq->pool != caller_pool)
1955 kick_pool(pwq->pool);
1968 * @pwq: pool_workqueue of interest
1970 * Decrement @pwq's nr_active and try to activate the first inactive work item.
1971 * For unbound workqueues, this function may temporarily drop @pwq->pool->lock.
1973 static void pwq_dec_nr_active(struct pool_workqueue *pwq)
1975 struct worker_pool *pool = pwq->pool;
1976 struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);
1981 * @pwq->nr_active should be decremented for both percpu and unbound
1984 pwq->nr_active--;
1988 * inactive work item on @pwq itself.
1991 pwq_activate_first_inactive(pwq, false);
1996 * If @pwq is for an unbound workqueue, it's more complicated because
1998 * pwq needs to wait for an nr_active count, it puts itself on
2004 * $nna->max may change as CPUs come online/offline and @pwq->wq's
2006 * larger than @pwq->wq->min_active which is above zero unless freezing.
2017 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
2018 * @pwq: pwq of interest
2022 * decrement nr_in_flight of its pwq and handle workqueue flushing.
2025 * For unbound workqueues, this function may temporarily drop @pwq->pool->lock
2032 static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)
2037 pwq_dec_nr_active(pwq);
2039 pwq->nr_in_flight[color]--;
2042 if (likely(pwq->flush_color != color))
2046 if (pwq->nr_in_flight[color])
2049 /* this pwq is done, clear flush_color */
2050 pwq->flush_color = -1;
2053 * If this was the last pwq, wake up the first flusher. It
2056 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
2057 complete(&pwq->wq->first_flusher->done);
2059 put_pwq(pwq);
2094 struct pool_workqueue *pwq;
2126 * work->data is guaranteed to point to pwq only while the work
2127 * item is queued on pwq->wq, and both updating work->data to point
2128 * to pwq on queueing and to pool on dequeueing are done under
2129 * pwq->pool->lock. This in turn guarantees that, if work->data
2130 * points to pwq which is associated with a locked pool, the work
2133 pwq = get_work_pwq(work);
2134 if (pwq && pwq->pool == pool) {
2141 * pwq->inactive_works since a queued barrier can't be
2146 * on the inactive_works list, will confuse pwq->nr_active
2154 move_linked_works(work, &pwq->pool->worklist, NULL);
2159 * work->data points to pwq iff queued. Let's point to pool. As
2166 pwq_dec_nr_in_flight(pwq, work_data);
2209 * @pwq: pwq @work belongs to
2214 * Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
2220 static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
2229 set_work_pwq(work, pwq, extra_flags);
2231 get_pwq(pwq);
2278 struct pool_workqueue *pwq;
2303 /* pwq which will be used unless @work is executing elsewhere */
2311 pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu));
2312 pool = pwq->pool;
2319 * For ordered workqueue, work items must be queued on the newest pwq
2332 pwq = worker->current_pwq;
2333 pool = pwq->pool;
2345 * pwq is determined and locked. For unbound pools, we could have raced
2346 * with pwq release and it could already be dead. If its refcnt is zero,
2347 * repeat pwq selection. Note that unbound pwqs never die without
2348 * another pwq replacing it in cpu_pwq or while work items are executing
2351 if (unlikely(!pwq->refcnt)) {
2358 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
2362 /* pwq determined, queue */
2363 trace_workqueue_queue_work(req_cpu, pwq, work);
2368 pwq->nr_in_flight[pwq->work_color]++;
2369 work_flags = work_color_to_flags(pwq->work_color);
2376 if (list_empty(&pwq->inactive_works) && pwq_tryinc_nr_active(pwq, false)) {
2381 insert_work(pwq, work, &pool->worklist, work_flags);
2385 insert_work(pwq, work, &pwq->inactive_works, work_flags);
3024 static void send_mayday(struct pool_workqueue *pwq)
3026 struct workqueue_struct *wq = pwq->wq;
3034 if (list_empty(&pwq->mayday_node)) {
3036 * If @pwq is for an unbound wq, its base ref may be put at
3037 * any time due to an attribute change. Pin @pwq until the
3040 get_pwq(pwq);
3041 list_add_tail(&pwq->mayday_node, &wq->maydays);
3043 pwq->stats[PWQ_STAT_MAYDAY]++;
3204 struct pool_workqueue *pwq = get_work_pwq(work);
3230 worker->current_pwq = pwq;
3241 strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);
3251 if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))
3270 pwq->stats[PWQ_STAT_STARTED]++;
3277 lock_map_acquire(pwq->wq->lockdep_map);
3311 lock_map_release(pwq->wq->lockdep_map);
3339 pwq->stats[PWQ_STAT_COMPLETED]++;
3359 pwq_dec_nr_in_flight(pwq, work_data);
3485 static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescuer)
3487 struct worker_pool *pool = pwq->pool;
3488 struct work_struct *cursor = &pwq->mayday_cursor;
3492 if (!pwq->nr_active)
3518 !list_empty(&pwq->wq->maydays))
3530 if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) {
3531 pwq->stats[PWQ_STAT_RESCUED]++;
3581 * pwq(s) queued. This can happen by non-rescuer workers consuming
3588 /* see whether any pwq is asking for help */
3592 struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
3594 struct worker_pool *pool = pwq->pool;
3598 list_del_init(&pwq->mayday_node);
3608 while (assign_rescuer_work(pwq, rescuer)) {
3616 if (++count > RESCUER_BATCH && !list_empty(&pwq->wq->maydays) &&
3617 pwq->nr_active && need_to_create_worker(pool)) {
3619 send_mayday(pwq);
3626 if (!list_empty(&pwq->mayday_cursor.entry) && list_empty(&pwq->mayday_node))
3627 list_del_init(&pwq->mayday_cursor.entry);
3643 put_pwq_unlocked(pwq);
3848 * @pwq: pwq to insert barrier into
3865 * underneath us, so we can't reliably determine pwq from @target.
3870 static void insert_wq_barrier(struct pool_workqueue *pwq,
3890 (pwq->wq->flags & WQ_BH) ? &bh_key : &thr_key);
3917 pwq->nr_in_flight[work_color]++;
3920 insert_work(pwq, &barr->work, head, work_flags);
3932 * -1. If no pwq has in-flight commands at the specified color, all
3933 * pwq->flush_color's stay at -1 and %false is returned. If any pwq
3934 * has in flight commands, its pwq->flush_color is set to
3935 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
3958 struct pool_workqueue *pwq;
3970 * of pwq iters, the pool is the same, only doing lock/unlock
3974 for_each_pwq(pwq, wq) {
3975 if (current_pool != pwq->pool) {
3978 current_pool = pwq->pool;
3983 WARN_ON_ONCE(pwq->flush_color != -1);
3985 if (pwq->nr_in_flight[flush_color]) {
3986 pwq->flush_color = flush_color;
3993 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
3994 pwq->work_color = work_color;
4211 struct pool_workqueue *pwq;
4227 for_each_pwq(pwq, wq) {
4230 raw_spin_lock_irq(&pwq->pool->lock);
4231 drained = pwq_is_empty(pwq);
4232 raw_spin_unlock_irq(&pwq->pool->lock);
4257 struct pool_workqueue *pwq;
4269 pwq = get_work_pwq(work);
4270 if (pwq) {
4271 if (unlikely(pwq->pool != pool))
4277 pwq = worker->current_pwq;
4280 wq = pwq->wq;
4283 insert_wq_barrier(pwq, barr, work, worker);
5067 * pwq->refcnt == pool->refcnt == 0
5174 * Scheduled on pwq_release_worker by put_pwq() when an unbound pwq hits zero
5179 struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
5181 struct workqueue_struct *wq = pwq->wq;
5182 struct worker_pool *pool = pwq->pool;
5186 * When @pwq is not linked, it doesn't hold any reference to the
5189 if (!list_empty(&pwq->pwqs_node)) {
5191 list_del_rcu(&pwq->pwqs_node);
5209 if (!list_empty(&pwq->pending_node)) {
5211 wq_node_nr_active(pwq->wq, pwq->pool->node);
5214 list_del_init(&pwq->pending_node);
5218 kfree_rcu(pwq, rcu);
5221 * If we're the last pwq going away, @wq is already dead and no one
5230 /* initialize newly allocated @pwq which is associated with @wq and @pool */
5231 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
5234 BUG_ON((unsigned long)pwq & ~WORK_STRUCT_PWQ_MASK);
5236 memset(pwq, 0, sizeof(*pwq));
5238 pwq->pool = pool;
5239 pwq->wq = wq;
5240 pwq->flush_color = -1;
5241 pwq->refcnt = 1;
5242 INIT_LIST_HEAD(&pwq->inactive_works);
5243 INIT_LIST_HEAD(&pwq->pending_node);
5244 INIT_LIST_HEAD(&pwq->pwqs_node);
5245 INIT_LIST_HEAD(&pwq->mayday_node);
5246 kthread_init_work(&pwq->release_work, pwq_release_workfn);
5251 * The cursor work should only be in the pwq->pool->worklist, and
5257 INIT_WORK(&pwq->mayday_cursor, mayday_cursor_func);
5258 atomic_long_set(&pwq->mayday_cursor.data, (unsigned long)pwq |
5262 /* sync @pwq with the current state of its associated wq and link it */
5263 static void link_pwq(struct pool_workqueue *pwq)
5265 struct workqueue_struct *wq = pwq->wq;
5270 if (!list_empty(&pwq->pwqs_node))
5274 pwq->work_color = wq->work_color;
5276 /* link in @pwq */
5277 list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs);
5280 /* obtain a pool matching @attr and create a pwq associating the pool and @wq */
5285 struct pool_workqueue *pwq;
5293 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
5294 if (!pwq) {
5299 init_pwq(pwq, wq, pool);
5300 return pwq;
5315 * @attrs: the wq_attrs of the default pwq of the target workqueue
5341 /* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
5343 int cpu, struct pool_workqueue *pwq)
5352 link_pwq(pwq);
5355 rcu_assign_pointer(*slot, pwq);
5408 * the default pwq covering whole @attrs->cpumask. Always create
5437 * For initialized ordered workqueues, there should only be one pwq
5440 * the old pwq's have completed.
5502 * a separate pwq to each CPU pod with possibles CPUs in @attrs->cpumask so that
5505 * itself back-to-back will stay on its current pwq.
5524 * unbound_wq_update_pwq - update a pwq slot for CPU hot[un]plug
5526 * @cpu: the CPU to update the pwq slot for
5544 struct pool_workqueue *old_pwq = NULL, *pwq;
5562 /* nothing to do if the target cpumask matches the current pwq */
5567 /* create a new pwq */
5568 pwq = alloc_unbound_pwq(wq, target_attrs);
5569 if (!pwq) {
5575 /* Install the new pwq. */
5577 old_pwq = install_unbound_pwq(wq, cpu, pwq);
5582 pwq = unbound_pwq(wq, -1);
5583 raw_spin_lock_irq(&pwq->pool->lock);
5584 get_pwq(pwq);
5585 raw_spin_unlock_irq(&pwq->pool->lock);
5586 old_pwq = install_unbound_pwq(wq, cpu, pwq);
5636 /* there should only be single pwq for ordering guarantee */
5650 struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
5652 if (pwq) {
5654 * Unlink pwq from wq->pwqs since link_pwq()
5658 if (!list_empty(&pwq->pwqs_node))
5659 list_del_rcu(&pwq->pwqs_node);
5660 kmem_cache_free(pwq_cache, pwq);
5768 * Round-robin through pwq's activating the first inactive work item
5772 struct pool_workqueue *pwq;
5775 for_each_pwq(pwq, wq) {
5779 raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
5780 if (pwq_activate_first_inactive(pwq, true)) {
5782 kick_pool(pwq->pool);
5784 raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
5890 * Failed alloc_and_link_pwqs() may leave pending pwq->release_work,
5978 static bool pwq_busy(struct pool_workqueue *pwq)
5983 if (pwq->nr_in_flight[i])
5986 if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
5988 if (!pwq_is_empty(pwq))
6013 struct pool_workqueue *pwq;
6044 for_each_pwq(pwq, wq) {
6045 raw_spin_lock_irq(&pwq->pool->lock);
6046 if (WARN_ON(pwq_busy(pwq))) {
6047 pr_warn("%s: %s has the following busy pwq\n",
6049 show_pwq(pwq);
6050 raw_spin_unlock_irq(&pwq->pool->lock);
6056 raw_spin_unlock_irq(&pwq->pool->lock);
6199 struct pool_workqueue *pwq;
6207 pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
6208 ret = !list_empty(&pwq->inactive_works);
6291 struct pool_workqueue *pwq = NULL;
6309 copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
6310 copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
6391 static void show_pwq(struct pool_workqueue *pwq)
6394 struct worker_pool *pool = pwq->pool;
6400 pr_info(" pwq %d:", pool->id);
6404 pwq->nr_active, pwq->refcnt,
6405 !list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
6408 if (worker->current_pwq == pwq) {
6418 if (worker->current_pwq != pwq)
6435 if (get_work_pwq(work) == pwq) {
6445 if (get_work_pwq(work) != pwq)
6455 if (!list_empty(&pwq->inactive_works)) {
6459 list_for_each_entry(work, &pwq->inactive_works, entry) {
6474 struct pool_workqueue *pwq;
6478 for_each_pwq(pwq, wq) {
6479 if (!pwq_is_empty(pwq)) {
6489 for_each_pwq(pwq, wq) {
6490 raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
6491 if (!pwq_is_empty(pwq)) {
6498 show_pwq(pwq);
6501 raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
6651 * are a lot of assumptions on strong associations among work, pwq and
6973 struct pool_workqueue *pwq;
6987 for_each_pwq(pwq, wq) {
6988 WARN_ON_ONCE(pwq->nr_active < 0);
6989 if (pwq->nr_active) {
8006 * An ordered wq should have only one pwq as ordering is