Lines Matching full:wq
176 * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
178 * PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
181 * WQ: wq->mutex protected.
183 * WR: wq->mutex protected for writes. RCU protected for reads.
185 * WO: wq->mutex protected for writes. Updated with WRITE_ONCE() and can be read
271 struct workqueue_struct *wq; /* I: the owning workqueue */
298 struct list_head pwqs_node; /* WR: node on wq->pwqs */
299 struct list_head mayday_node; /* MD: node on wq->maydays */
308 * grabbing wq->mutex.
318 struct list_head list; /* WQ: list of flushers */
319 int flush_color; /* WQ: flush color waiting for */
350 struct list_head pwqs; /* WR: all pwqs of this wq */
353 struct mutex mutex; /* protects this wq */
354 int work_color; /* WQ: current work color */
355 int flush_color; /* WQ: current flush color */
357 struct wq_flusher *first_flusher; /* WQ: first flusher */
358 struct list_head flusher_queue; /* WQ: flush waiters */
359 struct list_head flusher_overflow; /* WQ: flush overflow list */
364 int nr_drainers; /* WQ: drain in progress */
369 int saved_max_active; /* WQ: saved max_active */
370 int saved_min_active; /* WQ: saved min_active */
394 unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
460 static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
550 static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
607 * @wq: the target workqueue
609 * This must be called either with wq->mutex held or RCU read locked.
616 #define for_each_pwq(pwq, wq) \
617 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
618 lockdep_is_held(&(wq->mutex)))
740 unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
743 return per_cpu_ptr(wq->cpu_pwq, cpu);
745 return &wq->dfl_pwq;
749 static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
751 return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
753 lockdep_is_held(&wq->mutex));
758 * @wq: workqueue of interest
760 * @wq->unbound_attrs->cpumask contains the cpumask requested by the user which
764 static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq)
766 return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask;
1575 * @wq: workqueue of interest
1578 * Determine wq_node_nr_active to use for @wq on @node. Returns:
1586 static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq,
1589 if (!(wq->flags & WQ_UNBOUND))
1595 return wq->node_nr_active[node];
1600 * @wq: workqueue to update
1603 * Update @wq->node_nr_active[]->max. @wq must be unbound. max_active is
1605 * cpus. The result is always between @wq->min_active and max_active.
1607 static void wq_update_node_max_active(struct workqueue_struct *wq, int off_cpu)
1609 struct cpumask *effective = unbound_effective_cpumask(wq);
1610 int min_active = READ_ONCE(wq->min_active);
1611 int max_active = READ_ONCE(wq->max_active);
1614 lockdep_assert_held(&wq->mutex);
1626 /* If all CPUs of the wq get offline, use the default values */
1629 wq_node_nr_active(wq, node)->max = min_active;
1631 wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
1642 wq_node_nr_active(wq, node)->max =
1647 wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
1743 struct workqueue_struct *wq = pwq->wq;
1745 struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node);
1752 obtained = pwq->nr_active < READ_ONCE(wq->max_active);
1835 * @wq: workqueue_struct where its oldest pwq is to be unplugged
1852 * pwq's are linked into wq->pwqs with the oldest first, so the first one in
1855 static void unplug_oldest_pwq(struct workqueue_struct *wq)
1859 lockdep_assert_held(&wq->mutex);
1862 pwq = list_first_entry_or_null(&wq->pwqs, struct pool_workqueue,
1976 struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);
2004 * $nna->max may change as CPUs come online/offline and @pwq->wq's
2006 * larger than @pwq->wq->min_active which is above zero unless freezing.
2056 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
2057 complete(&pwq->wq->first_flusher->done);
2127 * item is queued on pwq->wq, and both updating work->data to point
2238 static bool is_chained_work(struct workqueue_struct *wq)
2244 * Return %true iff I'm a worker executing a work item on @wq. If
2247 return worker && worker->current_pwq->wq == wq;
2251 * When queueing an unbound work item to a wq, prefer local CPU if allowed
2275 static void __queue_work(int cpu, struct workqueue_struct *wq,
2292 * For a draining wq, only works from the same workqueue are
2294 * queues a new work item to a wq after destroy_workqueue(wq).
2296 if (unlikely(wq->flags & (__WQ_DESTROYING | __WQ_DRAINING) &&
2297 WARN_ONCE(!is_chained_work(wq), "workqueue: cannot queue %ps on wq %s\n",
2298 work->func, wq->name))) {
2305 if (wq->flags & WQ_UNBOUND)
2311 pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu));
2324 if (last_pool && last_pool != pool && !(wq->flags & __WQ_ORDERED)) {
2331 if (worker && worker->current_pwq->wq == wq) {
2352 if (wq->flags & WQ_UNBOUND) {
2359 wq->name, cpu);
2411 * @wq: workqueue to use
2422 bool queue_work_on(int cpu, struct workqueue_struct *wq,
2432 __queue_work(cpu, wq, work);
2473 * @wq: workqueue to use
2490 bool queue_work_node(int node, struct workqueue_struct *wq,
2505 WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
2513 __queue_work(cpu, wq, work);
2527 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
2531 static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
2548 __queue_work(cpu, wq, &dwork->work);
2553 dwork->wq = wq;
2574 * @wq: workqueue to use
2588 bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
2600 __queue_delayed_work(cpu, wq, dwork, delay);
2612 * @wq: workqueue to use
2627 bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
2636 __queue_delayed_work(cpu, wq, dwork, delay);
2649 __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
2655 * @wq: workqueue to use
2663 bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
2673 rwork->wq = wq;
3026 struct workqueue_struct *wq = pwq->wq;
3030 if (!wq->rescuer)
3036 * If @pwq is for an unbound wq, its base ref may be put at
3041 list_add_tail(&pwq->mayday_node, &wq->maydays);
3042 wake_up_process(wq->rescuer->task);
3053 raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
3238 * Record wq name for cmdline and debug reporting, may get
3241 strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);
3251 if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))
3277 lock_map_acquire(pwq->wq->lockdep_map);
3311 lock_map_release(pwq->wq->lockdep_map);
3518 !list_empty(&pwq->wq->maydays))
3565 struct workqueue_struct *wq = rescuer->rescue_wq;
3580 * shouldn't have any work pending, but @wq->maydays may still have
3583 * @wq->maydays processing before acting on should_stop so that the
3591 while (!list_empty(&wq->maydays)) {
3592 struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
3616 if (++count > RESCUER_BATCH && !list_empty(&pwq->wq->maydays) &&
3827 WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
3830 worker->current_pwq->wq->name, worker->current_func,
3890 (pwq->wq->flags & WQ_BH) ? &bh_key : &thr_key);
3925 * @wq: workqueue being flushed
3935 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
3938 * The caller should have initialized @wq->first_flusher prior to
3948 * mutex_lock(wq->mutex).
3954 static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
3962 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
3963 atomic_set(&wq->nr_pwqs_to_flush, 1);
3969 * sequentially to wq->pwqs by cpu index. So in the majority
3974 for_each_pwq(pwq, wq) {
3987 atomic_inc(&wq->nr_pwqs_to_flush);
4002 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
4003 complete(&wq->first_flusher->done);
4008 static void touch_wq_lockdep_map(struct workqueue_struct *wq)
4011 if (unlikely(!wq->lockdep_map))
4014 if (wq->flags & WQ_BH)
4017 lock_map_acquire(wq->lockdep_map);
4018 lock_map_release(wq->lockdep_map);
4020 if (wq->flags & WQ_BH)
4026 struct workqueue_struct *wq)
4029 if (wq->flags & WQ_BH)
4035 if (wq->flags & WQ_BH)
4042 * @wq: workqueue to flush
4047 void __flush_workqueue(struct workqueue_struct *wq)
4052 .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, (*wq->lockdep_map)),
4059 touch_wq_lockdep_map(wq);
4061 mutex_lock(&wq->mutex);
4066 next_color = work_next_color(wq->work_color);
4068 if (next_color != wq->flush_color) {
4074 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
4075 this_flusher.flush_color = wq->work_color;
4076 wq->work_color = next_color;
4078 if (!wq->first_flusher) {
4080 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
4082 wq->first_flusher = &this_flusher;
4084 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
4085 wq->work_color)) {
4087 wq->flush_color = next_color;
4088 wq->first_flusher = NULL;
4093 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
4094 list_add_tail(&this_flusher.list, &wq->flusher_queue);
4095 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
4103 list_add_tail(&this_flusher.list, &wq->flusher_overflow);
4106 check_flush_dependency(wq, NULL, false);
4108 mutex_unlock(&wq->mutex);
4118 if (READ_ONCE(wq->first_flusher) != &this_flusher)
4121 mutex_lock(&wq->mutex);
4124 if (wq->first_flusher != &this_flusher)
4127 WRITE_ONCE(wq->first_flusher, NULL);
4130 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
4136 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
4137 if (next->flush_color != wq->flush_color)
4143 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
4144 wq->flush_color != work_next_color(wq->work_color));
4147 wq->flush_color = work_next_color(wq->flush_color);
4150 if (!list_empty(&wq->flusher_overflow)) {
4157 list_for_each_entry(tmp, &wq->flusher_overflow, list)
4158 tmp->flush_color = wq->work_color;
4160 wq->work_color = work_next_color(wq->work_color);
4162 list_splice_tail_init(&wq->flusher_overflow,
4163 &wq->flusher_queue);
4164 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
4167 if (list_empty(&wq->flusher_queue)) {
4168 WARN_ON_ONCE(wq->flush_color != wq->work_color);
4176 WARN_ON_ONCE(wq->flush_color == wq->work_color);
4177 WARN_ON_ONCE(wq->flush_color != next->flush_color);
4180 wq->first_flusher = next;
4182 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
4189 wq->first_flusher = NULL;
4193 mutex_unlock(&wq->mutex);
4199 * @wq: workqueue to drain
4203 * work items on @wq can queue further work items on it. @wq is flushed
4208 void drain_workqueue(struct workqueue_struct *wq)
4215 * hotter than drain_workqueue() and already looks at @wq->flags.
4218 mutex_lock(&wq->mutex);
4219 if (!wq->nr_drainers++)
4220 wq->flags |= __WQ_DRAINING;
4221 mutex_unlock(&wq->mutex);
4223 __flush_workqueue(wq);
4225 mutex_lock(&wq->mutex);
4227 for_each_pwq(pwq, wq) {
4240 wq->name, __func__, flush_cnt);
4242 mutex_unlock(&wq->mutex);
4246 if (!--wq->nr_drainers)
4247 wq->flags &= ~__WQ_DRAINING;
4248 mutex_unlock(&wq->mutex);
4258 struct workqueue_struct *wq;
4280 wq = pwq->wq;
4281 check_flush_dependency(wq, work, from_cancel);
4286 touch_work_lockdep_map(work, wq);
4297 if (!from_cancel && (wq->saved_max_active == 1 || wq->rescuer))
4298 touch_wq_lockdep_map(wq);
4395 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
4769 * Unlike hash and equality test, copying shouldn't ignore wq-only
4770 * fields as copying is used for both pool and wq attrs. Instead,
4910 static void wq_init_lockdep(struct workqueue_struct *wq)
4914 lockdep_register_key(&wq->key);
4915 lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
4917 lock_name = wq->name;
4919 wq->lock_name = lock_name;
4920 wq->lockdep_map = &wq->__lockdep_map;
4921 lockdep_init_map(wq->lockdep_map, lock_name, &wq->key, 0);
4924 static void wq_unregister_lockdep(struct workqueue_struct *wq)
4926 if (wq->lockdep_map != &wq->__lockdep_map)
4929 lockdep_unregister_key(&wq->key);
4932 static void wq_free_lockdep(struct workqueue_struct *wq)
4934 if (wq->lockdep_map != &wq->__lockdep_map)
4937 if (wq->lock_name != wq->name)
4938 kfree(wq->lock_name);
4941 static void wq_init_lockdep(struct workqueue_struct *wq)
4945 static void wq_unregister_lockdep(struct workqueue_struct *wq)
4949 static void wq_free_lockdep(struct workqueue_struct *wq)
5008 struct workqueue_struct *wq =
5011 if (wq->flags & WQ_UNBOUND)
5012 free_node_nr_active(wq->node_nr_active);
5014 wq_free_lockdep(wq);
5015 free_percpu(wq->cpu_pwq);
5016 free_workqueue_attrs(wq->unbound_attrs);
5017 kfree(wq);
5181 struct workqueue_struct *wq = pwq->wq;
5187 * @wq, and @wq is invalid to access.
5190 mutex_lock(&wq->mutex);
5192 is_last = list_empty(&wq->pwqs);
5197 if (!is_last && (wq->flags & __WQ_ORDERED))
5198 unplug_oldest_pwq(wq);
5200 mutex_unlock(&wq->mutex);
5203 if (wq->flags & WQ_UNBOUND) {
5211 wq_node_nr_active(pwq->wq, pwq->pool->node);
5221 * If we're the last pwq going away, @wq is already dead and no one
5225 wq_unregister_lockdep(wq);
5226 call_rcu(&wq->rcu, rcu_free_wq);
5230 /* initialize newly allocated @pwq which is associated with @wq and @pool */
5231 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
5239 pwq->wq = wq;
5262 /* sync @pwq with the current state of its associated wq and link it */
5265 struct workqueue_struct *wq = pwq->wq;
5267 lockdep_assert_held(&wq->mutex);
5274 pwq->work_color = wq->work_color;
5277 list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs);
5280 /* obtain a pool matching @attr and create a pwq associating the pool and @wq */
5281 static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
5299 init_pwq(pwq, wq, pool);
5341 /* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
5342 static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
5345 struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
5349 lockdep_assert_held(&wq->mutex);
5361 struct workqueue_struct *wq; /* target workqueue */
5386 apply_wqattrs_prepare(struct workqueue_struct *wq,
5414 ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
5424 ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs);
5442 if ((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs))
5445 ctx->wq = wq;
5460 mutex_lock(&ctx->wq->mutex);
5462 copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
5466 ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
5468 ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);
5471 wq_update_node_max_active(ctx->wq, -1);
5473 mutex_unlock(&ctx->wq->mutex);
5476 static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
5482 if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
5485 ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);
5498 * @wq: the target workqueue
5501 * Apply @attrs to an unbound workqueue @wq. Unless disabled, this function maps
5511 int apply_workqueue_attrs(struct workqueue_struct *wq,
5517 ret = apply_workqueue_attrs_locked(wq, attrs);
5525 * @wq: the target workqueue
5533 * back to @wq->dfl_pwq which may not be optimal but is always correct.
5542 static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
5549 if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered)
5553 * We don't wanna alloc/free wq_attrs for each wq for each CPU.
5559 copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
5564 if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
5568 pwq = alloc_unbound_pwq(wq, target_attrs);
5571 wq->name);
5576 mutex_lock(&wq->mutex);
5577 old_pwq = install_unbound_pwq(wq, cpu, pwq);
5581 mutex_lock(&wq->mutex);
5582 pwq = unbound_pwq(wq, -1);
5586 old_pwq = install_unbound_pwq(wq, cpu, pwq);
5588 mutex_unlock(&wq->mutex);
5592 static int alloc_and_link_pwqs(struct workqueue_struct *wq)
5594 bool highpri = wq->flags & WQ_HIGHPRI;
5599 wq->cpu_pwq = alloc_percpu(struct pool_workqueue *);
5600 if (!wq->cpu_pwq)
5603 if (!(wq->flags & WQ_UNBOUND)) {
5606 if (wq->flags & WQ_BH)
5616 pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu);
5623 init_pwq(*pwq_p, wq, pool);
5625 mutex_lock(&wq->mutex);
5627 mutex_unlock(&wq->mutex);
5632 if (wq->flags & __WQ_ORDERED) {
5635 ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]);
5637 dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
5638 WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
5639 wq->pwqs.prev != &dfl_pwq->pwqs_node),
5640 "ordering guarantee broken for workqueue %s\n", wq->name);
5642 ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]);
5648 if (wq->cpu_pwq) {
5650 struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
5654 * Unlink pwq from wq->pwqs since link_pwq()
5655 * may have already added it. wq->mutex is not
5656 * needed as the wq has not been published yet.
5663 free_percpu(wq->cpu_pwq);
5664 wq->cpu_pwq = NULL;
5683 static int init_rescuer(struct workqueue_struct *wq)
5691 if (!(wq->flags & WQ_MEM_RECLAIM))
5696 pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n",
5697 wq->name);
5701 rescuer->rescue_wq = wq;
5707 pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe",
5708 wq->name, ERR_PTR(ret));
5713 wq->rescuer = rescuer;
5727 * wq_adjust_max_active - update a wq's max_active to the current setting
5728 * @wq: target workqueue
5730 * If @wq isn't freezing, set @wq->max_active to the saved_max_active and
5731 * activate inactive work items accordingly. If @wq is freezing, clear
5732 * @wq->max_active to zero.
5734 static void wq_adjust_max_active(struct workqueue_struct *wq)
5739 lockdep_assert_held(&wq->mutex);
5741 if ((wq->flags & WQ_FREEZABLE) && workqueue_freezing) {
5745 new_max = wq->saved_max_active;
5746 new_min = wq->saved_min_active;
5749 if (wq->max_active == new_max && wq->min_active == new_min)
5753 * Update @wq->max/min_active and then kick inactive work items if more
5758 WRITE_ONCE(wq->max_active, new_max);
5759 WRITE_ONCE(wq->min_active, new_min);
5761 if (wq->flags & WQ_UNBOUND)
5762 wq_update_node_max_active(wq, -1);
5775 for_each_pwq(pwq, wq) {
5794 struct workqueue_struct *wq;
5809 /* allocate wq and format name */
5811 wq_size = struct_size(wq, node_nr_active, nr_node_ids + 1);
5813 wq_size = sizeof(*wq);
5815 wq = kzalloc_noprof(wq_size, GFP_KERNEL);
5816 if (!wq)
5820 wq->unbound_attrs = alloc_workqueue_attrs_noprof();
5821 if (!wq->unbound_attrs)
5825 name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args);
5829 wq->name);
5839 max_active = wq_clamp_max_active(max_active, flags, wq->name);
5842 /* init wq */
5843 wq->flags = flags;
5844 wq->max_active = max_active;
5845 wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);
5846 wq->saved_max_active = wq->max_active;
5847 wq->saved_min_active = wq->min_active;
5848 mutex_init(&wq->mutex);
5849 atomic_set(&wq->nr_pwqs_to_flush, 0);
5850 INIT_LIST_HEAD(&wq->pwqs);
5851 INIT_LIST_HEAD(&wq->flusher_queue);
5852 INIT_LIST_HEAD(&wq->flusher_overflow);
5853 INIT_LIST_HEAD(&wq->maydays);
5855 INIT_LIST_HEAD(&wq->list);
5858 if (alloc_node_nr_active(wq->node_nr_active) < 0)
5868 if (alloc_and_link_pwqs(wq) < 0)
5871 mutex_lock(&wq->mutex);
5872 wq_adjust_max_active(wq);
5873 mutex_unlock(&wq->mutex);
5875 list_add_tail_rcu(&wq->list, &workqueues);
5877 if (wq_online && init_rescuer(wq) < 0)
5882 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
5885 return wq;
5892 * completes before calling kfree(wq).
5894 if (wq->flags & WQ_UNBOUND) {
5896 free_node_nr_active(wq->node_nr_active);
5899 free_workqueue_attrs(wq->unbound_attrs);
5900 kfree(wq);
5905 destroy_workqueue(wq);
5914 struct workqueue_struct *wq;
5918 wq = __alloc_workqueue(fmt, flags, max_active, args);
5920 if (!wq)
5923 wq_init_lockdep(wq);
5925 return wq;
5938 struct workqueue_struct *wq;
5943 wq = alloc_workqueue(fmt, flags, max_active, args);
5945 if (!wq)
5948 ret = devm_add_action_or_reset(dev, devm_workqueue_release, wq);
5952 return wq;
5962 struct workqueue_struct *wq;
5966 wq = __alloc_workqueue(fmt, flags, max_active, args);
5968 if (!wq)
5971 wq->lockdep_map = lockdep_map;
5973 return wq;
5986 if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
5996 * @wq: target workqueue
6011 void destroy_workqueue(struct workqueue_struct *wq)
6020 workqueue_sysfs_unregister(wq);
6023 mutex_lock(&wq->mutex);
6024 wq->flags |= __WQ_DESTROYING;
6025 mutex_unlock(&wq->mutex);
6028 drain_workqueue(wq);
6031 if (wq->rescuer) {
6033 kthread_stop(wq->rescuer->task);
6034 kfree(wq->rescuer);
6035 wq->rescuer = NULL;
6043 mutex_lock(&wq->mutex);
6044 for_each_pwq(pwq, wq) {
6048 __func__, wq->name);
6051 mutex_unlock(&wq->mutex);
6053 show_one_workqueue(wq);
6058 mutex_unlock(&wq->mutex);
6061 * wq list is used to freeze wq, remove from list after
6064 list_del_rcu(&wq->list);
6068 * We're the sole accessor of @wq. Directly access cpu_pwq and dfl_pwq
6069 * to put the base refs. @wq will be auto-destroyed from the last
6070 * pwq_put. RCU read lock prevents @wq from going away from under us.
6075 put_pwq_unlocked(unbound_pwq(wq, cpu));
6076 RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
6079 put_pwq_unlocked(unbound_pwq(wq, -1));
6080 RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);
6088 * @wq: target workqueue
6091 * Set max_active of @wq to @max_active. See the alloc_workqueue() function
6097 void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
6100 if (WARN_ON(wq->flags & WQ_BH))
6103 if (WARN_ON(wq->flags & __WQ_ORDERED))
6106 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
6108 mutex_lock(&wq->mutex);
6110 wq->saved_max_active = max_active;
6111 if (wq->flags & WQ_UNBOUND)
6112 wq->saved_min_active = min(wq->saved_min_active, max_active);
6114 wq_adjust_max_active(wq);
6116 mutex_unlock(&wq->mutex);
6122 * @wq: target unbound workqueue
6134 void workqueue_set_min_active(struct workqueue_struct *wq, int min_active)
6137 if (WARN_ON((wq->flags & (WQ_BH | WQ_UNBOUND | __WQ_ORDERED)) !=
6141 mutex_lock(&wq->mutex);
6142 wq->saved_min_active = clamp(min_active, 0, wq->saved_max_active);
6143 wq_adjust_max_active(wq);
6144 mutex_unlock(&wq->mutex);
6181 * @wq: target workqueue
6183 * Test whether @wq's cpu workqueue for @cpu is congested. There is
6197 bool workqueue_congested(int cpu, struct workqueue_struct *wq)
6207 pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
6292 struct workqueue_struct *wq = NULL;
6310 copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
6311 copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
6470 * @wq: workqueue whose state will be printed
6472 void show_one_workqueue(struct workqueue_struct *wq)
6478 for_each_pwq(pwq, wq) {
6487 pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
6489 for_each_pwq(pwq, wq) {
6568 struct workqueue_struct *wq;
6576 list_for_each_entry_rcu(wq, &workqueues, list)
6577 show_one_workqueue(wq);
6593 struct workqueue_struct *wq;
6599 list_for_each_entry_rcu(wq, &workqueues, list) {
6600 if (!(wq->flags & WQ_FREEZABLE))
6602 show_one_workqueue(wq);
6624 * ->desc tracks information (wq name or
6810 struct workqueue_struct *wq;
6831 list_for_each_entry(wq, &workqueues, list) {
6832 struct workqueue_attrs *attrs = wq->unbound_attrs;
6839 unbound_wq_update_pwq(wq, tcpu);
6841 mutex_lock(&wq->mutex);
6842 wq_update_node_max_active(wq, -1);
6843 mutex_unlock(&wq->mutex);
6853 struct workqueue_struct *wq;
6866 list_for_each_entry(wq, &workqueues, list) {
6867 struct workqueue_attrs *attrs = wq->unbound_attrs;
6874 unbound_wq_update_pwq(wq, tcpu);
6876 mutex_lock(&wq->mutex);
6877 wq_update_node_max_active(wq, cpu);
6878 mutex_unlock(&wq->mutex);
6936 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
6940 struct workqueue_struct *wq;
6947 list_for_each_entry(wq, &workqueues, list) {
6948 mutex_lock(&wq->mutex);
6949 wq_adjust_max_active(wq);
6950 mutex_unlock(&wq->mutex);
6972 struct workqueue_struct *wq;
6979 list_for_each_entry(wq, &workqueues, list) {
6980 if (!(wq->flags & WQ_FREEZABLE))
6987 for_each_pwq(pwq, wq) {
7009 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
7013 struct workqueue_struct *wq;
7023 list_for_each_entry(wq, &workqueues, list) {
7024 mutex_lock(&wq->mutex);
7025 wq_adjust_max_active(wq);
7026 mutex_unlock(&wq->mutex);
7038 struct workqueue_struct *wq;
7043 list_for_each_entry(wq, &workqueues, list) {
7044 if (!(wq->flags & WQ_UNBOUND) || (wq->flags & __WQ_DESTROYING))
7047 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask);
7070 list_for_each_entry(wq, &workqueues, list) {
7071 if (wq->rescuer && !wq->rescuer->pool)
7072 unbind_worker(wq->rescuer);
7135 struct workqueue_struct *wq;
7149 list_for_each_entry(wq, &workqueues, list) {
7151 unbound_wq_update_pwq(wq, cpu);
7189 struct workqueue_struct *wq;
7197 return wq_dev->wq;
7203 struct workqueue_struct *wq = dev_to_wq(dev);
7205 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
7212 struct workqueue_struct *wq = dev_to_wq(dev);
7214 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
7221 struct workqueue_struct *wq = dev_to_wq(dev);
7227 workqueue_set_max_active(wq, val);
7241 struct workqueue_struct *wq = dev_to_wq(dev);
7247 if (wq->flags & (WQ_BH | __WQ_ORDERED))
7261 struct workqueue_struct *wq = dev_to_wq(dev);
7264 mutex_lock(&wq->mutex);
7265 written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
7266 mutex_unlock(&wq->mutex);
7272 static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
7282 copy_workqueue_attrs(attrs, wq->unbound_attrs);
7289 struct workqueue_struct *wq = dev_to_wq(dev);
7295 attrs = wq_sysfs_prep_attrs(wq);
7301 ret = apply_workqueue_attrs_locked(wq, attrs);
7314 struct workqueue_struct *wq = dev_to_wq(dev);
7317 mutex_lock(&wq->mutex);
7319 cpumask_pr_args(wq->unbound_attrs->cpumask));
7320 mutex_unlock(&wq->mutex);
7328 struct workqueue_struct *wq = dev_to_wq(dev);
7334 attrs = wq_sysfs_prep_attrs(wq);
7340 ret = apply_workqueue_attrs_locked(wq, attrs);
7351 struct workqueue_struct *wq = dev_to_wq(dev);
7354 mutex_lock(&wq->mutex);
7355 if (wq->unbound_attrs->affn_scope == WQ_AFFN_DFL)
7361 wq_affn_names[wq->unbound_attrs->affn_scope]);
7362 mutex_unlock(&wq->mutex);
7371 struct workqueue_struct *wq = dev_to_wq(dev);
7380 attrs = wq_sysfs_prep_attrs(wq);
7383 ret = apply_workqueue_attrs_locked(wq, attrs);
7393 struct workqueue_struct *wq = dev_to_wq(dev);
7396 wq->unbound_attrs->affn_strict);
7403 struct workqueue_struct *wq = dev_to_wq(dev);
7411 attrs = wq_sysfs_prep_attrs(wq);
7414 ret = apply_workqueue_attrs_locked(wq, attrs);
7541 * @wq: the workqueue to register
7543 * Expose @wq in sysfs under /sys/bus/workqueue/devices.
7554 int workqueue_sysfs_register(struct workqueue_struct *wq)
7559 wq->wq_dev = wq_dev = kzalloc_obj(*wq_dev);
7563 wq_dev->wq = wq;
7566 dev_set_name(&wq_dev->dev, "%s", wq->name);
7577 wq->wq_dev = NULL;
7581 if (wq->flags & WQ_UNBOUND) {
7588 wq->wq_dev = NULL;
7601 * @wq: the workqueue to unregister
7603 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
7605 static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
7607 struct wq_device *wq_dev = wq->wq_dev;
7609 if (!wq->wq_dev)
7612 wq->wq_dev = NULL;
7616 static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
7997 /* create default unbound and ordered wq attrs */
8006 * An ordered wq should have only one pwq as ordering is
8089 struct workqueue_struct *wq;
8108 list_for_each_entry(wq, &workqueues, list) {
8109 WARN(init_rescuer(wq),
8111 wq->name);
8391 struct workqueue_struct *wq;
8410 list_for_each_entry(wq, &workqueues, list) {
8412 unbound_wq_update_pwq(wq, cpu);
8413 if (wq->flags & WQ_UNBOUND) {
8414 mutex_lock(&wq->mutex);
8415 wq_update_node_max_active(wq, -1);
8416 mutex_unlock(&wq->mutex);