xref: /linux/kernel/workqueue.c (revision f4ce1f3318ad4bc12463698696ebc36b145a6aa3)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * kernel/workqueue.c - generic async execution with shared worker pool
4  *
5  * Copyright (C) 2002		Ingo Molnar
6  *
7  *   Derived from the taskqueue/keventd code by:
8  *     David Woodhouse <dwmw2@infradead.org>
9  *     Andrew Morton
10  *     Kai Petzke <wpp@marie.physik.tu-berlin.de>
11  *     Theodore Ts'o <tytso@mit.edu>
12  *
13  * Made to use alloc_percpu by Christoph Lameter.
14  *
15  * Copyright (C) 2010		SUSE Linux Products GmbH
16  * Copyright (C) 2010		Tejun Heo <tj@kernel.org>
17  *
18  * This is the generic async execution mechanism.  Work items as are
19  * executed in process context.  The worker pool is shared and
20  * automatically managed.  There are two worker pools for each CPU (one for
21  * normal work items and the other for high priority ones) and some extra
22  * pools for workqueues which are not bound to any specific CPU - the
23  * number of these backing pools is dynamic.
24  *
25  * Please read Documentation/core-api/workqueue.rst for details.
26  */
27 
28 #include <linux/export.h>
29 #include <linux/kernel.h>
30 #include <linux/sched.h>
31 #include <linux/init.h>
32 #include <linux/interrupt.h>
33 #include <linux/signal.h>
34 #include <linux/completion.h>
35 #include <linux/workqueue.h>
36 #include <linux/slab.h>
37 #include <linux/cpu.h>
38 #include <linux/notifier.h>
39 #include <linux/kthread.h>
40 #include <linux/hardirq.h>
41 #include <linux/mempolicy.h>
42 #include <linux/freezer.h>
43 #include <linux/debug_locks.h>
44 #include <linux/lockdep.h>
45 #include <linux/idr.h>
46 #include <linux/jhash.h>
47 #include <linux/hashtable.h>
48 #include <linux/rculist.h>
49 #include <linux/nodemask.h>
50 #include <linux/moduleparam.h>
51 #include <linux/uaccess.h>
52 #include <linux/sched/isolation.h>
53 #include <linux/sched/debug.h>
54 #include <linux/nmi.h>
55 #include <linux/kvm_para.h>
56 #include <linux/delay.h>
57 #include <linux/irq_work.h>
58 
59 #include "workqueue_internal.h"
60 
61 enum worker_pool_flags {
62 	/*
63 	 * worker_pool flags
64 	 *
65 	 * A bound pool is either associated or disassociated with its CPU.
66 	 * While associated (!DISASSOCIATED), all workers are bound to the
67 	 * CPU and none has %WORKER_UNBOUND set and concurrency management
68 	 * is in effect.
69 	 *
70 	 * While DISASSOCIATED, the cpu may be offline and all workers have
71 	 * %WORKER_UNBOUND set and concurrency management disabled, and may
72 	 * be executing on any CPU.  The pool behaves as an unbound one.
73 	 *
74 	 * Note that DISASSOCIATED should be flipped only while holding
75 	 * wq_pool_attach_mutex to avoid changing binding state while
76 	 * worker_attach_to_pool() is in progress.
77 	 *
78 	 * As there can only be one concurrent BH execution context per CPU, a
79 	 * BH pool is per-CPU and always DISASSOCIATED.
80 	 */
81 	POOL_BH			= 1 << 0,	/* is a BH pool */
82 	POOL_MANAGER_ACTIVE	= 1 << 1,	/* being managed */
83 	POOL_DISASSOCIATED	= 1 << 2,	/* cpu can't serve workers */
84 	POOL_BH_DRAINING	= 1 << 3,	/* draining after CPU offline */
85 };
86 
87 enum worker_flags {
88 	/* worker flags */
89 	WORKER_DIE		= 1 << 1,	/* die die die */
90 	WORKER_IDLE		= 1 << 2,	/* is idle */
91 	WORKER_PREP		= 1 << 3,	/* preparing to run works */
92 	WORKER_CPU_INTENSIVE	= 1 << 6,	/* cpu intensive */
93 	WORKER_UNBOUND		= 1 << 7,	/* worker is unbound */
94 	WORKER_REBOUND		= 1 << 8,	/* worker was rebound */
95 
96 	WORKER_NOT_RUNNING	= WORKER_PREP | WORKER_CPU_INTENSIVE |
97 				  WORKER_UNBOUND | WORKER_REBOUND,
98 };
99 
100 enum work_cancel_flags {
101 	WORK_CANCEL_DELAYED	= 1 << 0,	/* canceling a delayed_work */
102 	WORK_CANCEL_DISABLE	= 1 << 1,	/* canceling to disable */
103 };
104 
105 enum wq_internal_consts {
106 	NR_STD_WORKER_POOLS	= 2,		/* # standard pools per cpu */
107 
108 	UNBOUND_POOL_HASH_ORDER	= 6,		/* hashed by pool->attrs */
109 	BUSY_WORKER_HASH_ORDER	= 6,		/* 64 pointers */
110 
111 	MAX_IDLE_WORKERS_RATIO	= 4,		/* 1/4 of busy can be idle */
112 	IDLE_WORKER_TIMEOUT	= 300 * HZ,	/* keep idle ones for 5 mins */
113 
114 	MAYDAY_INITIAL_TIMEOUT  = HZ / 100 >= 2 ? HZ / 100 : 2,
115 						/* call for help after 10ms
116 						   (min two ticks) */
117 	MAYDAY_INTERVAL		= HZ / 10,	/* and then every 100ms */
118 	CREATE_COOLDOWN		= HZ,		/* time to breath after fail */
119 
120 	/*
121 	 * Rescue workers are used only on emergencies and shared by
122 	 * all cpus.  Give MIN_NICE.
123 	 */
124 	RESCUER_NICE_LEVEL	= MIN_NICE,
125 	HIGHPRI_NICE_LEVEL	= MIN_NICE,
126 
127 	WQ_NAME_LEN		= 32,
128 	WORKER_ID_LEN		= 10 + WQ_NAME_LEN, /* "kworker/R-" + WQ_NAME_LEN */
129 };
130 
131 /*
132  * We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and
133  * MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because
134  * msecs_to_jiffies() can't be an initializer.
135  */
136 #define BH_WORKER_JIFFIES	msecs_to_jiffies(2)
137 #define BH_WORKER_RESTARTS	10
138 
139 /*
140  * Structure fields follow one of the following exclusion rules.
141  *
142  * I: Modifiable by initialization/destruction paths and read-only for
143  *    everyone else.
144  *
145  * P: Preemption protected.  Disabling preemption is enough and should
146  *    only be modified and accessed from the local cpu.
147  *
148  * L: pool->lock protected.  Access with pool->lock held.
149  *
150  * LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for
151  *     reads.
152  *
153  * K: Only modified by worker while holding pool->lock. Can be safely read by
154  *    self, while holding pool->lock or from IRQ context if %current is the
155  *    kworker.
156  *
157  * S: Only modified by worker self.
158  *
159  * A: wq_pool_attach_mutex protected.
160  *
161  * PL: wq_pool_mutex protected.
162  *
163  * PR: wq_pool_mutex protected for writes.  RCU protected for reads.
164  *
165  * PW: wq_pool_mutex and wq->mutex protected for writes.  Either for reads.
166  *
167  * PWR: wq_pool_mutex and wq->mutex protected for writes.  Either or
168  *      RCU for reads.
169  *
170  * WQ: wq->mutex protected.
171  *
172  * WR: wq->mutex protected for writes.  RCU protected for reads.
173  *
174  * WO: wq->mutex protected for writes. Updated with WRITE_ONCE() and can be read
175  *     with READ_ONCE() without locking.
176  *
177  * MD: wq_mayday_lock protected.
178  *
179  * WD: Used internally by the watchdog.
180  */
181 
182 /* struct worker is defined in workqueue_internal.h */
183 
184 struct worker_pool {
185 	raw_spinlock_t		lock;		/* the pool lock */
186 	int			cpu;		/* I: the associated cpu */
187 	int			node;		/* I: the associated node ID */
188 	int			id;		/* I: pool ID */
189 	unsigned int		flags;		/* L: flags */
190 
191 	unsigned long		watchdog_ts;	/* L: watchdog timestamp */
192 	bool			cpu_stall;	/* WD: stalled cpu bound pool */
193 
194 	/*
195 	 * The counter is incremented in a process context on the associated CPU
196 	 * w/ preemption disabled, and decremented or reset in the same context
197 	 * but w/ pool->lock held. The readers grab pool->lock and are
198 	 * guaranteed to see if the counter reached zero.
199 	 */
200 	int			nr_running;
201 
202 	struct list_head	worklist;	/* L: list of pending works */
203 
204 	int			nr_workers;	/* L: total number of workers */
205 	int			nr_idle;	/* L: currently idle workers */
206 
207 	struct list_head	idle_list;	/* L: list of idle workers */
208 	struct timer_list	idle_timer;	/* L: worker idle timeout */
209 	struct work_struct      idle_cull_work; /* L: worker idle cleanup */
210 
211 	struct timer_list	mayday_timer;	  /* L: SOS timer for workers */
212 
213 	/* a workers is either on busy_hash or idle_list, or the manager */
214 	DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
215 						/* L: hash of busy workers */
216 
217 	struct worker		*manager;	/* L: purely informational */
218 	struct list_head	workers;	/* A: attached workers */
219 
220 	struct ida		worker_ida;	/* worker IDs for task name */
221 
222 	struct workqueue_attrs	*attrs;		/* I: worker attributes */
223 	struct hlist_node	hash_node;	/* PL: unbound_pool_hash node */
224 	int			refcnt;		/* PL: refcnt for unbound pools */
225 
226 	/*
227 	 * Destruction of pool is RCU protected to allow dereferences
228 	 * from get_work_pool().
229 	 */
230 	struct rcu_head		rcu;
231 };
232 
233 /*
234  * Per-pool_workqueue statistics. These can be monitored using
235  * tools/workqueue/wq_monitor.py.
236  */
237 enum pool_workqueue_stats {
238 	PWQ_STAT_STARTED,	/* work items started execution */
239 	PWQ_STAT_COMPLETED,	/* work items completed execution */
240 	PWQ_STAT_CPU_TIME,	/* total CPU time consumed */
241 	PWQ_STAT_CPU_INTENSIVE,	/* wq_cpu_intensive_thresh_us violations */
242 	PWQ_STAT_CM_WAKEUP,	/* concurrency-management worker wakeups */
243 	PWQ_STAT_REPATRIATED,	/* unbound workers brought back into scope */
244 	PWQ_STAT_MAYDAY,	/* maydays to rescuer */
245 	PWQ_STAT_RESCUED,	/* linked work items executed by rescuer */
246 
247 	PWQ_NR_STATS,
248 };
249 
250 /*
251  * The per-pool workqueue.  While queued, bits below WORK_PWQ_SHIFT
252  * of work_struct->data are used for flags and the remaining high bits
253  * point to the pwq; thus, pwqs need to be aligned at two's power of the
254  * number of flag bits.
255  */
256 struct pool_workqueue {
257 	struct worker_pool	*pool;		/* I: the associated pool */
258 	struct workqueue_struct *wq;		/* I: the owning workqueue */
259 	int			work_color;	/* L: current color */
260 	int			flush_color;	/* L: flushing color */
261 	int			refcnt;		/* L: reference count */
262 	int			nr_in_flight[WORK_NR_COLORS];
263 						/* L: nr of in_flight works */
264 	bool			plugged;	/* L: execution suspended */
265 
266 	/*
267 	 * nr_active management and WORK_STRUCT_INACTIVE:
268 	 *
269 	 * When pwq->nr_active >= max_active, new work item is queued to
270 	 * pwq->inactive_works instead of pool->worklist and marked with
271 	 * WORK_STRUCT_INACTIVE.
272 	 *
273 	 * All work items marked with WORK_STRUCT_INACTIVE do not participate in
274 	 * nr_active and all work items in pwq->inactive_works are marked with
275 	 * WORK_STRUCT_INACTIVE. But not all WORK_STRUCT_INACTIVE work items are
276 	 * in pwq->inactive_works. Some of them are ready to run in
277 	 * pool->worklist or worker->scheduled. Those work itmes are only struct
278 	 * wq_barrier which is used for flush_work() and should not participate
279 	 * in nr_active. For non-barrier work item, it is marked with
280 	 * WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
281 	 */
282 	int			nr_active;	/* L: nr of active works */
283 	struct list_head	inactive_works;	/* L: inactive works */
284 	struct list_head	pending_node;	/* LN: node on wq_node_nr_active->pending_pwqs */
285 	struct list_head	pwqs_node;	/* WR: node on wq->pwqs */
286 	struct list_head	mayday_node;	/* MD: node on wq->maydays */
287 
288 	u64			stats[PWQ_NR_STATS];
289 
290 	/*
291 	 * Release of unbound pwq is punted to a kthread_worker. See put_pwq()
292 	 * and pwq_release_workfn() for details. pool_workqueue itself is also
293 	 * RCU protected so that the first pwq can be determined without
294 	 * grabbing wq->mutex.
295 	 */
296 	struct kthread_work	release_work;
297 	struct rcu_head		rcu;
298 } __aligned(1 << WORK_STRUCT_PWQ_SHIFT);
299 
300 /*
301  * Structure used to wait for workqueue flush.
302  */
303 struct wq_flusher {
304 	struct list_head	list;		/* WQ: list of flushers */
305 	int			flush_color;	/* WQ: flush color waiting for */
306 	struct completion	done;		/* flush completion */
307 };
308 
309 struct wq_device;
310 
311 /*
312  * Unlike in a per-cpu workqueue where max_active limits its concurrency level
313  * on each CPU, in an unbound workqueue, max_active applies to the whole system.
314  * As sharing a single nr_active across multiple sockets can be very expensive,
315  * the counting and enforcement is per NUMA node.
316  *
317  * The following struct is used to enforce per-node max_active. When a pwq wants
318  * to start executing a work item, it should increment ->nr using
319  * tryinc_node_nr_active(). If acquisition fails due to ->nr already being over
320  * ->max, the pwq is queued on ->pending_pwqs. As in-flight work items finish
321  * and decrement ->nr, node_activate_pending_pwq() activates the pending pwqs in
322  * round-robin order.
323  */
324 struct wq_node_nr_active {
325 	int			max;		/* per-node max_active */
326 	atomic_t		nr;		/* per-node nr_active */
327 	raw_spinlock_t		lock;		/* nests inside pool locks */
328 	struct list_head	pending_pwqs;	/* LN: pwqs with inactive works */
329 };
330 
331 /*
332  * The externally visible workqueue.  It relays the issued work items to
333  * the appropriate worker_pool through its pool_workqueues.
334  */
335 struct workqueue_struct {
336 	struct list_head	pwqs;		/* WR: all pwqs of this wq */
337 	struct list_head	list;		/* PR: list of all workqueues */
338 
339 	struct mutex		mutex;		/* protects this wq */
340 	int			work_color;	/* WQ: current work color */
341 	int			flush_color;	/* WQ: current flush color */
342 	atomic_t		nr_pwqs_to_flush; /* flush in progress */
343 	struct wq_flusher	*first_flusher;	/* WQ: first flusher */
344 	struct list_head	flusher_queue;	/* WQ: flush waiters */
345 	struct list_head	flusher_overflow; /* WQ: flush overflow list */
346 
347 	struct list_head	maydays;	/* MD: pwqs requesting rescue */
348 	struct worker		*rescuer;	/* MD: rescue worker */
349 
350 	int			nr_drainers;	/* WQ: drain in progress */
351 
352 	/* See alloc_workqueue() function comment for info on min/max_active */
353 	int			max_active;	/* WO: max active works */
354 	int			min_active;	/* WO: min active works */
355 	int			saved_max_active; /* WQ: saved max_active */
356 	int			saved_min_active; /* WQ: saved min_active */
357 
358 	struct workqueue_attrs	*unbound_attrs;	/* PW: only for unbound wqs */
359 	struct pool_workqueue __rcu *dfl_pwq;   /* PW: only for unbound wqs */
360 
361 #ifdef CONFIG_SYSFS
362 	struct wq_device	*wq_dev;	/* I: for sysfs interface */
363 #endif
364 #ifdef CONFIG_LOCKDEP
365 	char			*lock_name;
366 	struct lock_class_key	key;
367 	struct lockdep_map	__lockdep_map;
368 	struct lockdep_map	*lockdep_map;
369 #endif
370 	char			name[WQ_NAME_LEN]; /* I: workqueue name */
371 
372 	/*
373 	 * Destruction of workqueue_struct is RCU protected to allow walking
374 	 * the workqueues list without grabbing wq_pool_mutex.
375 	 * This is used to dump all workqueues from sysrq.
376 	 */
377 	struct rcu_head		rcu;
378 
379 	/* hot fields used during command issue, aligned to cacheline */
380 	unsigned int		flags ____cacheline_aligned; /* WQ: WQ_* flags */
381 	struct pool_workqueue __rcu * __percpu *cpu_pwq; /* I: per-cpu pwqs */
382 	struct wq_node_nr_active *node_nr_active[]; /* I: per-node nr_active */
383 };
384 
385 /*
386  * Each pod type describes how CPUs should be grouped for unbound workqueues.
387  * See the comment above workqueue_attrs->affn_scope.
388  */
389 struct wq_pod_type {
390 	int			nr_pods;	/* number of pods */
391 	cpumask_var_t		*pod_cpus;	/* pod -> cpus */
392 	int			*pod_node;	/* pod -> node */
393 	int			*cpu_pod;	/* cpu -> pod */
394 };
395 
396 struct work_offq_data {
397 	u32			pool_id;
398 	u32			disable;
399 	u32			flags;
400 };
401 
402 static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = {
403 	[WQ_AFFN_DFL]		= "default",
404 	[WQ_AFFN_CPU]		= "cpu",
405 	[WQ_AFFN_SMT]		= "smt",
406 	[WQ_AFFN_CACHE]		= "cache",
407 	[WQ_AFFN_NUMA]		= "numa",
408 	[WQ_AFFN_SYSTEM]	= "system",
409 };
410 
411 /*
412  * Per-cpu work items which run for longer than the following threshold are
413  * automatically considered CPU intensive and excluded from concurrency
414  * management to prevent them from noticeably delaying other per-cpu work items.
415  * ULONG_MAX indicates that the user hasn't overridden it with a boot parameter.
416  * The actual value is initialized in wq_cpu_intensive_thresh_init().
417  */
418 static unsigned long wq_cpu_intensive_thresh_us = ULONG_MAX;
419 module_param_named(cpu_intensive_thresh_us, wq_cpu_intensive_thresh_us, ulong, 0644);
420 #ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
421 static unsigned int wq_cpu_intensive_warning_thresh = 4;
422 module_param_named(cpu_intensive_warning_thresh, wq_cpu_intensive_warning_thresh, uint, 0644);
423 #endif
424 
425 /* see the comment above the definition of WQ_POWER_EFFICIENT */
426 static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
427 module_param_named(power_efficient, wq_power_efficient, bool, 0444);
428 
429 static bool wq_online;			/* can kworkers be created yet? */
430 static bool wq_topo_initialized __read_mostly = false;
431 
432 static struct kmem_cache *pwq_cache;
433 
434 static struct wq_pod_type wq_pod_types[WQ_AFFN_NR_TYPES];
435 static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE;
436 
437 /* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */
438 static struct workqueue_attrs *unbound_wq_update_pwq_attrs_buf;
439 
440 static DEFINE_MUTEX(wq_pool_mutex);	/* protects pools and workqueues list */
441 static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
442 static DEFINE_RAW_SPINLOCK(wq_mayday_lock);	/* protects wq->maydays list */
443 /* wait for manager to go away */
444 static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
445 
446 static LIST_HEAD(workqueues);		/* PR: list of all workqueues */
447 static bool workqueue_freezing;		/* PL: have wqs started freezing? */
448 
449 /* PL: mirror the cpu_online_mask excluding the CPU in the midst of hotplugging */
450 static cpumask_var_t wq_online_cpumask;
451 
452 /* PL&A: allowable cpus for unbound wqs and work items */
453 static cpumask_var_t wq_unbound_cpumask;
454 
455 /* PL: user requested unbound cpumask via sysfs */
456 static cpumask_var_t wq_requested_unbound_cpumask;
457 
458 /* PL: isolated cpumask to be excluded from unbound cpumask */
459 static cpumask_var_t wq_isolated_cpumask;
460 
461 /* for further constrain wq_unbound_cpumask by cmdline parameter*/
462 static struct cpumask wq_cmdline_cpumask __initdata;
463 
464 /* CPU where unbound work was last round robin scheduled from this CPU */
465 static DEFINE_PER_CPU(int, wq_rr_cpu_last);
466 
467 /*
468  * Local execution of unbound work items is no longer guaranteed.  The
469  * following always forces round-robin CPU selection on unbound work items
470  * to uncover usages which depend on it.
471  */
472 #ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
473 static bool wq_debug_force_rr_cpu = true;
474 #else
475 static bool wq_debug_force_rr_cpu = false;
476 #endif
477 module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
478 
479 /* to raise softirq for the BH worker pools on other CPUs */
480 static DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_work [NR_STD_WORKER_POOLS], bh_pool_irq_works);
481 
482 /* the BH worker pools */
483 static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], bh_worker_pools);
484 
485 /* the per-cpu worker pools */
486 static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools);
487 
488 static DEFINE_IDR(worker_pool_idr);	/* PR: idr of all pools */
489 
490 /* PL: hash of all unbound pools keyed by pool->attrs */
491 static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
492 
493 /* I: attributes used when instantiating standard unbound pools on demand */
494 static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
495 
496 /* I: attributes used when instantiating ordered pools on demand */
497 static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
498 
499 /*
500  * I: kthread_worker to release pwq's. pwq release needs to be bounced to a
501  * process context while holding a pool lock. Bounce to a dedicated kthread
502  * worker to avoid A-A deadlocks.
503  */
504 static struct kthread_worker *pwq_release_worker __ro_after_init;
505 
506 struct workqueue_struct *system_wq __ro_after_init;
507 EXPORT_SYMBOL(system_wq);
508 struct workqueue_struct *system_highpri_wq __ro_after_init;
509 EXPORT_SYMBOL_GPL(system_highpri_wq);
510 struct workqueue_struct *system_long_wq __ro_after_init;
511 EXPORT_SYMBOL_GPL(system_long_wq);
512 struct workqueue_struct *system_unbound_wq __ro_after_init;
513 EXPORT_SYMBOL_GPL(system_unbound_wq);
514 struct workqueue_struct *system_freezable_wq __ro_after_init;
515 EXPORT_SYMBOL_GPL(system_freezable_wq);
516 struct workqueue_struct *system_power_efficient_wq __ro_after_init;
517 EXPORT_SYMBOL_GPL(system_power_efficient_wq);
518 struct workqueue_struct *system_freezable_power_efficient_wq __ro_after_init;
519 EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
520 struct workqueue_struct *system_bh_wq;
521 EXPORT_SYMBOL_GPL(system_bh_wq);
522 struct workqueue_struct *system_bh_highpri_wq;
523 EXPORT_SYMBOL_GPL(system_bh_highpri_wq);
524 
525 static int worker_thread(void *__worker);
526 static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
527 static void show_pwq(struct pool_workqueue *pwq);
528 static void show_one_worker_pool(struct worker_pool *pool);
529 
530 #define CREATE_TRACE_POINTS
531 #include <trace/events/workqueue.h>
532 
533 #define assert_rcu_or_pool_mutex()					\
534 	RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() &&			\
535 			 !lockdep_is_held(&wq_pool_mutex),		\
536 			 "RCU or wq_pool_mutex should be held")
537 
538 #define assert_rcu_or_wq_mutex_or_pool_mutex(wq)			\
539 	RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() &&			\
540 			 !lockdep_is_held(&wq->mutex) &&		\
541 			 !lockdep_is_held(&wq_pool_mutex),		\
542 			 "RCU, wq->mutex or wq_pool_mutex should be held")
543 
544 #define for_each_bh_worker_pool(pool, cpu)				\
545 	for ((pool) = &per_cpu(bh_worker_pools, cpu)[0];		\
546 	     (pool) < &per_cpu(bh_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
547 	     (pool)++)
548 
549 #define for_each_cpu_worker_pool(pool, cpu)				\
550 	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0];		\
551 	     (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
552 	     (pool)++)
553 
554 /**
555  * for_each_pool - iterate through all worker_pools in the system
556  * @pool: iteration cursor
557  * @pi: integer used for iteration
558  *
559  * This must be called either with wq_pool_mutex held or RCU read
560  * locked.  If the pool needs to be used beyond the locking in effect, the
561  * caller is responsible for guaranteeing that the pool stays online.
562  *
563  * The if/else clause exists only for the lockdep assertion and can be
564  * ignored.
565  */
566 #define for_each_pool(pool, pi)						\
567 	idr_for_each_entry(&worker_pool_idr, pool, pi)			\
568 		if (({ assert_rcu_or_pool_mutex(); false; })) { }	\
569 		else
570 
571 /**
572  * for_each_pool_worker - iterate through all workers of a worker_pool
573  * @worker: iteration cursor
574  * @pool: worker_pool to iterate workers of
575  *
576  * This must be called with wq_pool_attach_mutex.
577  *
578  * The if/else clause exists only for the lockdep assertion and can be
579  * ignored.
580  */
581 #define for_each_pool_worker(worker, pool)				\
582 	list_for_each_entry((worker), &(pool)->workers, node)		\
583 		if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \
584 		else
585 
586 /**
587  * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
588  * @pwq: iteration cursor
589  * @wq: the target workqueue
590  *
591  * This must be called either with wq->mutex held or RCU read locked.
592  * If the pwq needs to be used beyond the locking in effect, the caller is
593  * responsible for guaranteeing that the pwq stays online.
594  *
595  * The if/else clause exists only for the lockdep assertion and can be
596  * ignored.
597  */
598 #define for_each_pwq(pwq, wq)						\
599 	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node,		\
600 				 lockdep_is_held(&(wq->mutex)))
601 
602 #ifdef CONFIG_DEBUG_OBJECTS_WORK
603 
604 static const struct debug_obj_descr work_debug_descr;
605 
work_debug_hint(void * addr)606 static void *work_debug_hint(void *addr)
607 {
608 	return ((struct work_struct *) addr)->func;
609 }
610 
work_is_static_object(void * addr)611 static bool work_is_static_object(void *addr)
612 {
613 	struct work_struct *work = addr;
614 
615 	return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
616 }
617 
618 /*
619  * fixup_init is called when:
620  * - an active object is initialized
621  */
work_fixup_init(void * addr,enum debug_obj_state state)622 static bool work_fixup_init(void *addr, enum debug_obj_state state)
623 {
624 	struct work_struct *work = addr;
625 
626 	switch (state) {
627 	case ODEBUG_STATE_ACTIVE:
628 		cancel_work_sync(work);
629 		debug_object_init(work, &work_debug_descr);
630 		return true;
631 	default:
632 		return false;
633 	}
634 }
635 
636 /*
637  * fixup_free is called when:
638  * - an active object is freed
639  */
work_fixup_free(void * addr,enum debug_obj_state state)640 static bool work_fixup_free(void *addr, enum debug_obj_state state)
641 {
642 	struct work_struct *work = addr;
643 
644 	switch (state) {
645 	case ODEBUG_STATE_ACTIVE:
646 		cancel_work_sync(work);
647 		debug_object_free(work, &work_debug_descr);
648 		return true;
649 	default:
650 		return false;
651 	}
652 }
653 
654 static const struct debug_obj_descr work_debug_descr = {
655 	.name		= "work_struct",
656 	.debug_hint	= work_debug_hint,
657 	.is_static_object = work_is_static_object,
658 	.fixup_init	= work_fixup_init,
659 	.fixup_free	= work_fixup_free,
660 };
661 
debug_work_activate(struct work_struct * work)662 static inline void debug_work_activate(struct work_struct *work)
663 {
664 	debug_object_activate(work, &work_debug_descr);
665 }
666 
debug_work_deactivate(struct work_struct * work)667 static inline void debug_work_deactivate(struct work_struct *work)
668 {
669 	debug_object_deactivate(work, &work_debug_descr);
670 }
671 
__init_work(struct work_struct * work,int onstack)672 void __init_work(struct work_struct *work, int onstack)
673 {
674 	if (onstack)
675 		debug_object_init_on_stack(work, &work_debug_descr);
676 	else
677 		debug_object_init(work, &work_debug_descr);
678 }
679 EXPORT_SYMBOL_GPL(__init_work);
680 
destroy_work_on_stack(struct work_struct * work)681 void destroy_work_on_stack(struct work_struct *work)
682 {
683 	debug_object_free(work, &work_debug_descr);
684 }
685 EXPORT_SYMBOL_GPL(destroy_work_on_stack);
686 
destroy_delayed_work_on_stack(struct delayed_work * work)687 void destroy_delayed_work_on_stack(struct delayed_work *work)
688 {
689 	destroy_timer_on_stack(&work->timer);
690 	debug_object_free(&work->work, &work_debug_descr);
691 }
692 EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
693 
694 #else
debug_work_activate(struct work_struct * work)695 static inline void debug_work_activate(struct work_struct *work) { }
debug_work_deactivate(struct work_struct * work)696 static inline void debug_work_deactivate(struct work_struct *work) { }
697 #endif
698 
699 /**
700  * worker_pool_assign_id - allocate ID and assign it to @pool
701  * @pool: the pool pointer of interest
702  *
703  * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
704  * successfully, -errno on failure.
705  */
worker_pool_assign_id(struct worker_pool * pool)706 static int worker_pool_assign_id(struct worker_pool *pool)
707 {
708 	int ret;
709 
710 	lockdep_assert_held(&wq_pool_mutex);
711 
712 	ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
713 			GFP_KERNEL);
714 	if (ret >= 0) {
715 		pool->id = ret;
716 		return 0;
717 	}
718 	return ret;
719 }
720 
721 static struct pool_workqueue __rcu **
unbound_pwq_slot(struct workqueue_struct * wq,int cpu)722 unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
723 {
724        if (cpu >= 0)
725                return per_cpu_ptr(wq->cpu_pwq, cpu);
726        else
727                return &wq->dfl_pwq;
728 }
729 
730 /* @cpu < 0 for dfl_pwq */
unbound_pwq(struct workqueue_struct * wq,int cpu)731 static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
732 {
733 	return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
734 				     lockdep_is_held(&wq_pool_mutex) ||
735 				     lockdep_is_held(&wq->mutex));
736 }
737 
738 /**
739  * unbound_effective_cpumask - effective cpumask of an unbound workqueue
740  * @wq: workqueue of interest
741  *
742  * @wq->unbound_attrs->cpumask contains the cpumask requested by the user which
743  * is masked with wq_unbound_cpumask to determine the effective cpumask. The
744  * default pwq is always mapped to the pool with the current effective cpumask.
745  */
unbound_effective_cpumask(struct workqueue_struct * wq)746 static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq)
747 {
748 	return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask;
749 }
750 
work_color_to_flags(int color)751 static unsigned int work_color_to_flags(int color)
752 {
753 	return color << WORK_STRUCT_COLOR_SHIFT;
754 }
755 
get_work_color(unsigned long work_data)756 static int get_work_color(unsigned long work_data)
757 {
758 	return (work_data >> WORK_STRUCT_COLOR_SHIFT) &
759 		((1 << WORK_STRUCT_COLOR_BITS) - 1);
760 }
761 
work_next_color(int color)762 static int work_next_color(int color)
763 {
764 	return (color + 1) % WORK_NR_COLORS;
765 }
766 
pool_offq_flags(struct worker_pool * pool)767 static unsigned long pool_offq_flags(struct worker_pool *pool)
768 {
769 	return (pool->flags & POOL_BH) ? WORK_OFFQ_BH : 0;
770 }
771 
772 /*
773  * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
774  * contain the pointer to the queued pwq.  Once execution starts, the flag
775  * is cleared and the high bits contain OFFQ flags and pool ID.
776  *
777  * set_work_pwq(), set_work_pool_and_clear_pending() and mark_work_canceling()
778  * can be used to set the pwq, pool or clear work->data. These functions should
779  * only be called while the work is owned - ie. while the PENDING bit is set.
780  *
781  * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
782  * corresponding to a work.  Pool is available once the work has been
783  * queued anywhere after initialization until it is sync canceled.  pwq is
784  * available only while the work item is queued.
785  */
set_work_data(struct work_struct * work,unsigned long data)786 static inline void set_work_data(struct work_struct *work, unsigned long data)
787 {
788 	WARN_ON_ONCE(!work_pending(work));
789 	atomic_long_set(&work->data, data | work_static(work));
790 }
791 
set_work_pwq(struct work_struct * work,struct pool_workqueue * pwq,unsigned long flags)792 static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
793 			 unsigned long flags)
794 {
795 	set_work_data(work, (unsigned long)pwq | WORK_STRUCT_PENDING |
796 		      WORK_STRUCT_PWQ | flags);
797 }
798 
set_work_pool_and_keep_pending(struct work_struct * work,int pool_id,unsigned long flags)799 static void set_work_pool_and_keep_pending(struct work_struct *work,
800 					   int pool_id, unsigned long flags)
801 {
802 	set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) |
803 		      WORK_STRUCT_PENDING | flags);
804 }
805 
set_work_pool_and_clear_pending(struct work_struct * work,int pool_id,unsigned long flags)806 static void set_work_pool_and_clear_pending(struct work_struct *work,
807 					    int pool_id, unsigned long flags)
808 {
809 	/*
810 	 * The following wmb is paired with the implied mb in
811 	 * test_and_set_bit(PENDING) and ensures all updates to @work made
812 	 * here are visible to and precede any updates by the next PENDING
813 	 * owner.
814 	 */
815 	smp_wmb();
816 	set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) |
817 		      flags);
818 	/*
819 	 * The following mb guarantees that previous clear of a PENDING bit
820 	 * will not be reordered with any speculative LOADS or STORES from
821 	 * work->current_func, which is executed afterwards.  This possible
822 	 * reordering can lead to a missed execution on attempt to queue
823 	 * the same @work.  E.g. consider this case:
824 	 *
825 	 *   CPU#0                         CPU#1
826 	 *   ----------------------------  --------------------------------
827 	 *
828 	 * 1  STORE event_indicated
829 	 * 2  queue_work_on() {
830 	 * 3    test_and_set_bit(PENDING)
831 	 * 4 }                             set_..._and_clear_pending() {
832 	 * 5                                 set_work_data() # clear bit
833 	 * 6                                 smp_mb()
834 	 * 7                               work->current_func() {
835 	 * 8				      LOAD event_indicated
836 	 *				   }
837 	 *
838 	 * Without an explicit full barrier speculative LOAD on line 8 can
839 	 * be executed before CPU#0 does STORE on line 1.  If that happens,
840 	 * CPU#0 observes the PENDING bit is still set and new execution of
841 	 * a @work is not queued in a hope, that CPU#1 will eventually
842 	 * finish the queued @work.  Meanwhile CPU#1 does not see
843 	 * event_indicated is set, because speculative LOAD was executed
844 	 * before actual STORE.
845 	 */
846 	smp_mb();
847 }
848 
work_struct_pwq(unsigned long data)849 static inline struct pool_workqueue *work_struct_pwq(unsigned long data)
850 {
851 	return (struct pool_workqueue *)(data & WORK_STRUCT_PWQ_MASK);
852 }
853 
get_work_pwq(struct work_struct * work)854 static struct pool_workqueue *get_work_pwq(struct work_struct *work)
855 {
856 	unsigned long data = atomic_long_read(&work->data);
857 
858 	if (data & WORK_STRUCT_PWQ)
859 		return work_struct_pwq(data);
860 	else
861 		return NULL;
862 }
863 
864 /**
865  * get_work_pool - return the worker_pool a given work was associated with
866  * @work: the work item of interest
867  *
868  * Pools are created and destroyed under wq_pool_mutex, and allows read
869  * access under RCU read lock.  As such, this function should be
870  * called under wq_pool_mutex or inside of a rcu_read_lock() region.
871  *
872  * All fields of the returned pool are accessible as long as the above
873  * mentioned locking is in effect.  If the returned pool needs to be used
874  * beyond the critical section, the caller is responsible for ensuring the
875  * returned pool is and stays online.
876  *
877  * Return: The worker_pool @work was last associated with.  %NULL if none.
878  */
get_work_pool(struct work_struct * work)879 static struct worker_pool *get_work_pool(struct work_struct *work)
880 {
881 	unsigned long data = atomic_long_read(&work->data);
882 	int pool_id;
883 
884 	assert_rcu_or_pool_mutex();
885 
886 	if (data & WORK_STRUCT_PWQ)
887 		return work_struct_pwq(data)->pool;
888 
889 	pool_id = data >> WORK_OFFQ_POOL_SHIFT;
890 	if (pool_id == WORK_OFFQ_POOL_NONE)
891 		return NULL;
892 
893 	return idr_find(&worker_pool_idr, pool_id);
894 }
895 
shift_and_mask(unsigned long v,u32 shift,u32 bits)896 static unsigned long shift_and_mask(unsigned long v, u32 shift, u32 bits)
897 {
898 	return (v >> shift) & ((1U << bits) - 1);
899 }
900 
work_offqd_unpack(struct work_offq_data * offqd,unsigned long data)901 static void work_offqd_unpack(struct work_offq_data *offqd, unsigned long data)
902 {
903 	WARN_ON_ONCE(data & WORK_STRUCT_PWQ);
904 
905 	offqd->pool_id = shift_and_mask(data, WORK_OFFQ_POOL_SHIFT,
906 					WORK_OFFQ_POOL_BITS);
907 	offqd->disable = shift_and_mask(data, WORK_OFFQ_DISABLE_SHIFT,
908 					WORK_OFFQ_DISABLE_BITS);
909 	offqd->flags = data & WORK_OFFQ_FLAG_MASK;
910 }
911 
work_offqd_pack_flags(struct work_offq_data * offqd)912 static unsigned long work_offqd_pack_flags(struct work_offq_data *offqd)
913 {
914 	return ((unsigned long)offqd->disable << WORK_OFFQ_DISABLE_SHIFT) |
915 		((unsigned long)offqd->flags);
916 }
917 
918 /*
919  * Policy functions.  These define the policies on how the global worker
920  * pools are managed.  Unless noted otherwise, these functions assume that
921  * they're being called with pool->lock held.
922  */
923 
924 /*
925  * Need to wake up a worker?  Called from anything but currently
926  * running workers.
927  *
928  * Note that, because unbound workers never contribute to nr_running, this
929  * function will always return %true for unbound pools as long as the
930  * worklist isn't empty.
931  */
need_more_worker(struct worker_pool * pool)932 static bool need_more_worker(struct worker_pool *pool)
933 {
934 	return !list_empty(&pool->worklist) && !pool->nr_running;
935 }
936 
937 /* Can I start working?  Called from busy but !running workers. */
may_start_working(struct worker_pool * pool)938 static bool may_start_working(struct worker_pool *pool)
939 {
940 	return pool->nr_idle;
941 }
942 
943 /* Do I need to keep working?  Called from currently running workers. */
keep_working(struct worker_pool * pool)944 static bool keep_working(struct worker_pool *pool)
945 {
946 	return !list_empty(&pool->worklist) && (pool->nr_running <= 1);
947 }
948 
949 /* Do we need a new worker?  Called from manager. */
need_to_create_worker(struct worker_pool * pool)950 static bool need_to_create_worker(struct worker_pool *pool)
951 {
952 	return need_more_worker(pool) && !may_start_working(pool);
953 }
954 
955 /* Do we have too many workers and should some go away? */
too_many_workers(struct worker_pool * pool)956 static bool too_many_workers(struct worker_pool *pool)
957 {
958 	bool managing = pool->flags & POOL_MANAGER_ACTIVE;
959 	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
960 	int nr_busy = pool->nr_workers - nr_idle;
961 
962 	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
963 }
964 
965 /**
966  * worker_set_flags - set worker flags and adjust nr_running accordingly
967  * @worker: self
968  * @flags: flags to set
969  *
970  * Set @flags in @worker->flags and adjust nr_running accordingly.
971  */
worker_set_flags(struct worker * worker,unsigned int flags)972 static inline void worker_set_flags(struct worker *worker, unsigned int flags)
973 {
974 	struct worker_pool *pool = worker->pool;
975 
976 	lockdep_assert_held(&pool->lock);
977 
978 	/* If transitioning into NOT_RUNNING, adjust nr_running. */
979 	if ((flags & WORKER_NOT_RUNNING) &&
980 	    !(worker->flags & WORKER_NOT_RUNNING)) {
981 		pool->nr_running--;
982 	}
983 
984 	worker->flags |= flags;
985 }
986 
987 /**
988  * worker_clr_flags - clear worker flags and adjust nr_running accordingly
989  * @worker: self
990  * @flags: flags to clear
991  *
992  * Clear @flags in @worker->flags and adjust nr_running accordingly.
993  */
worker_clr_flags(struct worker * worker,unsigned int flags)994 static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
995 {
996 	struct worker_pool *pool = worker->pool;
997 	unsigned int oflags = worker->flags;
998 
999 	lockdep_assert_held(&pool->lock);
1000 
1001 	worker->flags &= ~flags;
1002 
1003 	/*
1004 	 * If transitioning out of NOT_RUNNING, increment nr_running.  Note
1005 	 * that the nested NOT_RUNNING is not a noop.  NOT_RUNNING is mask
1006 	 * of multiple flags, not a single flag.
1007 	 */
1008 	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
1009 		if (!(worker->flags & WORKER_NOT_RUNNING))
1010 			pool->nr_running++;
1011 }
1012 
1013 /* Return the first idle worker.  Called with pool->lock held. */
first_idle_worker(struct worker_pool * pool)1014 static struct worker *first_idle_worker(struct worker_pool *pool)
1015 {
1016 	if (unlikely(list_empty(&pool->idle_list)))
1017 		return NULL;
1018 
1019 	return list_first_entry(&pool->idle_list, struct worker, entry);
1020 }
1021 
1022 /**
1023  * worker_enter_idle - enter idle state
1024  * @worker: worker which is entering idle state
1025  *
1026  * @worker is entering idle state.  Update stats and idle timer if
1027  * necessary.
1028  *
1029  * LOCKING:
1030  * raw_spin_lock_irq(pool->lock).
1031  */
worker_enter_idle(struct worker * worker)1032 static void worker_enter_idle(struct worker *worker)
1033 {
1034 	struct worker_pool *pool = worker->pool;
1035 
1036 	if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
1037 	    WARN_ON_ONCE(!list_empty(&worker->entry) &&
1038 			 (worker->hentry.next || worker->hentry.pprev)))
1039 		return;
1040 
1041 	/* can't use worker_set_flags(), also called from create_worker() */
1042 	worker->flags |= WORKER_IDLE;
1043 	pool->nr_idle++;
1044 	worker->last_active = jiffies;
1045 
1046 	/* idle_list is LIFO */
1047 	list_add(&worker->entry, &pool->idle_list);
1048 
1049 	if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1050 		mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1051 
1052 	/* Sanity check nr_running. */
1053 	WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running);
1054 }
1055 
1056 /**
1057  * worker_leave_idle - leave idle state
1058  * @worker: worker which is leaving idle state
1059  *
1060  * @worker is leaving idle state.  Update stats.
1061  *
1062  * LOCKING:
1063  * raw_spin_lock_irq(pool->lock).
1064  */
worker_leave_idle(struct worker * worker)1065 static void worker_leave_idle(struct worker *worker)
1066 {
1067 	struct worker_pool *pool = worker->pool;
1068 
1069 	if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1070 		return;
1071 	worker_clr_flags(worker, WORKER_IDLE);
1072 	pool->nr_idle--;
1073 	list_del_init(&worker->entry);
1074 }
1075 
1076 /**
1077  * find_worker_executing_work - find worker which is executing a work
1078  * @pool: pool of interest
1079  * @work: work to find worker for
1080  *
1081  * Find a worker which is executing @work on @pool by searching
1082  * @pool->busy_hash which is keyed by the address of @work.  For a worker
1083  * to match, its current execution should match the address of @work and
1084  * its work function.  This is to avoid unwanted dependency between
1085  * unrelated work executions through a work item being recycled while still
1086  * being executed.
1087  *
1088  * This is a bit tricky.  A work item may be freed once its execution
1089  * starts and nothing prevents the freed area from being recycled for
1090  * another work item.  If the same work item address ends up being reused
1091  * before the original execution finishes, workqueue will identify the
1092  * recycled work item as currently executing and make it wait until the
1093  * current execution finishes, introducing an unwanted dependency.
1094  *
1095  * This function checks the work item address and work function to avoid
1096  * false positives.  Note that this isn't complete as one may construct a
1097  * work function which can introduce dependency onto itself through a
1098  * recycled work item.  Well, if somebody wants to shoot oneself in the
1099  * foot that badly, there's only so much we can do, and if such deadlock
1100  * actually occurs, it should be easy to locate the culprit work function.
1101  *
1102  * CONTEXT:
1103  * raw_spin_lock_irq(pool->lock).
1104  *
1105  * Return:
1106  * Pointer to worker which is executing @work if found, %NULL
1107  * otherwise.
1108  */
find_worker_executing_work(struct worker_pool * pool,struct work_struct * work)1109 static struct worker *find_worker_executing_work(struct worker_pool *pool,
1110 						 struct work_struct *work)
1111 {
1112 	struct worker *worker;
1113 
1114 	hash_for_each_possible(pool->busy_hash, worker, hentry,
1115 			       (unsigned long)work)
1116 		if (worker->current_work == work &&
1117 		    worker->current_func == work->func)
1118 			return worker;
1119 
1120 	return NULL;
1121 }
1122 
1123 /**
1124  * move_linked_works - move linked works to a list
1125  * @work: start of series of works to be scheduled
1126  * @head: target list to append @work to
1127  * @nextp: out parameter for nested worklist walking
1128  *
1129  * Schedule linked works starting from @work to @head. Work series to be
1130  * scheduled starts at @work and includes any consecutive work with
1131  * WORK_STRUCT_LINKED set in its predecessor. See assign_work() for details on
1132  * @nextp.
1133  *
1134  * CONTEXT:
1135  * raw_spin_lock_irq(pool->lock).
1136  */
move_linked_works(struct work_struct * work,struct list_head * head,struct work_struct ** nextp)1137 static void move_linked_works(struct work_struct *work, struct list_head *head,
1138 			      struct work_struct **nextp)
1139 {
1140 	struct work_struct *n;
1141 
1142 	/*
1143 	 * Linked worklist will always end before the end of the list,
1144 	 * use NULL for list head.
1145 	 */
1146 	list_for_each_entry_safe_from(work, n, NULL, entry) {
1147 		list_move_tail(&work->entry, head);
1148 		if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1149 			break;
1150 	}
1151 
1152 	/*
1153 	 * If we're already inside safe list traversal and have moved
1154 	 * multiple works to the scheduled queue, the next position
1155 	 * needs to be updated.
1156 	 */
1157 	if (nextp)
1158 		*nextp = n;
1159 }
1160 
1161 /**
1162  * assign_work - assign a work item and its linked work items to a worker
1163  * @work: work to assign
1164  * @worker: worker to assign to
1165  * @nextp: out parameter for nested worklist walking
1166  *
1167  * Assign @work and its linked work items to @worker. If @work is already being
1168  * executed by another worker in the same pool, it'll be punted there.
1169  *
1170  * If @nextp is not NULL, it's updated to point to the next work of the last
1171  * scheduled work. This allows assign_work() to be nested inside
1172  * list_for_each_entry_safe().
1173  *
1174  * Returns %true if @work was successfully assigned to @worker. %false if @work
1175  * was punted to another worker already executing it.
1176  */
assign_work(struct work_struct * work,struct worker * worker,struct work_struct ** nextp)1177 static bool assign_work(struct work_struct *work, struct worker *worker,
1178 			struct work_struct **nextp)
1179 {
1180 	struct worker_pool *pool = worker->pool;
1181 	struct worker *collision;
1182 
1183 	lockdep_assert_held(&pool->lock);
1184 
1185 	/*
1186 	 * A single work shouldn't be executed concurrently by multiple workers.
1187 	 * __queue_work() ensures that @work doesn't jump to a different pool
1188 	 * while still running in the previous pool. Here, we should ensure that
1189 	 * @work is not executed concurrently by multiple workers from the same
1190 	 * pool. Check whether anyone is already processing the work. If so,
1191 	 * defer the work to the currently executing one.
1192 	 */
1193 	collision = find_worker_executing_work(pool, work);
1194 	if (unlikely(collision)) {
1195 		move_linked_works(work, &collision->scheduled, nextp);
1196 		return false;
1197 	}
1198 
1199 	move_linked_works(work, &worker->scheduled, nextp);
1200 	return true;
1201 }
1202 
bh_pool_irq_work(struct worker_pool * pool)1203 static struct irq_work *bh_pool_irq_work(struct worker_pool *pool)
1204 {
1205 	int high = pool->attrs->nice == HIGHPRI_NICE_LEVEL ? 1 : 0;
1206 
1207 	return &per_cpu(bh_pool_irq_works, pool->cpu)[high];
1208 }
1209 
kick_bh_pool(struct worker_pool * pool)1210 static void kick_bh_pool(struct worker_pool *pool)
1211 {
1212 #ifdef CONFIG_SMP
1213 	/* see drain_dead_softirq_workfn() for BH_DRAINING */
1214 	if (unlikely(pool->cpu != smp_processor_id() &&
1215 		     !(pool->flags & POOL_BH_DRAINING))) {
1216 		irq_work_queue_on(bh_pool_irq_work(pool), pool->cpu);
1217 		return;
1218 	}
1219 #endif
1220 	if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
1221 		raise_softirq_irqoff(HI_SOFTIRQ);
1222 	else
1223 		raise_softirq_irqoff(TASKLET_SOFTIRQ);
1224 }
1225 
1226 /**
1227  * kick_pool - wake up an idle worker if necessary
1228  * @pool: pool to kick
1229  *
1230  * @pool may have pending work items. Wake up worker if necessary. Returns
1231  * whether a worker was woken up.
1232  */
kick_pool(struct worker_pool * pool)1233 static bool kick_pool(struct worker_pool *pool)
1234 {
1235 	struct worker *worker = first_idle_worker(pool);
1236 	struct task_struct *p;
1237 
1238 	lockdep_assert_held(&pool->lock);
1239 
1240 	if (!need_more_worker(pool) || !worker)
1241 		return false;
1242 
1243 	if (pool->flags & POOL_BH) {
1244 		kick_bh_pool(pool);
1245 		return true;
1246 	}
1247 
1248 	p = worker->task;
1249 
1250 #ifdef CONFIG_SMP
1251 	/*
1252 	 * Idle @worker is about to execute @work and waking up provides an
1253 	 * opportunity to migrate @worker at a lower cost by setting the task's
1254 	 * wake_cpu field. Let's see if we want to move @worker to improve
1255 	 * execution locality.
1256 	 *
1257 	 * We're waking the worker that went idle the latest and there's some
1258 	 * chance that @worker is marked idle but hasn't gone off CPU yet. If
1259 	 * so, setting the wake_cpu won't do anything. As this is a best-effort
1260 	 * optimization and the race window is narrow, let's leave as-is for
1261 	 * now. If this becomes pronounced, we can skip over workers which are
1262 	 * still on cpu when picking an idle worker.
1263 	 *
1264 	 * If @pool has non-strict affinity, @worker might have ended up outside
1265 	 * its affinity scope. Repatriate.
1266 	 */
1267 	if (!pool->attrs->affn_strict &&
1268 	    !cpumask_test_cpu(p->wake_cpu, pool->attrs->__pod_cpumask)) {
1269 		struct work_struct *work = list_first_entry(&pool->worklist,
1270 						struct work_struct, entry);
1271 		int wake_cpu = cpumask_any_and_distribute(pool->attrs->__pod_cpumask,
1272 							  cpu_online_mask);
1273 		if (wake_cpu < nr_cpu_ids) {
1274 			p->wake_cpu = wake_cpu;
1275 			get_work_pwq(work)->stats[PWQ_STAT_REPATRIATED]++;
1276 		}
1277 	}
1278 #endif
1279 	wake_up_process(p);
1280 	return true;
1281 }
1282 
1283 #ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
1284 
1285 /*
1286  * Concurrency-managed per-cpu work items that hog CPU for longer than
1287  * wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism,
1288  * which prevents them from stalling other concurrency-managed work items. If a
1289  * work function keeps triggering this mechanism, it's likely that the work item
1290  * should be using an unbound workqueue instead.
1291  *
1292  * wq_cpu_intensive_report() tracks work functions which trigger such conditions
1293  * and report them so that they can be examined and converted to use unbound
1294  * workqueues as appropriate. To avoid flooding the console, each violating work
1295  * function is tracked and reported with exponential backoff.
1296  */
1297 #define WCI_MAX_ENTS 128
1298 
1299 struct wci_ent {
1300 	work_func_t		func;
1301 	atomic64_t		cnt;
1302 	struct hlist_node	hash_node;
1303 };
1304 
1305 static struct wci_ent wci_ents[WCI_MAX_ENTS];
1306 static int wci_nr_ents;
1307 static DEFINE_RAW_SPINLOCK(wci_lock);
1308 static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS));
1309 
wci_find_ent(work_func_t func)1310 static struct wci_ent *wci_find_ent(work_func_t func)
1311 {
1312 	struct wci_ent *ent;
1313 
1314 	hash_for_each_possible_rcu(wci_hash, ent, hash_node,
1315 				   (unsigned long)func) {
1316 		if (ent->func == func)
1317 			return ent;
1318 	}
1319 	return NULL;
1320 }
1321 
wq_cpu_intensive_report(work_func_t func)1322 static void wq_cpu_intensive_report(work_func_t func)
1323 {
1324 	struct wci_ent *ent;
1325 
1326 restart:
1327 	ent = wci_find_ent(func);
1328 	if (ent) {
1329 		u64 cnt;
1330 
1331 		/*
1332 		 * Start reporting from the warning_thresh and back off
1333 		 * exponentially.
1334 		 */
1335 		cnt = atomic64_inc_return_relaxed(&ent->cnt);
1336 		if (wq_cpu_intensive_warning_thresh &&
1337 		    cnt >= wq_cpu_intensive_warning_thresh &&
1338 		    is_power_of_2(cnt + 1 - wq_cpu_intensive_warning_thresh))
1339 			printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n",
1340 					ent->func, wq_cpu_intensive_thresh_us,
1341 					atomic64_read(&ent->cnt));
1342 		return;
1343 	}
1344 
1345 	/*
1346 	 * @func is a new violation. Allocate a new entry for it. If wcn_ents[]
1347 	 * is exhausted, something went really wrong and we probably made enough
1348 	 * noise already.
1349 	 */
1350 	if (wci_nr_ents >= WCI_MAX_ENTS)
1351 		return;
1352 
1353 	raw_spin_lock(&wci_lock);
1354 
1355 	if (wci_nr_ents >= WCI_MAX_ENTS) {
1356 		raw_spin_unlock(&wci_lock);
1357 		return;
1358 	}
1359 
1360 	if (wci_find_ent(func)) {
1361 		raw_spin_unlock(&wci_lock);
1362 		goto restart;
1363 	}
1364 
1365 	ent = &wci_ents[wci_nr_ents++];
1366 	ent->func = func;
1367 	atomic64_set(&ent->cnt, 0);
1368 	hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func);
1369 
1370 	raw_spin_unlock(&wci_lock);
1371 
1372 	goto restart;
1373 }
1374 
1375 #else	/* CONFIG_WQ_CPU_INTENSIVE_REPORT */
wq_cpu_intensive_report(work_func_t func)1376 static void wq_cpu_intensive_report(work_func_t func) {}
1377 #endif	/* CONFIG_WQ_CPU_INTENSIVE_REPORT */
1378 
1379 /**
1380  * wq_worker_running - a worker is running again
1381  * @task: task waking up
1382  *
1383  * This function is called when a worker returns from schedule()
1384  */
wq_worker_running(struct task_struct * task)1385 void wq_worker_running(struct task_struct *task)
1386 {
1387 	struct worker *worker = kthread_data(task);
1388 
1389 	if (!READ_ONCE(worker->sleeping))
1390 		return;
1391 
1392 	/*
1393 	 * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
1394 	 * and the nr_running increment below, we may ruin the nr_running reset
1395 	 * and leave with an unexpected pool->nr_running == 1 on the newly unbound
1396 	 * pool. Protect against such race.
1397 	 */
1398 	preempt_disable();
1399 	if (!(worker->flags & WORKER_NOT_RUNNING))
1400 		worker->pool->nr_running++;
1401 	preempt_enable();
1402 
1403 	/*
1404 	 * CPU intensive auto-detection cares about how long a work item hogged
1405 	 * CPU without sleeping. Reset the starting timestamp on wakeup.
1406 	 */
1407 	worker->current_at = worker->task->se.sum_exec_runtime;
1408 
1409 	WRITE_ONCE(worker->sleeping, 0);
1410 }
1411 
1412 /**
1413  * wq_worker_sleeping - a worker is going to sleep
1414  * @task: task going to sleep
1415  *
1416  * This function is called from schedule() when a busy worker is
1417  * going to sleep.
1418  */
wq_worker_sleeping(struct task_struct * task)1419 void wq_worker_sleeping(struct task_struct *task)
1420 {
1421 	struct worker *worker = kthread_data(task);
1422 	struct worker_pool *pool;
1423 
1424 	/*
1425 	 * Rescuers, which may not have all the fields set up like normal
1426 	 * workers, also reach here, let's not access anything before
1427 	 * checking NOT_RUNNING.
1428 	 */
1429 	if (worker->flags & WORKER_NOT_RUNNING)
1430 		return;
1431 
1432 	pool = worker->pool;
1433 
1434 	/* Return if preempted before wq_worker_running() was reached */
1435 	if (READ_ONCE(worker->sleeping))
1436 		return;
1437 
1438 	WRITE_ONCE(worker->sleeping, 1);
1439 	raw_spin_lock_irq(&pool->lock);
1440 
1441 	/*
1442 	 * Recheck in case unbind_workers() preempted us. We don't
1443 	 * want to decrement nr_running after the worker is unbound
1444 	 * and nr_running has been reset.
1445 	 */
1446 	if (worker->flags & WORKER_NOT_RUNNING) {
1447 		raw_spin_unlock_irq(&pool->lock);
1448 		return;
1449 	}
1450 
1451 	pool->nr_running--;
1452 	if (kick_pool(pool))
1453 		worker->current_pwq->stats[PWQ_STAT_CM_WAKEUP]++;
1454 
1455 	raw_spin_unlock_irq(&pool->lock);
1456 }
1457 
1458 /**
1459  * wq_worker_tick - a scheduler tick occurred while a kworker is running
1460  * @task: task currently running
1461  *
1462  * Called from sched_tick(). We're in the IRQ context and the current
1463  * worker's fields which follow the 'K' locking rule can be accessed safely.
1464  */
wq_worker_tick(struct task_struct * task)1465 void wq_worker_tick(struct task_struct *task)
1466 {
1467 	struct worker *worker = kthread_data(task);
1468 	struct pool_workqueue *pwq = worker->current_pwq;
1469 	struct worker_pool *pool = worker->pool;
1470 
1471 	if (!pwq)
1472 		return;
1473 
1474 	pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;
1475 
1476 	if (!wq_cpu_intensive_thresh_us)
1477 		return;
1478 
1479 	/*
1480 	 * If the current worker is concurrency managed and hogged the CPU for
1481 	 * longer than wq_cpu_intensive_thresh_us, it's automatically marked
1482 	 * CPU_INTENSIVE to avoid stalling other concurrency-managed work items.
1483 	 *
1484 	 * Set @worker->sleeping means that @worker is in the process of
1485 	 * switching out voluntarily and won't be contributing to
1486 	 * @pool->nr_running until it wakes up. As wq_worker_sleeping() also
1487 	 * decrements ->nr_running, setting CPU_INTENSIVE here can lead to
1488 	 * double decrements. The task is releasing the CPU anyway. Let's skip.
1489 	 * We probably want to make this prettier in the future.
1490 	 */
1491 	if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) ||
1492 	    worker->task->se.sum_exec_runtime - worker->current_at <
1493 	    wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
1494 		return;
1495 
1496 	raw_spin_lock(&pool->lock);
1497 
1498 	worker_set_flags(worker, WORKER_CPU_INTENSIVE);
1499 	wq_cpu_intensive_report(worker->current_func);
1500 	pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;
1501 
1502 	if (kick_pool(pool))
1503 		pwq->stats[PWQ_STAT_CM_WAKEUP]++;
1504 
1505 	raw_spin_unlock(&pool->lock);
1506 }
1507 
1508 /**
1509  * wq_worker_last_func - retrieve worker's last work function
1510  * @task: Task to retrieve last work function of.
1511  *
1512  * Determine the last function a worker executed. This is called from
1513  * the scheduler to get a worker's last known identity.
1514  *
1515  * CONTEXT:
1516  * raw_spin_lock_irq(rq->lock)
1517  *
1518  * This function is called during schedule() when a kworker is going
1519  * to sleep. It's used by psi to identify aggregation workers during
1520  * dequeuing, to allow periodic aggregation to shut-off when that
1521  * worker is the last task in the system or cgroup to go to sleep.
1522  *
1523  * As this function doesn't involve any workqueue-related locking, it
1524  * only returns stable values when called from inside the scheduler's
1525  * queuing and dequeuing paths, when @task, which must be a kworker,
1526  * is guaranteed to not be processing any works.
1527  *
1528  * Return:
1529  * The last work function %current executed as a worker, NULL if it
1530  * hasn't executed any work yet.
1531  */
wq_worker_last_func(struct task_struct * task)1532 work_func_t wq_worker_last_func(struct task_struct *task)
1533 {
1534 	struct worker *worker = kthread_data(task);
1535 
1536 	return worker->last_func;
1537 }
1538 
1539 /**
1540  * wq_node_nr_active - Determine wq_node_nr_active to use
1541  * @wq: workqueue of interest
1542  * @node: NUMA node, can be %NUMA_NO_NODE
1543  *
1544  * Determine wq_node_nr_active to use for @wq on @node. Returns:
1545  *
1546  * - %NULL for per-cpu workqueues as they don't need to use shared nr_active.
1547  *
1548  * - node_nr_active[nr_node_ids] if @node is %NUMA_NO_NODE.
1549  *
1550  * - Otherwise, node_nr_active[@node].
1551  */
wq_node_nr_active(struct workqueue_struct * wq,int node)1552 static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq,
1553 						   int node)
1554 {
1555 	if (!(wq->flags & WQ_UNBOUND))
1556 		return NULL;
1557 
1558 	if (node == NUMA_NO_NODE)
1559 		node = nr_node_ids;
1560 
1561 	return wq->node_nr_active[node];
1562 }
1563 
1564 /**
1565  * wq_update_node_max_active - Update per-node max_actives to use
1566  * @wq: workqueue to update
1567  * @off_cpu: CPU that's going down, -1 if a CPU is not going down
1568  *
1569  * Update @wq->node_nr_active[]->max. @wq must be unbound. max_active is
1570  * distributed among nodes according to the proportions of numbers of online
1571  * cpus. The result is always between @wq->min_active and max_active.
1572  */
wq_update_node_max_active(struct workqueue_struct * wq,int off_cpu)1573 static void wq_update_node_max_active(struct workqueue_struct *wq, int off_cpu)
1574 {
1575 	struct cpumask *effective = unbound_effective_cpumask(wq);
1576 	int min_active = READ_ONCE(wq->min_active);
1577 	int max_active = READ_ONCE(wq->max_active);
1578 	int total_cpus, node;
1579 
1580 	lockdep_assert_held(&wq->mutex);
1581 
1582 	if (!wq_topo_initialized)
1583 		return;
1584 
1585 	if (off_cpu >= 0 && !cpumask_test_cpu(off_cpu, effective))
1586 		off_cpu = -1;
1587 
1588 	total_cpus = cpumask_weight_and(effective, cpu_online_mask);
1589 	if (off_cpu >= 0)
1590 		total_cpus--;
1591 
1592 	/* If all CPUs of the wq get offline, use the default values */
1593 	if (unlikely(!total_cpus)) {
1594 		for_each_node(node)
1595 			wq_node_nr_active(wq, node)->max = min_active;
1596 
1597 		wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
1598 		return;
1599 	}
1600 
1601 	for_each_node(node) {
1602 		int node_cpus;
1603 
1604 		node_cpus = cpumask_weight_and(effective, cpumask_of_node(node));
1605 		if (off_cpu >= 0 && cpu_to_node(off_cpu) == node)
1606 			node_cpus--;
1607 
1608 		wq_node_nr_active(wq, node)->max =
1609 			clamp(DIV_ROUND_UP(max_active * node_cpus, total_cpus),
1610 			      min_active, max_active);
1611 	}
1612 
1613 	wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
1614 }
1615 
1616 /**
1617  * get_pwq - get an extra reference on the specified pool_workqueue
1618  * @pwq: pool_workqueue to get
1619  *
1620  * Obtain an extra reference on @pwq.  The caller should guarantee that
1621  * @pwq has positive refcnt and be holding the matching pool->lock.
1622  */
get_pwq(struct pool_workqueue * pwq)1623 static void get_pwq(struct pool_workqueue *pwq)
1624 {
1625 	lockdep_assert_held(&pwq->pool->lock);
1626 	WARN_ON_ONCE(pwq->refcnt <= 0);
1627 	pwq->refcnt++;
1628 }
1629 
1630 /**
1631  * put_pwq - put a pool_workqueue reference
1632  * @pwq: pool_workqueue to put
1633  *
1634  * Drop a reference of @pwq.  If its refcnt reaches zero, schedule its
1635  * destruction.  The caller should be holding the matching pool->lock.
1636  */
put_pwq(struct pool_workqueue * pwq)1637 static void put_pwq(struct pool_workqueue *pwq)
1638 {
1639 	lockdep_assert_held(&pwq->pool->lock);
1640 	if (likely(--pwq->refcnt))
1641 		return;
1642 	/*
1643 	 * @pwq can't be released under pool->lock, bounce to a dedicated
1644 	 * kthread_worker to avoid A-A deadlocks.
1645 	 */
1646 	kthread_queue_work(pwq_release_worker, &pwq->release_work);
1647 }
1648 
1649 /**
1650  * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1651  * @pwq: pool_workqueue to put (can be %NULL)
1652  *
1653  * put_pwq() with locking.  This function also allows %NULL @pwq.
1654  */
put_pwq_unlocked(struct pool_workqueue * pwq)1655 static void put_pwq_unlocked(struct pool_workqueue *pwq)
1656 {
1657 	if (pwq) {
1658 		/*
1659 		 * As both pwqs and pools are RCU protected, the
1660 		 * following lock operations are safe.
1661 		 */
1662 		raw_spin_lock_irq(&pwq->pool->lock);
1663 		put_pwq(pwq);
1664 		raw_spin_unlock_irq(&pwq->pool->lock);
1665 	}
1666 }
1667 
pwq_is_empty(struct pool_workqueue * pwq)1668 static bool pwq_is_empty(struct pool_workqueue *pwq)
1669 {
1670 	return !pwq->nr_active && list_empty(&pwq->inactive_works);
1671 }
1672 
__pwq_activate_work(struct pool_workqueue * pwq,struct work_struct * work)1673 static void __pwq_activate_work(struct pool_workqueue *pwq,
1674 				struct work_struct *work)
1675 {
1676 	unsigned long *wdb = work_data_bits(work);
1677 
1678 	WARN_ON_ONCE(!(*wdb & WORK_STRUCT_INACTIVE));
1679 	trace_workqueue_activate_work(work);
1680 	if (list_empty(&pwq->pool->worklist))
1681 		pwq->pool->watchdog_ts = jiffies;
1682 	move_linked_works(work, &pwq->pool->worklist, NULL);
1683 	__clear_bit(WORK_STRUCT_INACTIVE_BIT, wdb);
1684 }
1685 
tryinc_node_nr_active(struct wq_node_nr_active * nna)1686 static bool tryinc_node_nr_active(struct wq_node_nr_active *nna)
1687 {
1688 	int max = READ_ONCE(nna->max);
1689 
1690 	while (true) {
1691 		int old, tmp;
1692 
1693 		old = atomic_read(&nna->nr);
1694 		if (old >= max)
1695 			return false;
1696 		tmp = atomic_cmpxchg_relaxed(&nna->nr, old, old + 1);
1697 		if (tmp == old)
1698 			return true;
1699 	}
1700 }
1701 
1702 /**
1703  * pwq_tryinc_nr_active - Try to increment nr_active for a pwq
1704  * @pwq: pool_workqueue of interest
1705  * @fill: max_active may have increased, try to increase concurrency level
1706  *
1707  * Try to increment nr_active for @pwq. Returns %true if an nr_active count is
1708  * successfully obtained. %false otherwise.
1709  */
pwq_tryinc_nr_active(struct pool_workqueue * pwq,bool fill)1710 static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
1711 {
1712 	struct workqueue_struct *wq = pwq->wq;
1713 	struct worker_pool *pool = pwq->pool;
1714 	struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node);
1715 	bool obtained = false;
1716 
1717 	lockdep_assert_held(&pool->lock);
1718 
1719 	if (!nna) {
1720 		/* BH or per-cpu workqueue, pwq->nr_active is sufficient */
1721 		obtained = pwq->nr_active < READ_ONCE(wq->max_active);
1722 		goto out;
1723 	}
1724 
1725 	if (unlikely(pwq->plugged))
1726 		return false;
1727 
1728 	/*
1729 	 * Unbound workqueue uses per-node shared nr_active $nna. If @pwq is
1730 	 * already waiting on $nna, pwq_dec_nr_active() will maintain the
1731 	 * concurrency level. Don't jump the line.
1732 	 *
1733 	 * We need to ignore the pending test after max_active has increased as
1734 	 * pwq_dec_nr_active() can only maintain the concurrency level but not
1735 	 * increase it. This is indicated by @fill.
1736 	 */
1737 	if (!list_empty(&pwq->pending_node) && likely(!fill))
1738 		goto out;
1739 
1740 	obtained = tryinc_node_nr_active(nna);
1741 	if (obtained)
1742 		goto out;
1743 
1744 	/*
1745 	 * Lockless acquisition failed. Lock, add ourself to $nna->pending_pwqs
1746 	 * and try again. The smp_mb() is paired with the implied memory barrier
1747 	 * of atomic_dec_return() in pwq_dec_nr_active() to ensure that either
1748 	 * we see the decremented $nna->nr or they see non-empty
1749 	 * $nna->pending_pwqs.
1750 	 */
1751 	raw_spin_lock(&nna->lock);
1752 
1753 	if (list_empty(&pwq->pending_node))
1754 		list_add_tail(&pwq->pending_node, &nna->pending_pwqs);
1755 	else if (likely(!fill))
1756 		goto out_unlock;
1757 
1758 	smp_mb();
1759 
1760 	obtained = tryinc_node_nr_active(nna);
1761 
1762 	/*
1763 	 * If @fill, @pwq might have already been pending. Being spuriously
1764 	 * pending in cold paths doesn't affect anything. Let's leave it be.
1765 	 */
1766 	if (obtained && likely(!fill))
1767 		list_del_init(&pwq->pending_node);
1768 
1769 out_unlock:
1770 	raw_spin_unlock(&nna->lock);
1771 out:
1772 	if (obtained)
1773 		pwq->nr_active++;
1774 	return obtained;
1775 }
1776 
1777 /**
1778  * pwq_activate_first_inactive - Activate the first inactive work item on a pwq
1779  * @pwq: pool_workqueue of interest
1780  * @fill: max_active may have increased, try to increase concurrency level
1781  *
1782  * Activate the first inactive work item of @pwq if available and allowed by
1783  * max_active limit.
1784  *
1785  * Returns %true if an inactive work item has been activated. %false if no
1786  * inactive work item is found or max_active limit is reached.
1787  */
pwq_activate_first_inactive(struct pool_workqueue * pwq,bool fill)1788 static bool pwq_activate_first_inactive(struct pool_workqueue *pwq, bool fill)
1789 {
1790 	struct work_struct *work =
1791 		list_first_entry_or_null(&pwq->inactive_works,
1792 					 struct work_struct, entry);
1793 
1794 	if (work && pwq_tryinc_nr_active(pwq, fill)) {
1795 		__pwq_activate_work(pwq, work);
1796 		return true;
1797 	} else {
1798 		return false;
1799 	}
1800 }
1801 
1802 /**
1803  * unplug_oldest_pwq - unplug the oldest pool_workqueue
1804  * @wq: workqueue_struct where its oldest pwq is to be unplugged
1805  *
1806  * This function should only be called for ordered workqueues where only the
1807  * oldest pwq is unplugged, the others are plugged to suspend execution to
1808  * ensure proper work item ordering::
1809  *
1810  *    dfl_pwq --------------+     [P] - plugged
1811  *                          |
1812  *                          v
1813  *    pwqs -> A -> B [P] -> C [P] (newest)
1814  *            |    |        |
1815  *            1    3        5
1816  *            |    |        |
1817  *            2    4        6
1818  *
1819  * When the oldest pwq is drained and removed, this function should be called
1820  * to unplug the next oldest one to start its work item execution. Note that
1821  * pwq's are linked into wq->pwqs with the oldest first, so the first one in
1822  * the list is the oldest.
1823  */
unplug_oldest_pwq(struct workqueue_struct * wq)1824 static void unplug_oldest_pwq(struct workqueue_struct *wq)
1825 {
1826 	struct pool_workqueue *pwq;
1827 
1828 	lockdep_assert_held(&wq->mutex);
1829 
1830 	/* Caller should make sure that pwqs isn't empty before calling */
1831 	pwq = list_first_entry_or_null(&wq->pwqs, struct pool_workqueue,
1832 				       pwqs_node);
1833 	raw_spin_lock_irq(&pwq->pool->lock);
1834 	if (pwq->plugged) {
1835 		pwq->plugged = false;
1836 		if (pwq_activate_first_inactive(pwq, true))
1837 			kick_pool(pwq->pool);
1838 	}
1839 	raw_spin_unlock_irq(&pwq->pool->lock);
1840 }
1841 
1842 /**
1843  * node_activate_pending_pwq - Activate a pending pwq on a wq_node_nr_active
1844  * @nna: wq_node_nr_active to activate a pending pwq for
1845  * @caller_pool: worker_pool the caller is locking
1846  *
1847  * Activate a pwq in @nna->pending_pwqs. Called with @caller_pool locked.
1848  * @caller_pool may be unlocked and relocked to lock other worker_pools.
1849  */
node_activate_pending_pwq(struct wq_node_nr_active * nna,struct worker_pool * caller_pool)1850 static void node_activate_pending_pwq(struct wq_node_nr_active *nna,
1851 				      struct worker_pool *caller_pool)
1852 {
1853 	struct worker_pool *locked_pool = caller_pool;
1854 	struct pool_workqueue *pwq;
1855 	struct work_struct *work;
1856 
1857 	lockdep_assert_held(&caller_pool->lock);
1858 
1859 	raw_spin_lock(&nna->lock);
1860 retry:
1861 	pwq = list_first_entry_or_null(&nna->pending_pwqs,
1862 				       struct pool_workqueue, pending_node);
1863 	if (!pwq)
1864 		goto out_unlock;
1865 
1866 	/*
1867 	 * If @pwq is for a different pool than @locked_pool, we need to lock
1868 	 * @pwq->pool->lock. Let's trylock first. If unsuccessful, do the unlock
1869 	 * / lock dance. For that, we also need to release @nna->lock as it's
1870 	 * nested inside pool locks.
1871 	 */
1872 	if (pwq->pool != locked_pool) {
1873 		raw_spin_unlock(&locked_pool->lock);
1874 		locked_pool = pwq->pool;
1875 		if (!raw_spin_trylock(&locked_pool->lock)) {
1876 			raw_spin_unlock(&nna->lock);
1877 			raw_spin_lock(&locked_pool->lock);
1878 			raw_spin_lock(&nna->lock);
1879 			goto retry;
1880 		}
1881 	}
1882 
1883 	/*
1884 	 * $pwq may not have any inactive work items due to e.g. cancellations.
1885 	 * Drop it from pending_pwqs and see if there's another one.
1886 	 */
1887 	work = list_first_entry_or_null(&pwq->inactive_works,
1888 					struct work_struct, entry);
1889 	if (!work) {
1890 		list_del_init(&pwq->pending_node);
1891 		goto retry;
1892 	}
1893 
1894 	/*
1895 	 * Acquire an nr_active count and activate the inactive work item. If
1896 	 * $pwq still has inactive work items, rotate it to the end of the
1897 	 * pending_pwqs so that we round-robin through them. This means that
1898 	 * inactive work items are not activated in queueing order which is fine
1899 	 * given that there has never been any ordering across different pwqs.
1900 	 */
1901 	if (likely(tryinc_node_nr_active(nna))) {
1902 		pwq->nr_active++;
1903 		__pwq_activate_work(pwq, work);
1904 
1905 		if (list_empty(&pwq->inactive_works))
1906 			list_del_init(&pwq->pending_node);
1907 		else
1908 			list_move_tail(&pwq->pending_node, &nna->pending_pwqs);
1909 
1910 		/* if activating a foreign pool, make sure it's running */
1911 		if (pwq->pool != caller_pool)
1912 			kick_pool(pwq->pool);
1913 	}
1914 
1915 out_unlock:
1916 	raw_spin_unlock(&nna->lock);
1917 	if (locked_pool != caller_pool) {
1918 		raw_spin_unlock(&locked_pool->lock);
1919 		raw_spin_lock(&caller_pool->lock);
1920 	}
1921 }
1922 
1923 /**
1924  * pwq_dec_nr_active - Retire an active count
1925  * @pwq: pool_workqueue of interest
1926  *
1927  * Decrement @pwq's nr_active and try to activate the first inactive work item.
1928  * For unbound workqueues, this function may temporarily drop @pwq->pool->lock.
1929  */
pwq_dec_nr_active(struct pool_workqueue * pwq)1930 static void pwq_dec_nr_active(struct pool_workqueue *pwq)
1931 {
1932 	struct worker_pool *pool = pwq->pool;
1933 	struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);
1934 
1935 	lockdep_assert_held(&pool->lock);
1936 
1937 	/*
1938 	 * @pwq->nr_active should be decremented for both percpu and unbound
1939 	 * workqueues.
1940 	 */
1941 	pwq->nr_active--;
1942 
1943 	/*
1944 	 * For a percpu workqueue, it's simple. Just need to kick the first
1945 	 * inactive work item on @pwq itself.
1946 	 */
1947 	if (!nna) {
1948 		pwq_activate_first_inactive(pwq, false);
1949 		return;
1950 	}
1951 
1952 	/*
1953 	 * If @pwq is for an unbound workqueue, it's more complicated because
1954 	 * multiple pwqs and pools may be sharing the nr_active count. When a
1955 	 * pwq needs to wait for an nr_active count, it puts itself on
1956 	 * $nna->pending_pwqs. The following atomic_dec_return()'s implied
1957 	 * memory barrier is paired with smp_mb() in pwq_tryinc_nr_active() to
1958 	 * guarantee that either we see non-empty pending_pwqs or they see
1959 	 * decremented $nna->nr.
1960 	 *
1961 	 * $nna->max may change as CPUs come online/offline and @pwq->wq's
1962 	 * max_active gets updated. However, it is guaranteed to be equal to or
1963 	 * larger than @pwq->wq->min_active which is above zero unless freezing.
1964 	 * This maintains the forward progress guarantee.
1965 	 */
1966 	if (atomic_dec_return(&nna->nr) >= READ_ONCE(nna->max))
1967 		return;
1968 
1969 	if (!list_empty(&nna->pending_pwqs))
1970 		node_activate_pending_pwq(nna, pool);
1971 }
1972 
1973 /**
1974  * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1975  * @pwq: pwq of interest
1976  * @work_data: work_data of work which left the queue
1977  *
1978  * A work either has completed or is removed from pending queue,
1979  * decrement nr_in_flight of its pwq and handle workqueue flushing.
1980  *
1981  * NOTE:
1982  * For unbound workqueues, this function may temporarily drop @pwq->pool->lock
1983  * and thus should be called after all other state updates for the in-flight
1984  * work item is complete.
1985  *
1986  * CONTEXT:
1987  * raw_spin_lock_irq(pool->lock).
1988  */
pwq_dec_nr_in_flight(struct pool_workqueue * pwq,unsigned long work_data)1989 static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)
1990 {
1991 	int color = get_work_color(work_data);
1992 
1993 	if (!(work_data & WORK_STRUCT_INACTIVE))
1994 		pwq_dec_nr_active(pwq);
1995 
1996 	pwq->nr_in_flight[color]--;
1997 
1998 	/* is flush in progress and are we at the flushing tip? */
1999 	if (likely(pwq->flush_color != color))
2000 		goto out_put;
2001 
2002 	/* are there still in-flight works? */
2003 	if (pwq->nr_in_flight[color])
2004 		goto out_put;
2005 
2006 	/* this pwq is done, clear flush_color */
2007 	pwq->flush_color = -1;
2008 
2009 	/*
2010 	 * If this was the last pwq, wake up the first flusher.  It
2011 	 * will handle the rest.
2012 	 */
2013 	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
2014 		complete(&pwq->wq->first_flusher->done);
2015 out_put:
2016 	put_pwq(pwq);
2017 }
2018 
2019 /**
2020  * try_to_grab_pending - steal work item from worklist and disable irq
2021  * @work: work item to steal
2022  * @cflags: %WORK_CANCEL_ flags
2023  * @irq_flags: place to store irq state
2024  *
2025  * Try to grab PENDING bit of @work.  This function can handle @work in any
2026  * stable state - idle, on timer or on worklist.
2027  *
2028  * Return:
2029  *
2030  *  ========	================================================================
2031  *  1		if @work was pending and we successfully stole PENDING
2032  *  0		if @work was idle and we claimed PENDING
2033  *  -EAGAIN	if PENDING couldn't be grabbed at the moment, safe to busy-retry
2034  *  ========	================================================================
2035  *
2036  * Note:
2037  * On >= 0 return, the caller owns @work's PENDING bit.  To avoid getting
2038  * interrupted while holding PENDING and @work off queue, irq must be
2039  * disabled on entry.  This, combined with delayed_work->timer being
2040  * irqsafe, ensures that we return -EAGAIN for finite short period of time.
2041  *
2042  * On successful return, >= 0, irq is disabled and the caller is
2043  * responsible for releasing it using local_irq_restore(*@irq_flags).
2044  *
2045  * This function is safe to call from any context including IRQ handler.
2046  */
try_to_grab_pending(struct work_struct * work,u32 cflags,unsigned long * irq_flags)2047 static int try_to_grab_pending(struct work_struct *work, u32 cflags,
2048 			       unsigned long *irq_flags)
2049 {
2050 	struct worker_pool *pool;
2051 	struct pool_workqueue *pwq;
2052 
2053 	local_irq_save(*irq_flags);
2054 
2055 	/* try to steal the timer if it exists */
2056 	if (cflags & WORK_CANCEL_DELAYED) {
2057 		struct delayed_work *dwork = to_delayed_work(work);
2058 
2059 		/*
2060 		 * dwork->timer is irqsafe.  If del_timer() fails, it's
2061 		 * guaranteed that the timer is not queued anywhere and not
2062 		 * running on the local CPU.
2063 		 */
2064 		if (likely(del_timer(&dwork->timer)))
2065 			return 1;
2066 	}
2067 
2068 	/* try to claim PENDING the normal way */
2069 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
2070 		return 0;
2071 
2072 	rcu_read_lock();
2073 	/*
2074 	 * The queueing is in progress, or it is already queued. Try to
2075 	 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
2076 	 */
2077 	pool = get_work_pool(work);
2078 	if (!pool)
2079 		goto fail;
2080 
2081 	raw_spin_lock(&pool->lock);
2082 	/*
2083 	 * work->data is guaranteed to point to pwq only while the work
2084 	 * item is queued on pwq->wq, and both updating work->data to point
2085 	 * to pwq on queueing and to pool on dequeueing are done under
2086 	 * pwq->pool->lock.  This in turn guarantees that, if work->data
2087 	 * points to pwq which is associated with a locked pool, the work
2088 	 * item is currently queued on that pool.
2089 	 */
2090 	pwq = get_work_pwq(work);
2091 	if (pwq && pwq->pool == pool) {
2092 		unsigned long work_data = *work_data_bits(work);
2093 
2094 		debug_work_deactivate(work);
2095 
2096 		/*
2097 		 * A cancelable inactive work item must be in the
2098 		 * pwq->inactive_works since a queued barrier can't be
2099 		 * canceled (see the comments in insert_wq_barrier()).
2100 		 *
2101 		 * An inactive work item cannot be deleted directly because
2102 		 * it might have linked barrier work items which, if left
2103 		 * on the inactive_works list, will confuse pwq->nr_active
2104 		 * management later on and cause stall.  Move the linked
2105 		 * barrier work items to the worklist when deleting the grabbed
2106 		 * item. Also keep WORK_STRUCT_INACTIVE in work_data, so that
2107 		 * it doesn't participate in nr_active management in later
2108 		 * pwq_dec_nr_in_flight().
2109 		 */
2110 		if (work_data & WORK_STRUCT_INACTIVE)
2111 			move_linked_works(work, &pwq->pool->worklist, NULL);
2112 
2113 		list_del_init(&work->entry);
2114 
2115 		/*
2116 		 * work->data points to pwq iff queued. Let's point to pool. As
2117 		 * this destroys work->data needed by the next step, stash it.
2118 		 */
2119 		set_work_pool_and_keep_pending(work, pool->id,
2120 					       pool_offq_flags(pool));
2121 
2122 		/* must be the last step, see the function comment */
2123 		pwq_dec_nr_in_flight(pwq, work_data);
2124 
2125 		raw_spin_unlock(&pool->lock);
2126 		rcu_read_unlock();
2127 		return 1;
2128 	}
2129 	raw_spin_unlock(&pool->lock);
2130 fail:
2131 	rcu_read_unlock();
2132 	local_irq_restore(*irq_flags);
2133 	return -EAGAIN;
2134 }
2135 
2136 /**
2137  * work_grab_pending - steal work item from worklist and disable irq
2138  * @work: work item to steal
2139  * @cflags: %WORK_CANCEL_ flags
2140  * @irq_flags: place to store IRQ state
2141  *
2142  * Grab PENDING bit of @work. @work can be in any stable state - idle, on timer
2143  * or on worklist.
2144  *
2145  * Can be called from any context. IRQ is disabled on return with IRQ state
2146  * stored in *@irq_flags. The caller is responsible for re-enabling it using
2147  * local_irq_restore().
2148  *
2149  * Returns %true if @work was pending. %false if idle.
2150  */
work_grab_pending(struct work_struct * work,u32 cflags,unsigned long * irq_flags)2151 static bool work_grab_pending(struct work_struct *work, u32 cflags,
2152 			      unsigned long *irq_flags)
2153 {
2154 	int ret;
2155 
2156 	while (true) {
2157 		ret = try_to_grab_pending(work, cflags, irq_flags);
2158 		if (ret >= 0)
2159 			return ret;
2160 		cpu_relax();
2161 	}
2162 }
2163 
2164 /**
2165  * insert_work - insert a work into a pool
2166  * @pwq: pwq @work belongs to
2167  * @work: work to insert
2168  * @head: insertion point
2169  * @extra_flags: extra WORK_STRUCT_* flags to set
2170  *
2171  * Insert @work which belongs to @pwq after @head.  @extra_flags is or'd to
2172  * work_struct flags.
2173  *
2174  * CONTEXT:
2175  * raw_spin_lock_irq(pool->lock).
2176  */
insert_work(struct pool_workqueue * pwq,struct work_struct * work,struct list_head * head,unsigned int extra_flags)2177 static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
2178 			struct list_head *head, unsigned int extra_flags)
2179 {
2180 	debug_work_activate(work);
2181 
2182 	/* record the work call stack in order to print it in KASAN reports */
2183 	kasan_record_aux_stack(work);
2184 
2185 	/* we own @work, set data and link */
2186 	set_work_pwq(work, pwq, extra_flags);
2187 	list_add_tail(&work->entry, head);
2188 	get_pwq(pwq);
2189 }
2190 
2191 /*
2192  * Test whether @work is being queued from another work executing on the
2193  * same workqueue.
2194  */
is_chained_work(struct workqueue_struct * wq)2195 static bool is_chained_work(struct workqueue_struct *wq)
2196 {
2197 	struct worker *worker;
2198 
2199 	worker = current_wq_worker();
2200 	/*
2201 	 * Return %true iff I'm a worker executing a work item on @wq.  If
2202 	 * I'm @worker, it's safe to dereference it without locking.
2203 	 */
2204 	return worker && worker->current_pwq->wq == wq;
2205 }
2206 
2207 /*
2208  * When queueing an unbound work item to a wq, prefer local CPU if allowed
2209  * by wq_unbound_cpumask.  Otherwise, round robin among the allowed ones to
2210  * avoid perturbing sensitive tasks.
2211  */
wq_select_unbound_cpu(int cpu)2212 static int wq_select_unbound_cpu(int cpu)
2213 {
2214 	int new_cpu;
2215 
2216 	if (likely(!wq_debug_force_rr_cpu)) {
2217 		if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
2218 			return cpu;
2219 	} else {
2220 		pr_warn_once("workqueue: round-robin CPU selection forced, expect performance impact\n");
2221 	}
2222 
2223 	new_cpu = __this_cpu_read(wq_rr_cpu_last);
2224 	new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
2225 	if (unlikely(new_cpu >= nr_cpu_ids)) {
2226 		new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
2227 		if (unlikely(new_cpu >= nr_cpu_ids))
2228 			return cpu;
2229 	}
2230 	__this_cpu_write(wq_rr_cpu_last, new_cpu);
2231 
2232 	return new_cpu;
2233 }
2234 
__queue_work(int cpu,struct workqueue_struct * wq,struct work_struct * work)2235 static void __queue_work(int cpu, struct workqueue_struct *wq,
2236 			 struct work_struct *work)
2237 {
2238 	struct pool_workqueue *pwq;
2239 	struct worker_pool *last_pool, *pool;
2240 	unsigned int work_flags;
2241 	unsigned int req_cpu = cpu;
2242 
2243 	/*
2244 	 * While a work item is PENDING && off queue, a task trying to
2245 	 * steal the PENDING will busy-loop waiting for it to either get
2246 	 * queued or lose PENDING.  Grabbing PENDING and queueing should
2247 	 * happen with IRQ disabled.
2248 	 */
2249 	lockdep_assert_irqs_disabled();
2250 
2251 	/*
2252 	 * For a draining wq, only works from the same workqueue are
2253 	 * allowed. The __WQ_DESTROYING helps to spot the issue that
2254 	 * queues a new work item to a wq after destroy_workqueue(wq).
2255 	 */
2256 	if (unlikely(wq->flags & (__WQ_DESTROYING | __WQ_DRAINING) &&
2257 		     WARN_ONCE(!is_chained_work(wq), "workqueue: cannot queue %ps on wq %s\n",
2258 			       work->func, wq->name))) {
2259 		return;
2260 	}
2261 	rcu_read_lock();
2262 retry:
2263 	/* pwq which will be used unless @work is executing elsewhere */
2264 	if (req_cpu == WORK_CPU_UNBOUND) {
2265 		if (wq->flags & WQ_UNBOUND)
2266 			cpu = wq_select_unbound_cpu(raw_smp_processor_id());
2267 		else
2268 			cpu = raw_smp_processor_id();
2269 	}
2270 
2271 	pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu));
2272 	pool = pwq->pool;
2273 
2274 	/*
2275 	 * If @work was previously on a different pool, it might still be
2276 	 * running there, in which case the work needs to be queued on that
2277 	 * pool to guarantee non-reentrancy.
2278 	 *
2279 	 * For ordered workqueue, work items must be queued on the newest pwq
2280 	 * for accurate order management.  Guaranteed order also guarantees
2281 	 * non-reentrancy.  See the comments above unplug_oldest_pwq().
2282 	 */
2283 	last_pool = get_work_pool(work);
2284 	if (last_pool && last_pool != pool && !(wq->flags & __WQ_ORDERED)) {
2285 		struct worker *worker;
2286 
2287 		raw_spin_lock(&last_pool->lock);
2288 
2289 		worker = find_worker_executing_work(last_pool, work);
2290 
2291 		if (worker && worker->current_pwq->wq == wq) {
2292 			pwq = worker->current_pwq;
2293 			pool = pwq->pool;
2294 			WARN_ON_ONCE(pool != last_pool);
2295 		} else {
2296 			/* meh... not running there, queue here */
2297 			raw_spin_unlock(&last_pool->lock);
2298 			raw_spin_lock(&pool->lock);
2299 		}
2300 	} else {
2301 		raw_spin_lock(&pool->lock);
2302 	}
2303 
2304 	/*
2305 	 * pwq is determined and locked. For unbound pools, we could have raced
2306 	 * with pwq release and it could already be dead. If its refcnt is zero,
2307 	 * repeat pwq selection. Note that unbound pwqs never die without
2308 	 * another pwq replacing it in cpu_pwq or while work items are executing
2309 	 * on it, so the retrying is guaranteed to make forward-progress.
2310 	 */
2311 	if (unlikely(!pwq->refcnt)) {
2312 		if (wq->flags & WQ_UNBOUND) {
2313 			raw_spin_unlock(&pool->lock);
2314 			cpu_relax();
2315 			goto retry;
2316 		}
2317 		/* oops */
2318 		WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
2319 			  wq->name, cpu);
2320 	}
2321 
2322 	/* pwq determined, queue */
2323 	trace_workqueue_queue_work(req_cpu, pwq, work);
2324 
2325 	if (WARN_ON(!list_empty(&work->entry)))
2326 		goto out;
2327 
2328 	pwq->nr_in_flight[pwq->work_color]++;
2329 	work_flags = work_color_to_flags(pwq->work_color);
2330 
2331 	/*
2332 	 * Limit the number of concurrently active work items to max_active.
2333 	 * @work must also queue behind existing inactive work items to maintain
2334 	 * ordering when max_active changes. See wq_adjust_max_active().
2335 	 */
2336 	if (list_empty(&pwq->inactive_works) && pwq_tryinc_nr_active(pwq, false)) {
2337 		if (list_empty(&pool->worklist))
2338 			pool->watchdog_ts = jiffies;
2339 
2340 		trace_workqueue_activate_work(work);
2341 		insert_work(pwq, work, &pool->worklist, work_flags);
2342 		kick_pool(pool);
2343 	} else {
2344 		work_flags |= WORK_STRUCT_INACTIVE;
2345 		insert_work(pwq, work, &pwq->inactive_works, work_flags);
2346 	}
2347 
2348 out:
2349 	raw_spin_unlock(&pool->lock);
2350 	rcu_read_unlock();
2351 }
2352 
clear_pending_if_disabled(struct work_struct * work)2353 static bool clear_pending_if_disabled(struct work_struct *work)
2354 {
2355 	unsigned long data = *work_data_bits(work);
2356 	struct work_offq_data offqd;
2357 
2358 	if (likely((data & WORK_STRUCT_PWQ) ||
2359 		   !(data & WORK_OFFQ_DISABLE_MASK)))
2360 		return false;
2361 
2362 	work_offqd_unpack(&offqd, data);
2363 	set_work_pool_and_clear_pending(work, offqd.pool_id,
2364 					work_offqd_pack_flags(&offqd));
2365 	return true;
2366 }
2367 
2368 /**
2369  * queue_work_on - queue work on specific cpu
2370  * @cpu: CPU number to execute work on
2371  * @wq: workqueue to use
2372  * @work: work to queue
2373  *
2374  * We queue the work to a specific CPU, the caller must ensure it
2375  * can't go away.  Callers that fail to ensure that the specified
2376  * CPU cannot go away will execute on a randomly chosen CPU.
2377  * But note well that callers specifying a CPU that never has been
2378  * online will get a splat.
2379  *
2380  * Return: %false if @work was already on a queue, %true otherwise.
2381  */
queue_work_on(int cpu,struct workqueue_struct * wq,struct work_struct * work)2382 bool queue_work_on(int cpu, struct workqueue_struct *wq,
2383 		   struct work_struct *work)
2384 {
2385 	bool ret = false;
2386 	unsigned long irq_flags;
2387 
2388 	local_irq_save(irq_flags);
2389 
2390 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2391 	    !clear_pending_if_disabled(work)) {
2392 		__queue_work(cpu, wq, work);
2393 		ret = true;
2394 	}
2395 
2396 	local_irq_restore(irq_flags);
2397 	return ret;
2398 }
2399 EXPORT_SYMBOL(queue_work_on);
2400 
2401 /**
2402  * select_numa_node_cpu - Select a CPU based on NUMA node
2403  * @node: NUMA node ID that we want to select a CPU from
2404  *
2405  * This function will attempt to find a "random" cpu available on a given
2406  * node. If there are no CPUs available on the given node it will return
2407  * WORK_CPU_UNBOUND indicating that we should just schedule to any
2408  * available CPU if we need to schedule this work.
2409  */
select_numa_node_cpu(int node)2410 static int select_numa_node_cpu(int node)
2411 {
2412 	int cpu;
2413 
2414 	/* Delay binding to CPU if node is not valid or online */
2415 	if (node < 0 || node >= MAX_NUMNODES || !node_online(node))
2416 		return WORK_CPU_UNBOUND;
2417 
2418 	/* Use local node/cpu if we are already there */
2419 	cpu = raw_smp_processor_id();
2420 	if (node == cpu_to_node(cpu))
2421 		return cpu;
2422 
2423 	/* Use "random" otherwise know as "first" online CPU of node */
2424 	cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
2425 
2426 	/* If CPU is valid return that, otherwise just defer */
2427 	return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
2428 }
2429 
2430 /**
2431  * queue_work_node - queue work on a "random" cpu for a given NUMA node
2432  * @node: NUMA node that we are targeting the work for
2433  * @wq: workqueue to use
2434  * @work: work to queue
2435  *
2436  * We queue the work to a "random" CPU within a given NUMA node. The basic
2437  * idea here is to provide a way to somehow associate work with a given
2438  * NUMA node.
2439  *
2440  * This function will only make a best effort attempt at getting this onto
2441  * the right NUMA node. If no node is requested or the requested node is
2442  * offline then we just fall back to standard queue_work behavior.
2443  *
2444  * Currently the "random" CPU ends up being the first available CPU in the
2445  * intersection of cpu_online_mask and the cpumask of the node, unless we
2446  * are running on the node. In that case we just use the current CPU.
2447  *
2448  * Return: %false if @work was already on a queue, %true otherwise.
2449  */
queue_work_node(int node,struct workqueue_struct * wq,struct work_struct * work)2450 bool queue_work_node(int node, struct workqueue_struct *wq,
2451 		     struct work_struct *work)
2452 {
2453 	unsigned long irq_flags;
2454 	bool ret = false;
2455 
2456 	/*
2457 	 * This current implementation is specific to unbound workqueues.
2458 	 * Specifically we only return the first available CPU for a given
2459 	 * node instead of cycling through individual CPUs within the node.
2460 	 *
2461 	 * If this is used with a per-cpu workqueue then the logic in
2462 	 * workqueue_select_cpu_near would need to be updated to allow for
2463 	 * some round robin type logic.
2464 	 */
2465 	WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
2466 
2467 	local_irq_save(irq_flags);
2468 
2469 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2470 	    !clear_pending_if_disabled(work)) {
2471 		int cpu = select_numa_node_cpu(node);
2472 
2473 		__queue_work(cpu, wq, work);
2474 		ret = true;
2475 	}
2476 
2477 	local_irq_restore(irq_flags);
2478 	return ret;
2479 }
2480 EXPORT_SYMBOL_GPL(queue_work_node);
2481 
delayed_work_timer_fn(struct timer_list * t)2482 void delayed_work_timer_fn(struct timer_list *t)
2483 {
2484 	struct delayed_work *dwork = from_timer(dwork, t, timer);
2485 
2486 	/* should have been called from irqsafe timer with irq already off */
2487 	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
2488 }
2489 EXPORT_SYMBOL(delayed_work_timer_fn);
2490 
__queue_delayed_work(int cpu,struct workqueue_struct * wq,struct delayed_work * dwork,unsigned long delay)2491 static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
2492 				struct delayed_work *dwork, unsigned long delay)
2493 {
2494 	struct timer_list *timer = &dwork->timer;
2495 	struct work_struct *work = &dwork->work;
2496 
2497 	WARN_ON_ONCE(!wq);
2498 	WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
2499 	WARN_ON_ONCE(timer_pending(timer));
2500 	WARN_ON_ONCE(!list_empty(&work->entry));
2501 
2502 	/*
2503 	 * If @delay is 0, queue @dwork->work immediately.  This is for
2504 	 * both optimization and correctness.  The earliest @timer can
2505 	 * expire is on the closest next tick and delayed_work users depend
2506 	 * on that there's no such delay when @delay is 0.
2507 	 */
2508 	if (!delay) {
2509 		__queue_work(cpu, wq, &dwork->work);
2510 		return;
2511 	}
2512 
2513 	WARN_ON_ONCE(cpu != WORK_CPU_UNBOUND && !cpu_online(cpu));
2514 	dwork->wq = wq;
2515 	dwork->cpu = cpu;
2516 	timer->expires = jiffies + delay;
2517 
2518 	if (housekeeping_enabled(HK_TYPE_TIMER)) {
2519 		/* If the current cpu is a housekeeping cpu, use it. */
2520 		cpu = smp_processor_id();
2521 		if (!housekeeping_test_cpu(cpu, HK_TYPE_TIMER))
2522 			cpu = housekeeping_any_cpu(HK_TYPE_TIMER);
2523 		add_timer_on(timer, cpu);
2524 	} else {
2525 		if (likely(cpu == WORK_CPU_UNBOUND))
2526 			add_timer_global(timer);
2527 		else
2528 			add_timer_on(timer, cpu);
2529 	}
2530 }
2531 
2532 /**
2533  * queue_delayed_work_on - queue work on specific CPU after delay
2534  * @cpu: CPU number to execute work on
2535  * @wq: workqueue to use
2536  * @dwork: work to queue
2537  * @delay: number of jiffies to wait before queueing
2538  *
2539  * We queue the delayed_work to a specific CPU, for non-zero delays the
2540  * caller must ensure it is online and can't go away. Callers that fail
2541  * to ensure this, may get @dwork->timer queued to an offlined CPU and
2542  * this will prevent queueing of @dwork->work unless the offlined CPU
2543  * becomes online again.
2544  *
2545  * Return: %false if @work was already on a queue, %true otherwise.  If
2546  * @delay is zero and @dwork is idle, it will be scheduled for immediate
2547  * execution.
2548  */
queue_delayed_work_on(int cpu,struct workqueue_struct * wq,struct delayed_work * dwork,unsigned long delay)2549 bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
2550 			   struct delayed_work *dwork, unsigned long delay)
2551 {
2552 	struct work_struct *work = &dwork->work;
2553 	bool ret = false;
2554 	unsigned long irq_flags;
2555 
2556 	/* read the comment in __queue_work() */
2557 	local_irq_save(irq_flags);
2558 
2559 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2560 	    !clear_pending_if_disabled(work)) {
2561 		__queue_delayed_work(cpu, wq, dwork, delay);
2562 		ret = true;
2563 	}
2564 
2565 	local_irq_restore(irq_flags);
2566 	return ret;
2567 }
2568 EXPORT_SYMBOL(queue_delayed_work_on);
2569 
2570 /**
2571  * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
2572  * @cpu: CPU number to execute work on
2573  * @wq: workqueue to use
2574  * @dwork: work to queue
2575  * @delay: number of jiffies to wait before queueing
2576  *
2577  * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
2578  * modify @dwork's timer so that it expires after @delay.  If @delay is
2579  * zero, @work is guaranteed to be scheduled immediately regardless of its
2580  * current state.
2581  *
2582  * Return: %false if @dwork was idle and queued, %true if @dwork was
2583  * pending and its timer was modified.
2584  *
2585  * This function is safe to call from any context including IRQ handler.
2586  * See try_to_grab_pending() for details.
2587  */
mod_delayed_work_on(int cpu,struct workqueue_struct * wq,struct delayed_work * dwork,unsigned long delay)2588 bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
2589 			 struct delayed_work *dwork, unsigned long delay)
2590 {
2591 	unsigned long irq_flags;
2592 	bool ret;
2593 
2594 	ret = work_grab_pending(&dwork->work, WORK_CANCEL_DELAYED, &irq_flags);
2595 
2596 	if (!clear_pending_if_disabled(&dwork->work))
2597 		__queue_delayed_work(cpu, wq, dwork, delay);
2598 
2599 	local_irq_restore(irq_flags);
2600 	return ret;
2601 }
2602 EXPORT_SYMBOL_GPL(mod_delayed_work_on);
2603 
rcu_work_rcufn(struct rcu_head * rcu)2604 static void rcu_work_rcufn(struct rcu_head *rcu)
2605 {
2606 	struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu);
2607 
2608 	/* read the comment in __queue_work() */
2609 	local_irq_disable();
2610 	__queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
2611 	local_irq_enable();
2612 }
2613 
2614 /**
2615  * queue_rcu_work - queue work after a RCU grace period
2616  * @wq: workqueue to use
2617  * @rwork: work to queue
2618  *
2619  * Return: %false if @rwork was already pending, %true otherwise.  Note
2620  * that a full RCU grace period is guaranteed only after a %true return.
2621  * While @rwork is guaranteed to be executed after a %false return, the
2622  * execution may happen before a full RCU grace period has passed.
2623  */
queue_rcu_work(struct workqueue_struct * wq,struct rcu_work * rwork)2624 bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
2625 {
2626 	struct work_struct *work = &rwork->work;
2627 
2628 	/*
2629 	 * rcu_work can't be canceled or disabled. Warn if the user reached
2630 	 * inside @rwork and disabled the inner work.
2631 	 */
2632 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
2633 	    !WARN_ON_ONCE(clear_pending_if_disabled(work))) {
2634 		rwork->wq = wq;
2635 		call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);
2636 		return true;
2637 	}
2638 
2639 	return false;
2640 }
2641 EXPORT_SYMBOL(queue_rcu_work);
2642 
alloc_worker(int node)2643 static struct worker *alloc_worker(int node)
2644 {
2645 	struct worker *worker;
2646 
2647 	worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
2648 	if (worker) {
2649 		INIT_LIST_HEAD(&worker->entry);
2650 		INIT_LIST_HEAD(&worker->scheduled);
2651 		INIT_LIST_HEAD(&worker->node);
2652 		/* on creation a worker is in !idle && prep state */
2653 		worker->flags = WORKER_PREP;
2654 	}
2655 	return worker;
2656 }
2657 
pool_allowed_cpus(struct worker_pool * pool)2658 static cpumask_t *pool_allowed_cpus(struct worker_pool *pool)
2659 {
2660 	if (pool->cpu < 0 && pool->attrs->affn_strict)
2661 		return pool->attrs->__pod_cpumask;
2662 	else
2663 		return pool->attrs->cpumask;
2664 }
2665 
2666 /**
2667  * worker_attach_to_pool() - attach a worker to a pool
2668  * @worker: worker to be attached
2669  * @pool: the target pool
2670  *
2671  * Attach @worker to @pool.  Once attached, the %WORKER_UNBOUND flag and
2672  * cpu-binding of @worker are kept coordinated with the pool across
2673  * cpu-[un]hotplugs.
2674  */
worker_attach_to_pool(struct worker * worker,struct worker_pool * pool)2675 static void worker_attach_to_pool(struct worker *worker,
2676 				  struct worker_pool *pool)
2677 {
2678 	mutex_lock(&wq_pool_attach_mutex);
2679 
2680 	/*
2681 	 * The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains stable
2682 	 * across this function. See the comments above the flag definition for
2683 	 * details. BH workers are, while per-CPU, always DISASSOCIATED.
2684 	 */
2685 	if (pool->flags & POOL_DISASSOCIATED) {
2686 		worker->flags |= WORKER_UNBOUND;
2687 	} else {
2688 		WARN_ON_ONCE(pool->flags & POOL_BH);
2689 		kthread_set_per_cpu(worker->task, pool->cpu);
2690 	}
2691 
2692 	if (worker->rescue_wq)
2693 		set_cpus_allowed_ptr(worker->task, pool_allowed_cpus(pool));
2694 
2695 	list_add_tail(&worker->node, &pool->workers);
2696 	worker->pool = pool;
2697 
2698 	mutex_unlock(&wq_pool_attach_mutex);
2699 }
2700 
unbind_worker(struct worker * worker)2701 static void unbind_worker(struct worker *worker)
2702 {
2703 	lockdep_assert_held(&wq_pool_attach_mutex);
2704 
2705 	kthread_set_per_cpu(worker->task, -1);
2706 	if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))
2707 		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0);
2708 	else
2709 		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0);
2710 }
2711 
2712 
detach_worker(struct worker * worker)2713 static void detach_worker(struct worker *worker)
2714 {
2715 	lockdep_assert_held(&wq_pool_attach_mutex);
2716 
2717 	unbind_worker(worker);
2718 	list_del(&worker->node);
2719 }
2720 
2721 /**
2722  * worker_detach_from_pool() - detach a worker from its pool
2723  * @worker: worker which is attached to its pool
2724  *
2725  * Undo the attaching which had been done in worker_attach_to_pool().  The
2726  * caller worker shouldn't access to the pool after detached except it has
2727  * other reference to the pool.
2728  */
worker_detach_from_pool(struct worker * worker)2729 static void worker_detach_from_pool(struct worker *worker)
2730 {
2731 	struct worker_pool *pool = worker->pool;
2732 
2733 	/* there is one permanent BH worker per CPU which should never detach */
2734 	WARN_ON_ONCE(pool->flags & POOL_BH);
2735 
2736 	mutex_lock(&wq_pool_attach_mutex);
2737 	detach_worker(worker);
2738 	worker->pool = NULL;
2739 	mutex_unlock(&wq_pool_attach_mutex);
2740 
2741 	/* clear leftover flags without pool->lock after it is detached */
2742 	worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);
2743 }
2744 
format_worker_id(char * buf,size_t size,struct worker * worker,struct worker_pool * pool)2745 static int format_worker_id(char *buf, size_t size, struct worker *worker,
2746 			    struct worker_pool *pool)
2747 {
2748 	if (worker->rescue_wq)
2749 		return scnprintf(buf, size, "kworker/R-%s",
2750 				 worker->rescue_wq->name);
2751 
2752 	if (pool) {
2753 		if (pool->cpu >= 0)
2754 			return scnprintf(buf, size, "kworker/%d:%d%s",
2755 					 pool->cpu, worker->id,
2756 					 pool->attrs->nice < 0  ? "H" : "");
2757 		else
2758 			return scnprintf(buf, size, "kworker/u%d:%d",
2759 					 pool->id, worker->id);
2760 	} else {
2761 		return scnprintf(buf, size, "kworker/dying");
2762 	}
2763 }
2764 
2765 /**
2766  * create_worker - create a new workqueue worker
2767  * @pool: pool the new worker will belong to
2768  *
2769  * Create and start a new worker which is attached to @pool.
2770  *
2771  * CONTEXT:
2772  * Might sleep.  Does GFP_KERNEL allocations.
2773  *
2774  * Return:
2775  * Pointer to the newly created worker.
2776  */
create_worker(struct worker_pool * pool)2777 static struct worker *create_worker(struct worker_pool *pool)
2778 {
2779 	struct worker *worker;
2780 	int id;
2781 
2782 	/* ID is needed to determine kthread name */
2783 	id = ida_alloc(&pool->worker_ida, GFP_KERNEL);
2784 	if (id < 0) {
2785 		pr_err_once("workqueue: Failed to allocate a worker ID: %pe\n",
2786 			    ERR_PTR(id));
2787 		return NULL;
2788 	}
2789 
2790 	worker = alloc_worker(pool->node);
2791 	if (!worker) {
2792 		pr_err_once("workqueue: Failed to allocate a worker\n");
2793 		goto fail;
2794 	}
2795 
2796 	worker->id = id;
2797 
2798 	if (!(pool->flags & POOL_BH)) {
2799 		char id_buf[WORKER_ID_LEN];
2800 
2801 		format_worker_id(id_buf, sizeof(id_buf), worker, pool);
2802 		worker->task = kthread_create_on_node(worker_thread, worker,
2803 						      pool->node, "%s", id_buf);
2804 		if (IS_ERR(worker->task)) {
2805 			if (PTR_ERR(worker->task) == -EINTR) {
2806 				pr_err("workqueue: Interrupted when creating a worker thread \"%s\"\n",
2807 				       id_buf);
2808 			} else {
2809 				pr_err_once("workqueue: Failed to create a worker thread: %pe",
2810 					    worker->task);
2811 			}
2812 			goto fail;
2813 		}
2814 
2815 		set_user_nice(worker->task, pool->attrs->nice);
2816 		kthread_bind_mask(worker->task, pool_allowed_cpus(pool));
2817 	}
2818 
2819 	/* successful, attach the worker to the pool */
2820 	worker_attach_to_pool(worker, pool);
2821 
2822 	/* start the newly created worker */
2823 	raw_spin_lock_irq(&pool->lock);
2824 
2825 	worker->pool->nr_workers++;
2826 	worker_enter_idle(worker);
2827 
2828 	/*
2829 	 * @worker is waiting on a completion in kthread() and will trigger hung
2830 	 * check if not woken up soon. As kick_pool() is noop if @pool is empty,
2831 	 * wake it up explicitly.
2832 	 */
2833 	if (worker->task)
2834 		wake_up_process(worker->task);
2835 
2836 	raw_spin_unlock_irq(&pool->lock);
2837 
2838 	return worker;
2839 
2840 fail:
2841 	ida_free(&pool->worker_ida, id);
2842 	kfree(worker);
2843 	return NULL;
2844 }
2845 
detach_dying_workers(struct list_head * cull_list)2846 static void detach_dying_workers(struct list_head *cull_list)
2847 {
2848 	struct worker *worker;
2849 
2850 	list_for_each_entry(worker, cull_list, entry)
2851 		detach_worker(worker);
2852 }
2853 
reap_dying_workers(struct list_head * cull_list)2854 static void reap_dying_workers(struct list_head *cull_list)
2855 {
2856 	struct worker *worker, *tmp;
2857 
2858 	list_for_each_entry_safe(worker, tmp, cull_list, entry) {
2859 		list_del_init(&worker->entry);
2860 		kthread_stop_put(worker->task);
2861 		kfree(worker);
2862 	}
2863 }
2864 
2865 /**
2866  * set_worker_dying - Tag a worker for destruction
2867  * @worker: worker to be destroyed
2868  * @list: transfer worker away from its pool->idle_list and into list
2869  *
2870  * Tag @worker for destruction and adjust @pool stats accordingly.  The worker
2871  * should be idle.
2872  *
2873  * CONTEXT:
2874  * raw_spin_lock_irq(pool->lock).
2875  */
set_worker_dying(struct worker * worker,struct list_head * list)2876 static void set_worker_dying(struct worker *worker, struct list_head *list)
2877 {
2878 	struct worker_pool *pool = worker->pool;
2879 
2880 	lockdep_assert_held(&pool->lock);
2881 	lockdep_assert_held(&wq_pool_attach_mutex);
2882 
2883 	/* sanity check frenzy */
2884 	if (WARN_ON(worker->current_work) ||
2885 	    WARN_ON(!list_empty(&worker->scheduled)) ||
2886 	    WARN_ON(!(worker->flags & WORKER_IDLE)))
2887 		return;
2888 
2889 	pool->nr_workers--;
2890 	pool->nr_idle--;
2891 
2892 	worker->flags |= WORKER_DIE;
2893 
2894 	list_move(&worker->entry, list);
2895 
2896 	/* get an extra task struct reference for later kthread_stop_put() */
2897 	get_task_struct(worker->task);
2898 }
2899 
2900 /**
2901  * idle_worker_timeout - check if some idle workers can now be deleted.
2902  * @t: The pool's idle_timer that just expired
2903  *
2904  * The timer is armed in worker_enter_idle(). Note that it isn't disarmed in
2905  * worker_leave_idle(), as a worker flicking between idle and active while its
2906  * pool is at the too_many_workers() tipping point would cause too much timer
2907  * housekeeping overhead. Since IDLE_WORKER_TIMEOUT is long enough, we just let
2908  * it expire and re-evaluate things from there.
2909  */
idle_worker_timeout(struct timer_list * t)2910 static void idle_worker_timeout(struct timer_list *t)
2911 {
2912 	struct worker_pool *pool = from_timer(pool, t, idle_timer);
2913 	bool do_cull = false;
2914 
2915 	if (work_pending(&pool->idle_cull_work))
2916 		return;
2917 
2918 	raw_spin_lock_irq(&pool->lock);
2919 
2920 	if (too_many_workers(pool)) {
2921 		struct worker *worker;
2922 		unsigned long expires;
2923 
2924 		/* idle_list is kept in LIFO order, check the last one */
2925 		worker = list_last_entry(&pool->idle_list, struct worker, entry);
2926 		expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2927 		do_cull = !time_before(jiffies, expires);
2928 
2929 		if (!do_cull)
2930 			mod_timer(&pool->idle_timer, expires);
2931 	}
2932 	raw_spin_unlock_irq(&pool->lock);
2933 
2934 	if (do_cull)
2935 		queue_work(system_unbound_wq, &pool->idle_cull_work);
2936 }
2937 
2938 /**
2939  * idle_cull_fn - cull workers that have been idle for too long.
2940  * @work: the pool's work for handling these idle workers
2941  *
2942  * This goes through a pool's idle workers and gets rid of those that have been
2943  * idle for at least IDLE_WORKER_TIMEOUT seconds.
2944  *
2945  * We don't want to disturb isolated CPUs because of a pcpu kworker being
2946  * culled, so this also resets worker affinity. This requires a sleepable
2947  * context, hence the split between timer callback and work item.
2948  */
idle_cull_fn(struct work_struct * work)2949 static void idle_cull_fn(struct work_struct *work)
2950 {
2951 	struct worker_pool *pool = container_of(work, struct worker_pool, idle_cull_work);
2952 	LIST_HEAD(cull_list);
2953 
2954 	/*
2955 	 * Grabbing wq_pool_attach_mutex here ensures an already-running worker
2956 	 * cannot proceed beyong set_pf_worker() in its self-destruct path.
2957 	 * This is required as a previously-preempted worker could run after
2958 	 * set_worker_dying() has happened but before detach_dying_workers() did.
2959 	 */
2960 	mutex_lock(&wq_pool_attach_mutex);
2961 	raw_spin_lock_irq(&pool->lock);
2962 
2963 	while (too_many_workers(pool)) {
2964 		struct worker *worker;
2965 		unsigned long expires;
2966 
2967 		worker = list_last_entry(&pool->idle_list, struct worker, entry);
2968 		expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2969 
2970 		if (time_before(jiffies, expires)) {
2971 			mod_timer(&pool->idle_timer, expires);
2972 			break;
2973 		}
2974 
2975 		set_worker_dying(worker, &cull_list);
2976 	}
2977 
2978 	raw_spin_unlock_irq(&pool->lock);
2979 	detach_dying_workers(&cull_list);
2980 	mutex_unlock(&wq_pool_attach_mutex);
2981 
2982 	reap_dying_workers(&cull_list);
2983 }
2984 
send_mayday(struct work_struct * work)2985 static void send_mayday(struct work_struct *work)
2986 {
2987 	struct pool_workqueue *pwq = get_work_pwq(work);
2988 	struct workqueue_struct *wq = pwq->wq;
2989 
2990 	lockdep_assert_held(&wq_mayday_lock);
2991 
2992 	if (!wq->rescuer)
2993 		return;
2994 
2995 	/* mayday mayday mayday */
2996 	if (list_empty(&pwq->mayday_node)) {
2997 		/*
2998 		 * If @pwq is for an unbound wq, its base ref may be put at
2999 		 * any time due to an attribute change.  Pin @pwq until the
3000 		 * rescuer is done with it.
3001 		 */
3002 		get_pwq(pwq);
3003 		list_add_tail(&pwq->mayday_node, &wq->maydays);
3004 		wake_up_process(wq->rescuer->task);
3005 		pwq->stats[PWQ_STAT_MAYDAY]++;
3006 	}
3007 }
3008 
pool_mayday_timeout(struct timer_list * t)3009 static void pool_mayday_timeout(struct timer_list *t)
3010 {
3011 	struct worker_pool *pool = from_timer(pool, t, mayday_timer);
3012 	struct work_struct *work;
3013 
3014 	raw_spin_lock_irq(&pool->lock);
3015 	raw_spin_lock(&wq_mayday_lock);		/* for wq->maydays */
3016 
3017 	if (need_to_create_worker(pool)) {
3018 		/*
3019 		 * We've been trying to create a new worker but
3020 		 * haven't been successful.  We might be hitting an
3021 		 * allocation deadlock.  Send distress signals to
3022 		 * rescuers.
3023 		 */
3024 		list_for_each_entry(work, &pool->worklist, entry)
3025 			send_mayday(work);
3026 	}
3027 
3028 	raw_spin_unlock(&wq_mayday_lock);
3029 	raw_spin_unlock_irq(&pool->lock);
3030 
3031 	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
3032 }
3033 
3034 /**
3035  * maybe_create_worker - create a new worker if necessary
3036  * @pool: pool to create a new worker for
3037  *
3038  * Create a new worker for @pool if necessary.  @pool is guaranteed to
3039  * have at least one idle worker on return from this function.  If
3040  * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
3041  * sent to all rescuers with works scheduled on @pool to resolve
3042  * possible allocation deadlock.
3043  *
3044  * On return, need_to_create_worker() is guaranteed to be %false and
3045  * may_start_working() %true.
3046  *
3047  * LOCKING:
3048  * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
3049  * multiple times.  Does GFP_KERNEL allocations.  Called only from
3050  * manager.
3051  */
maybe_create_worker(struct worker_pool * pool)3052 static void maybe_create_worker(struct worker_pool *pool)
3053 __releases(&pool->lock)
3054 __acquires(&pool->lock)
3055 {
3056 restart:
3057 	raw_spin_unlock_irq(&pool->lock);
3058 
3059 	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
3060 	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
3061 
3062 	while (true) {
3063 		if (create_worker(pool) || !need_to_create_worker(pool))
3064 			break;
3065 
3066 		schedule_timeout_interruptible(CREATE_COOLDOWN);
3067 
3068 		if (!need_to_create_worker(pool))
3069 			break;
3070 	}
3071 
3072 	del_timer_sync(&pool->mayday_timer);
3073 	raw_spin_lock_irq(&pool->lock);
3074 	/*
3075 	 * This is necessary even after a new worker was just successfully
3076 	 * created as @pool->lock was dropped and the new worker might have
3077 	 * already become busy.
3078 	 */
3079 	if (need_to_create_worker(pool))
3080 		goto restart;
3081 }
3082 
3083 /**
3084  * manage_workers - manage worker pool
3085  * @worker: self
3086  *
3087  * Assume the manager role and manage the worker pool @worker belongs
3088  * to.  At any given time, there can be only zero or one manager per
3089  * pool.  The exclusion is handled automatically by this function.
3090  *
3091  * The caller can safely start processing works on false return.  On
3092  * true return, it's guaranteed that need_to_create_worker() is false
3093  * and may_start_working() is true.
3094  *
3095  * CONTEXT:
3096  * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
3097  * multiple times.  Does GFP_KERNEL allocations.
3098  *
3099  * Return:
3100  * %false if the pool doesn't need management and the caller can safely
3101  * start processing works, %true if management function was performed and
3102  * the conditions that the caller verified before calling the function may
3103  * no longer be true.
3104  */
manage_workers(struct worker * worker)3105 static bool manage_workers(struct worker *worker)
3106 {
3107 	struct worker_pool *pool = worker->pool;
3108 
3109 	if (pool->flags & POOL_MANAGER_ACTIVE)
3110 		return false;
3111 
3112 	pool->flags |= POOL_MANAGER_ACTIVE;
3113 	pool->manager = worker;
3114 
3115 	maybe_create_worker(pool);
3116 
3117 	pool->manager = NULL;
3118 	pool->flags &= ~POOL_MANAGER_ACTIVE;
3119 	rcuwait_wake_up(&manager_wait);
3120 	return true;
3121 }
3122 
3123 /**
3124  * process_one_work - process single work
3125  * @worker: self
3126  * @work: work to process
3127  *
3128  * Process @work.  This function contains all the logics necessary to
3129  * process a single work including synchronization against and
3130  * interaction with other workers on the same cpu, queueing and
3131  * flushing.  As long as context requirement is met, any worker can
3132  * call this function to process a work.
3133  *
3134  * CONTEXT:
3135  * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
3136  */
process_one_work(struct worker * worker,struct work_struct * work)3137 static void process_one_work(struct worker *worker, struct work_struct *work)
3138 __releases(&pool->lock)
3139 __acquires(&pool->lock)
3140 {
3141 	struct pool_workqueue *pwq = get_work_pwq(work);
3142 	struct worker_pool *pool = worker->pool;
3143 	unsigned long work_data;
3144 	int lockdep_start_depth, rcu_start_depth;
3145 	bool bh_draining = pool->flags & POOL_BH_DRAINING;
3146 #ifdef CONFIG_LOCKDEP
3147 	/*
3148 	 * It is permissible to free the struct work_struct from
3149 	 * inside the function that is called from it, this we need to
3150 	 * take into account for lockdep too.  To avoid bogus "held
3151 	 * lock freed" warnings as well as problems when looking into
3152 	 * work->lockdep_map, make a copy and use that here.
3153 	 */
3154 	struct lockdep_map lockdep_map;
3155 
3156 	lockdep_copy_map(&lockdep_map, &work->lockdep_map);
3157 #endif
3158 	/* ensure we're on the correct CPU */
3159 	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
3160 		     raw_smp_processor_id() != pool->cpu);
3161 
3162 	/* claim and dequeue */
3163 	debug_work_deactivate(work);
3164 	hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
3165 	worker->current_work = work;
3166 	worker->current_func = work->func;
3167 	worker->current_pwq = pwq;
3168 	if (worker->task)
3169 		worker->current_at = worker->task->se.sum_exec_runtime;
3170 	work_data = *work_data_bits(work);
3171 	worker->current_color = get_work_color(work_data);
3172 
3173 	/*
3174 	 * Record wq name for cmdline and debug reporting, may get
3175 	 * overridden through set_worker_desc().
3176 	 */
3177 	strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);
3178 
3179 	list_del_init(&work->entry);
3180 
3181 	/*
3182 	 * CPU intensive works don't participate in concurrency management.
3183 	 * They're the scheduler's responsibility.  This takes @worker out
3184 	 * of concurrency management and the next code block will chain
3185 	 * execution of the pending work items.
3186 	 */
3187 	if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))
3188 		worker_set_flags(worker, WORKER_CPU_INTENSIVE);
3189 
3190 	/*
3191 	 * Kick @pool if necessary. It's always noop for per-cpu worker pools
3192 	 * since nr_running would always be >= 1 at this point. This is used to
3193 	 * chain execution of the pending work items for WORKER_NOT_RUNNING
3194 	 * workers such as the UNBOUND and CPU_INTENSIVE ones.
3195 	 */
3196 	kick_pool(pool);
3197 
3198 	/*
3199 	 * Record the last pool and clear PENDING which should be the last
3200 	 * update to @work.  Also, do this inside @pool->lock so that
3201 	 * PENDING and queued state changes happen together while IRQ is
3202 	 * disabled.
3203 	 */
3204 	set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool));
3205 
3206 	pwq->stats[PWQ_STAT_STARTED]++;
3207 	raw_spin_unlock_irq(&pool->lock);
3208 
3209 	rcu_start_depth = rcu_preempt_depth();
3210 	lockdep_start_depth = lockdep_depth(current);
3211 	/* see drain_dead_softirq_workfn() */
3212 	if (!bh_draining)
3213 		lock_map_acquire(pwq->wq->lockdep_map);
3214 	lock_map_acquire(&lockdep_map);
3215 	/*
3216 	 * Strictly speaking we should mark the invariant state without holding
3217 	 * any locks, that is, before these two lock_map_acquire()'s.
3218 	 *
3219 	 * However, that would result in:
3220 	 *
3221 	 *   A(W1)
3222 	 *   WFC(C)
3223 	 *		A(W1)
3224 	 *		C(C)
3225 	 *
3226 	 * Which would create W1->C->W1 dependencies, even though there is no
3227 	 * actual deadlock possible. There are two solutions, using a
3228 	 * read-recursive acquire on the work(queue) 'locks', but this will then
3229 	 * hit the lockdep limitation on recursive locks, or simply discard
3230 	 * these locks.
3231 	 *
3232 	 * AFAICT there is no possible deadlock scenario between the
3233 	 * flush_work() and complete() primitives (except for single-threaded
3234 	 * workqueues), so hiding them isn't a problem.
3235 	 */
3236 	lockdep_invariant_state(true);
3237 	trace_workqueue_execute_start(work);
3238 	worker->current_func(work);
3239 	/*
3240 	 * While we must be careful to not use "work" after this, the trace
3241 	 * point will only record its address.
3242 	 */
3243 	trace_workqueue_execute_end(work, worker->current_func);
3244 	pwq->stats[PWQ_STAT_COMPLETED]++;
3245 	lock_map_release(&lockdep_map);
3246 	if (!bh_draining)
3247 		lock_map_release(pwq->wq->lockdep_map);
3248 
3249 	if (unlikely((worker->task && in_atomic()) ||
3250 		     lockdep_depth(current) != lockdep_start_depth ||
3251 		     rcu_preempt_depth() != rcu_start_depth)) {
3252 		pr_err("BUG: workqueue leaked atomic, lock or RCU: %s[%d]\n"
3253 		       "     preempt=0x%08x lock=%d->%d RCU=%d->%d workfn=%ps\n",
3254 		       current->comm, task_pid_nr(current), preempt_count(),
3255 		       lockdep_start_depth, lockdep_depth(current),
3256 		       rcu_start_depth, rcu_preempt_depth(),
3257 		       worker->current_func);
3258 		debug_show_held_locks(current);
3259 		dump_stack();
3260 	}
3261 
3262 	/*
3263 	 * The following prevents a kworker from hogging CPU on !PREEMPTION
3264 	 * kernels, where a requeueing work item waiting for something to
3265 	 * happen could deadlock with stop_machine as such work item could
3266 	 * indefinitely requeue itself while all other CPUs are trapped in
3267 	 * stop_machine. At the same time, report a quiescent RCU state so
3268 	 * the same condition doesn't freeze RCU.
3269 	 */
3270 	if (worker->task)
3271 		cond_resched();
3272 
3273 	raw_spin_lock_irq(&pool->lock);
3274 
3275 	/*
3276 	 * In addition to %WQ_CPU_INTENSIVE, @worker may also have been marked
3277 	 * CPU intensive by wq_worker_tick() if @work hogged CPU longer than
3278 	 * wq_cpu_intensive_thresh_us. Clear it.
3279 	 */
3280 	worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
3281 
3282 	/* tag the worker for identification in schedule() */
3283 	worker->last_func = worker->current_func;
3284 
3285 	/* we're done with it, release */
3286 	hash_del(&worker->hentry);
3287 	worker->current_work = NULL;
3288 	worker->current_func = NULL;
3289 	worker->current_pwq = NULL;
3290 	worker->current_color = INT_MAX;
3291 
3292 	/* must be the last step, see the function comment */
3293 	pwq_dec_nr_in_flight(pwq, work_data);
3294 }
3295 
3296 /**
3297  * process_scheduled_works - process scheduled works
3298  * @worker: self
3299  *
3300  * Process all scheduled works.  Please note that the scheduled list
3301  * may change while processing a work, so this function repeatedly
3302  * fetches a work from the top and executes it.
3303  *
3304  * CONTEXT:
3305  * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
3306  * multiple times.
3307  */
process_scheduled_works(struct worker * worker)3308 static void process_scheduled_works(struct worker *worker)
3309 {
3310 	struct work_struct *work;
3311 	bool first = true;
3312 
3313 	while ((work = list_first_entry_or_null(&worker->scheduled,
3314 						struct work_struct, entry))) {
3315 		if (first) {
3316 			worker->pool->watchdog_ts = jiffies;
3317 			first = false;
3318 		}
3319 		process_one_work(worker, work);
3320 	}
3321 }
3322 
set_pf_worker(bool val)3323 static void set_pf_worker(bool val)
3324 {
3325 	mutex_lock(&wq_pool_attach_mutex);
3326 	if (val)
3327 		current->flags |= PF_WQ_WORKER;
3328 	else
3329 		current->flags &= ~PF_WQ_WORKER;
3330 	mutex_unlock(&wq_pool_attach_mutex);
3331 }
3332 
3333 /**
3334  * worker_thread - the worker thread function
3335  * @__worker: self
3336  *
3337  * The worker thread function.  All workers belong to a worker_pool -
3338  * either a per-cpu one or dynamic unbound one.  These workers process all
3339  * work items regardless of their specific target workqueue.  The only
3340  * exception is work items which belong to workqueues with a rescuer which
3341  * will be explained in rescuer_thread().
3342  *
3343  * Return: 0
3344  */
worker_thread(void * __worker)3345 static int worker_thread(void *__worker)
3346 {
3347 	struct worker *worker = __worker;
3348 	struct worker_pool *pool = worker->pool;
3349 
3350 	/* tell the scheduler that this is a workqueue worker */
3351 	set_pf_worker(true);
3352 woke_up:
3353 	raw_spin_lock_irq(&pool->lock);
3354 
3355 	/* am I supposed to die? */
3356 	if (unlikely(worker->flags & WORKER_DIE)) {
3357 		raw_spin_unlock_irq(&pool->lock);
3358 		set_pf_worker(false);
3359 		/*
3360 		 * The worker is dead and PF_WQ_WORKER is cleared, worker->pool
3361 		 * shouldn't be accessed, reset it to NULL in case otherwise.
3362 		 */
3363 		worker->pool = NULL;
3364 		ida_free(&pool->worker_ida, worker->id);
3365 		return 0;
3366 	}
3367 
3368 	worker_leave_idle(worker);
3369 recheck:
3370 	/* no more worker necessary? */
3371 	if (!need_more_worker(pool))
3372 		goto sleep;
3373 
3374 	/* do we need to manage? */
3375 	if (unlikely(!may_start_working(pool)) && manage_workers(worker))
3376 		goto recheck;
3377 
3378 	/*
3379 	 * ->scheduled list can only be filled while a worker is
3380 	 * preparing to process a work or actually processing it.
3381 	 * Make sure nobody diddled with it while I was sleeping.
3382 	 */
3383 	WARN_ON_ONCE(!list_empty(&worker->scheduled));
3384 
3385 	/*
3386 	 * Finish PREP stage.  We're guaranteed to have at least one idle
3387 	 * worker or that someone else has already assumed the manager
3388 	 * role.  This is where @worker starts participating in concurrency
3389 	 * management if applicable and concurrency management is restored
3390 	 * after being rebound.  See rebind_workers() for details.
3391 	 */
3392 	worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
3393 
3394 	do {
3395 		struct work_struct *work =
3396 			list_first_entry(&pool->worklist,
3397 					 struct work_struct, entry);
3398 
3399 		if (assign_work(work, worker, NULL))
3400 			process_scheduled_works(worker);
3401 	} while (keep_working(pool));
3402 
3403 	worker_set_flags(worker, WORKER_PREP);
3404 sleep:
3405 	/*
3406 	 * pool->lock is held and there's no work to process and no need to
3407 	 * manage, sleep.  Workers are woken up only while holding
3408 	 * pool->lock or from local cpu, so setting the current state
3409 	 * before releasing pool->lock is enough to prevent losing any
3410 	 * event.
3411 	 */
3412 	worker_enter_idle(worker);
3413 	__set_current_state(TASK_IDLE);
3414 	raw_spin_unlock_irq(&pool->lock);
3415 	schedule();
3416 	goto woke_up;
3417 }
3418 
3419 /**
3420  * rescuer_thread - the rescuer thread function
3421  * @__rescuer: self
3422  *
3423  * Workqueue rescuer thread function.  There's one rescuer for each
3424  * workqueue which has WQ_MEM_RECLAIM set.
3425  *
3426  * Regular work processing on a pool may block trying to create a new
3427  * worker which uses GFP_KERNEL allocation which has slight chance of
3428  * developing into deadlock if some works currently on the same queue
3429  * need to be processed to satisfy the GFP_KERNEL allocation.  This is
3430  * the problem rescuer solves.
3431  *
3432  * When such condition is possible, the pool summons rescuers of all
3433  * workqueues which have works queued on the pool and let them process
3434  * those works so that forward progress can be guaranteed.
3435  *
3436  * This should happen rarely.
3437  *
3438  * Return: 0
3439  */
rescuer_thread(void * __rescuer)3440 static int rescuer_thread(void *__rescuer)
3441 {
3442 	struct worker *rescuer = __rescuer;
3443 	struct workqueue_struct *wq = rescuer->rescue_wq;
3444 	bool should_stop;
3445 
3446 	set_user_nice(current, RESCUER_NICE_LEVEL);
3447 
3448 	/*
3449 	 * Mark rescuer as worker too.  As WORKER_PREP is never cleared, it
3450 	 * doesn't participate in concurrency management.
3451 	 */
3452 	set_pf_worker(true);
3453 repeat:
3454 	set_current_state(TASK_IDLE);
3455 
3456 	/*
3457 	 * By the time the rescuer is requested to stop, the workqueue
3458 	 * shouldn't have any work pending, but @wq->maydays may still have
3459 	 * pwq(s) queued.  This can happen by non-rescuer workers consuming
3460 	 * all the work items before the rescuer got to them.  Go through
3461 	 * @wq->maydays processing before acting on should_stop so that the
3462 	 * list is always empty on exit.
3463 	 */
3464 	should_stop = kthread_should_stop();
3465 
3466 	/* see whether any pwq is asking for help */
3467 	raw_spin_lock_irq(&wq_mayday_lock);
3468 
3469 	while (!list_empty(&wq->maydays)) {
3470 		struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
3471 					struct pool_workqueue, mayday_node);
3472 		struct worker_pool *pool = pwq->pool;
3473 		struct work_struct *work, *n;
3474 
3475 		__set_current_state(TASK_RUNNING);
3476 		list_del_init(&pwq->mayday_node);
3477 
3478 		raw_spin_unlock_irq(&wq_mayday_lock);
3479 
3480 		worker_attach_to_pool(rescuer, pool);
3481 
3482 		raw_spin_lock_irq(&pool->lock);
3483 
3484 		/*
3485 		 * Slurp in all works issued via this workqueue and
3486 		 * process'em.
3487 		 */
3488 		WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
3489 		list_for_each_entry_safe(work, n, &pool->worklist, entry) {
3490 			if (get_work_pwq(work) == pwq &&
3491 			    assign_work(work, rescuer, &n))
3492 				pwq->stats[PWQ_STAT_RESCUED]++;
3493 		}
3494 
3495 		if (!list_empty(&rescuer->scheduled)) {
3496 			process_scheduled_works(rescuer);
3497 
3498 			/*
3499 			 * The above execution of rescued work items could
3500 			 * have created more to rescue through
3501 			 * pwq_activate_first_inactive() or chained
3502 			 * queueing.  Let's put @pwq back on mayday list so
3503 			 * that such back-to-back work items, which may be
3504 			 * being used to relieve memory pressure, don't
3505 			 * incur MAYDAY_INTERVAL delay inbetween.
3506 			 */
3507 			if (pwq->nr_active && need_to_create_worker(pool)) {
3508 				raw_spin_lock(&wq_mayday_lock);
3509 				/*
3510 				 * Queue iff we aren't racing destruction
3511 				 * and somebody else hasn't queued it already.
3512 				 */
3513 				if (wq->rescuer && list_empty(&pwq->mayday_node)) {
3514 					get_pwq(pwq);
3515 					list_add_tail(&pwq->mayday_node, &wq->maydays);
3516 				}
3517 				raw_spin_unlock(&wq_mayday_lock);
3518 			}
3519 		}
3520 
3521 		/*
3522 		 * Leave this pool. Notify regular workers; otherwise, we end up
3523 		 * with 0 concurrency and stalling the execution.
3524 		 */
3525 		kick_pool(pool);
3526 
3527 		raw_spin_unlock_irq(&pool->lock);
3528 
3529 		worker_detach_from_pool(rescuer);
3530 
3531 		/*
3532 		 * Put the reference grabbed by send_mayday().  @pool might
3533 		 * go away any time after it.
3534 		 */
3535 		put_pwq_unlocked(pwq);
3536 
3537 		raw_spin_lock_irq(&wq_mayday_lock);
3538 	}
3539 
3540 	raw_spin_unlock_irq(&wq_mayday_lock);
3541 
3542 	if (should_stop) {
3543 		__set_current_state(TASK_RUNNING);
3544 		set_pf_worker(false);
3545 		return 0;
3546 	}
3547 
3548 	/* rescuers should never participate in concurrency management */
3549 	WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
3550 	schedule();
3551 	goto repeat;
3552 }
3553 
bh_worker(struct worker * worker)3554 static void bh_worker(struct worker *worker)
3555 {
3556 	struct worker_pool *pool = worker->pool;
3557 	int nr_restarts = BH_WORKER_RESTARTS;
3558 	unsigned long end = jiffies + BH_WORKER_JIFFIES;
3559 
3560 	raw_spin_lock_irq(&pool->lock);
3561 	worker_leave_idle(worker);
3562 
3563 	/*
3564 	 * This function follows the structure of worker_thread(). See there for
3565 	 * explanations on each step.
3566 	 */
3567 	if (!need_more_worker(pool))
3568 		goto done;
3569 
3570 	WARN_ON_ONCE(!list_empty(&worker->scheduled));
3571 	worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
3572 
3573 	do {
3574 		struct work_struct *work =
3575 			list_first_entry(&pool->worklist,
3576 					 struct work_struct, entry);
3577 
3578 		if (assign_work(work, worker, NULL))
3579 			process_scheduled_works(worker);
3580 	} while (keep_working(pool) &&
3581 		 --nr_restarts && time_before(jiffies, end));
3582 
3583 	worker_set_flags(worker, WORKER_PREP);
3584 done:
3585 	worker_enter_idle(worker);
3586 	kick_pool(pool);
3587 	raw_spin_unlock_irq(&pool->lock);
3588 }
3589 
3590 /*
3591  * TODO: Convert all tasklet users to workqueue and use softirq directly.
3592  *
3593  * This is currently called from tasklet[_hi]action() and thus is also called
3594  * whenever there are tasklets to run. Let's do an early exit if there's nothing
3595  * queued. Once conversion from tasklet is complete, the need_more_worker() test
3596  * can be dropped.
3597  *
3598  * After full conversion, we'll add worker->softirq_action, directly use the
3599  * softirq action and obtain the worker pointer from the softirq_action pointer.
3600  */
workqueue_softirq_action(bool highpri)3601 void workqueue_softirq_action(bool highpri)
3602 {
3603 	struct worker_pool *pool =
3604 		&per_cpu(bh_worker_pools, smp_processor_id())[highpri];
3605 	if (need_more_worker(pool))
3606 		bh_worker(list_first_entry(&pool->workers, struct worker, node));
3607 }
3608 
3609 struct wq_drain_dead_softirq_work {
3610 	struct work_struct	work;
3611 	struct worker_pool	*pool;
3612 	struct completion	done;
3613 };
3614 
drain_dead_softirq_workfn(struct work_struct * work)3615 static void drain_dead_softirq_workfn(struct work_struct *work)
3616 {
3617 	struct wq_drain_dead_softirq_work *dead_work =
3618 		container_of(work, struct wq_drain_dead_softirq_work, work);
3619 	struct worker_pool *pool = dead_work->pool;
3620 	bool repeat;
3621 
3622 	/*
3623 	 * @pool's CPU is dead and we want to execute its still pending work
3624 	 * items from this BH work item which is running on a different CPU. As
3625 	 * its CPU is dead, @pool can't be kicked and, as work execution path
3626 	 * will be nested, a lockdep annotation needs to be suppressed. Mark
3627 	 * @pool with %POOL_BH_DRAINING for the special treatments.
3628 	 */
3629 	raw_spin_lock_irq(&pool->lock);
3630 	pool->flags |= POOL_BH_DRAINING;
3631 	raw_spin_unlock_irq(&pool->lock);
3632 
3633 	bh_worker(list_first_entry(&pool->workers, struct worker, node));
3634 
3635 	raw_spin_lock_irq(&pool->lock);
3636 	pool->flags &= ~POOL_BH_DRAINING;
3637 	repeat = need_more_worker(pool);
3638 	raw_spin_unlock_irq(&pool->lock);
3639 
3640 	/*
3641 	 * bh_worker() might hit consecutive execution limit and bail. If there
3642 	 * still are pending work items, reschedule self and return so that we
3643 	 * don't hog this CPU's BH.
3644 	 */
3645 	if (repeat) {
3646 		if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
3647 			queue_work(system_bh_highpri_wq, work);
3648 		else
3649 			queue_work(system_bh_wq, work);
3650 	} else {
3651 		complete(&dead_work->done);
3652 	}
3653 }
3654 
3655 /*
3656  * @cpu is dead. Drain the remaining BH work items on the current CPU. It's
3657  * possible to allocate dead_work per CPU and avoid flushing. However, then we
3658  * have to worry about draining overlapping with CPU coming back online or
3659  * nesting (one CPU's dead_work queued on another CPU which is also dead and so
3660  * on). Let's keep it simple and drain them synchronously. These are BH work
3661  * items which shouldn't be requeued on the same pool. Shouldn't take long.
3662  */
workqueue_softirq_dead(unsigned int cpu)3663 void workqueue_softirq_dead(unsigned int cpu)
3664 {
3665 	int i;
3666 
3667 	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
3668 		struct worker_pool *pool = &per_cpu(bh_worker_pools, cpu)[i];
3669 		struct wq_drain_dead_softirq_work dead_work;
3670 
3671 		if (!need_more_worker(pool))
3672 			continue;
3673 
3674 		INIT_WORK_ONSTACK(&dead_work.work, drain_dead_softirq_workfn);
3675 		dead_work.pool = pool;
3676 		init_completion(&dead_work.done);
3677 
3678 		if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
3679 			queue_work(system_bh_highpri_wq, &dead_work.work);
3680 		else
3681 			queue_work(system_bh_wq, &dead_work.work);
3682 
3683 		wait_for_completion(&dead_work.done);
3684 		destroy_work_on_stack(&dead_work.work);
3685 	}
3686 }
3687 
3688 /**
3689  * check_flush_dependency - check for flush dependency sanity
3690  * @target_wq: workqueue being flushed
3691  * @target_work: work item being flushed (NULL for workqueue flushes)
3692  * @from_cancel: are we called from the work cancel path
3693  *
3694  * %current is trying to flush the whole @target_wq or @target_work on it.
3695  * If this is not the cancel path (which implies work being flushed is either
3696  * already running, or will not be at all), check if @target_wq doesn't have
3697  * %WQ_MEM_RECLAIM and verify that %current is not reclaiming memory or running
3698  * on a workqueue which doesn't have %WQ_MEM_RECLAIM as that can break forward-
3699  * progress guarantee leading to a deadlock.
3700  */
check_flush_dependency(struct workqueue_struct * target_wq,struct work_struct * target_work,bool from_cancel)3701 static void check_flush_dependency(struct workqueue_struct *target_wq,
3702 				   struct work_struct *target_work,
3703 				   bool from_cancel)
3704 {
3705 	work_func_t target_func;
3706 	struct worker *worker;
3707 
3708 	if (from_cancel || target_wq->flags & WQ_MEM_RECLAIM)
3709 		return;
3710 
3711 	worker = current_wq_worker();
3712 	target_func = target_work ? target_work->func : NULL;
3713 
3714 	WARN_ONCE(current->flags & PF_MEMALLOC,
3715 		  "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
3716 		  current->pid, current->comm, target_wq->name, target_func);
3717 	WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
3718 			      (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),
3719 		  "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
3720 		  worker->current_pwq->wq->name, worker->current_func,
3721 		  target_wq->name, target_func);
3722 }
3723 
3724 struct wq_barrier {
3725 	struct work_struct	work;
3726 	struct completion	done;
3727 	struct task_struct	*task;	/* purely informational */
3728 };
3729 
wq_barrier_func(struct work_struct * work)3730 static void wq_barrier_func(struct work_struct *work)
3731 {
3732 	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
3733 	complete(&barr->done);
3734 }
3735 
3736 /**
3737  * insert_wq_barrier - insert a barrier work
3738  * @pwq: pwq to insert barrier into
3739  * @barr: wq_barrier to insert
3740  * @target: target work to attach @barr to
3741  * @worker: worker currently executing @target, NULL if @target is not executing
3742  *
3743  * @barr is linked to @target such that @barr is completed only after
3744  * @target finishes execution.  Please note that the ordering
3745  * guarantee is observed only with respect to @target and on the local
3746  * cpu.
3747  *
3748  * Currently, a queued barrier can't be canceled.  This is because
3749  * try_to_grab_pending() can't determine whether the work to be
3750  * grabbed is at the head of the queue and thus can't clear LINKED
3751  * flag of the previous work while there must be a valid next work
3752  * after a work with LINKED flag set.
3753  *
3754  * Note that when @worker is non-NULL, @target may be modified
3755  * underneath us, so we can't reliably determine pwq from @target.
3756  *
3757  * CONTEXT:
3758  * raw_spin_lock_irq(pool->lock).
3759  */
insert_wq_barrier(struct pool_workqueue * pwq,struct wq_barrier * barr,struct work_struct * target,struct worker * worker)3760 static void insert_wq_barrier(struct pool_workqueue *pwq,
3761 			      struct wq_barrier *barr,
3762 			      struct work_struct *target, struct worker *worker)
3763 {
3764 	static __maybe_unused struct lock_class_key bh_key, thr_key;
3765 	unsigned int work_flags = 0;
3766 	unsigned int work_color;
3767 	struct list_head *head;
3768 
3769 	/*
3770 	 * debugobject calls are safe here even with pool->lock locked
3771 	 * as we know for sure that this will not trigger any of the
3772 	 * checks and call back into the fixup functions where we
3773 	 * might deadlock.
3774 	 *
3775 	 * BH and threaded workqueues need separate lockdep keys to avoid
3776 	 * spuriously triggering "inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W}
3777 	 * usage".
3778 	 */
3779 	INIT_WORK_ONSTACK_KEY(&barr->work, wq_barrier_func,
3780 			      (pwq->wq->flags & WQ_BH) ? &bh_key : &thr_key);
3781 	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
3782 
3783 	init_completion_map(&barr->done, &target->lockdep_map);
3784 
3785 	barr->task = current;
3786 
3787 	/* The barrier work item does not participate in nr_active. */
3788 	work_flags |= WORK_STRUCT_INACTIVE;
3789 
3790 	/*
3791 	 * If @target is currently being executed, schedule the
3792 	 * barrier to the worker; otherwise, put it after @target.
3793 	 */
3794 	if (worker) {
3795 		head = worker->scheduled.next;
3796 		work_color = worker->current_color;
3797 	} else {
3798 		unsigned long *bits = work_data_bits(target);
3799 
3800 		head = target->entry.next;
3801 		/* there can already be other linked works, inherit and set */
3802 		work_flags |= *bits & WORK_STRUCT_LINKED;
3803 		work_color = get_work_color(*bits);
3804 		__set_bit(WORK_STRUCT_LINKED_BIT, bits);
3805 	}
3806 
3807 	pwq->nr_in_flight[work_color]++;
3808 	work_flags |= work_color_to_flags(work_color);
3809 
3810 	insert_work(pwq, &barr->work, head, work_flags);
3811 }
3812 
3813 /**
3814  * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
3815  * @wq: workqueue being flushed
3816  * @flush_color: new flush color, < 0 for no-op
3817  * @work_color: new work color, < 0 for no-op
3818  *
3819  * Prepare pwqs for workqueue flushing.
3820  *
3821  * If @flush_color is non-negative, flush_color on all pwqs should be
3822  * -1.  If no pwq has in-flight commands at the specified color, all
3823  * pwq->flush_color's stay at -1 and %false is returned.  If any pwq
3824  * has in flight commands, its pwq->flush_color is set to
3825  * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
3826  * wakeup logic is armed and %true is returned.
3827  *
3828  * The caller should have initialized @wq->first_flusher prior to
3829  * calling this function with non-negative @flush_color.  If
3830  * @flush_color is negative, no flush color update is done and %false
3831  * is returned.
3832  *
3833  * If @work_color is non-negative, all pwqs should have the same
3834  * work_color which is previous to @work_color and all will be
3835  * advanced to @work_color.
3836  *
3837  * CONTEXT:
3838  * mutex_lock(wq->mutex).
3839  *
3840  * Return:
3841  * %true if @flush_color >= 0 and there's something to flush.  %false
3842  * otherwise.
3843  */
flush_workqueue_prep_pwqs(struct workqueue_struct * wq,int flush_color,int work_color)3844 static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
3845 				      int flush_color, int work_color)
3846 {
3847 	bool wait = false;
3848 	struct pool_workqueue *pwq;
3849 	struct worker_pool *current_pool = NULL;
3850 
3851 	if (flush_color >= 0) {
3852 		WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
3853 		atomic_set(&wq->nr_pwqs_to_flush, 1);
3854 	}
3855 
3856 	/*
3857 	 * For unbound workqueue, pwqs will map to only a few pools.
3858 	 * Most of the time, pwqs within the same pool will be linked
3859 	 * sequentially to wq->pwqs by cpu index. So in the majority
3860 	 * of pwq iters, the pool is the same, only doing lock/unlock
3861 	 * if the pool has changed. This can largely reduce expensive
3862 	 * lock operations.
3863 	 */
3864 	for_each_pwq(pwq, wq) {
3865 		if (current_pool != pwq->pool) {
3866 			if (likely(current_pool))
3867 				raw_spin_unlock_irq(&current_pool->lock);
3868 			current_pool = pwq->pool;
3869 			raw_spin_lock_irq(&current_pool->lock);
3870 		}
3871 
3872 		if (flush_color >= 0) {
3873 			WARN_ON_ONCE(pwq->flush_color != -1);
3874 
3875 			if (pwq->nr_in_flight[flush_color]) {
3876 				pwq->flush_color = flush_color;
3877 				atomic_inc(&wq->nr_pwqs_to_flush);
3878 				wait = true;
3879 			}
3880 		}
3881 
3882 		if (work_color >= 0) {
3883 			WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
3884 			pwq->work_color = work_color;
3885 		}
3886 
3887 	}
3888 
3889 	if (current_pool)
3890 		raw_spin_unlock_irq(&current_pool->lock);
3891 
3892 	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
3893 		complete(&wq->first_flusher->done);
3894 
3895 	return wait;
3896 }
3897 
touch_wq_lockdep_map(struct workqueue_struct * wq)3898 static void touch_wq_lockdep_map(struct workqueue_struct *wq)
3899 {
3900 #ifdef CONFIG_LOCKDEP
3901 	if (unlikely(!wq->lockdep_map))
3902 		return;
3903 
3904 	if (wq->flags & WQ_BH)
3905 		local_bh_disable();
3906 
3907 	lock_map_acquire(wq->lockdep_map);
3908 	lock_map_release(wq->lockdep_map);
3909 
3910 	if (wq->flags & WQ_BH)
3911 		local_bh_enable();
3912 #endif
3913 }
3914 
touch_work_lockdep_map(struct work_struct * work,struct workqueue_struct * wq)3915 static void touch_work_lockdep_map(struct work_struct *work,
3916 				   struct workqueue_struct *wq)
3917 {
3918 #ifdef CONFIG_LOCKDEP
3919 	if (wq->flags & WQ_BH)
3920 		local_bh_disable();
3921 
3922 	lock_map_acquire(&work->lockdep_map);
3923 	lock_map_release(&work->lockdep_map);
3924 
3925 	if (wq->flags & WQ_BH)
3926 		local_bh_enable();
3927 #endif
3928 }
3929 
3930 /**
3931  * __flush_workqueue - ensure that any scheduled work has run to completion.
3932  * @wq: workqueue to flush
3933  *
3934  * This function sleeps until all work items which were queued on entry
3935  * have finished execution, but it is not livelocked by new incoming ones.
3936  */
__flush_workqueue(struct workqueue_struct * wq)3937 void __flush_workqueue(struct workqueue_struct *wq)
3938 {
3939 	struct wq_flusher this_flusher = {
3940 		.list = LIST_HEAD_INIT(this_flusher.list),
3941 		.flush_color = -1,
3942 		.done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, (*wq->lockdep_map)),
3943 	};
3944 	int next_color;
3945 
3946 	if (WARN_ON(!wq_online))
3947 		return;
3948 
3949 	touch_wq_lockdep_map(wq);
3950 
3951 	mutex_lock(&wq->mutex);
3952 
3953 	/*
3954 	 * Start-to-wait phase
3955 	 */
3956 	next_color = work_next_color(wq->work_color);
3957 
3958 	if (next_color != wq->flush_color) {
3959 		/*
3960 		 * Color space is not full.  The current work_color
3961 		 * becomes our flush_color and work_color is advanced
3962 		 * by one.
3963 		 */
3964 		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
3965 		this_flusher.flush_color = wq->work_color;
3966 		wq->work_color = next_color;
3967 
3968 		if (!wq->first_flusher) {
3969 			/* no flush in progress, become the first flusher */
3970 			WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
3971 
3972 			wq->first_flusher = &this_flusher;
3973 
3974 			if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
3975 						       wq->work_color)) {
3976 				/* nothing to flush, done */
3977 				wq->flush_color = next_color;
3978 				wq->first_flusher = NULL;
3979 				goto out_unlock;
3980 			}
3981 		} else {
3982 			/* wait in queue */
3983 			WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
3984 			list_add_tail(&this_flusher.list, &wq->flusher_queue);
3985 			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
3986 		}
3987 	} else {
3988 		/*
3989 		 * Oops, color space is full, wait on overflow queue.
3990 		 * The next flush completion will assign us
3991 		 * flush_color and transfer to flusher_queue.
3992 		 */
3993 		list_add_tail(&this_flusher.list, &wq->flusher_overflow);
3994 	}
3995 
3996 	check_flush_dependency(wq, NULL, false);
3997 
3998 	mutex_unlock(&wq->mutex);
3999 
4000 	wait_for_completion(&this_flusher.done);
4001 
4002 	/*
4003 	 * Wake-up-and-cascade phase
4004 	 *
4005 	 * First flushers are responsible for cascading flushes and
4006 	 * handling overflow.  Non-first flushers can simply return.
4007 	 */
4008 	if (READ_ONCE(wq->first_flusher) != &this_flusher)
4009 		return;
4010 
4011 	mutex_lock(&wq->mutex);
4012 
4013 	/* we might have raced, check again with mutex held */
4014 	if (wq->first_flusher != &this_flusher)
4015 		goto out_unlock;
4016 
4017 	WRITE_ONCE(wq->first_flusher, NULL);
4018 
4019 	WARN_ON_ONCE(!list_empty(&this_flusher.list));
4020 	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
4021 
4022 	while (true) {
4023 		struct wq_flusher *next, *tmp;
4024 
4025 		/* complete all the flushers sharing the current flush color */
4026 		list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
4027 			if (next->flush_color != wq->flush_color)
4028 				break;
4029 			list_del_init(&next->list);
4030 			complete(&next->done);
4031 		}
4032 
4033 		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
4034 			     wq->flush_color != work_next_color(wq->work_color));
4035 
4036 		/* this flush_color is finished, advance by one */
4037 		wq->flush_color = work_next_color(wq->flush_color);
4038 
4039 		/* one color has been freed, handle overflow queue */
4040 		if (!list_empty(&wq->flusher_overflow)) {
4041 			/*
4042 			 * Assign the same color to all overflowed
4043 			 * flushers, advance work_color and append to
4044 			 * flusher_queue.  This is the start-to-wait
4045 			 * phase for these overflowed flushers.
4046 			 */
4047 			list_for_each_entry(tmp, &wq->flusher_overflow, list)
4048 				tmp->flush_color = wq->work_color;
4049 
4050 			wq->work_color = work_next_color(wq->work_color);
4051 
4052 			list_splice_tail_init(&wq->flusher_overflow,
4053 					      &wq->flusher_queue);
4054 			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
4055 		}
4056 
4057 		if (list_empty(&wq->flusher_queue)) {
4058 			WARN_ON_ONCE(wq->flush_color != wq->work_color);
4059 			break;
4060 		}
4061 
4062 		/*
4063 		 * Need to flush more colors.  Make the next flusher
4064 		 * the new first flusher and arm pwqs.
4065 		 */
4066 		WARN_ON_ONCE(wq->flush_color == wq->work_color);
4067 		WARN_ON_ONCE(wq->flush_color != next->flush_color);
4068 
4069 		list_del_init(&next->list);
4070 		wq->first_flusher = next;
4071 
4072 		if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
4073 			break;
4074 
4075 		/*
4076 		 * Meh... this color is already done, clear first
4077 		 * flusher and repeat cascading.
4078 		 */
4079 		wq->first_flusher = NULL;
4080 	}
4081 
4082 out_unlock:
4083 	mutex_unlock(&wq->mutex);
4084 }
4085 EXPORT_SYMBOL(__flush_workqueue);
4086 
4087 /**
4088  * drain_workqueue - drain a workqueue
4089  * @wq: workqueue to drain
4090  *
4091  * Wait until the workqueue becomes empty.  While draining is in progress,
4092  * only chain queueing is allowed.  IOW, only currently pending or running
4093  * work items on @wq can queue further work items on it.  @wq is flushed
4094  * repeatedly until it becomes empty.  The number of flushing is determined
4095  * by the depth of chaining and should be relatively short.  Whine if it
4096  * takes too long.
4097  */
drain_workqueue(struct workqueue_struct * wq)4098 void drain_workqueue(struct workqueue_struct *wq)
4099 {
4100 	unsigned int flush_cnt = 0;
4101 	struct pool_workqueue *pwq;
4102 
4103 	/*
4104 	 * __queue_work() needs to test whether there are drainers, is much
4105 	 * hotter than drain_workqueue() and already looks at @wq->flags.
4106 	 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
4107 	 */
4108 	mutex_lock(&wq->mutex);
4109 	if (!wq->nr_drainers++)
4110 		wq->flags |= __WQ_DRAINING;
4111 	mutex_unlock(&wq->mutex);
4112 reflush:
4113 	__flush_workqueue(wq);
4114 
4115 	mutex_lock(&wq->mutex);
4116 
4117 	for_each_pwq(pwq, wq) {
4118 		bool drained;
4119 
4120 		raw_spin_lock_irq(&pwq->pool->lock);
4121 		drained = pwq_is_empty(pwq);
4122 		raw_spin_unlock_irq(&pwq->pool->lock);
4123 
4124 		if (drained)
4125 			continue;
4126 
4127 		if (++flush_cnt == 10 ||
4128 		    (flush_cnt % 100 == 0 && flush_cnt <= 1000))
4129 			pr_warn("workqueue %s: %s() isn't complete after %u tries\n",
4130 				wq->name, __func__, flush_cnt);
4131 
4132 		mutex_unlock(&wq->mutex);
4133 		goto reflush;
4134 	}
4135 
4136 	if (!--wq->nr_drainers)
4137 		wq->flags &= ~__WQ_DRAINING;
4138 	mutex_unlock(&wq->mutex);
4139 }
4140 EXPORT_SYMBOL_GPL(drain_workqueue);
4141 
start_flush_work(struct work_struct * work,struct wq_barrier * barr,bool from_cancel)4142 static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,
4143 			     bool from_cancel)
4144 {
4145 	struct worker *worker = NULL;
4146 	struct worker_pool *pool;
4147 	struct pool_workqueue *pwq;
4148 	struct workqueue_struct *wq;
4149 
4150 	rcu_read_lock();
4151 	pool = get_work_pool(work);
4152 	if (!pool) {
4153 		rcu_read_unlock();
4154 		return false;
4155 	}
4156 
4157 	raw_spin_lock_irq(&pool->lock);
4158 	/* see the comment in try_to_grab_pending() with the same code */
4159 	pwq = get_work_pwq(work);
4160 	if (pwq) {
4161 		if (unlikely(pwq->pool != pool))
4162 			goto already_gone;
4163 	} else {
4164 		worker = find_worker_executing_work(pool, work);
4165 		if (!worker)
4166 			goto already_gone;
4167 		pwq = worker->current_pwq;
4168 	}
4169 
4170 	wq = pwq->wq;
4171 	check_flush_dependency(wq, work, from_cancel);
4172 
4173 	insert_wq_barrier(pwq, barr, work, worker);
4174 	raw_spin_unlock_irq(&pool->lock);
4175 
4176 	touch_work_lockdep_map(work, wq);
4177 
4178 	/*
4179 	 * Force a lock recursion deadlock when using flush_work() inside a
4180 	 * single-threaded or rescuer equipped workqueue.
4181 	 *
4182 	 * For single threaded workqueues the deadlock happens when the work
4183 	 * is after the work issuing the flush_work(). For rescuer equipped
4184 	 * workqueues the deadlock happens when the rescuer stalls, blocking
4185 	 * forward progress.
4186 	 */
4187 	if (!from_cancel && (wq->saved_max_active == 1 || wq->rescuer))
4188 		touch_wq_lockdep_map(wq);
4189 
4190 	rcu_read_unlock();
4191 	return true;
4192 already_gone:
4193 	raw_spin_unlock_irq(&pool->lock);
4194 	rcu_read_unlock();
4195 	return false;
4196 }
4197 
__flush_work(struct work_struct * work,bool from_cancel)4198 static bool __flush_work(struct work_struct *work, bool from_cancel)
4199 {
4200 	struct wq_barrier barr;
4201 
4202 	if (WARN_ON(!wq_online))
4203 		return false;
4204 
4205 	if (WARN_ON(!work->func))
4206 		return false;
4207 
4208 	if (!start_flush_work(work, &barr, from_cancel))
4209 		return false;
4210 
4211 	/*
4212 	 * start_flush_work() returned %true. If @from_cancel is set, we know
4213 	 * that @work must have been executing during start_flush_work() and
4214 	 * can't currently be queued. Its data must contain OFFQ bits. If @work
4215 	 * was queued on a BH workqueue, we also know that it was running in the
4216 	 * BH context and thus can be busy-waited.
4217 	 */
4218 	if (from_cancel) {
4219 		unsigned long data = *work_data_bits(work);
4220 
4221 		if (!WARN_ON_ONCE(data & WORK_STRUCT_PWQ) &&
4222 		    (data & WORK_OFFQ_BH)) {
4223 			/*
4224 			 * On RT, prevent a live lock when %current preempted
4225 			 * soft interrupt processing or prevents ksoftirqd from
4226 			 * running by keeping flipping BH. If the BH work item
4227 			 * runs on a different CPU then this has no effect other
4228 			 * than doing the BH disable/enable dance for nothing.
4229 			 * This is copied from
4230 			 * kernel/softirq.c::tasklet_unlock_spin_wait().
4231 			 */
4232 			while (!try_wait_for_completion(&barr.done)) {
4233 				if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
4234 					local_bh_disable();
4235 					local_bh_enable();
4236 				} else {
4237 					cpu_relax();
4238 				}
4239 			}
4240 			goto out_destroy;
4241 		}
4242 	}
4243 
4244 	wait_for_completion(&barr.done);
4245 
4246 out_destroy:
4247 	destroy_work_on_stack(&barr.work);
4248 	return true;
4249 }
4250 
4251 /**
4252  * flush_work - wait for a work to finish executing the last queueing instance
4253  * @work: the work to flush
4254  *
4255  * Wait until @work has finished execution.  @work is guaranteed to be idle
4256  * on return if it hasn't been requeued since flush started.
4257  *
4258  * Return:
4259  * %true if flush_work() waited for the work to finish execution,
4260  * %false if it was already idle.
4261  */
flush_work(struct work_struct * work)4262 bool flush_work(struct work_struct *work)
4263 {
4264 	might_sleep();
4265 	return __flush_work(work, false);
4266 }
4267 EXPORT_SYMBOL_GPL(flush_work);
4268 
4269 /**
4270  * flush_delayed_work - wait for a dwork to finish executing the last queueing
4271  * @dwork: the delayed work to flush
4272  *
4273  * Delayed timer is cancelled and the pending work is queued for
4274  * immediate execution.  Like flush_work(), this function only
4275  * considers the last queueing instance of @dwork.
4276  *
4277  * Return:
4278  * %true if flush_work() waited for the work to finish execution,
4279  * %false if it was already idle.
4280  */
flush_delayed_work(struct delayed_work * dwork)4281 bool flush_delayed_work(struct delayed_work *dwork)
4282 {
4283 	local_irq_disable();
4284 	if (del_timer_sync(&dwork->timer))
4285 		__queue_work(dwork->cpu, dwork->wq, &dwork->work);
4286 	local_irq_enable();
4287 	return flush_work(&dwork->work);
4288 }
4289 EXPORT_SYMBOL(flush_delayed_work);
4290 
4291 /**
4292  * flush_rcu_work - wait for a rwork to finish executing the last queueing
4293  * @rwork: the rcu work to flush
4294  *
4295  * Return:
4296  * %true if flush_rcu_work() waited for the work to finish execution,
4297  * %false if it was already idle.
4298  */
flush_rcu_work(struct rcu_work * rwork)4299 bool flush_rcu_work(struct rcu_work *rwork)
4300 {
4301 	if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {
4302 		rcu_barrier();
4303 		flush_work(&rwork->work);
4304 		return true;
4305 	} else {
4306 		return flush_work(&rwork->work);
4307 	}
4308 }
4309 EXPORT_SYMBOL(flush_rcu_work);
4310 
work_offqd_disable(struct work_offq_data * offqd)4311 static void work_offqd_disable(struct work_offq_data *offqd)
4312 {
4313 	const unsigned long max = (1lu << WORK_OFFQ_DISABLE_BITS) - 1;
4314 
4315 	if (likely(offqd->disable < max))
4316 		offqd->disable++;
4317 	else
4318 		WARN_ONCE(true, "workqueue: work disable count overflowed\n");
4319 }
4320 
work_offqd_enable(struct work_offq_data * offqd)4321 static void work_offqd_enable(struct work_offq_data *offqd)
4322 {
4323 	if (likely(offqd->disable > 0))
4324 		offqd->disable--;
4325 	else
4326 		WARN_ONCE(true, "workqueue: work disable count underflowed\n");
4327 }
4328 
__cancel_work(struct work_struct * work,u32 cflags)4329 static bool __cancel_work(struct work_struct *work, u32 cflags)
4330 {
4331 	struct work_offq_data offqd;
4332 	unsigned long irq_flags;
4333 	int ret;
4334 
4335 	ret = work_grab_pending(work, cflags, &irq_flags);
4336 
4337 	work_offqd_unpack(&offqd, *work_data_bits(work));
4338 
4339 	if (cflags & WORK_CANCEL_DISABLE)
4340 		work_offqd_disable(&offqd);
4341 
4342 	set_work_pool_and_clear_pending(work, offqd.pool_id,
4343 					work_offqd_pack_flags(&offqd));
4344 	local_irq_restore(irq_flags);
4345 	return ret;
4346 }
4347 
__cancel_work_sync(struct work_struct * work,u32 cflags)4348 static bool __cancel_work_sync(struct work_struct *work, u32 cflags)
4349 {
4350 	bool ret;
4351 
4352 	ret = __cancel_work(work, cflags | WORK_CANCEL_DISABLE);
4353 
4354 	if (*work_data_bits(work) & WORK_OFFQ_BH)
4355 		WARN_ON_ONCE(in_hardirq());
4356 	else
4357 		might_sleep();
4358 
4359 	/*
4360 	 * Skip __flush_work() during early boot when we know that @work isn't
4361 	 * executing. This allows canceling during early boot.
4362 	 */
4363 	if (wq_online)
4364 		__flush_work(work, true);
4365 
4366 	if (!(cflags & WORK_CANCEL_DISABLE))
4367 		enable_work(work);
4368 
4369 	return ret;
4370 }
4371 
4372 /*
4373  * See cancel_delayed_work()
4374  */
cancel_work(struct work_struct * work)4375 bool cancel_work(struct work_struct *work)
4376 {
4377 	return __cancel_work(work, 0);
4378 }
4379 EXPORT_SYMBOL(cancel_work);
4380 
4381 /**
4382  * cancel_work_sync - cancel a work and wait for it to finish
4383  * @work: the work to cancel
4384  *
4385  * Cancel @work and wait for its execution to finish. This function can be used
4386  * even if the work re-queues itself or migrates to another workqueue. On return
4387  * from this function, @work is guaranteed to be not pending or executing on any
4388  * CPU as long as there aren't racing enqueues.
4389  *
4390  * cancel_work_sync(&delayed_work->work) must not be used for delayed_work's.
4391  * Use cancel_delayed_work_sync() instead.
4392  *
4393  * Must be called from a sleepable context if @work was last queued on a non-BH
4394  * workqueue. Can also be called from non-hardirq atomic contexts including BH
4395  * if @work was last queued on a BH workqueue.
4396  *
4397  * Returns %true if @work was pending, %false otherwise.
4398  */
cancel_work_sync(struct work_struct * work)4399 bool cancel_work_sync(struct work_struct *work)
4400 {
4401 	return __cancel_work_sync(work, 0);
4402 }
4403 EXPORT_SYMBOL_GPL(cancel_work_sync);
4404 
4405 /**
4406  * cancel_delayed_work - cancel a delayed work
4407  * @dwork: delayed_work to cancel
4408  *
4409  * Kill off a pending delayed_work.
4410  *
4411  * Return: %true if @dwork was pending and canceled; %false if it wasn't
4412  * pending.
4413  *
4414  * Note:
4415  * The work callback function may still be running on return, unless
4416  * it returns %true and the work doesn't re-arm itself.  Explicitly flush or
4417  * use cancel_delayed_work_sync() to wait on it.
4418  *
4419  * This function is safe to call from any context including IRQ handler.
4420  */
cancel_delayed_work(struct delayed_work * dwork)4421 bool cancel_delayed_work(struct delayed_work *dwork)
4422 {
4423 	return __cancel_work(&dwork->work, WORK_CANCEL_DELAYED);
4424 }
4425 EXPORT_SYMBOL(cancel_delayed_work);
4426 
4427 /**
4428  * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
4429  * @dwork: the delayed work cancel
4430  *
4431  * This is cancel_work_sync() for delayed works.
4432  *
4433  * Return:
4434  * %true if @dwork was pending, %false otherwise.
4435  */
cancel_delayed_work_sync(struct delayed_work * dwork)4436 bool cancel_delayed_work_sync(struct delayed_work *dwork)
4437 {
4438 	return __cancel_work_sync(&dwork->work, WORK_CANCEL_DELAYED);
4439 }
4440 EXPORT_SYMBOL(cancel_delayed_work_sync);
4441 
4442 /**
4443  * disable_work - Disable and cancel a work item
4444  * @work: work item to disable
4445  *
4446  * Disable @work by incrementing its disable count and cancel it if currently
4447  * pending. As long as the disable count is non-zero, any attempt to queue @work
4448  * will fail and return %false. The maximum supported disable depth is 2 to the
4449  * power of %WORK_OFFQ_DISABLE_BITS, currently 65536.
4450  *
4451  * Can be called from any context. Returns %true if @work was pending, %false
4452  * otherwise.
4453  */
disable_work(struct work_struct * work)4454 bool disable_work(struct work_struct *work)
4455 {
4456 	return __cancel_work(work, WORK_CANCEL_DISABLE);
4457 }
4458 EXPORT_SYMBOL_GPL(disable_work);
4459 
4460 /**
4461  * disable_work_sync - Disable, cancel and drain a work item
4462  * @work: work item to disable
4463  *
4464  * Similar to disable_work() but also wait for @work to finish if currently
4465  * executing.
4466  *
4467  * Must be called from a sleepable context if @work was last queued on a non-BH
4468  * workqueue. Can also be called from non-hardirq atomic contexts including BH
4469  * if @work was last queued on a BH workqueue.
4470  *
4471  * Returns %true if @work was pending, %false otherwise.
4472  */
disable_work_sync(struct work_struct * work)4473 bool disable_work_sync(struct work_struct *work)
4474 {
4475 	return __cancel_work_sync(work, WORK_CANCEL_DISABLE);
4476 }
4477 EXPORT_SYMBOL_GPL(disable_work_sync);
4478 
4479 /**
4480  * enable_work - Enable a work item
4481  * @work: work item to enable
4482  *
4483  * Undo disable_work[_sync]() by decrementing @work's disable count. @work can
4484  * only be queued if its disable count is 0.
4485  *
4486  * Can be called from any context. Returns %true if the disable count reached 0.
4487  * Otherwise, %false.
4488  */
enable_work(struct work_struct * work)4489 bool enable_work(struct work_struct *work)
4490 {
4491 	struct work_offq_data offqd;
4492 	unsigned long irq_flags;
4493 
4494 	work_grab_pending(work, 0, &irq_flags);
4495 
4496 	work_offqd_unpack(&offqd, *work_data_bits(work));
4497 	work_offqd_enable(&offqd);
4498 	set_work_pool_and_clear_pending(work, offqd.pool_id,
4499 					work_offqd_pack_flags(&offqd));
4500 	local_irq_restore(irq_flags);
4501 
4502 	return !offqd.disable;
4503 }
4504 EXPORT_SYMBOL_GPL(enable_work);
4505 
4506 /**
4507  * disable_delayed_work - Disable and cancel a delayed work item
4508  * @dwork: delayed work item to disable
4509  *
4510  * disable_work() for delayed work items.
4511  */
disable_delayed_work(struct delayed_work * dwork)4512 bool disable_delayed_work(struct delayed_work *dwork)
4513 {
4514 	return __cancel_work(&dwork->work,
4515 			     WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE);
4516 }
4517 EXPORT_SYMBOL_GPL(disable_delayed_work);
4518 
4519 /**
4520  * disable_delayed_work_sync - Disable, cancel and drain a delayed work item
4521  * @dwork: delayed work item to disable
4522  *
4523  * disable_work_sync() for delayed work items.
4524  */
disable_delayed_work_sync(struct delayed_work * dwork)4525 bool disable_delayed_work_sync(struct delayed_work *dwork)
4526 {
4527 	return __cancel_work_sync(&dwork->work,
4528 				  WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE);
4529 }
4530 EXPORT_SYMBOL_GPL(disable_delayed_work_sync);
4531 
4532 /**
4533  * enable_delayed_work - Enable a delayed work item
4534  * @dwork: delayed work item to enable
4535  *
4536  * enable_work() for delayed work items.
4537  */
enable_delayed_work(struct delayed_work * dwork)4538 bool enable_delayed_work(struct delayed_work *dwork)
4539 {
4540 	return enable_work(&dwork->work);
4541 }
4542 EXPORT_SYMBOL_GPL(enable_delayed_work);
4543 
4544 /**
4545  * schedule_on_each_cpu - execute a function synchronously on each online CPU
4546  * @func: the function to call
4547  *
4548  * schedule_on_each_cpu() executes @func on each online CPU using the
4549  * system workqueue and blocks until all CPUs have completed.
4550  * schedule_on_each_cpu() is very slow.
4551  *
4552  * Return:
4553  * 0 on success, -errno on failure.
4554  */
schedule_on_each_cpu(work_func_t func)4555 int schedule_on_each_cpu(work_func_t func)
4556 {
4557 	int cpu;
4558 	struct work_struct __percpu *works;
4559 
4560 	works = alloc_percpu(struct work_struct);
4561 	if (!works)
4562 		return -ENOMEM;
4563 
4564 	cpus_read_lock();
4565 
4566 	for_each_online_cpu(cpu) {
4567 		struct work_struct *work = per_cpu_ptr(works, cpu);
4568 
4569 		INIT_WORK(work, func);
4570 		schedule_work_on(cpu, work);
4571 	}
4572 
4573 	for_each_online_cpu(cpu)
4574 		flush_work(per_cpu_ptr(works, cpu));
4575 
4576 	cpus_read_unlock();
4577 	free_percpu(works);
4578 	return 0;
4579 }
4580 
4581 /**
4582  * execute_in_process_context - reliably execute the routine with user context
4583  * @fn:		the function to execute
4584  * @ew:		guaranteed storage for the execute work structure (must
4585  *		be available when the work executes)
4586  *
4587  * Executes the function immediately if process context is available,
4588  * otherwise schedules the function for delayed execution.
4589  *
4590  * Return:	0 - function was executed
4591  *		1 - function was scheduled for execution
4592  */
execute_in_process_context(work_func_t fn,struct execute_work * ew)4593 int execute_in_process_context(work_func_t fn, struct execute_work *ew)
4594 {
4595 	if (!in_interrupt()) {
4596 		fn(&ew->work);
4597 		return 0;
4598 	}
4599 
4600 	INIT_WORK(&ew->work, fn);
4601 	schedule_work(&ew->work);
4602 
4603 	return 1;
4604 }
4605 EXPORT_SYMBOL_GPL(execute_in_process_context);
4606 
4607 /**
4608  * free_workqueue_attrs - free a workqueue_attrs
4609  * @attrs: workqueue_attrs to free
4610  *
4611  * Undo alloc_workqueue_attrs().
4612  */
free_workqueue_attrs(struct workqueue_attrs * attrs)4613 void free_workqueue_attrs(struct workqueue_attrs *attrs)
4614 {
4615 	if (attrs) {
4616 		free_cpumask_var(attrs->cpumask);
4617 		free_cpumask_var(attrs->__pod_cpumask);
4618 		kfree(attrs);
4619 	}
4620 }
4621 
4622 /**
4623  * alloc_workqueue_attrs - allocate a workqueue_attrs
4624  *
4625  * Allocate a new workqueue_attrs, initialize with default settings and
4626  * return it.
4627  *
4628  * Return: The allocated new workqueue_attr on success. %NULL on failure.
4629  */
alloc_workqueue_attrs(void)4630 struct workqueue_attrs *alloc_workqueue_attrs(void)
4631 {
4632 	struct workqueue_attrs *attrs;
4633 
4634 	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
4635 	if (!attrs)
4636 		goto fail;
4637 	if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
4638 		goto fail;
4639 	if (!alloc_cpumask_var(&attrs->__pod_cpumask, GFP_KERNEL))
4640 		goto fail;
4641 
4642 	cpumask_copy(attrs->cpumask, cpu_possible_mask);
4643 	attrs->affn_scope = WQ_AFFN_DFL;
4644 	return attrs;
4645 fail:
4646 	free_workqueue_attrs(attrs);
4647 	return NULL;
4648 }
4649 
copy_workqueue_attrs(struct workqueue_attrs * to,const struct workqueue_attrs * from)4650 static void copy_workqueue_attrs(struct workqueue_attrs *to,
4651 				 const struct workqueue_attrs *from)
4652 {
4653 	to->nice = from->nice;
4654 	cpumask_copy(to->cpumask, from->cpumask);
4655 	cpumask_copy(to->__pod_cpumask, from->__pod_cpumask);
4656 	to->affn_strict = from->affn_strict;
4657 
4658 	/*
4659 	 * Unlike hash and equality test, copying shouldn't ignore wq-only
4660 	 * fields as copying is used for both pool and wq attrs. Instead,
4661 	 * get_unbound_pool() explicitly clears the fields.
4662 	 */
4663 	to->affn_scope = from->affn_scope;
4664 	to->ordered = from->ordered;
4665 }
4666 
4667 /*
4668  * Some attrs fields are workqueue-only. Clear them for worker_pool's. See the
4669  * comments in 'struct workqueue_attrs' definition.
4670  */
wqattrs_clear_for_pool(struct workqueue_attrs * attrs)4671 static void wqattrs_clear_for_pool(struct workqueue_attrs *attrs)
4672 {
4673 	attrs->affn_scope = WQ_AFFN_NR_TYPES;
4674 	attrs->ordered = false;
4675 	if (attrs->affn_strict)
4676 		cpumask_copy(attrs->cpumask, cpu_possible_mask);
4677 }
4678 
4679 /* hash value of the content of @attr */
wqattrs_hash(const struct workqueue_attrs * attrs)4680 static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
4681 {
4682 	u32 hash = 0;
4683 
4684 	hash = jhash_1word(attrs->nice, hash);
4685 	hash = jhash_1word(attrs->affn_strict, hash);
4686 	hash = jhash(cpumask_bits(attrs->__pod_cpumask),
4687 		     BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
4688 	if (!attrs->affn_strict)
4689 		hash = jhash(cpumask_bits(attrs->cpumask),
4690 			     BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
4691 	return hash;
4692 }
4693 
4694 /* content equality test */
wqattrs_equal(const struct workqueue_attrs * a,const struct workqueue_attrs * b)4695 static bool wqattrs_equal(const struct workqueue_attrs *a,
4696 			  const struct workqueue_attrs *b)
4697 {
4698 	if (a->nice != b->nice)
4699 		return false;
4700 	if (a->affn_strict != b->affn_strict)
4701 		return false;
4702 	if (!cpumask_equal(a->__pod_cpumask, b->__pod_cpumask))
4703 		return false;
4704 	if (!a->affn_strict && !cpumask_equal(a->cpumask, b->cpumask))
4705 		return false;
4706 	return true;
4707 }
4708 
4709 /* Update @attrs with actually available CPUs */
wqattrs_actualize_cpumask(struct workqueue_attrs * attrs,const cpumask_t * unbound_cpumask)4710 static void wqattrs_actualize_cpumask(struct workqueue_attrs *attrs,
4711 				      const cpumask_t *unbound_cpumask)
4712 {
4713 	/*
4714 	 * Calculate the effective CPU mask of @attrs given @unbound_cpumask. If
4715 	 * @attrs->cpumask doesn't overlap with @unbound_cpumask, we fallback to
4716 	 * @unbound_cpumask.
4717 	 */
4718 	cpumask_and(attrs->cpumask, attrs->cpumask, unbound_cpumask);
4719 	if (unlikely(cpumask_empty(attrs->cpumask)))
4720 		cpumask_copy(attrs->cpumask, unbound_cpumask);
4721 }
4722 
4723 /* find wq_pod_type to use for @attrs */
4724 static const struct wq_pod_type *
wqattrs_pod_type(const struct workqueue_attrs * attrs)4725 wqattrs_pod_type(const struct workqueue_attrs *attrs)
4726 {
4727 	enum wq_affn_scope scope;
4728 	struct wq_pod_type *pt;
4729 
4730 	/* to synchronize access to wq_affn_dfl */
4731 	lockdep_assert_held(&wq_pool_mutex);
4732 
4733 	if (attrs->affn_scope == WQ_AFFN_DFL)
4734 		scope = wq_affn_dfl;
4735 	else
4736 		scope = attrs->affn_scope;
4737 
4738 	pt = &wq_pod_types[scope];
4739 
4740 	if (!WARN_ON_ONCE(attrs->affn_scope == WQ_AFFN_NR_TYPES) &&
4741 	    likely(pt->nr_pods))
4742 		return pt;
4743 
4744 	/*
4745 	 * Before workqueue_init_topology(), only SYSTEM is available which is
4746 	 * initialized in workqueue_init_early().
4747 	 */
4748 	pt = &wq_pod_types[WQ_AFFN_SYSTEM];
4749 	BUG_ON(!pt->nr_pods);
4750 	return pt;
4751 }
4752 
4753 /**
4754  * init_worker_pool - initialize a newly zalloc'd worker_pool
4755  * @pool: worker_pool to initialize
4756  *
4757  * Initialize a newly zalloc'd @pool.  It also allocates @pool->attrs.
4758  *
4759  * Return: 0 on success, -errno on failure.  Even on failure, all fields
4760  * inside @pool proper are initialized and put_unbound_pool() can be called
4761  * on @pool safely to release it.
4762  */
init_worker_pool(struct worker_pool * pool)4763 static int init_worker_pool(struct worker_pool *pool)
4764 {
4765 	raw_spin_lock_init(&pool->lock);
4766 	pool->id = -1;
4767 	pool->cpu = -1;
4768 	pool->node = NUMA_NO_NODE;
4769 	pool->flags |= POOL_DISASSOCIATED;
4770 	pool->watchdog_ts = jiffies;
4771 	INIT_LIST_HEAD(&pool->worklist);
4772 	INIT_LIST_HEAD(&pool->idle_list);
4773 	hash_init(pool->busy_hash);
4774 
4775 	timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
4776 	INIT_WORK(&pool->idle_cull_work, idle_cull_fn);
4777 
4778 	timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);
4779 
4780 	INIT_LIST_HEAD(&pool->workers);
4781 
4782 	ida_init(&pool->worker_ida);
4783 	INIT_HLIST_NODE(&pool->hash_node);
4784 	pool->refcnt = 1;
4785 
4786 	/* shouldn't fail above this point */
4787 	pool->attrs = alloc_workqueue_attrs();
4788 	if (!pool->attrs)
4789 		return -ENOMEM;
4790 
4791 	wqattrs_clear_for_pool(pool->attrs);
4792 
4793 	return 0;
4794 }
4795 
4796 #ifdef CONFIG_LOCKDEP
wq_init_lockdep(struct workqueue_struct * wq)4797 static void wq_init_lockdep(struct workqueue_struct *wq)
4798 {
4799 	char *lock_name;
4800 
4801 	lockdep_register_key(&wq->key);
4802 	lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
4803 	if (!lock_name)
4804 		lock_name = wq->name;
4805 
4806 	wq->lock_name = lock_name;
4807 	wq->lockdep_map = &wq->__lockdep_map;
4808 	lockdep_init_map(wq->lockdep_map, lock_name, &wq->key, 0);
4809 }
4810 
wq_unregister_lockdep(struct workqueue_struct * wq)4811 static void wq_unregister_lockdep(struct workqueue_struct *wq)
4812 {
4813 	if (wq->lockdep_map != &wq->__lockdep_map)
4814 		return;
4815 
4816 	lockdep_unregister_key(&wq->key);
4817 }
4818 
wq_free_lockdep(struct workqueue_struct * wq)4819 static void wq_free_lockdep(struct workqueue_struct *wq)
4820 {
4821 	if (wq->lockdep_map != &wq->__lockdep_map)
4822 		return;
4823 
4824 	if (wq->lock_name != wq->name)
4825 		kfree(wq->lock_name);
4826 }
4827 #else
wq_init_lockdep(struct workqueue_struct * wq)4828 static void wq_init_lockdep(struct workqueue_struct *wq)
4829 {
4830 }
4831 
wq_unregister_lockdep(struct workqueue_struct * wq)4832 static void wq_unregister_lockdep(struct workqueue_struct *wq)
4833 {
4834 }
4835 
wq_free_lockdep(struct workqueue_struct * wq)4836 static void wq_free_lockdep(struct workqueue_struct *wq)
4837 {
4838 }
4839 #endif
4840 
free_node_nr_active(struct wq_node_nr_active ** nna_ar)4841 static void free_node_nr_active(struct wq_node_nr_active **nna_ar)
4842 {
4843 	int node;
4844 
4845 	for_each_node(node) {
4846 		kfree(nna_ar[node]);
4847 		nna_ar[node] = NULL;
4848 	}
4849 
4850 	kfree(nna_ar[nr_node_ids]);
4851 	nna_ar[nr_node_ids] = NULL;
4852 }
4853 
init_node_nr_active(struct wq_node_nr_active * nna)4854 static void init_node_nr_active(struct wq_node_nr_active *nna)
4855 {
4856 	nna->max = WQ_DFL_MIN_ACTIVE;
4857 	atomic_set(&nna->nr, 0);
4858 	raw_spin_lock_init(&nna->lock);
4859 	INIT_LIST_HEAD(&nna->pending_pwqs);
4860 }
4861 
4862 /*
4863  * Each node's nr_active counter will be accessed mostly from its own node and
4864  * should be allocated in the node.
4865  */
alloc_node_nr_active(struct wq_node_nr_active ** nna_ar)4866 static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar)
4867 {
4868 	struct wq_node_nr_active *nna;
4869 	int node;
4870 
4871 	for_each_node(node) {
4872 		nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node);
4873 		if (!nna)
4874 			goto err_free;
4875 		init_node_nr_active(nna);
4876 		nna_ar[node] = nna;
4877 	}
4878 
4879 	/* [nr_node_ids] is used as the fallback */
4880 	nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE);
4881 	if (!nna)
4882 		goto err_free;
4883 	init_node_nr_active(nna);
4884 	nna_ar[nr_node_ids] = nna;
4885 
4886 	return 0;
4887 
4888 err_free:
4889 	free_node_nr_active(nna_ar);
4890 	return -ENOMEM;
4891 }
4892 
rcu_free_wq(struct rcu_head * rcu)4893 static void rcu_free_wq(struct rcu_head *rcu)
4894 {
4895 	struct workqueue_struct *wq =
4896 		container_of(rcu, struct workqueue_struct, rcu);
4897 
4898 	if (wq->flags & WQ_UNBOUND)
4899 		free_node_nr_active(wq->node_nr_active);
4900 
4901 	wq_free_lockdep(wq);
4902 	free_percpu(wq->cpu_pwq);
4903 	free_workqueue_attrs(wq->unbound_attrs);
4904 	kfree(wq);
4905 }
4906 
rcu_free_pool(struct rcu_head * rcu)4907 static void rcu_free_pool(struct rcu_head *rcu)
4908 {
4909 	struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
4910 
4911 	ida_destroy(&pool->worker_ida);
4912 	free_workqueue_attrs(pool->attrs);
4913 	kfree(pool);
4914 }
4915 
4916 /**
4917  * put_unbound_pool - put a worker_pool
4918  * @pool: worker_pool to put
4919  *
4920  * Put @pool.  If its refcnt reaches zero, it gets destroyed in RCU
4921  * safe manner.  get_unbound_pool() calls this function on its failure path
4922  * and this function should be able to release pools which went through,
4923  * successfully or not, init_worker_pool().
4924  *
4925  * Should be called with wq_pool_mutex held.
4926  */
put_unbound_pool(struct worker_pool * pool)4927 static void put_unbound_pool(struct worker_pool *pool)
4928 {
4929 	struct worker *worker;
4930 	LIST_HEAD(cull_list);
4931 
4932 	lockdep_assert_held(&wq_pool_mutex);
4933 
4934 	if (--pool->refcnt)
4935 		return;
4936 
4937 	/* sanity checks */
4938 	if (WARN_ON(!(pool->cpu < 0)) ||
4939 	    WARN_ON(!list_empty(&pool->worklist)))
4940 		return;
4941 
4942 	/* release id and unhash */
4943 	if (pool->id >= 0)
4944 		idr_remove(&worker_pool_idr, pool->id);
4945 	hash_del(&pool->hash_node);
4946 
4947 	/*
4948 	 * Become the manager and destroy all workers.  This prevents
4949 	 * @pool's workers from blocking on attach_mutex.  We're the last
4950 	 * manager and @pool gets freed with the flag set.
4951 	 *
4952 	 * Having a concurrent manager is quite unlikely to happen as we can
4953 	 * only get here with
4954 	 *   pwq->refcnt == pool->refcnt == 0
4955 	 * which implies no work queued to the pool, which implies no worker can
4956 	 * become the manager. However a worker could have taken the role of
4957 	 * manager before the refcnts dropped to 0, since maybe_create_worker()
4958 	 * drops pool->lock
4959 	 */
4960 	while (true) {
4961 		rcuwait_wait_event(&manager_wait,
4962 				   !(pool->flags & POOL_MANAGER_ACTIVE),
4963 				   TASK_UNINTERRUPTIBLE);
4964 
4965 		mutex_lock(&wq_pool_attach_mutex);
4966 		raw_spin_lock_irq(&pool->lock);
4967 		if (!(pool->flags & POOL_MANAGER_ACTIVE)) {
4968 			pool->flags |= POOL_MANAGER_ACTIVE;
4969 			break;
4970 		}
4971 		raw_spin_unlock_irq(&pool->lock);
4972 		mutex_unlock(&wq_pool_attach_mutex);
4973 	}
4974 
4975 	while ((worker = first_idle_worker(pool)))
4976 		set_worker_dying(worker, &cull_list);
4977 	WARN_ON(pool->nr_workers || pool->nr_idle);
4978 	raw_spin_unlock_irq(&pool->lock);
4979 
4980 	detach_dying_workers(&cull_list);
4981 
4982 	mutex_unlock(&wq_pool_attach_mutex);
4983 
4984 	reap_dying_workers(&cull_list);
4985 
4986 	/* shut down the timers */
4987 	del_timer_sync(&pool->idle_timer);
4988 	cancel_work_sync(&pool->idle_cull_work);
4989 	del_timer_sync(&pool->mayday_timer);
4990 
4991 	/* RCU protected to allow dereferences from get_work_pool() */
4992 	call_rcu(&pool->rcu, rcu_free_pool);
4993 }
4994 
4995 /**
4996  * get_unbound_pool - get a worker_pool with the specified attributes
4997  * @attrs: the attributes of the worker_pool to get
4998  *
4999  * Obtain a worker_pool which has the same attributes as @attrs, bump the
5000  * reference count and return it.  If there already is a matching
5001  * worker_pool, it will be used; otherwise, this function attempts to
5002  * create a new one.
5003  *
5004  * Should be called with wq_pool_mutex held.
5005  *
5006  * Return: On success, a worker_pool with the same attributes as @attrs.
5007  * On failure, %NULL.
5008  */
get_unbound_pool(const struct workqueue_attrs * attrs)5009 static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
5010 {
5011 	struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_NUMA];
5012 	u32 hash = wqattrs_hash(attrs);
5013 	struct worker_pool *pool;
5014 	int pod, node = NUMA_NO_NODE;
5015 
5016 	lockdep_assert_held(&wq_pool_mutex);
5017 
5018 	/* do we already have a matching pool? */
5019 	hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
5020 		if (wqattrs_equal(pool->attrs, attrs)) {
5021 			pool->refcnt++;
5022 			return pool;
5023 		}
5024 	}
5025 
5026 	/* If __pod_cpumask is contained inside a NUMA pod, that's our node */
5027 	for (pod = 0; pod < pt->nr_pods; pod++) {
5028 		if (cpumask_subset(attrs->__pod_cpumask, pt->pod_cpus[pod])) {
5029 			node = pt->pod_node[pod];
5030 			break;
5031 		}
5032 	}
5033 
5034 	/* nope, create a new one */
5035 	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, node);
5036 	if (!pool || init_worker_pool(pool) < 0)
5037 		goto fail;
5038 
5039 	pool->node = node;
5040 	copy_workqueue_attrs(pool->attrs, attrs);
5041 	wqattrs_clear_for_pool(pool->attrs);
5042 
5043 	if (worker_pool_assign_id(pool) < 0)
5044 		goto fail;
5045 
5046 	/* create and start the initial worker */
5047 	if (wq_online && !create_worker(pool))
5048 		goto fail;
5049 
5050 	/* install */
5051 	hash_add(unbound_pool_hash, &pool->hash_node, hash);
5052 
5053 	return pool;
5054 fail:
5055 	if (pool)
5056 		put_unbound_pool(pool);
5057 	return NULL;
5058 }
5059 
5060 /*
5061  * Scheduled on pwq_release_worker by put_pwq() when an unbound pwq hits zero
5062  * refcnt and needs to be destroyed.
5063  */
pwq_release_workfn(struct kthread_work * work)5064 static void pwq_release_workfn(struct kthread_work *work)
5065 {
5066 	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
5067 						  release_work);
5068 	struct workqueue_struct *wq = pwq->wq;
5069 	struct worker_pool *pool = pwq->pool;
5070 	bool is_last = false;
5071 
5072 	/*
5073 	 * When @pwq is not linked, it doesn't hold any reference to the
5074 	 * @wq, and @wq is invalid to access.
5075 	 */
5076 	if (!list_empty(&pwq->pwqs_node)) {
5077 		mutex_lock(&wq->mutex);
5078 		list_del_rcu(&pwq->pwqs_node);
5079 		is_last = list_empty(&wq->pwqs);
5080 
5081 		/*
5082 		 * For ordered workqueue with a plugged dfl_pwq, restart it now.
5083 		 */
5084 		if (!is_last && (wq->flags & __WQ_ORDERED))
5085 			unplug_oldest_pwq(wq);
5086 
5087 		mutex_unlock(&wq->mutex);
5088 	}
5089 
5090 	if (wq->flags & WQ_UNBOUND) {
5091 		mutex_lock(&wq_pool_mutex);
5092 		put_unbound_pool(pool);
5093 		mutex_unlock(&wq_pool_mutex);
5094 	}
5095 
5096 	if (!list_empty(&pwq->pending_node)) {
5097 		struct wq_node_nr_active *nna =
5098 			wq_node_nr_active(pwq->wq, pwq->pool->node);
5099 
5100 		raw_spin_lock_irq(&nna->lock);
5101 		list_del_init(&pwq->pending_node);
5102 		raw_spin_unlock_irq(&nna->lock);
5103 	}
5104 
5105 	kfree_rcu(pwq, rcu);
5106 
5107 	/*
5108 	 * If we're the last pwq going away, @wq is already dead and no one
5109 	 * is gonna access it anymore.  Schedule RCU free.
5110 	 */
5111 	if (is_last) {
5112 		wq_unregister_lockdep(wq);
5113 		call_rcu(&wq->rcu, rcu_free_wq);
5114 	}
5115 }
5116 
5117 /* initialize newly allocated @pwq which is associated with @wq and @pool */
init_pwq(struct pool_workqueue * pwq,struct workqueue_struct * wq,struct worker_pool * pool)5118 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
5119 		     struct worker_pool *pool)
5120 {
5121 	BUG_ON((unsigned long)pwq & ~WORK_STRUCT_PWQ_MASK);
5122 
5123 	memset(pwq, 0, sizeof(*pwq));
5124 
5125 	pwq->pool = pool;
5126 	pwq->wq = wq;
5127 	pwq->flush_color = -1;
5128 	pwq->refcnt = 1;
5129 	INIT_LIST_HEAD(&pwq->inactive_works);
5130 	INIT_LIST_HEAD(&pwq->pending_node);
5131 	INIT_LIST_HEAD(&pwq->pwqs_node);
5132 	INIT_LIST_HEAD(&pwq->mayday_node);
5133 	kthread_init_work(&pwq->release_work, pwq_release_workfn);
5134 }
5135 
5136 /* sync @pwq with the current state of its associated wq and link it */
link_pwq(struct pool_workqueue * pwq)5137 static void link_pwq(struct pool_workqueue *pwq)
5138 {
5139 	struct workqueue_struct *wq = pwq->wq;
5140 
5141 	lockdep_assert_held(&wq->mutex);
5142 
5143 	/* may be called multiple times, ignore if already linked */
5144 	if (!list_empty(&pwq->pwqs_node))
5145 		return;
5146 
5147 	/* set the matching work_color */
5148 	pwq->work_color = wq->work_color;
5149 
5150 	/* link in @pwq */
5151 	list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs);
5152 }
5153 
5154 /* obtain a pool matching @attr and create a pwq associating the pool and @wq */
alloc_unbound_pwq(struct workqueue_struct * wq,const struct workqueue_attrs * attrs)5155 static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
5156 					const struct workqueue_attrs *attrs)
5157 {
5158 	struct worker_pool *pool;
5159 	struct pool_workqueue *pwq;
5160 
5161 	lockdep_assert_held(&wq_pool_mutex);
5162 
5163 	pool = get_unbound_pool(attrs);
5164 	if (!pool)
5165 		return NULL;
5166 
5167 	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
5168 	if (!pwq) {
5169 		put_unbound_pool(pool);
5170 		return NULL;
5171 	}
5172 
5173 	init_pwq(pwq, wq, pool);
5174 	return pwq;
5175 }
5176 
apply_wqattrs_lock(void)5177 static void apply_wqattrs_lock(void)
5178 {
5179 	mutex_lock(&wq_pool_mutex);
5180 }
5181 
apply_wqattrs_unlock(void)5182 static void apply_wqattrs_unlock(void)
5183 {
5184 	mutex_unlock(&wq_pool_mutex);
5185 }
5186 
5187 /**
5188  * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod
5189  * @attrs: the wq_attrs of the default pwq of the target workqueue
5190  * @cpu: the target CPU
5191  *
5192  * Calculate the cpumask a workqueue with @attrs should use on @pod.
5193  * The result is stored in @attrs->__pod_cpumask.
5194  *
5195  * If pod affinity is not enabled, @attrs->cpumask is always used. If enabled
5196  * and @pod has online CPUs requested by @attrs, the returned cpumask is the
5197  * intersection of the possible CPUs of @pod and @attrs->cpumask.
5198  *
5199  * The caller is responsible for ensuring that the cpumask of @pod stays stable.
5200  */
wq_calc_pod_cpumask(struct workqueue_attrs * attrs,int cpu)5201 static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu)
5202 {
5203 	const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
5204 	int pod = pt->cpu_pod[cpu];
5205 
5206 	/* calculate possible CPUs in @pod that @attrs wants */
5207 	cpumask_and(attrs->__pod_cpumask, pt->pod_cpus[pod], attrs->cpumask);
5208 	/* does @pod have any online CPUs @attrs wants? */
5209 	if (!cpumask_intersects(attrs->__pod_cpumask, wq_online_cpumask)) {
5210 		cpumask_copy(attrs->__pod_cpumask, attrs->cpumask);
5211 		return;
5212 	}
5213 }
5214 
5215 /* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
install_unbound_pwq(struct workqueue_struct * wq,int cpu,struct pool_workqueue * pwq)5216 static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
5217 					int cpu, struct pool_workqueue *pwq)
5218 {
5219 	struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
5220 	struct pool_workqueue *old_pwq;
5221 
5222 	lockdep_assert_held(&wq_pool_mutex);
5223 	lockdep_assert_held(&wq->mutex);
5224 
5225 	/* link_pwq() can handle duplicate calls */
5226 	link_pwq(pwq);
5227 
5228 	old_pwq = rcu_access_pointer(*slot);
5229 	rcu_assign_pointer(*slot, pwq);
5230 	return old_pwq;
5231 }
5232 
5233 /* context to store the prepared attrs & pwqs before applying */
5234 struct apply_wqattrs_ctx {
5235 	struct workqueue_struct	*wq;		/* target workqueue */
5236 	struct workqueue_attrs	*attrs;		/* attrs to apply */
5237 	struct list_head	list;		/* queued for batching commit */
5238 	struct pool_workqueue	*dfl_pwq;
5239 	struct pool_workqueue	*pwq_tbl[];
5240 };
5241 
5242 /* free the resources after success or abort */
apply_wqattrs_cleanup(struct apply_wqattrs_ctx * ctx)5243 static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
5244 {
5245 	if (ctx) {
5246 		int cpu;
5247 
5248 		for_each_possible_cpu(cpu)
5249 			put_pwq_unlocked(ctx->pwq_tbl[cpu]);
5250 		put_pwq_unlocked(ctx->dfl_pwq);
5251 
5252 		free_workqueue_attrs(ctx->attrs);
5253 
5254 		kfree(ctx);
5255 	}
5256 }
5257 
5258 /* allocate the attrs and pwqs for later installation */
5259 static struct apply_wqattrs_ctx *
apply_wqattrs_prepare(struct workqueue_struct * wq,const struct workqueue_attrs * attrs,const cpumask_var_t unbound_cpumask)5260 apply_wqattrs_prepare(struct workqueue_struct *wq,
5261 		      const struct workqueue_attrs *attrs,
5262 		      const cpumask_var_t unbound_cpumask)
5263 {
5264 	struct apply_wqattrs_ctx *ctx;
5265 	struct workqueue_attrs *new_attrs;
5266 	int cpu;
5267 
5268 	lockdep_assert_held(&wq_pool_mutex);
5269 
5270 	if (WARN_ON(attrs->affn_scope < 0 ||
5271 		    attrs->affn_scope >= WQ_AFFN_NR_TYPES))
5272 		return ERR_PTR(-EINVAL);
5273 
5274 	ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_cpu_ids), GFP_KERNEL);
5275 
5276 	new_attrs = alloc_workqueue_attrs();
5277 	if (!ctx || !new_attrs)
5278 		goto out_free;
5279 
5280 	/*
5281 	 * If something goes wrong during CPU up/down, we'll fall back to
5282 	 * the default pwq covering whole @attrs->cpumask.  Always create
5283 	 * it even if we don't use it immediately.
5284 	 */
5285 	copy_workqueue_attrs(new_attrs, attrs);
5286 	wqattrs_actualize_cpumask(new_attrs, unbound_cpumask);
5287 	cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
5288 	ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
5289 	if (!ctx->dfl_pwq)
5290 		goto out_free;
5291 
5292 	for_each_possible_cpu(cpu) {
5293 		if (new_attrs->ordered) {
5294 			ctx->dfl_pwq->refcnt++;
5295 			ctx->pwq_tbl[cpu] = ctx->dfl_pwq;
5296 		} else {
5297 			wq_calc_pod_cpumask(new_attrs, cpu);
5298 			ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs);
5299 			if (!ctx->pwq_tbl[cpu])
5300 				goto out_free;
5301 		}
5302 	}
5303 
5304 	/* save the user configured attrs and sanitize it. */
5305 	copy_workqueue_attrs(new_attrs, attrs);
5306 	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
5307 	cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
5308 	ctx->attrs = new_attrs;
5309 
5310 	/*
5311 	 * For initialized ordered workqueues, there should only be one pwq
5312 	 * (dfl_pwq). Set the plugged flag of ctx->dfl_pwq to suspend execution
5313 	 * of newly queued work items until execution of older work items in
5314 	 * the old pwq's have completed.
5315 	 */
5316 	if ((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs))
5317 		ctx->dfl_pwq->plugged = true;
5318 
5319 	ctx->wq = wq;
5320 	return ctx;
5321 
5322 out_free:
5323 	free_workqueue_attrs(new_attrs);
5324 	apply_wqattrs_cleanup(ctx);
5325 	return ERR_PTR(-ENOMEM);
5326 }
5327 
5328 /* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
apply_wqattrs_commit(struct apply_wqattrs_ctx * ctx)5329 static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
5330 {
5331 	int cpu;
5332 
5333 	/* all pwqs have been created successfully, let's install'em */
5334 	mutex_lock(&ctx->wq->mutex);
5335 
5336 	copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
5337 
5338 	/* save the previous pwqs and install the new ones */
5339 	for_each_possible_cpu(cpu)
5340 		ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
5341 							ctx->pwq_tbl[cpu]);
5342 	ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);
5343 
5344 	/* update node_nr_active->max */
5345 	wq_update_node_max_active(ctx->wq, -1);
5346 
5347 	/* rescuer needs to respect wq cpumask changes */
5348 	if (ctx->wq->rescuer)
5349 		set_cpus_allowed_ptr(ctx->wq->rescuer->task,
5350 				     unbound_effective_cpumask(ctx->wq));
5351 
5352 	mutex_unlock(&ctx->wq->mutex);
5353 }
5354 
apply_workqueue_attrs_locked(struct workqueue_struct * wq,const struct workqueue_attrs * attrs)5355 static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
5356 					const struct workqueue_attrs *attrs)
5357 {
5358 	struct apply_wqattrs_ctx *ctx;
5359 
5360 	/* only unbound workqueues can change attributes */
5361 	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
5362 		return -EINVAL;
5363 
5364 	ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);
5365 	if (IS_ERR(ctx))
5366 		return PTR_ERR(ctx);
5367 
5368 	/* the ctx has been prepared successfully, let's commit it */
5369 	apply_wqattrs_commit(ctx);
5370 	apply_wqattrs_cleanup(ctx);
5371 
5372 	return 0;
5373 }
5374 
5375 /**
5376  * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
5377  * @wq: the target workqueue
5378  * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
5379  *
5380  * Apply @attrs to an unbound workqueue @wq. Unless disabled, this function maps
5381  * a separate pwq to each CPU pod with possibles CPUs in @attrs->cpumask so that
5382  * work items are affine to the pod it was issued on. Older pwqs are released as
5383  * in-flight work items finish. Note that a work item which repeatedly requeues
5384  * itself back-to-back will stay on its current pwq.
5385  *
5386  * Performs GFP_KERNEL allocations.
5387  *
5388  * Return: 0 on success and -errno on failure.
5389  */
apply_workqueue_attrs(struct workqueue_struct * wq,const struct workqueue_attrs * attrs)5390 int apply_workqueue_attrs(struct workqueue_struct *wq,
5391 			  const struct workqueue_attrs *attrs)
5392 {
5393 	int ret;
5394 
5395 	mutex_lock(&wq_pool_mutex);
5396 	ret = apply_workqueue_attrs_locked(wq, attrs);
5397 	mutex_unlock(&wq_pool_mutex);
5398 
5399 	return ret;
5400 }
5401 
5402 /**
5403  * unbound_wq_update_pwq - update a pwq slot for CPU hot[un]plug
5404  * @wq: the target workqueue
5405  * @cpu: the CPU to update the pwq slot for
5406  *
5407  * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
5408  * %CPU_DOWN_FAILED.  @cpu is in the same pod of the CPU being hot[un]plugged.
5409  *
5410  *
5411  * If pod affinity can't be adjusted due to memory allocation failure, it falls
5412  * back to @wq->dfl_pwq which may not be optimal but is always correct.
5413  *
5414  * Note that when the last allowed CPU of a pod goes offline for a workqueue
5415  * with a cpumask spanning multiple pods, the workers which were already
5416  * executing the work items for the workqueue will lose their CPU affinity and
5417  * may execute on any CPU. This is similar to how per-cpu workqueues behave on
5418  * CPU_DOWN. If a workqueue user wants strict affinity, it's the user's
5419  * responsibility to flush the work item from CPU_DOWN_PREPARE.
5420  */
unbound_wq_update_pwq(struct workqueue_struct * wq,int cpu)5421 static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
5422 {
5423 	struct pool_workqueue *old_pwq = NULL, *pwq;
5424 	struct workqueue_attrs *target_attrs;
5425 
5426 	lockdep_assert_held(&wq_pool_mutex);
5427 
5428 	if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered)
5429 		return;
5430 
5431 	/*
5432 	 * We don't wanna alloc/free wq_attrs for each wq for each CPU.
5433 	 * Let's use a preallocated one.  The following buf is protected by
5434 	 * CPU hotplug exclusion.
5435 	 */
5436 	target_attrs = unbound_wq_update_pwq_attrs_buf;
5437 
5438 	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
5439 	wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask);
5440 
5441 	/* nothing to do if the target cpumask matches the current pwq */
5442 	wq_calc_pod_cpumask(target_attrs, cpu);
5443 	if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
5444 		return;
5445 
5446 	/* create a new pwq */
5447 	pwq = alloc_unbound_pwq(wq, target_attrs);
5448 	if (!pwq) {
5449 		pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n",
5450 			wq->name);
5451 		goto use_dfl_pwq;
5452 	}
5453 
5454 	/* Install the new pwq. */
5455 	mutex_lock(&wq->mutex);
5456 	old_pwq = install_unbound_pwq(wq, cpu, pwq);
5457 	goto out_unlock;
5458 
5459 use_dfl_pwq:
5460 	mutex_lock(&wq->mutex);
5461 	pwq = unbound_pwq(wq, -1);
5462 	raw_spin_lock_irq(&pwq->pool->lock);
5463 	get_pwq(pwq);
5464 	raw_spin_unlock_irq(&pwq->pool->lock);
5465 	old_pwq = install_unbound_pwq(wq, cpu, pwq);
5466 out_unlock:
5467 	mutex_unlock(&wq->mutex);
5468 	put_pwq_unlocked(old_pwq);
5469 }
5470 
alloc_and_link_pwqs(struct workqueue_struct * wq)5471 static int alloc_and_link_pwqs(struct workqueue_struct *wq)
5472 {
5473 	bool highpri = wq->flags & WQ_HIGHPRI;
5474 	int cpu, ret;
5475 
5476 	lockdep_assert_held(&wq_pool_mutex);
5477 
5478 	wq->cpu_pwq = alloc_percpu(struct pool_workqueue *);
5479 	if (!wq->cpu_pwq)
5480 		goto enomem;
5481 
5482 	if (!(wq->flags & WQ_UNBOUND)) {
5483 		struct worker_pool __percpu *pools;
5484 
5485 		if (wq->flags & WQ_BH)
5486 			pools = bh_worker_pools;
5487 		else
5488 			pools = cpu_worker_pools;
5489 
5490 		for_each_possible_cpu(cpu) {
5491 			struct pool_workqueue **pwq_p;
5492 			struct worker_pool *pool;
5493 
5494 			pool = &(per_cpu_ptr(pools, cpu)[highpri]);
5495 			pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu);
5496 
5497 			*pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL,
5498 						       pool->node);
5499 			if (!*pwq_p)
5500 				goto enomem;
5501 
5502 			init_pwq(*pwq_p, wq, pool);
5503 
5504 			mutex_lock(&wq->mutex);
5505 			link_pwq(*pwq_p);
5506 			mutex_unlock(&wq->mutex);
5507 		}
5508 		return 0;
5509 	}
5510 
5511 	if (wq->flags & __WQ_ORDERED) {
5512 		struct pool_workqueue *dfl_pwq;
5513 
5514 		ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]);
5515 		/* there should only be single pwq for ordering guarantee */
5516 		dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
5517 		WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
5518 			      wq->pwqs.prev != &dfl_pwq->pwqs_node),
5519 		     "ordering guarantee broken for workqueue %s\n", wq->name);
5520 	} else {
5521 		ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]);
5522 	}
5523 
5524 	return ret;
5525 
5526 enomem:
5527 	if (wq->cpu_pwq) {
5528 		for_each_possible_cpu(cpu) {
5529 			struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
5530 
5531 			if (pwq)
5532 				kmem_cache_free(pwq_cache, pwq);
5533 		}
5534 		free_percpu(wq->cpu_pwq);
5535 		wq->cpu_pwq = NULL;
5536 	}
5537 	return -ENOMEM;
5538 }
5539 
wq_clamp_max_active(int max_active,unsigned int flags,const char * name)5540 static int wq_clamp_max_active(int max_active, unsigned int flags,
5541 			       const char *name)
5542 {
5543 	if (max_active < 1 || max_active > WQ_MAX_ACTIVE)
5544 		pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
5545 			max_active, name, 1, WQ_MAX_ACTIVE);
5546 
5547 	return clamp_val(max_active, 1, WQ_MAX_ACTIVE);
5548 }
5549 
5550 /*
5551  * Workqueues which may be used during memory reclaim should have a rescuer
5552  * to guarantee forward progress.
5553  */
init_rescuer(struct workqueue_struct * wq)5554 static int init_rescuer(struct workqueue_struct *wq)
5555 {
5556 	struct worker *rescuer;
5557 	char id_buf[WORKER_ID_LEN];
5558 	int ret;
5559 
5560 	lockdep_assert_held(&wq_pool_mutex);
5561 
5562 	if (!(wq->flags & WQ_MEM_RECLAIM))
5563 		return 0;
5564 
5565 	rescuer = alloc_worker(NUMA_NO_NODE);
5566 	if (!rescuer) {
5567 		pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n",
5568 		       wq->name);
5569 		return -ENOMEM;
5570 	}
5571 
5572 	rescuer->rescue_wq = wq;
5573 	format_worker_id(id_buf, sizeof(id_buf), rescuer, NULL);
5574 
5575 	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", id_buf);
5576 	if (IS_ERR(rescuer->task)) {
5577 		ret = PTR_ERR(rescuer->task);
5578 		pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe",
5579 		       wq->name, ERR_PTR(ret));
5580 		kfree(rescuer);
5581 		return ret;
5582 	}
5583 
5584 	wq->rescuer = rescuer;
5585 	if (wq->flags & WQ_UNBOUND)
5586 		kthread_bind_mask(rescuer->task, unbound_effective_cpumask(wq));
5587 	else
5588 		kthread_bind_mask(rescuer->task, cpu_possible_mask);
5589 	wake_up_process(rescuer->task);
5590 
5591 	return 0;
5592 }
5593 
5594 /**
5595  * wq_adjust_max_active - update a wq's max_active to the current setting
5596  * @wq: target workqueue
5597  *
5598  * If @wq isn't freezing, set @wq->max_active to the saved_max_active and
5599  * activate inactive work items accordingly. If @wq is freezing, clear
5600  * @wq->max_active to zero.
5601  */
wq_adjust_max_active(struct workqueue_struct * wq)5602 static void wq_adjust_max_active(struct workqueue_struct *wq)
5603 {
5604 	bool activated;
5605 	int new_max, new_min;
5606 
5607 	lockdep_assert_held(&wq->mutex);
5608 
5609 	if ((wq->flags & WQ_FREEZABLE) && workqueue_freezing) {
5610 		new_max = 0;
5611 		new_min = 0;
5612 	} else {
5613 		new_max = wq->saved_max_active;
5614 		new_min = wq->saved_min_active;
5615 	}
5616 
5617 	if (wq->max_active == new_max && wq->min_active == new_min)
5618 		return;
5619 
5620 	/*
5621 	 * Update @wq->max/min_active and then kick inactive work items if more
5622 	 * active work items are allowed. This doesn't break work item ordering
5623 	 * because new work items are always queued behind existing inactive
5624 	 * work items if there are any.
5625 	 */
5626 	WRITE_ONCE(wq->max_active, new_max);
5627 	WRITE_ONCE(wq->min_active, new_min);
5628 
5629 	if (wq->flags & WQ_UNBOUND)
5630 		wq_update_node_max_active(wq, -1);
5631 
5632 	if (new_max == 0)
5633 		return;
5634 
5635 	/*
5636 	 * Round-robin through pwq's activating the first inactive work item
5637 	 * until max_active is filled.
5638 	 */
5639 	do {
5640 		struct pool_workqueue *pwq;
5641 
5642 		activated = false;
5643 		for_each_pwq(pwq, wq) {
5644 			unsigned long irq_flags;
5645 
5646 			/* can be called during early boot w/ irq disabled */
5647 			raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
5648 			if (pwq_activate_first_inactive(pwq, true)) {
5649 				activated = true;
5650 				kick_pool(pwq->pool);
5651 			}
5652 			raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
5653 		}
5654 	} while (activated);
5655 }
5656 
5657 __printf(1, 0)
__alloc_workqueue(const char * fmt,unsigned int flags,int max_active,va_list args)5658 static struct workqueue_struct *__alloc_workqueue(const char *fmt,
5659 						  unsigned int flags,
5660 						  int max_active, va_list args)
5661 {
5662 	struct workqueue_struct *wq;
5663 	size_t wq_size;
5664 	int name_len;
5665 
5666 	if (flags & WQ_BH) {
5667 		if (WARN_ON_ONCE(flags & ~__WQ_BH_ALLOWS))
5668 			return NULL;
5669 		if (WARN_ON_ONCE(max_active))
5670 			return NULL;
5671 	}
5672 
5673 	/* see the comment above the definition of WQ_POWER_EFFICIENT */
5674 	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
5675 		flags |= WQ_UNBOUND;
5676 
5677 	/* allocate wq and format name */
5678 	if (flags & WQ_UNBOUND)
5679 		wq_size = struct_size(wq, node_nr_active, nr_node_ids + 1);
5680 	else
5681 		wq_size = sizeof(*wq);
5682 
5683 	wq = kzalloc(wq_size, GFP_KERNEL);
5684 	if (!wq)
5685 		return NULL;
5686 
5687 	if (flags & WQ_UNBOUND) {
5688 		wq->unbound_attrs = alloc_workqueue_attrs();
5689 		if (!wq->unbound_attrs)
5690 			goto err_free_wq;
5691 	}
5692 
5693 	name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args);
5694 
5695 	if (name_len >= WQ_NAME_LEN)
5696 		pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n",
5697 			     wq->name);
5698 
5699 	if (flags & WQ_BH) {
5700 		/*
5701 		 * BH workqueues always share a single execution context per CPU
5702 		 * and don't impose any max_active limit.
5703 		 */
5704 		max_active = INT_MAX;
5705 	} else {
5706 		max_active = max_active ?: WQ_DFL_ACTIVE;
5707 		max_active = wq_clamp_max_active(max_active, flags, wq->name);
5708 	}
5709 
5710 	/* init wq */
5711 	wq->flags = flags;
5712 	wq->max_active = max_active;
5713 	wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);
5714 	wq->saved_max_active = wq->max_active;
5715 	wq->saved_min_active = wq->min_active;
5716 	mutex_init(&wq->mutex);
5717 	atomic_set(&wq->nr_pwqs_to_flush, 0);
5718 	INIT_LIST_HEAD(&wq->pwqs);
5719 	INIT_LIST_HEAD(&wq->flusher_queue);
5720 	INIT_LIST_HEAD(&wq->flusher_overflow);
5721 	INIT_LIST_HEAD(&wq->maydays);
5722 
5723 	INIT_LIST_HEAD(&wq->list);
5724 
5725 	if (flags & WQ_UNBOUND) {
5726 		if (alloc_node_nr_active(wq->node_nr_active) < 0)
5727 			goto err_free_wq;
5728 	}
5729 
5730 	/*
5731 	 * wq_pool_mutex protects the workqueues list, allocations of PWQs,
5732 	 * and the global freeze state.
5733 	 */
5734 	apply_wqattrs_lock();
5735 
5736 	if (alloc_and_link_pwqs(wq) < 0)
5737 		goto err_unlock_free_node_nr_active;
5738 
5739 	mutex_lock(&wq->mutex);
5740 	wq_adjust_max_active(wq);
5741 	mutex_unlock(&wq->mutex);
5742 
5743 	list_add_tail_rcu(&wq->list, &workqueues);
5744 
5745 	if (wq_online && init_rescuer(wq) < 0)
5746 		goto err_unlock_destroy;
5747 
5748 	apply_wqattrs_unlock();
5749 
5750 	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
5751 		goto err_destroy;
5752 
5753 	return wq;
5754 
5755 err_unlock_free_node_nr_active:
5756 	apply_wqattrs_unlock();
5757 	/*
5758 	 * Failed alloc_and_link_pwqs() may leave pending pwq->release_work,
5759 	 * flushing the pwq_release_worker ensures that the pwq_release_workfn()
5760 	 * completes before calling kfree(wq).
5761 	 */
5762 	if (wq->flags & WQ_UNBOUND) {
5763 		kthread_flush_worker(pwq_release_worker);
5764 		free_node_nr_active(wq->node_nr_active);
5765 	}
5766 err_free_wq:
5767 	free_workqueue_attrs(wq->unbound_attrs);
5768 	kfree(wq);
5769 	return NULL;
5770 err_unlock_destroy:
5771 	apply_wqattrs_unlock();
5772 err_destroy:
5773 	destroy_workqueue(wq);
5774 	return NULL;
5775 }
5776 
5777 __printf(1, 4)
alloc_workqueue(const char * fmt,unsigned int flags,int max_active,...)5778 struct workqueue_struct *alloc_workqueue(const char *fmt,
5779 					 unsigned int flags,
5780 					 int max_active, ...)
5781 {
5782 	struct workqueue_struct *wq;
5783 	va_list args;
5784 
5785 	va_start(args, max_active);
5786 	wq = __alloc_workqueue(fmt, flags, max_active, args);
5787 	va_end(args);
5788 	if (!wq)
5789 		return NULL;
5790 
5791 	wq_init_lockdep(wq);
5792 
5793 	return wq;
5794 }
5795 EXPORT_SYMBOL_GPL(alloc_workqueue);
5796 
5797 #ifdef CONFIG_LOCKDEP
5798 __printf(1, 5)
5799 struct workqueue_struct *
alloc_workqueue_lockdep_map(const char * fmt,unsigned int flags,int max_active,struct lockdep_map * lockdep_map,...)5800 alloc_workqueue_lockdep_map(const char *fmt, unsigned int flags,
5801 			    int max_active, struct lockdep_map *lockdep_map, ...)
5802 {
5803 	struct workqueue_struct *wq;
5804 	va_list args;
5805 
5806 	va_start(args, lockdep_map);
5807 	wq = __alloc_workqueue(fmt, flags, max_active, args);
5808 	va_end(args);
5809 	if (!wq)
5810 		return NULL;
5811 
5812 	wq->lockdep_map = lockdep_map;
5813 
5814 	return wq;
5815 }
5816 EXPORT_SYMBOL_GPL(alloc_workqueue_lockdep_map);
5817 #endif
5818 
pwq_busy(struct pool_workqueue * pwq)5819 static bool pwq_busy(struct pool_workqueue *pwq)
5820 {
5821 	int i;
5822 
5823 	for (i = 0; i < WORK_NR_COLORS; i++)
5824 		if (pwq->nr_in_flight[i])
5825 			return true;
5826 
5827 	if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
5828 		return true;
5829 	if (!pwq_is_empty(pwq))
5830 		return true;
5831 
5832 	return false;
5833 }
5834 
5835 /**
5836  * destroy_workqueue - safely terminate a workqueue
5837  * @wq: target workqueue
5838  *
5839  * Safely destroy a workqueue. All work currently pending will be done first.
5840  */
destroy_workqueue(struct workqueue_struct * wq)5841 void destroy_workqueue(struct workqueue_struct *wq)
5842 {
5843 	struct pool_workqueue *pwq;
5844 	int cpu;
5845 
5846 	/*
5847 	 * Remove it from sysfs first so that sanity check failure doesn't
5848 	 * lead to sysfs name conflicts.
5849 	 */
5850 	workqueue_sysfs_unregister(wq);
5851 
5852 	/* mark the workqueue destruction is in progress */
5853 	mutex_lock(&wq->mutex);
5854 	wq->flags |= __WQ_DESTROYING;
5855 	mutex_unlock(&wq->mutex);
5856 
5857 	/* drain it before proceeding with destruction */
5858 	drain_workqueue(wq);
5859 
5860 	/* kill rescuer, if sanity checks fail, leave it w/o rescuer */
5861 	if (wq->rescuer) {
5862 		struct worker *rescuer = wq->rescuer;
5863 
5864 		/* this prevents new queueing */
5865 		raw_spin_lock_irq(&wq_mayday_lock);
5866 		wq->rescuer = NULL;
5867 		raw_spin_unlock_irq(&wq_mayday_lock);
5868 
5869 		/* rescuer will empty maydays list before exiting */
5870 		kthread_stop(rescuer->task);
5871 		kfree(rescuer);
5872 	}
5873 
5874 	/*
5875 	 * Sanity checks - grab all the locks so that we wait for all
5876 	 * in-flight operations which may do put_pwq().
5877 	 */
5878 	mutex_lock(&wq_pool_mutex);
5879 	mutex_lock(&wq->mutex);
5880 	for_each_pwq(pwq, wq) {
5881 		raw_spin_lock_irq(&pwq->pool->lock);
5882 		if (WARN_ON(pwq_busy(pwq))) {
5883 			pr_warn("%s: %s has the following busy pwq\n",
5884 				__func__, wq->name);
5885 			show_pwq(pwq);
5886 			raw_spin_unlock_irq(&pwq->pool->lock);
5887 			mutex_unlock(&wq->mutex);
5888 			mutex_unlock(&wq_pool_mutex);
5889 			show_one_workqueue(wq);
5890 			return;
5891 		}
5892 		raw_spin_unlock_irq(&pwq->pool->lock);
5893 	}
5894 	mutex_unlock(&wq->mutex);
5895 
5896 	/*
5897 	 * wq list is used to freeze wq, remove from list after
5898 	 * flushing is complete in case freeze races us.
5899 	 */
5900 	list_del_rcu(&wq->list);
5901 	mutex_unlock(&wq_pool_mutex);
5902 
5903 	/*
5904 	 * We're the sole accessor of @wq. Directly access cpu_pwq and dfl_pwq
5905 	 * to put the base refs. @wq will be auto-destroyed from the last
5906 	 * pwq_put. RCU read lock prevents @wq from going away from under us.
5907 	 */
5908 	rcu_read_lock();
5909 
5910 	for_each_possible_cpu(cpu) {
5911 		put_pwq_unlocked(unbound_pwq(wq, cpu));
5912 		RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
5913 	}
5914 
5915 	put_pwq_unlocked(unbound_pwq(wq, -1));
5916 	RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);
5917 
5918 	rcu_read_unlock();
5919 }
5920 EXPORT_SYMBOL_GPL(destroy_workqueue);
5921 
5922 /**
5923  * workqueue_set_max_active - adjust max_active of a workqueue
5924  * @wq: target workqueue
5925  * @max_active: new max_active value.
5926  *
5927  * Set max_active of @wq to @max_active. See the alloc_workqueue() function
5928  * comment.
5929  *
5930  * CONTEXT:
5931  * Don't call from IRQ context.
5932  */
workqueue_set_max_active(struct workqueue_struct * wq,int max_active)5933 void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
5934 {
5935 	/* max_active doesn't mean anything for BH workqueues */
5936 	if (WARN_ON(wq->flags & WQ_BH))
5937 		return;
5938 	/* disallow meddling with max_active for ordered workqueues */
5939 	if (WARN_ON(wq->flags & __WQ_ORDERED))
5940 		return;
5941 
5942 	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
5943 
5944 	mutex_lock(&wq->mutex);
5945 
5946 	wq->saved_max_active = max_active;
5947 	if (wq->flags & WQ_UNBOUND)
5948 		wq->saved_min_active = min(wq->saved_min_active, max_active);
5949 
5950 	wq_adjust_max_active(wq);
5951 
5952 	mutex_unlock(&wq->mutex);
5953 }
5954 EXPORT_SYMBOL_GPL(workqueue_set_max_active);
5955 
5956 /**
5957  * workqueue_set_min_active - adjust min_active of an unbound workqueue
5958  * @wq: target unbound workqueue
5959  * @min_active: new min_active value
5960  *
5961  * Set min_active of an unbound workqueue. Unlike other types of workqueues, an
5962  * unbound workqueue is not guaranteed to be able to process max_active
5963  * interdependent work items. Instead, an unbound workqueue is guaranteed to be
5964  * able to process min_active number of interdependent work items which is
5965  * %WQ_DFL_MIN_ACTIVE by default.
5966  *
5967  * Use this function to adjust the min_active value between 0 and the current
5968  * max_active.
5969  */
workqueue_set_min_active(struct workqueue_struct * wq,int min_active)5970 void workqueue_set_min_active(struct workqueue_struct *wq, int min_active)
5971 {
5972 	/* min_active is only meaningful for non-ordered unbound workqueues */
5973 	if (WARN_ON((wq->flags & (WQ_BH | WQ_UNBOUND | __WQ_ORDERED)) !=
5974 		    WQ_UNBOUND))
5975 		return;
5976 
5977 	mutex_lock(&wq->mutex);
5978 	wq->saved_min_active = clamp(min_active, 0, wq->saved_max_active);
5979 	wq_adjust_max_active(wq);
5980 	mutex_unlock(&wq->mutex);
5981 }
5982 
5983 /**
5984  * current_work - retrieve %current task's work struct
5985  *
5986  * Determine if %current task is a workqueue worker and what it's working on.
5987  * Useful to find out the context that the %current task is running in.
5988  *
5989  * Return: work struct if %current task is a workqueue worker, %NULL otherwise.
5990  */
current_work(void)5991 struct work_struct *current_work(void)
5992 {
5993 	struct worker *worker = current_wq_worker();
5994 
5995 	return worker ? worker->current_work : NULL;
5996 }
5997 EXPORT_SYMBOL(current_work);
5998 
5999 /**
6000  * current_is_workqueue_rescuer - is %current workqueue rescuer?
6001  *
6002  * Determine whether %current is a workqueue rescuer.  Can be used from
6003  * work functions to determine whether it's being run off the rescuer task.
6004  *
6005  * Return: %true if %current is a workqueue rescuer. %false otherwise.
6006  */
current_is_workqueue_rescuer(void)6007 bool current_is_workqueue_rescuer(void)
6008 {
6009 	struct worker *worker = current_wq_worker();
6010 
6011 	return worker && worker->rescue_wq;
6012 }
6013 
6014 /**
6015  * workqueue_congested - test whether a workqueue is congested
6016  * @cpu: CPU in question
6017  * @wq: target workqueue
6018  *
6019  * Test whether @wq's cpu workqueue for @cpu is congested.  There is
6020  * no synchronization around this function and the test result is
6021  * unreliable and only useful as advisory hints or for debugging.
6022  *
6023  * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
6024  *
6025  * With the exception of ordered workqueues, all workqueues have per-cpu
6026  * pool_workqueues, each with its own congested state. A workqueue being
6027  * congested on one CPU doesn't mean that the workqueue is contested on any
6028  * other CPUs.
6029  *
6030  * Return:
6031  * %true if congested, %false otherwise.
6032  */
workqueue_congested(int cpu,struct workqueue_struct * wq)6033 bool workqueue_congested(int cpu, struct workqueue_struct *wq)
6034 {
6035 	struct pool_workqueue *pwq;
6036 	bool ret;
6037 
6038 	rcu_read_lock();
6039 	preempt_disable();
6040 
6041 	if (cpu == WORK_CPU_UNBOUND)
6042 		cpu = smp_processor_id();
6043 
6044 	pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
6045 	ret = !list_empty(&pwq->inactive_works);
6046 
6047 	preempt_enable();
6048 	rcu_read_unlock();
6049 
6050 	return ret;
6051 }
6052 EXPORT_SYMBOL_GPL(workqueue_congested);
6053 
6054 /**
6055  * work_busy - test whether a work is currently pending or running
6056  * @work: the work to be tested
6057  *
6058  * Test whether @work is currently pending or running.  There is no
6059  * synchronization around this function and the test result is
6060  * unreliable and only useful as advisory hints or for debugging.
6061  *
6062  * Return:
6063  * OR'd bitmask of WORK_BUSY_* bits.
6064  */
work_busy(struct work_struct * work)6065 unsigned int work_busy(struct work_struct *work)
6066 {
6067 	struct worker_pool *pool;
6068 	unsigned long irq_flags;
6069 	unsigned int ret = 0;
6070 
6071 	if (work_pending(work))
6072 		ret |= WORK_BUSY_PENDING;
6073 
6074 	rcu_read_lock();
6075 	pool = get_work_pool(work);
6076 	if (pool) {
6077 		raw_spin_lock_irqsave(&pool->lock, irq_flags);
6078 		if (find_worker_executing_work(pool, work))
6079 			ret |= WORK_BUSY_RUNNING;
6080 		raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
6081 	}
6082 	rcu_read_unlock();
6083 
6084 	return ret;
6085 }
6086 EXPORT_SYMBOL_GPL(work_busy);
6087 
6088 /**
6089  * set_worker_desc - set description for the current work item
6090  * @fmt: printf-style format string
6091  * @...: arguments for the format string
6092  *
6093  * This function can be called by a running work function to describe what
6094  * the work item is about.  If the worker task gets dumped, this
6095  * information will be printed out together to help debugging.  The
6096  * description can be at most WORKER_DESC_LEN including the trailing '\0'.
6097  */
set_worker_desc(const char * fmt,...)6098 void set_worker_desc(const char *fmt, ...)
6099 {
6100 	struct worker *worker = current_wq_worker();
6101 	va_list args;
6102 
6103 	if (worker) {
6104 		va_start(args, fmt);
6105 		vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
6106 		va_end(args);
6107 	}
6108 }
6109 EXPORT_SYMBOL_GPL(set_worker_desc);
6110 
6111 /**
6112  * print_worker_info - print out worker information and description
6113  * @log_lvl: the log level to use when printing
6114  * @task: target task
6115  *
6116  * If @task is a worker and currently executing a work item, print out the
6117  * name of the workqueue being serviced and worker description set with
6118  * set_worker_desc() by the currently executing work item.
6119  *
6120  * This function can be safely called on any task as long as the
6121  * task_struct itself is accessible.  While safe, this function isn't
6122  * synchronized and may print out mixups or garbages of limited length.
6123  */
print_worker_info(const char * log_lvl,struct task_struct * task)6124 void print_worker_info(const char *log_lvl, struct task_struct *task)
6125 {
6126 	work_func_t *fn = NULL;
6127 	char name[WQ_NAME_LEN] = { };
6128 	char desc[WORKER_DESC_LEN] = { };
6129 	struct pool_workqueue *pwq = NULL;
6130 	struct workqueue_struct *wq = NULL;
6131 	struct worker *worker;
6132 
6133 	if (!(task->flags & PF_WQ_WORKER))
6134 		return;
6135 
6136 	/*
6137 	 * This function is called without any synchronization and @task
6138 	 * could be in any state.  Be careful with dereferences.
6139 	 */
6140 	worker = kthread_probe_data(task);
6141 
6142 	/*
6143 	 * Carefully copy the associated workqueue's workfn, name and desc.
6144 	 * Keep the original last '\0' in case the original is garbage.
6145 	 */
6146 	copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
6147 	copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
6148 	copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
6149 	copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
6150 	copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);
6151 
6152 	if (fn || name[0] || desc[0]) {
6153 		printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
6154 		if (strcmp(name, desc))
6155 			pr_cont(" (%s)", desc);
6156 		pr_cont("\n");
6157 	}
6158 }
6159 
pr_cont_pool_info(struct worker_pool * pool)6160 static void pr_cont_pool_info(struct worker_pool *pool)
6161 {
6162 	pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
6163 	if (pool->node != NUMA_NO_NODE)
6164 		pr_cont(" node=%d", pool->node);
6165 	pr_cont(" flags=0x%x", pool->flags);
6166 	if (pool->flags & POOL_BH)
6167 		pr_cont(" bh%s",
6168 			pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
6169 	else
6170 		pr_cont(" nice=%d", pool->attrs->nice);
6171 }
6172 
pr_cont_worker_id(struct worker * worker)6173 static void pr_cont_worker_id(struct worker *worker)
6174 {
6175 	struct worker_pool *pool = worker->pool;
6176 
6177 	if (pool->flags & WQ_BH)
6178 		pr_cont("bh%s",
6179 			pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
6180 	else
6181 		pr_cont("%d%s", task_pid_nr(worker->task),
6182 			worker->rescue_wq ? "(RESCUER)" : "");
6183 }
6184 
6185 struct pr_cont_work_struct {
6186 	bool comma;
6187 	work_func_t func;
6188 	long ctr;
6189 };
6190 
pr_cont_work_flush(bool comma,work_func_t func,struct pr_cont_work_struct * pcwsp)6191 static void pr_cont_work_flush(bool comma, work_func_t func, struct pr_cont_work_struct *pcwsp)
6192 {
6193 	if (!pcwsp->ctr)
6194 		goto out_record;
6195 	if (func == pcwsp->func) {
6196 		pcwsp->ctr++;
6197 		return;
6198 	}
6199 	if (pcwsp->ctr == 1)
6200 		pr_cont("%s %ps", pcwsp->comma ? "," : "", pcwsp->func);
6201 	else
6202 		pr_cont("%s %ld*%ps", pcwsp->comma ? "," : "", pcwsp->ctr, pcwsp->func);
6203 	pcwsp->ctr = 0;
6204 out_record:
6205 	if ((long)func == -1L)
6206 		return;
6207 	pcwsp->comma = comma;
6208 	pcwsp->func = func;
6209 	pcwsp->ctr = 1;
6210 }
6211 
pr_cont_work(bool comma,struct work_struct * work,struct pr_cont_work_struct * pcwsp)6212 static void pr_cont_work(bool comma, struct work_struct *work, struct pr_cont_work_struct *pcwsp)
6213 {
6214 	if (work->func == wq_barrier_func) {
6215 		struct wq_barrier *barr;
6216 
6217 		barr = container_of(work, struct wq_barrier, work);
6218 
6219 		pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
6220 		pr_cont("%s BAR(%d)", comma ? "," : "",
6221 			task_pid_nr(barr->task));
6222 	} else {
6223 		if (!comma)
6224 			pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
6225 		pr_cont_work_flush(comma, work->func, pcwsp);
6226 	}
6227 }
6228 
show_pwq(struct pool_workqueue * pwq)6229 static void show_pwq(struct pool_workqueue *pwq)
6230 {
6231 	struct pr_cont_work_struct pcws = { .ctr = 0, };
6232 	struct worker_pool *pool = pwq->pool;
6233 	struct work_struct *work;
6234 	struct worker *worker;
6235 	bool has_in_flight = false, has_pending = false;
6236 	int bkt;
6237 
6238 	pr_info("  pwq %d:", pool->id);
6239 	pr_cont_pool_info(pool);
6240 
6241 	pr_cont(" active=%d refcnt=%d%s\n",
6242 		pwq->nr_active, pwq->refcnt,
6243 		!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
6244 
6245 	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
6246 		if (worker->current_pwq == pwq) {
6247 			has_in_flight = true;
6248 			break;
6249 		}
6250 	}
6251 	if (has_in_flight) {
6252 		bool comma = false;
6253 
6254 		pr_info("    in-flight:");
6255 		hash_for_each(pool->busy_hash, bkt, worker, hentry) {
6256 			if (worker->current_pwq != pwq)
6257 				continue;
6258 
6259 			pr_cont(" %s", comma ? "," : "");
6260 			pr_cont_worker_id(worker);
6261 			pr_cont(":%ps", worker->current_func);
6262 			list_for_each_entry(work, &worker->scheduled, entry)
6263 				pr_cont_work(false, work, &pcws);
6264 			pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
6265 			comma = true;
6266 		}
6267 		pr_cont("\n");
6268 	}
6269 
6270 	list_for_each_entry(work, &pool->worklist, entry) {
6271 		if (get_work_pwq(work) == pwq) {
6272 			has_pending = true;
6273 			break;
6274 		}
6275 	}
6276 	if (has_pending) {
6277 		bool comma = false;
6278 
6279 		pr_info("    pending:");
6280 		list_for_each_entry(work, &pool->worklist, entry) {
6281 			if (get_work_pwq(work) != pwq)
6282 				continue;
6283 
6284 			pr_cont_work(comma, work, &pcws);
6285 			comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
6286 		}
6287 		pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
6288 		pr_cont("\n");
6289 	}
6290 
6291 	if (!list_empty(&pwq->inactive_works)) {
6292 		bool comma = false;
6293 
6294 		pr_info("    inactive:");
6295 		list_for_each_entry(work, &pwq->inactive_works, entry) {
6296 			pr_cont_work(comma, work, &pcws);
6297 			comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
6298 		}
6299 		pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
6300 		pr_cont("\n");
6301 	}
6302 }
6303 
6304 /**
6305  * show_one_workqueue - dump state of specified workqueue
6306  * @wq: workqueue whose state will be printed
6307  */
show_one_workqueue(struct workqueue_struct * wq)6308 void show_one_workqueue(struct workqueue_struct *wq)
6309 {
6310 	struct pool_workqueue *pwq;
6311 	bool idle = true;
6312 	unsigned long irq_flags;
6313 
6314 	for_each_pwq(pwq, wq) {
6315 		if (!pwq_is_empty(pwq)) {
6316 			idle = false;
6317 			break;
6318 		}
6319 	}
6320 	if (idle) /* Nothing to print for idle workqueue */
6321 		return;
6322 
6323 	pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
6324 
6325 	for_each_pwq(pwq, wq) {
6326 		raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
6327 		if (!pwq_is_empty(pwq)) {
6328 			/*
6329 			 * Defer printing to avoid deadlocks in console
6330 			 * drivers that queue work while holding locks
6331 			 * also taken in their write paths.
6332 			 */
6333 			printk_deferred_enter();
6334 			show_pwq(pwq);
6335 			printk_deferred_exit();
6336 		}
6337 		raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
6338 		/*
6339 		 * We could be printing a lot from atomic context, e.g.
6340 		 * sysrq-t -> show_all_workqueues(). Avoid triggering
6341 		 * hard lockup.
6342 		 */
6343 		touch_nmi_watchdog();
6344 	}
6345 
6346 }
6347 
6348 /**
6349  * show_one_worker_pool - dump state of specified worker pool
6350  * @pool: worker pool whose state will be printed
6351  */
show_one_worker_pool(struct worker_pool * pool)6352 static void show_one_worker_pool(struct worker_pool *pool)
6353 {
6354 	struct worker *worker;
6355 	bool first = true;
6356 	unsigned long irq_flags;
6357 	unsigned long hung = 0;
6358 
6359 	raw_spin_lock_irqsave(&pool->lock, irq_flags);
6360 	if (pool->nr_workers == pool->nr_idle)
6361 		goto next_pool;
6362 
6363 	/* How long the first pending work is waiting for a worker. */
6364 	if (!list_empty(&pool->worklist))
6365 		hung = jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000;
6366 
6367 	/*
6368 	 * Defer printing to avoid deadlocks in console drivers that
6369 	 * queue work while holding locks also taken in their write
6370 	 * paths.
6371 	 */
6372 	printk_deferred_enter();
6373 	pr_info("pool %d:", pool->id);
6374 	pr_cont_pool_info(pool);
6375 	pr_cont(" hung=%lus workers=%d", hung, pool->nr_workers);
6376 	if (pool->manager)
6377 		pr_cont(" manager: %d",
6378 			task_pid_nr(pool->manager->task));
6379 	list_for_each_entry(worker, &pool->idle_list, entry) {
6380 		pr_cont(" %s", first ? "idle: " : "");
6381 		pr_cont_worker_id(worker);
6382 		first = false;
6383 	}
6384 	pr_cont("\n");
6385 	printk_deferred_exit();
6386 next_pool:
6387 	raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
6388 	/*
6389 	 * We could be printing a lot from atomic context, e.g.
6390 	 * sysrq-t -> show_all_workqueues(). Avoid triggering
6391 	 * hard lockup.
6392 	 */
6393 	touch_nmi_watchdog();
6394 
6395 }
6396 
6397 /**
6398  * show_all_workqueues - dump workqueue state
6399  *
6400  * Called from a sysrq handler and prints out all busy workqueues and pools.
6401  */
show_all_workqueues(void)6402 void show_all_workqueues(void)
6403 {
6404 	struct workqueue_struct *wq;
6405 	struct worker_pool *pool;
6406 	int pi;
6407 
6408 	rcu_read_lock();
6409 
6410 	pr_info("Showing busy workqueues and worker pools:\n");
6411 
6412 	list_for_each_entry_rcu(wq, &workqueues, list)
6413 		show_one_workqueue(wq);
6414 
6415 	for_each_pool(pool, pi)
6416 		show_one_worker_pool(pool);
6417 
6418 	rcu_read_unlock();
6419 }
6420 
6421 /**
6422  * show_freezable_workqueues - dump freezable workqueue state
6423  *
6424  * Called from try_to_freeze_tasks() and prints out all freezable workqueues
6425  * still busy.
6426  */
show_freezable_workqueues(void)6427 void show_freezable_workqueues(void)
6428 {
6429 	struct workqueue_struct *wq;
6430 
6431 	rcu_read_lock();
6432 
6433 	pr_info("Showing freezable workqueues that are still busy:\n");
6434 
6435 	list_for_each_entry_rcu(wq, &workqueues, list) {
6436 		if (!(wq->flags & WQ_FREEZABLE))
6437 			continue;
6438 		show_one_workqueue(wq);
6439 	}
6440 
6441 	rcu_read_unlock();
6442 }
6443 
6444 /* used to show worker information through /proc/PID/{comm,stat,status} */
wq_worker_comm(char * buf,size_t size,struct task_struct * task)6445 void wq_worker_comm(char *buf, size_t size, struct task_struct *task)
6446 {
6447 	/* stabilize PF_WQ_WORKER and worker pool association */
6448 	mutex_lock(&wq_pool_attach_mutex);
6449 
6450 	if (task->flags & PF_WQ_WORKER) {
6451 		struct worker *worker = kthread_data(task);
6452 		struct worker_pool *pool = worker->pool;
6453 		int off;
6454 
6455 		off = format_worker_id(buf, size, worker, pool);
6456 
6457 		if (pool) {
6458 			raw_spin_lock_irq(&pool->lock);
6459 			/*
6460 			 * ->desc tracks information (wq name or
6461 			 * set_worker_desc()) for the latest execution.  If
6462 			 * current, prepend '+', otherwise '-'.
6463 			 */
6464 			if (worker->desc[0] != '\0') {
6465 				if (worker->current_work)
6466 					scnprintf(buf + off, size - off, "+%s",
6467 						  worker->desc);
6468 				else
6469 					scnprintf(buf + off, size - off, "-%s",
6470 						  worker->desc);
6471 			}
6472 			raw_spin_unlock_irq(&pool->lock);
6473 		}
6474 	} else {
6475 		strscpy(buf, task->comm, size);
6476 	}
6477 
6478 	mutex_unlock(&wq_pool_attach_mutex);
6479 }
6480 
6481 #ifdef CONFIG_SMP
6482 
6483 /*
6484  * CPU hotplug.
6485  *
6486  * There are two challenges in supporting CPU hotplug.  Firstly, there
6487  * are a lot of assumptions on strong associations among work, pwq and
6488  * pool which make migrating pending and scheduled works very
6489  * difficult to implement without impacting hot paths.  Secondly,
6490  * worker pools serve mix of short, long and very long running works making
6491  * blocked draining impractical.
6492  *
6493  * This is solved by allowing the pools to be disassociated from the CPU
6494  * running as an unbound one and allowing it to be reattached later if the
6495  * cpu comes back online.
6496  */
6497 
unbind_workers(int cpu)6498 static void unbind_workers(int cpu)
6499 {
6500 	struct worker_pool *pool;
6501 	struct worker *worker;
6502 
6503 	for_each_cpu_worker_pool(pool, cpu) {
6504 		mutex_lock(&wq_pool_attach_mutex);
6505 		raw_spin_lock_irq(&pool->lock);
6506 
6507 		/*
6508 		 * We've blocked all attach/detach operations. Make all workers
6509 		 * unbound and set DISASSOCIATED.  Before this, all workers
6510 		 * must be on the cpu.  After this, they may become diasporas.
6511 		 * And the preemption disabled section in their sched callbacks
6512 		 * are guaranteed to see WORKER_UNBOUND since the code here
6513 		 * is on the same cpu.
6514 		 */
6515 		for_each_pool_worker(worker, pool)
6516 			worker->flags |= WORKER_UNBOUND;
6517 
6518 		pool->flags |= POOL_DISASSOCIATED;
6519 
6520 		/*
6521 		 * The handling of nr_running in sched callbacks are disabled
6522 		 * now.  Zap nr_running.  After this, nr_running stays zero and
6523 		 * need_more_worker() and keep_working() are always true as
6524 		 * long as the worklist is not empty.  This pool now behaves as
6525 		 * an unbound (in terms of concurrency management) pool which
6526 		 * are served by workers tied to the pool.
6527 		 */
6528 		pool->nr_running = 0;
6529 
6530 		/*
6531 		 * With concurrency management just turned off, a busy
6532 		 * worker blocking could lead to lengthy stalls.  Kick off
6533 		 * unbound chain execution of currently pending work items.
6534 		 */
6535 		kick_pool(pool);
6536 
6537 		raw_spin_unlock_irq(&pool->lock);
6538 
6539 		for_each_pool_worker(worker, pool)
6540 			unbind_worker(worker);
6541 
6542 		mutex_unlock(&wq_pool_attach_mutex);
6543 	}
6544 }
6545 
6546 /**
6547  * rebind_workers - rebind all workers of a pool to the associated CPU
6548  * @pool: pool of interest
6549  *
6550  * @pool->cpu is coming online.  Rebind all workers to the CPU.
6551  */
rebind_workers(struct worker_pool * pool)6552 static void rebind_workers(struct worker_pool *pool)
6553 {
6554 	struct worker *worker;
6555 
6556 	lockdep_assert_held(&wq_pool_attach_mutex);
6557 
6558 	/*
6559 	 * Restore CPU affinity of all workers.  As all idle workers should
6560 	 * be on the run-queue of the associated CPU before any local
6561 	 * wake-ups for concurrency management happen, restore CPU affinity
6562 	 * of all workers first and then clear UNBOUND.  As we're called
6563 	 * from CPU_ONLINE, the following shouldn't fail.
6564 	 */
6565 	for_each_pool_worker(worker, pool) {
6566 		kthread_set_per_cpu(worker->task, pool->cpu);
6567 		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
6568 						  pool_allowed_cpus(pool)) < 0);
6569 	}
6570 
6571 	raw_spin_lock_irq(&pool->lock);
6572 
6573 	pool->flags &= ~POOL_DISASSOCIATED;
6574 
6575 	for_each_pool_worker(worker, pool) {
6576 		unsigned int worker_flags = worker->flags;
6577 
6578 		/*
6579 		 * We want to clear UNBOUND but can't directly call
6580 		 * worker_clr_flags() or adjust nr_running.  Atomically
6581 		 * replace UNBOUND with another NOT_RUNNING flag REBOUND.
6582 		 * @worker will clear REBOUND using worker_clr_flags() when
6583 		 * it initiates the next execution cycle thus restoring
6584 		 * concurrency management.  Note that when or whether
6585 		 * @worker clears REBOUND doesn't affect correctness.
6586 		 *
6587 		 * WRITE_ONCE() is necessary because @worker->flags may be
6588 		 * tested without holding any lock in
6589 		 * wq_worker_running().  Without it, NOT_RUNNING test may
6590 		 * fail incorrectly leading to premature concurrency
6591 		 * management operations.
6592 		 */
6593 		WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
6594 		worker_flags |= WORKER_REBOUND;
6595 		worker_flags &= ~WORKER_UNBOUND;
6596 		WRITE_ONCE(worker->flags, worker_flags);
6597 	}
6598 
6599 	raw_spin_unlock_irq(&pool->lock);
6600 }
6601 
6602 /**
6603  * restore_unbound_workers_cpumask - restore cpumask of unbound workers
6604  * @pool: unbound pool of interest
6605  * @cpu: the CPU which is coming up
6606  *
6607  * An unbound pool may end up with a cpumask which doesn't have any online
6608  * CPUs.  When a worker of such pool get scheduled, the scheduler resets
6609  * its cpus_allowed.  If @cpu is in @pool's cpumask which didn't have any
6610  * online CPU before, cpus_allowed of all its workers should be restored.
6611  */
restore_unbound_workers_cpumask(struct worker_pool * pool,int cpu)6612 static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
6613 {
6614 	static cpumask_t cpumask;
6615 	struct worker *worker;
6616 
6617 	lockdep_assert_held(&wq_pool_attach_mutex);
6618 
6619 	/* is @cpu allowed for @pool? */
6620 	if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
6621 		return;
6622 
6623 	cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
6624 
6625 	/* as we're called from CPU_ONLINE, the following shouldn't fail */
6626 	for_each_pool_worker(worker, pool)
6627 		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
6628 }
6629 
workqueue_prepare_cpu(unsigned int cpu)6630 int workqueue_prepare_cpu(unsigned int cpu)
6631 {
6632 	struct worker_pool *pool;
6633 
6634 	for_each_cpu_worker_pool(pool, cpu) {
6635 		if (pool->nr_workers)
6636 			continue;
6637 		if (!create_worker(pool))
6638 			return -ENOMEM;
6639 	}
6640 	return 0;
6641 }
6642 
workqueue_online_cpu(unsigned int cpu)6643 int workqueue_online_cpu(unsigned int cpu)
6644 {
6645 	struct worker_pool *pool;
6646 	struct workqueue_struct *wq;
6647 	int pi;
6648 
6649 	mutex_lock(&wq_pool_mutex);
6650 
6651 	cpumask_set_cpu(cpu, wq_online_cpumask);
6652 
6653 	for_each_pool(pool, pi) {
6654 		/* BH pools aren't affected by hotplug */
6655 		if (pool->flags & POOL_BH)
6656 			continue;
6657 
6658 		mutex_lock(&wq_pool_attach_mutex);
6659 		if (pool->cpu == cpu)
6660 			rebind_workers(pool);
6661 		else if (pool->cpu < 0)
6662 			restore_unbound_workers_cpumask(pool, cpu);
6663 		mutex_unlock(&wq_pool_attach_mutex);
6664 	}
6665 
6666 	/* update pod affinity of unbound workqueues */
6667 	list_for_each_entry(wq, &workqueues, list) {
6668 		struct workqueue_attrs *attrs = wq->unbound_attrs;
6669 
6670 		if (attrs) {
6671 			const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
6672 			int tcpu;
6673 
6674 			for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]])
6675 				unbound_wq_update_pwq(wq, tcpu);
6676 
6677 			mutex_lock(&wq->mutex);
6678 			wq_update_node_max_active(wq, -1);
6679 			mutex_unlock(&wq->mutex);
6680 		}
6681 	}
6682 
6683 	mutex_unlock(&wq_pool_mutex);
6684 	return 0;
6685 }
6686 
workqueue_offline_cpu(unsigned int cpu)6687 int workqueue_offline_cpu(unsigned int cpu)
6688 {
6689 	struct workqueue_struct *wq;
6690 
6691 	/* unbinding per-cpu workers should happen on the local CPU */
6692 	if (WARN_ON(cpu != smp_processor_id()))
6693 		return -1;
6694 
6695 	unbind_workers(cpu);
6696 
6697 	/* update pod affinity of unbound workqueues */
6698 	mutex_lock(&wq_pool_mutex);
6699 
6700 	cpumask_clear_cpu(cpu, wq_online_cpumask);
6701 
6702 	list_for_each_entry(wq, &workqueues, list) {
6703 		struct workqueue_attrs *attrs = wq->unbound_attrs;
6704 
6705 		if (attrs) {
6706 			const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
6707 			int tcpu;
6708 
6709 			for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]])
6710 				unbound_wq_update_pwq(wq, tcpu);
6711 
6712 			mutex_lock(&wq->mutex);
6713 			wq_update_node_max_active(wq, cpu);
6714 			mutex_unlock(&wq->mutex);
6715 		}
6716 	}
6717 	mutex_unlock(&wq_pool_mutex);
6718 
6719 	return 0;
6720 }
6721 
6722 struct work_for_cpu {
6723 	struct work_struct work;
6724 	long (*fn)(void *);
6725 	void *arg;
6726 	long ret;
6727 };
6728 
work_for_cpu_fn(struct work_struct * work)6729 static void work_for_cpu_fn(struct work_struct *work)
6730 {
6731 	struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
6732 
6733 	wfc->ret = wfc->fn(wfc->arg);
6734 }
6735 
6736 /**
6737  * work_on_cpu_key - run a function in thread context on a particular cpu
6738  * @cpu: the cpu to run on
6739  * @fn: the function to run
6740  * @arg: the function arg
6741  * @key: The lock class key for lock debugging purposes
6742  *
6743  * It is up to the caller to ensure that the cpu doesn't go offline.
6744  * The caller must not hold any locks which would prevent @fn from completing.
6745  *
6746  * Return: The value @fn returns.
6747  */
work_on_cpu_key(int cpu,long (* fn)(void *),void * arg,struct lock_class_key * key)6748 long work_on_cpu_key(int cpu, long (*fn)(void *),
6749 		     void *arg, struct lock_class_key *key)
6750 {
6751 	struct work_for_cpu wfc = { .fn = fn, .arg = arg };
6752 
6753 	INIT_WORK_ONSTACK_KEY(&wfc.work, work_for_cpu_fn, key);
6754 	schedule_work_on(cpu, &wfc.work);
6755 	flush_work(&wfc.work);
6756 	destroy_work_on_stack(&wfc.work);
6757 	return wfc.ret;
6758 }
6759 EXPORT_SYMBOL_GPL(work_on_cpu_key);
6760 
6761 /**
6762  * work_on_cpu_safe_key - run a function in thread context on a particular cpu
6763  * @cpu: the cpu to run on
6764  * @fn:  the function to run
6765  * @arg: the function argument
6766  * @key: The lock class key for lock debugging purposes
6767  *
6768  * Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
6769  * any locks which would prevent @fn from completing.
6770  *
6771  * Return: The value @fn returns.
6772  */
work_on_cpu_safe_key(int cpu,long (* fn)(void *),void * arg,struct lock_class_key * key)6773 long work_on_cpu_safe_key(int cpu, long (*fn)(void *),
6774 			  void *arg, struct lock_class_key *key)
6775 {
6776 	long ret = -ENODEV;
6777 
6778 	cpus_read_lock();
6779 	if (cpu_online(cpu))
6780 		ret = work_on_cpu_key(cpu, fn, arg, key);
6781 	cpus_read_unlock();
6782 	return ret;
6783 }
6784 EXPORT_SYMBOL_GPL(work_on_cpu_safe_key);
6785 #endif /* CONFIG_SMP */
6786 
6787 #ifdef CONFIG_FREEZER
6788 
6789 /**
6790  * freeze_workqueues_begin - begin freezing workqueues
6791  *
6792  * Start freezing workqueues.  After this function returns, all freezable
6793  * workqueues will queue new works to their inactive_works list instead of
6794  * pool->worklist.
6795  *
6796  * CONTEXT:
6797  * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
6798  */
freeze_workqueues_begin(void)6799 void freeze_workqueues_begin(void)
6800 {
6801 	struct workqueue_struct *wq;
6802 
6803 	mutex_lock(&wq_pool_mutex);
6804 
6805 	WARN_ON_ONCE(workqueue_freezing);
6806 	workqueue_freezing = true;
6807 
6808 	list_for_each_entry(wq, &workqueues, list) {
6809 		mutex_lock(&wq->mutex);
6810 		wq_adjust_max_active(wq);
6811 		mutex_unlock(&wq->mutex);
6812 	}
6813 
6814 	mutex_unlock(&wq_pool_mutex);
6815 }
6816 
6817 /**
6818  * freeze_workqueues_busy - are freezable workqueues still busy?
6819  *
6820  * Check whether freezing is complete.  This function must be called
6821  * between freeze_workqueues_begin() and thaw_workqueues().
6822  *
6823  * CONTEXT:
6824  * Grabs and releases wq_pool_mutex.
6825  *
6826  * Return:
6827  * %true if some freezable workqueues are still busy.  %false if freezing
6828  * is complete.
6829  */
freeze_workqueues_busy(void)6830 bool freeze_workqueues_busy(void)
6831 {
6832 	bool busy = false;
6833 	struct workqueue_struct *wq;
6834 	struct pool_workqueue *pwq;
6835 
6836 	mutex_lock(&wq_pool_mutex);
6837 
6838 	WARN_ON_ONCE(!workqueue_freezing);
6839 
6840 	list_for_each_entry(wq, &workqueues, list) {
6841 		if (!(wq->flags & WQ_FREEZABLE))
6842 			continue;
6843 		/*
6844 		 * nr_active is monotonically decreasing.  It's safe
6845 		 * to peek without lock.
6846 		 */
6847 		rcu_read_lock();
6848 		for_each_pwq(pwq, wq) {
6849 			WARN_ON_ONCE(pwq->nr_active < 0);
6850 			if (pwq->nr_active) {
6851 				busy = true;
6852 				rcu_read_unlock();
6853 				goto out_unlock;
6854 			}
6855 		}
6856 		rcu_read_unlock();
6857 	}
6858 out_unlock:
6859 	mutex_unlock(&wq_pool_mutex);
6860 	return busy;
6861 }
6862 
6863 /**
6864  * thaw_workqueues - thaw workqueues
6865  *
6866  * Thaw workqueues.  Normal queueing is restored and all collected
6867  * frozen works are transferred to their respective pool worklists.
6868  *
6869  * CONTEXT:
6870  * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
6871  */
thaw_workqueues(void)6872 void thaw_workqueues(void)
6873 {
6874 	struct workqueue_struct *wq;
6875 
6876 	mutex_lock(&wq_pool_mutex);
6877 
6878 	if (!workqueue_freezing)
6879 		goto out_unlock;
6880 
6881 	workqueue_freezing = false;
6882 
6883 	/* restore max_active and repopulate worklist */
6884 	list_for_each_entry(wq, &workqueues, list) {
6885 		mutex_lock(&wq->mutex);
6886 		wq_adjust_max_active(wq);
6887 		mutex_unlock(&wq->mutex);
6888 	}
6889 
6890 out_unlock:
6891 	mutex_unlock(&wq_pool_mutex);
6892 }
6893 #endif /* CONFIG_FREEZER */
6894 
workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask)6895 static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask)
6896 {
6897 	LIST_HEAD(ctxs);
6898 	int ret = 0;
6899 	struct workqueue_struct *wq;
6900 	struct apply_wqattrs_ctx *ctx, *n;
6901 
6902 	lockdep_assert_held(&wq_pool_mutex);
6903 
6904 	list_for_each_entry(wq, &workqueues, list) {
6905 		if (!(wq->flags & WQ_UNBOUND) || (wq->flags & __WQ_DESTROYING))
6906 			continue;
6907 
6908 		ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask);
6909 		if (IS_ERR(ctx)) {
6910 			ret = PTR_ERR(ctx);
6911 			break;
6912 		}
6913 
6914 		list_add_tail(&ctx->list, &ctxs);
6915 	}
6916 
6917 	list_for_each_entry_safe(ctx, n, &ctxs, list) {
6918 		if (!ret)
6919 			apply_wqattrs_commit(ctx);
6920 		apply_wqattrs_cleanup(ctx);
6921 	}
6922 
6923 	if (!ret) {
6924 		mutex_lock(&wq_pool_attach_mutex);
6925 		cpumask_copy(wq_unbound_cpumask, unbound_cpumask);
6926 		mutex_unlock(&wq_pool_attach_mutex);
6927 	}
6928 	return ret;
6929 }
6930 
6931 /**
6932  * workqueue_unbound_exclude_cpumask - Exclude given CPUs from unbound cpumask
6933  * @exclude_cpumask: the cpumask to be excluded from wq_unbound_cpumask
6934  *
6935  * This function can be called from cpuset code to provide a set of isolated
6936  * CPUs that should be excluded from wq_unbound_cpumask.
6937  */
workqueue_unbound_exclude_cpumask(cpumask_var_t exclude_cpumask)6938 int workqueue_unbound_exclude_cpumask(cpumask_var_t exclude_cpumask)
6939 {
6940 	cpumask_var_t cpumask;
6941 	int ret = 0;
6942 
6943 	if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
6944 		return -ENOMEM;
6945 
6946 	mutex_lock(&wq_pool_mutex);
6947 
6948 	/*
6949 	 * If the operation fails, it will fall back to
6950 	 * wq_requested_unbound_cpumask which is initially set to
6951 	 * (HK_TYPE_WQ ∩ HK_TYPE_DOMAIN) house keeping mask and rewritten
6952 	 * by any subsequent write to workqueue/cpumask sysfs file.
6953 	 */
6954 	if (!cpumask_andnot(cpumask, wq_requested_unbound_cpumask, exclude_cpumask))
6955 		cpumask_copy(cpumask, wq_requested_unbound_cpumask);
6956 	if (!cpumask_equal(cpumask, wq_unbound_cpumask))
6957 		ret = workqueue_apply_unbound_cpumask(cpumask);
6958 
6959 	/* Save the current isolated cpumask & export it via sysfs */
6960 	if (!ret)
6961 		cpumask_copy(wq_isolated_cpumask, exclude_cpumask);
6962 
6963 	mutex_unlock(&wq_pool_mutex);
6964 	free_cpumask_var(cpumask);
6965 	return ret;
6966 }
6967 
parse_affn_scope(const char * val)6968 static int parse_affn_scope(const char *val)
6969 {
6970 	int i;
6971 
6972 	for (i = 0; i < ARRAY_SIZE(wq_affn_names); i++) {
6973 		if (!strncasecmp(val, wq_affn_names[i], strlen(wq_affn_names[i])))
6974 			return i;
6975 	}
6976 	return -EINVAL;
6977 }
6978 
wq_affn_dfl_set(const char * val,const struct kernel_param * kp)6979 static int wq_affn_dfl_set(const char *val, const struct kernel_param *kp)
6980 {
6981 	struct workqueue_struct *wq;
6982 	int affn, cpu;
6983 
6984 	affn = parse_affn_scope(val);
6985 	if (affn < 0)
6986 		return affn;
6987 	if (affn == WQ_AFFN_DFL)
6988 		return -EINVAL;
6989 
6990 	cpus_read_lock();
6991 	mutex_lock(&wq_pool_mutex);
6992 
6993 	wq_affn_dfl = affn;
6994 
6995 	list_for_each_entry(wq, &workqueues, list) {
6996 		for_each_online_cpu(cpu)
6997 			unbound_wq_update_pwq(wq, cpu);
6998 	}
6999 
7000 	mutex_unlock(&wq_pool_mutex);
7001 	cpus_read_unlock();
7002 
7003 	return 0;
7004 }
7005 
wq_affn_dfl_get(char * buffer,const struct kernel_param * kp)7006 static int wq_affn_dfl_get(char *buffer, const struct kernel_param *kp)
7007 {
7008 	return scnprintf(buffer, PAGE_SIZE, "%s\n", wq_affn_names[wq_affn_dfl]);
7009 }
7010 
7011 static const struct kernel_param_ops wq_affn_dfl_ops = {
7012 	.set	= wq_affn_dfl_set,
7013 	.get	= wq_affn_dfl_get,
7014 };
7015 
7016 module_param_cb(default_affinity_scope, &wq_affn_dfl_ops, NULL, 0644);
7017 
7018 #ifdef CONFIG_SYSFS
7019 /*
7020  * Workqueues with WQ_SYSFS flag set is visible to userland via
7021  * /sys/bus/workqueue/devices/WQ_NAME.  All visible workqueues have the
7022  * following attributes.
7023  *
7024  *  per_cpu		RO bool	: whether the workqueue is per-cpu or unbound
7025  *  max_active		RW int	: maximum number of in-flight work items
7026  *
7027  * Unbound workqueues have the following extra attributes.
7028  *
7029  *  nice		RW int	: nice value of the workers
7030  *  cpumask		RW mask	: bitmask of allowed CPUs for the workers
7031  *  affinity_scope	RW str  : worker CPU affinity scope (cache, numa, none)
7032  *  affinity_strict	RW bool : worker CPU affinity is strict
7033  */
7034 struct wq_device {
7035 	struct workqueue_struct		*wq;
7036 	struct device			dev;
7037 };
7038 
dev_to_wq(struct device * dev)7039 static struct workqueue_struct *dev_to_wq(struct device *dev)
7040 {
7041 	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
7042 
7043 	return wq_dev->wq;
7044 }
7045 
per_cpu_show(struct device * dev,struct device_attribute * attr,char * buf)7046 static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
7047 			    char *buf)
7048 {
7049 	struct workqueue_struct *wq = dev_to_wq(dev);
7050 
7051 	return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
7052 }
7053 static DEVICE_ATTR_RO(per_cpu);
7054 
max_active_show(struct device * dev,struct device_attribute * attr,char * buf)7055 static ssize_t max_active_show(struct device *dev,
7056 			       struct device_attribute *attr, char *buf)
7057 {
7058 	struct workqueue_struct *wq = dev_to_wq(dev);
7059 
7060 	return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
7061 }
7062 
max_active_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)7063 static ssize_t max_active_store(struct device *dev,
7064 				struct device_attribute *attr, const char *buf,
7065 				size_t count)
7066 {
7067 	struct workqueue_struct *wq = dev_to_wq(dev);
7068 	int val;
7069 
7070 	if (sscanf(buf, "%d", &val) != 1 || val <= 0)
7071 		return -EINVAL;
7072 
7073 	workqueue_set_max_active(wq, val);
7074 	return count;
7075 }
7076 static DEVICE_ATTR_RW(max_active);
7077 
7078 static struct attribute *wq_sysfs_attrs[] = {
7079 	&dev_attr_per_cpu.attr,
7080 	&dev_attr_max_active.attr,
7081 	NULL,
7082 };
7083 ATTRIBUTE_GROUPS(wq_sysfs);
7084 
wq_nice_show(struct device * dev,struct device_attribute * attr,char * buf)7085 static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
7086 			    char *buf)
7087 {
7088 	struct workqueue_struct *wq = dev_to_wq(dev);
7089 	int written;
7090 
7091 	mutex_lock(&wq->mutex);
7092 	written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
7093 	mutex_unlock(&wq->mutex);
7094 
7095 	return written;
7096 }
7097 
7098 /* prepare workqueue_attrs for sysfs store operations */
wq_sysfs_prep_attrs(struct workqueue_struct * wq)7099 static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
7100 {
7101 	struct workqueue_attrs *attrs;
7102 
7103 	lockdep_assert_held(&wq_pool_mutex);
7104 
7105 	attrs = alloc_workqueue_attrs();
7106 	if (!attrs)
7107 		return NULL;
7108 
7109 	copy_workqueue_attrs(attrs, wq->unbound_attrs);
7110 	return attrs;
7111 }
7112 
wq_nice_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)7113 static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
7114 			     const char *buf, size_t count)
7115 {
7116 	struct workqueue_struct *wq = dev_to_wq(dev);
7117 	struct workqueue_attrs *attrs;
7118 	int ret = -ENOMEM;
7119 
7120 	apply_wqattrs_lock();
7121 
7122 	attrs = wq_sysfs_prep_attrs(wq);
7123 	if (!attrs)
7124 		goto out_unlock;
7125 
7126 	if (sscanf(buf, "%d", &attrs->nice) == 1 &&
7127 	    attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
7128 		ret = apply_workqueue_attrs_locked(wq, attrs);
7129 	else
7130 		ret = -EINVAL;
7131 
7132 out_unlock:
7133 	apply_wqattrs_unlock();
7134 	free_workqueue_attrs(attrs);
7135 	return ret ?: count;
7136 }
7137 
wq_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)7138 static ssize_t wq_cpumask_show(struct device *dev,
7139 			       struct device_attribute *attr, char *buf)
7140 {
7141 	struct workqueue_struct *wq = dev_to_wq(dev);
7142 	int written;
7143 
7144 	mutex_lock(&wq->mutex);
7145 	written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
7146 			    cpumask_pr_args(wq->unbound_attrs->cpumask));
7147 	mutex_unlock(&wq->mutex);
7148 	return written;
7149 }
7150 
wq_cpumask_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)7151 static ssize_t wq_cpumask_store(struct device *dev,
7152 				struct device_attribute *attr,
7153 				const char *buf, size_t count)
7154 {
7155 	struct workqueue_struct *wq = dev_to_wq(dev);
7156 	struct workqueue_attrs *attrs;
7157 	int ret = -ENOMEM;
7158 
7159 	apply_wqattrs_lock();
7160 
7161 	attrs = wq_sysfs_prep_attrs(wq);
7162 	if (!attrs)
7163 		goto out_unlock;
7164 
7165 	ret = cpumask_parse(buf, attrs->cpumask);
7166 	if (!ret)
7167 		ret = apply_workqueue_attrs_locked(wq, attrs);
7168 
7169 out_unlock:
7170 	apply_wqattrs_unlock();
7171 	free_workqueue_attrs(attrs);
7172 	return ret ?: count;
7173 }
7174 
wq_affn_scope_show(struct device * dev,struct device_attribute * attr,char * buf)7175 static ssize_t wq_affn_scope_show(struct device *dev,
7176 				  struct device_attribute *attr, char *buf)
7177 {
7178 	struct workqueue_struct *wq = dev_to_wq(dev);
7179 	int written;
7180 
7181 	mutex_lock(&wq->mutex);
7182 	if (wq->unbound_attrs->affn_scope == WQ_AFFN_DFL)
7183 		written = scnprintf(buf, PAGE_SIZE, "%s (%s)\n",
7184 				    wq_affn_names[WQ_AFFN_DFL],
7185 				    wq_affn_names[wq_affn_dfl]);
7186 	else
7187 		written = scnprintf(buf, PAGE_SIZE, "%s\n",
7188 				    wq_affn_names[wq->unbound_attrs->affn_scope]);
7189 	mutex_unlock(&wq->mutex);
7190 
7191 	return written;
7192 }
7193 
wq_affn_scope_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)7194 static ssize_t wq_affn_scope_store(struct device *dev,
7195 				   struct device_attribute *attr,
7196 				   const char *buf, size_t count)
7197 {
7198 	struct workqueue_struct *wq = dev_to_wq(dev);
7199 	struct workqueue_attrs *attrs;
7200 	int affn, ret = -ENOMEM;
7201 
7202 	affn = parse_affn_scope(buf);
7203 	if (affn < 0)
7204 		return affn;
7205 
7206 	apply_wqattrs_lock();
7207 	attrs = wq_sysfs_prep_attrs(wq);
7208 	if (attrs) {
7209 		attrs->affn_scope = affn;
7210 		ret = apply_workqueue_attrs_locked(wq, attrs);
7211 	}
7212 	apply_wqattrs_unlock();
7213 	free_workqueue_attrs(attrs);
7214 	return ret ?: count;
7215 }
7216 
wq_affinity_strict_show(struct device * dev,struct device_attribute * attr,char * buf)7217 static ssize_t wq_affinity_strict_show(struct device *dev,
7218 				       struct device_attribute *attr, char *buf)
7219 {
7220 	struct workqueue_struct *wq = dev_to_wq(dev);
7221 
7222 	return scnprintf(buf, PAGE_SIZE, "%d\n",
7223 			 wq->unbound_attrs->affn_strict);
7224 }
7225 
wq_affinity_strict_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)7226 static ssize_t wq_affinity_strict_store(struct device *dev,
7227 					struct device_attribute *attr,
7228 					const char *buf, size_t count)
7229 {
7230 	struct workqueue_struct *wq = dev_to_wq(dev);
7231 	struct workqueue_attrs *attrs;
7232 	int v, ret = -ENOMEM;
7233 
7234 	if (sscanf(buf, "%d", &v) != 1)
7235 		return -EINVAL;
7236 
7237 	apply_wqattrs_lock();
7238 	attrs = wq_sysfs_prep_attrs(wq);
7239 	if (attrs) {
7240 		attrs->affn_strict = (bool)v;
7241 		ret = apply_workqueue_attrs_locked(wq, attrs);
7242 	}
7243 	apply_wqattrs_unlock();
7244 	free_workqueue_attrs(attrs);
7245 	return ret ?: count;
7246 }
7247 
7248 static struct device_attribute wq_sysfs_unbound_attrs[] = {
7249 	__ATTR(nice, 0644, wq_nice_show, wq_nice_store),
7250 	__ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
7251 	__ATTR(affinity_scope, 0644, wq_affn_scope_show, wq_affn_scope_store),
7252 	__ATTR(affinity_strict, 0644, wq_affinity_strict_show, wq_affinity_strict_store),
7253 	__ATTR_NULL,
7254 };
7255 
7256 static const struct bus_type wq_subsys = {
7257 	.name				= "workqueue",
7258 	.dev_groups			= wq_sysfs_groups,
7259 };
7260 
7261 /**
7262  *  workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
7263  *  @cpumask: the cpumask to set
7264  *
7265  *  The low-level workqueues cpumask is a global cpumask that limits
7266  *  the affinity of all unbound workqueues.  This function check the @cpumask
7267  *  and apply it to all unbound workqueues and updates all pwqs of them.
7268  *
7269  *  Return:	0	- Success
7270  *		-EINVAL	- Invalid @cpumask
7271  *		-ENOMEM	- Failed to allocate memory for attrs or pwqs.
7272  */
workqueue_set_unbound_cpumask(cpumask_var_t cpumask)7273 static int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
7274 {
7275 	int ret = -EINVAL;
7276 
7277 	/*
7278 	 * Not excluding isolated cpus on purpose.
7279 	 * If the user wishes to include them, we allow that.
7280 	 */
7281 	cpumask_and(cpumask, cpumask, cpu_possible_mask);
7282 	if (!cpumask_empty(cpumask)) {
7283 		ret = 0;
7284 		apply_wqattrs_lock();
7285 		if (!cpumask_equal(cpumask, wq_unbound_cpumask))
7286 			ret = workqueue_apply_unbound_cpumask(cpumask);
7287 		if (!ret)
7288 			cpumask_copy(wq_requested_unbound_cpumask, cpumask);
7289 		apply_wqattrs_unlock();
7290 	}
7291 
7292 	return ret;
7293 }
7294 
__wq_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf,cpumask_var_t mask)7295 static ssize_t __wq_cpumask_show(struct device *dev,
7296 		struct device_attribute *attr, char *buf, cpumask_var_t mask)
7297 {
7298 	int written;
7299 
7300 	mutex_lock(&wq_pool_mutex);
7301 	written = scnprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(mask));
7302 	mutex_unlock(&wq_pool_mutex);
7303 
7304 	return written;
7305 }
7306 
cpumask_requested_show(struct device * dev,struct device_attribute * attr,char * buf)7307 static ssize_t cpumask_requested_show(struct device *dev,
7308 		struct device_attribute *attr, char *buf)
7309 {
7310 	return __wq_cpumask_show(dev, attr, buf, wq_requested_unbound_cpumask);
7311 }
7312 static DEVICE_ATTR_RO(cpumask_requested);
7313 
cpumask_isolated_show(struct device * dev,struct device_attribute * attr,char * buf)7314 static ssize_t cpumask_isolated_show(struct device *dev,
7315 		struct device_attribute *attr, char *buf)
7316 {
7317 	return __wq_cpumask_show(dev, attr, buf, wq_isolated_cpumask);
7318 }
7319 static DEVICE_ATTR_RO(cpumask_isolated);
7320 
cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)7321 static ssize_t cpumask_show(struct device *dev,
7322 		struct device_attribute *attr, char *buf)
7323 {
7324 	return __wq_cpumask_show(dev, attr, buf, wq_unbound_cpumask);
7325 }
7326 
cpumask_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)7327 static ssize_t cpumask_store(struct device *dev,
7328 		struct device_attribute *attr, const char *buf, size_t count)
7329 {
7330 	cpumask_var_t cpumask;
7331 	int ret;
7332 
7333 	if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
7334 		return -ENOMEM;
7335 
7336 	ret = cpumask_parse(buf, cpumask);
7337 	if (!ret)
7338 		ret = workqueue_set_unbound_cpumask(cpumask);
7339 
7340 	free_cpumask_var(cpumask);
7341 	return ret ? ret : count;
7342 }
7343 static DEVICE_ATTR_RW(cpumask);
7344 
7345 static struct attribute *wq_sysfs_cpumask_attrs[] = {
7346 	&dev_attr_cpumask.attr,
7347 	&dev_attr_cpumask_requested.attr,
7348 	&dev_attr_cpumask_isolated.attr,
7349 	NULL,
7350 };
7351 ATTRIBUTE_GROUPS(wq_sysfs_cpumask);
7352 
wq_sysfs_init(void)7353 static int __init wq_sysfs_init(void)
7354 {
7355 	return subsys_virtual_register(&wq_subsys, wq_sysfs_cpumask_groups);
7356 }
7357 core_initcall(wq_sysfs_init);
7358 
wq_device_release(struct device * dev)7359 static void wq_device_release(struct device *dev)
7360 {
7361 	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
7362 
7363 	kfree(wq_dev);
7364 }
7365 
7366 /**
7367  * workqueue_sysfs_register - make a workqueue visible in sysfs
7368  * @wq: the workqueue to register
7369  *
7370  * Expose @wq in sysfs under /sys/bus/workqueue/devices.
7371  * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
7372  * which is the preferred method.
7373  *
7374  * Workqueue user should use this function directly iff it wants to apply
7375  * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
7376  * apply_workqueue_attrs() may race against userland updating the
7377  * attributes.
7378  *
7379  * Return: 0 on success, -errno on failure.
7380  */
workqueue_sysfs_register(struct workqueue_struct * wq)7381 int workqueue_sysfs_register(struct workqueue_struct *wq)
7382 {
7383 	struct wq_device *wq_dev;
7384 	int ret;
7385 
7386 	/*
7387 	 * Adjusting max_active breaks ordering guarantee.  Disallow exposing
7388 	 * ordered workqueues.
7389 	 */
7390 	if (WARN_ON(wq->flags & __WQ_ORDERED))
7391 		return -EINVAL;
7392 
7393 	wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
7394 	if (!wq_dev)
7395 		return -ENOMEM;
7396 
7397 	wq_dev->wq = wq;
7398 	wq_dev->dev.bus = &wq_subsys;
7399 	wq_dev->dev.release = wq_device_release;
7400 	dev_set_name(&wq_dev->dev, "%s", wq->name);
7401 
7402 	/*
7403 	 * unbound_attrs are created separately.  Suppress uevent until
7404 	 * everything is ready.
7405 	 */
7406 	dev_set_uevent_suppress(&wq_dev->dev, true);
7407 
7408 	ret = device_register(&wq_dev->dev);
7409 	if (ret) {
7410 		put_device(&wq_dev->dev);
7411 		wq->wq_dev = NULL;
7412 		return ret;
7413 	}
7414 
7415 	if (wq->flags & WQ_UNBOUND) {
7416 		struct device_attribute *attr;
7417 
7418 		for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
7419 			ret = device_create_file(&wq_dev->dev, attr);
7420 			if (ret) {
7421 				device_unregister(&wq_dev->dev);
7422 				wq->wq_dev = NULL;
7423 				return ret;
7424 			}
7425 		}
7426 	}
7427 
7428 	dev_set_uevent_suppress(&wq_dev->dev, false);
7429 	kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
7430 	return 0;
7431 }
7432 
7433 /**
7434  * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
7435  * @wq: the workqueue to unregister
7436  *
7437  * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
7438  */
workqueue_sysfs_unregister(struct workqueue_struct * wq)7439 static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
7440 {
7441 	struct wq_device *wq_dev = wq->wq_dev;
7442 
7443 	if (!wq->wq_dev)
7444 		return;
7445 
7446 	wq->wq_dev = NULL;
7447 	device_unregister(&wq_dev->dev);
7448 }
7449 #else	/* CONFIG_SYSFS */
workqueue_sysfs_unregister(struct workqueue_struct * wq)7450 static void workqueue_sysfs_unregister(struct workqueue_struct *wq)	{ }
7451 #endif	/* CONFIG_SYSFS */
7452 
7453 /*
7454  * Workqueue watchdog.
7455  *
7456  * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
7457  * flush dependency, a concurrency managed work item which stays RUNNING
7458  * indefinitely.  Workqueue stalls can be very difficult to debug as the
7459  * usual warning mechanisms don't trigger and internal workqueue state is
7460  * largely opaque.
7461  *
7462  * Workqueue watchdog monitors all worker pools periodically and dumps
7463  * state if some pools failed to make forward progress for a while where
7464  * forward progress is defined as the first item on ->worklist changing.
7465  *
7466  * This mechanism is controlled through the kernel parameter
7467  * "workqueue.watchdog_thresh" which can be updated at runtime through the
7468  * corresponding sysfs parameter file.
7469  */
7470 #ifdef CONFIG_WQ_WATCHDOG
7471 
7472 static unsigned long wq_watchdog_thresh = 30;
7473 static struct timer_list wq_watchdog_timer;
7474 
7475 static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
7476 static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
7477 
7478 static unsigned int wq_panic_on_stall;
7479 module_param_named(panic_on_stall, wq_panic_on_stall, uint, 0644);
7480 
7481 /*
7482  * Show workers that might prevent the processing of pending work items.
7483  * The only candidates are CPU-bound workers in the running state.
7484  * Pending work items should be handled by another idle worker
7485  * in all other situations.
7486  */
show_cpu_pool_hog(struct worker_pool * pool)7487 static void show_cpu_pool_hog(struct worker_pool *pool)
7488 {
7489 	struct worker *worker;
7490 	unsigned long irq_flags;
7491 	int bkt;
7492 
7493 	raw_spin_lock_irqsave(&pool->lock, irq_flags);
7494 
7495 	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
7496 		if (task_is_running(worker->task)) {
7497 			/*
7498 			 * Defer printing to avoid deadlocks in console
7499 			 * drivers that queue work while holding locks
7500 			 * also taken in their write paths.
7501 			 */
7502 			printk_deferred_enter();
7503 
7504 			pr_info("pool %d:\n", pool->id);
7505 			sched_show_task(worker->task);
7506 
7507 			printk_deferred_exit();
7508 		}
7509 	}
7510 
7511 	raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
7512 }
7513 
show_cpu_pools_hogs(void)7514 static void show_cpu_pools_hogs(void)
7515 {
7516 	struct worker_pool *pool;
7517 	int pi;
7518 
7519 	pr_info("Showing backtraces of running workers in stalled CPU-bound worker pools:\n");
7520 
7521 	rcu_read_lock();
7522 
7523 	for_each_pool(pool, pi) {
7524 		if (pool->cpu_stall)
7525 			show_cpu_pool_hog(pool);
7526 
7527 	}
7528 
7529 	rcu_read_unlock();
7530 }
7531 
panic_on_wq_watchdog(void)7532 static void panic_on_wq_watchdog(void)
7533 {
7534 	static unsigned int wq_stall;
7535 
7536 	if (wq_panic_on_stall) {
7537 		wq_stall++;
7538 		BUG_ON(wq_stall >= wq_panic_on_stall);
7539 	}
7540 }
7541 
wq_watchdog_reset_touched(void)7542 static void wq_watchdog_reset_touched(void)
7543 {
7544 	int cpu;
7545 
7546 	wq_watchdog_touched = jiffies;
7547 	for_each_possible_cpu(cpu)
7548 		per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
7549 }
7550 
wq_watchdog_timer_fn(struct timer_list * unused)7551 static void wq_watchdog_timer_fn(struct timer_list *unused)
7552 {
7553 	unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
7554 	bool lockup_detected = false;
7555 	bool cpu_pool_stall = false;
7556 	unsigned long now = jiffies;
7557 	struct worker_pool *pool;
7558 	int pi;
7559 
7560 	if (!thresh)
7561 		return;
7562 
7563 	rcu_read_lock();
7564 
7565 	for_each_pool(pool, pi) {
7566 		unsigned long pool_ts, touched, ts;
7567 
7568 		pool->cpu_stall = false;
7569 		if (list_empty(&pool->worklist))
7570 			continue;
7571 
7572 		/*
7573 		 * If a virtual machine is stopped by the host it can look to
7574 		 * the watchdog like a stall.
7575 		 */
7576 		kvm_check_and_clear_guest_paused();
7577 
7578 		/* get the latest of pool and touched timestamps */
7579 		if (pool->cpu >= 0)
7580 			touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
7581 		else
7582 			touched = READ_ONCE(wq_watchdog_touched);
7583 		pool_ts = READ_ONCE(pool->watchdog_ts);
7584 
7585 		if (time_after(pool_ts, touched))
7586 			ts = pool_ts;
7587 		else
7588 			ts = touched;
7589 
7590 		/* did we stall? */
7591 		if (time_after(now, ts + thresh)) {
7592 			lockup_detected = true;
7593 			if (pool->cpu >= 0 && !(pool->flags & POOL_BH)) {
7594 				pool->cpu_stall = true;
7595 				cpu_pool_stall = true;
7596 			}
7597 			pr_emerg("BUG: workqueue lockup - pool");
7598 			pr_cont_pool_info(pool);
7599 			pr_cont(" stuck for %us!\n",
7600 				jiffies_to_msecs(now - pool_ts) / 1000);
7601 		}
7602 
7603 
7604 	}
7605 
7606 	rcu_read_unlock();
7607 
7608 	if (lockup_detected)
7609 		show_all_workqueues();
7610 
7611 	if (cpu_pool_stall)
7612 		show_cpu_pools_hogs();
7613 
7614 	if (lockup_detected)
7615 		panic_on_wq_watchdog();
7616 
7617 	wq_watchdog_reset_touched();
7618 	mod_timer(&wq_watchdog_timer, jiffies + thresh);
7619 }
7620 
wq_watchdog_touch(int cpu)7621 notrace void wq_watchdog_touch(int cpu)
7622 {
7623 	unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
7624 	unsigned long touch_ts = READ_ONCE(wq_watchdog_touched);
7625 	unsigned long now = jiffies;
7626 
7627 	if (cpu >= 0)
7628 		per_cpu(wq_watchdog_touched_cpu, cpu) = now;
7629 	else
7630 		WARN_ONCE(1, "%s should be called with valid CPU", __func__);
7631 
7632 	/* Don't unnecessarily store to global cacheline */
7633 	if (time_after(now, touch_ts + thresh / 4))
7634 		WRITE_ONCE(wq_watchdog_touched, jiffies);
7635 }
7636 
wq_watchdog_set_thresh(unsigned long thresh)7637 static void wq_watchdog_set_thresh(unsigned long thresh)
7638 {
7639 	wq_watchdog_thresh = 0;
7640 	del_timer_sync(&wq_watchdog_timer);
7641 
7642 	if (thresh) {
7643 		wq_watchdog_thresh = thresh;
7644 		wq_watchdog_reset_touched();
7645 		mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
7646 	}
7647 }
7648 
wq_watchdog_param_set_thresh(const char * val,const struct kernel_param * kp)7649 static int wq_watchdog_param_set_thresh(const char *val,
7650 					const struct kernel_param *kp)
7651 {
7652 	unsigned long thresh;
7653 	int ret;
7654 
7655 	ret = kstrtoul(val, 0, &thresh);
7656 	if (ret)
7657 		return ret;
7658 
7659 	if (system_wq)
7660 		wq_watchdog_set_thresh(thresh);
7661 	else
7662 		wq_watchdog_thresh = thresh;
7663 
7664 	return 0;
7665 }
7666 
7667 static const struct kernel_param_ops wq_watchdog_thresh_ops = {
7668 	.set	= wq_watchdog_param_set_thresh,
7669 	.get	= param_get_ulong,
7670 };
7671 
7672 module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
7673 		0644);
7674 
wq_watchdog_init(void)7675 static void wq_watchdog_init(void)
7676 {
7677 	timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
7678 	wq_watchdog_set_thresh(wq_watchdog_thresh);
7679 }
7680 
7681 #else	/* CONFIG_WQ_WATCHDOG */
7682 
wq_watchdog_init(void)7683 static inline void wq_watchdog_init(void) { }
7684 
7685 #endif	/* CONFIG_WQ_WATCHDOG */
7686 
bh_pool_kick_normal(struct irq_work * irq_work)7687 static void bh_pool_kick_normal(struct irq_work *irq_work)
7688 {
7689 	raise_softirq_irqoff(TASKLET_SOFTIRQ);
7690 }
7691 
bh_pool_kick_highpri(struct irq_work * irq_work)7692 static void bh_pool_kick_highpri(struct irq_work *irq_work)
7693 {
7694 	raise_softirq_irqoff(HI_SOFTIRQ);
7695 }
7696 
restrict_unbound_cpumask(const char * name,const struct cpumask * mask)7697 static void __init restrict_unbound_cpumask(const char *name, const struct cpumask *mask)
7698 {
7699 	if (!cpumask_intersects(wq_unbound_cpumask, mask)) {
7700 		pr_warn("workqueue: Restricting unbound_cpumask (%*pb) with %s (%*pb) leaves no CPU, ignoring\n",
7701 			cpumask_pr_args(wq_unbound_cpumask), name, cpumask_pr_args(mask));
7702 		return;
7703 	}
7704 
7705 	cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, mask);
7706 }
7707 
init_cpu_worker_pool(struct worker_pool * pool,int cpu,int nice)7708 static void __init init_cpu_worker_pool(struct worker_pool *pool, int cpu, int nice)
7709 {
7710 	BUG_ON(init_worker_pool(pool));
7711 	pool->cpu = cpu;
7712 	cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
7713 	cpumask_copy(pool->attrs->__pod_cpumask, cpumask_of(cpu));
7714 	pool->attrs->nice = nice;
7715 	pool->attrs->affn_strict = true;
7716 	pool->node = cpu_to_node(cpu);
7717 
7718 	/* alloc pool ID */
7719 	mutex_lock(&wq_pool_mutex);
7720 	BUG_ON(worker_pool_assign_id(pool));
7721 	mutex_unlock(&wq_pool_mutex);
7722 }
7723 
7724 /**
7725  * workqueue_init_early - early init for workqueue subsystem
7726  *
7727  * This is the first step of three-staged workqueue subsystem initialization and
7728  * invoked as soon as the bare basics - memory allocation, cpumasks and idr are
7729  * up. It sets up all the data structures and system workqueues and allows early
7730  * boot code to create workqueues and queue/cancel work items. Actual work item
7731  * execution starts only after kthreads can be created and scheduled right
7732  * before early initcalls.
7733  */
workqueue_init_early(void)7734 void __init workqueue_init_early(void)
7735 {
7736 	struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM];
7737 	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
7738 	void (*irq_work_fns[2])(struct irq_work *) = { bh_pool_kick_normal,
7739 						       bh_pool_kick_highpri };
7740 	int i, cpu;
7741 
7742 	BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
7743 
7744 	BUG_ON(!alloc_cpumask_var(&wq_online_cpumask, GFP_KERNEL));
7745 	BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
7746 	BUG_ON(!alloc_cpumask_var(&wq_requested_unbound_cpumask, GFP_KERNEL));
7747 	BUG_ON(!zalloc_cpumask_var(&wq_isolated_cpumask, GFP_KERNEL));
7748 
7749 	cpumask_copy(wq_online_cpumask, cpu_online_mask);
7750 	cpumask_copy(wq_unbound_cpumask, cpu_possible_mask);
7751 	restrict_unbound_cpumask("HK_TYPE_WQ", housekeeping_cpumask(HK_TYPE_WQ));
7752 	restrict_unbound_cpumask("HK_TYPE_DOMAIN", housekeeping_cpumask(HK_TYPE_DOMAIN));
7753 	if (!cpumask_empty(&wq_cmdline_cpumask))
7754 		restrict_unbound_cpumask("workqueue.unbound_cpus", &wq_cmdline_cpumask);
7755 
7756 	cpumask_copy(wq_requested_unbound_cpumask, wq_unbound_cpumask);
7757 
7758 	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
7759 
7760 	unbound_wq_update_pwq_attrs_buf = alloc_workqueue_attrs();
7761 	BUG_ON(!unbound_wq_update_pwq_attrs_buf);
7762 
7763 	/*
7764 	 * If nohz_full is enabled, set power efficient workqueue as unbound.
7765 	 * This allows workqueue items to be moved to HK CPUs.
7766 	 */
7767 	if (housekeeping_enabled(HK_TYPE_TICK))
7768 		wq_power_efficient = true;
7769 
7770 	/* initialize WQ_AFFN_SYSTEM pods */
7771 	pt->pod_cpus = kcalloc(1, sizeof(pt->pod_cpus[0]), GFP_KERNEL);
7772 	pt->pod_node = kcalloc(1, sizeof(pt->pod_node[0]), GFP_KERNEL);
7773 	pt->cpu_pod = kcalloc(nr_cpu_ids, sizeof(pt->cpu_pod[0]), GFP_KERNEL);
7774 	BUG_ON(!pt->pod_cpus || !pt->pod_node || !pt->cpu_pod);
7775 
7776 	BUG_ON(!zalloc_cpumask_var_node(&pt->pod_cpus[0], GFP_KERNEL, NUMA_NO_NODE));
7777 
7778 	pt->nr_pods = 1;
7779 	cpumask_copy(pt->pod_cpus[0], cpu_possible_mask);
7780 	pt->pod_node[0] = NUMA_NO_NODE;
7781 	pt->cpu_pod[0] = 0;
7782 
7783 	/* initialize BH and CPU pools */
7784 	for_each_possible_cpu(cpu) {
7785 		struct worker_pool *pool;
7786 
7787 		i = 0;
7788 		for_each_bh_worker_pool(pool, cpu) {
7789 			init_cpu_worker_pool(pool, cpu, std_nice[i]);
7790 			pool->flags |= POOL_BH;
7791 			init_irq_work(bh_pool_irq_work(pool), irq_work_fns[i]);
7792 			i++;
7793 		}
7794 
7795 		i = 0;
7796 		for_each_cpu_worker_pool(pool, cpu)
7797 			init_cpu_worker_pool(pool, cpu, std_nice[i++]);
7798 	}
7799 
7800 	/* create default unbound and ordered wq attrs */
7801 	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
7802 		struct workqueue_attrs *attrs;
7803 
7804 		BUG_ON(!(attrs = alloc_workqueue_attrs()));
7805 		attrs->nice = std_nice[i];
7806 		unbound_std_wq_attrs[i] = attrs;
7807 
7808 		/*
7809 		 * An ordered wq should have only one pwq as ordering is
7810 		 * guaranteed by max_active which is enforced by pwqs.
7811 		 */
7812 		BUG_ON(!(attrs = alloc_workqueue_attrs()));
7813 		attrs->nice = std_nice[i];
7814 		attrs->ordered = true;
7815 		ordered_wq_attrs[i] = attrs;
7816 	}
7817 
7818 	system_wq = alloc_workqueue("events", 0, 0);
7819 	system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
7820 	system_long_wq = alloc_workqueue("events_long", 0, 0);
7821 	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
7822 					    WQ_MAX_ACTIVE);
7823 	system_freezable_wq = alloc_workqueue("events_freezable",
7824 					      WQ_FREEZABLE, 0);
7825 	system_power_efficient_wq = alloc_workqueue("events_power_efficient",
7826 					      WQ_POWER_EFFICIENT, 0);
7827 	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_pwr_efficient",
7828 					      WQ_FREEZABLE | WQ_POWER_EFFICIENT,
7829 					      0);
7830 	system_bh_wq = alloc_workqueue("events_bh", WQ_BH, 0);
7831 	system_bh_highpri_wq = alloc_workqueue("events_bh_highpri",
7832 					       WQ_BH | WQ_HIGHPRI, 0);
7833 	BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
7834 	       !system_unbound_wq || !system_freezable_wq ||
7835 	       !system_power_efficient_wq ||
7836 	       !system_freezable_power_efficient_wq ||
7837 	       !system_bh_wq || !system_bh_highpri_wq);
7838 }
7839 
wq_cpu_intensive_thresh_init(void)7840 static void __init wq_cpu_intensive_thresh_init(void)
7841 {
7842 	unsigned long thresh;
7843 	unsigned long bogo;
7844 
7845 	pwq_release_worker = kthread_run_worker(0, "pool_workqueue_release");
7846 	BUG_ON(IS_ERR(pwq_release_worker));
7847 
7848 	/* if the user set it to a specific value, keep it */
7849 	if (wq_cpu_intensive_thresh_us != ULONG_MAX)
7850 		return;
7851 
7852 	/*
7853 	 * The default of 10ms is derived from the fact that most modern (as of
7854 	 * 2023) processors can do a lot in 10ms and that it's just below what
7855 	 * most consider human-perceivable. However, the kernel also runs on a
7856 	 * lot slower CPUs including microcontrollers where the threshold is way
7857 	 * too low.
7858 	 *
7859 	 * Let's scale up the threshold upto 1 second if BogoMips is below 4000.
7860 	 * This is by no means accurate but it doesn't have to be. The mechanism
7861 	 * is still useful even when the threshold is fully scaled up. Also, as
7862 	 * the reports would usually be applicable to everyone, some machines
7863 	 * operating on longer thresholds won't significantly diminish their
7864 	 * usefulness.
7865 	 */
7866 	thresh = 10 * USEC_PER_MSEC;
7867 
7868 	/* see init/calibrate.c for lpj -> BogoMIPS calculation */
7869 	bogo = max_t(unsigned long, loops_per_jiffy / 500000 * HZ, 1);
7870 	if (bogo < 4000)
7871 		thresh = min_t(unsigned long, thresh * 4000 / bogo, USEC_PER_SEC);
7872 
7873 	pr_debug("wq_cpu_intensive_thresh: lpj=%lu BogoMIPS=%lu thresh_us=%lu\n",
7874 		 loops_per_jiffy, bogo, thresh);
7875 
7876 	wq_cpu_intensive_thresh_us = thresh;
7877 }
7878 
7879 /**
7880  * workqueue_init - bring workqueue subsystem fully online
7881  *
7882  * This is the second step of three-staged workqueue subsystem initialization
7883  * and invoked as soon as kthreads can be created and scheduled. Workqueues have
7884  * been created and work items queued on them, but there are no kworkers
7885  * executing the work items yet. Populate the worker pools with the initial
7886  * workers and enable future kworker creations.
7887  */
workqueue_init(void)7888 void __init workqueue_init(void)
7889 {
7890 	struct workqueue_struct *wq;
7891 	struct worker_pool *pool;
7892 	int cpu, bkt;
7893 
7894 	wq_cpu_intensive_thresh_init();
7895 
7896 	mutex_lock(&wq_pool_mutex);
7897 
7898 	/*
7899 	 * Per-cpu pools created earlier could be missing node hint. Fix them
7900 	 * up. Also, create a rescuer for workqueues that requested it.
7901 	 */
7902 	for_each_possible_cpu(cpu) {
7903 		for_each_bh_worker_pool(pool, cpu)
7904 			pool->node = cpu_to_node(cpu);
7905 		for_each_cpu_worker_pool(pool, cpu)
7906 			pool->node = cpu_to_node(cpu);
7907 	}
7908 
7909 	list_for_each_entry(wq, &workqueues, list) {
7910 		WARN(init_rescuer(wq),
7911 		     "workqueue: failed to create early rescuer for %s",
7912 		     wq->name);
7913 	}
7914 
7915 	mutex_unlock(&wq_pool_mutex);
7916 
7917 	/*
7918 	 * Create the initial workers. A BH pool has one pseudo worker that
7919 	 * represents the shared BH execution context and thus doesn't get
7920 	 * affected by hotplug events. Create the BH pseudo workers for all
7921 	 * possible CPUs here.
7922 	 */
7923 	for_each_possible_cpu(cpu)
7924 		for_each_bh_worker_pool(pool, cpu)
7925 			BUG_ON(!create_worker(pool));
7926 
7927 	for_each_online_cpu(cpu) {
7928 		for_each_cpu_worker_pool(pool, cpu) {
7929 			pool->flags &= ~POOL_DISASSOCIATED;
7930 			BUG_ON(!create_worker(pool));
7931 		}
7932 	}
7933 
7934 	hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
7935 		BUG_ON(!create_worker(pool));
7936 
7937 	wq_online = true;
7938 	wq_watchdog_init();
7939 }
7940 
7941 /*
7942  * Initialize @pt by first initializing @pt->cpu_pod[] with pod IDs according to
7943  * @cpu_shares_pod(). Each subset of CPUs that share a pod is assigned a unique
7944  * and consecutive pod ID. The rest of @pt is initialized accordingly.
7945  */
init_pod_type(struct wq_pod_type * pt,bool (* cpus_share_pod)(int,int))7946 static void __init init_pod_type(struct wq_pod_type *pt,
7947 				 bool (*cpus_share_pod)(int, int))
7948 {
7949 	int cur, pre, cpu, pod;
7950 
7951 	pt->nr_pods = 0;
7952 
7953 	/* init @pt->cpu_pod[] according to @cpus_share_pod() */
7954 	pt->cpu_pod = kcalloc(nr_cpu_ids, sizeof(pt->cpu_pod[0]), GFP_KERNEL);
7955 	BUG_ON(!pt->cpu_pod);
7956 
7957 	for_each_possible_cpu(cur) {
7958 		for_each_possible_cpu(pre) {
7959 			if (pre >= cur) {
7960 				pt->cpu_pod[cur] = pt->nr_pods++;
7961 				break;
7962 			}
7963 			if (cpus_share_pod(cur, pre)) {
7964 				pt->cpu_pod[cur] = pt->cpu_pod[pre];
7965 				break;
7966 			}
7967 		}
7968 	}
7969 
7970 	/* init the rest to match @pt->cpu_pod[] */
7971 	pt->pod_cpus = kcalloc(pt->nr_pods, sizeof(pt->pod_cpus[0]), GFP_KERNEL);
7972 	pt->pod_node = kcalloc(pt->nr_pods, sizeof(pt->pod_node[0]), GFP_KERNEL);
7973 	BUG_ON(!pt->pod_cpus || !pt->pod_node);
7974 
7975 	for (pod = 0; pod < pt->nr_pods; pod++)
7976 		BUG_ON(!zalloc_cpumask_var(&pt->pod_cpus[pod], GFP_KERNEL));
7977 
7978 	for_each_possible_cpu(cpu) {
7979 		cpumask_set_cpu(cpu, pt->pod_cpus[pt->cpu_pod[cpu]]);
7980 		pt->pod_node[pt->cpu_pod[cpu]] = cpu_to_node(cpu);
7981 	}
7982 }
7983 
cpus_dont_share(int cpu0,int cpu1)7984 static bool __init cpus_dont_share(int cpu0, int cpu1)
7985 {
7986 	return false;
7987 }
7988 
cpus_share_smt(int cpu0,int cpu1)7989 static bool __init cpus_share_smt(int cpu0, int cpu1)
7990 {
7991 #ifdef CONFIG_SCHED_SMT
7992 	return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1));
7993 #else
7994 	return false;
7995 #endif
7996 }
7997 
cpus_share_numa(int cpu0,int cpu1)7998 static bool __init cpus_share_numa(int cpu0, int cpu1)
7999 {
8000 	return cpu_to_node(cpu0) == cpu_to_node(cpu1);
8001 }
8002 
8003 /**
8004  * workqueue_init_topology - initialize CPU pods for unbound workqueues
8005  *
8006  * This is the third step of three-staged workqueue subsystem initialization and
8007  * invoked after SMP and topology information are fully initialized. It
8008  * initializes the unbound CPU pods accordingly.
8009  */
workqueue_init_topology(void)8010 void __init workqueue_init_topology(void)
8011 {
8012 	struct workqueue_struct *wq;
8013 	int cpu;
8014 
8015 	init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share);
8016 	init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt);
8017 	init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache);
8018 	init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa);
8019 
8020 	wq_topo_initialized = true;
8021 
8022 	mutex_lock(&wq_pool_mutex);
8023 
8024 	/*
8025 	 * Workqueues allocated earlier would have all CPUs sharing the default
8026 	 * worker pool. Explicitly call unbound_wq_update_pwq() on all workqueue
8027 	 * and CPU combinations to apply per-pod sharing.
8028 	 */
8029 	list_for_each_entry(wq, &workqueues, list) {
8030 		for_each_online_cpu(cpu)
8031 			unbound_wq_update_pwq(wq, cpu);
8032 		if (wq->flags & WQ_UNBOUND) {
8033 			mutex_lock(&wq->mutex);
8034 			wq_update_node_max_active(wq, -1);
8035 			mutex_unlock(&wq->mutex);
8036 		}
8037 	}
8038 
8039 	mutex_unlock(&wq_pool_mutex);
8040 }
8041 
__warn_flushing_systemwide_wq(void)8042 void __warn_flushing_systemwide_wq(void)
8043 {
8044 	pr_warn("WARNING: Flushing system-wide workqueues will be prohibited in near future.\n");
8045 	dump_stack();
8046 }
8047 EXPORT_SYMBOL(__warn_flushing_systemwide_wq);
8048 
workqueue_unbound_cpus_setup(char * str)8049 static int __init workqueue_unbound_cpus_setup(char *str)
8050 {
8051 	if (cpulist_parse(str, &wq_cmdline_cpumask) < 0) {
8052 		cpumask_clear(&wq_cmdline_cpumask);
8053 		pr_warn("workqueue.unbound_cpus: incorrect CPU range, using default\n");
8054 	}
8055 
8056 	return 1;
8057 }
8058 __setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup);
8059