1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/workqueue.c - generic async execution with shared worker pool 4 * 5 * Copyright (C) 2002 Ingo Molnar 6 * 7 * Derived from the taskqueue/keventd code by: 8 * David Woodhouse <dwmw2@infradead.org> 9 * Andrew Morton 10 * Kai Petzke <wpp@marie.physik.tu-berlin.de> 11 * Theodore Ts'o <tytso@mit.edu> 12 * 13 * Made to use alloc_percpu by Christoph Lameter. 14 * 15 * Copyright (C) 2010 SUSE Linux Products GmbH 16 * Copyright (C) 2010 Tejun Heo <tj@kernel.org> 17 * 18 * This is the generic async execution mechanism. Work items as are 19 * executed in process context. The worker pool is shared and 20 * automatically managed. There are two worker pools for each CPU (one for 21 * normal work items and the other for high priority ones) and some extra 22 * pools for workqueues which are not bound to any specific CPU - the 23 * number of these backing pools is dynamic. 24 * 25 * Please read Documentation/core-api/workqueue.rst for details. 26 */ 27 28 #include <linux/export.h> 29 #include <linux/kernel.h> 30 #include <linux/sched.h> 31 #include <linux/init.h> 32 #include <linux/interrupt.h> 33 #include <linux/signal.h> 34 #include <linux/completion.h> 35 #include <linux/workqueue.h> 36 #include <linux/slab.h> 37 #include <linux/cpu.h> 38 #include <linux/notifier.h> 39 #include <linux/kthread.h> 40 #include <linux/hardirq.h> 41 #include <linux/mempolicy.h> 42 #include <linux/freezer.h> 43 #include <linux/debug_locks.h> 44 #include <linux/device/devres.h> 45 #include <linux/lockdep.h> 46 #include <linux/idr.h> 47 #include <linux/jhash.h> 48 #include <linux/hashtable.h> 49 #include <linux/rculist.h> 50 #include <linux/nodemask.h> 51 #include <linux/moduleparam.h> 52 #include <linux/uaccess.h> 53 #include <linux/sched/isolation.h> 54 #include <linux/sched/debug.h> 55 #include <linux/nmi.h> 56 #include <linux/kvm_para.h> 57 #include <linux/delay.h> 58 #include <linux/irq_work.h> 59 60 #include "workqueue_internal.h" 61 62 enum worker_pool_flags { 63 /* 64 * worker_pool flags 65 * 66 * A bound pool is either associated or disassociated with its CPU. 67 * While associated (!DISASSOCIATED), all workers are bound to the 68 * CPU and none has %WORKER_UNBOUND set and concurrency management 69 * is in effect. 70 * 71 * While DISASSOCIATED, the cpu may be offline and all workers have 72 * %WORKER_UNBOUND set and concurrency management disabled, and may 73 * be executing on any CPU. The pool behaves as an unbound one. 74 * 75 * Note that DISASSOCIATED should be flipped only while holding 76 * wq_pool_attach_mutex to avoid changing binding state while 77 * worker_attach_to_pool() is in progress. 78 * 79 * As there can only be one concurrent BH execution context per CPU, a 80 * BH pool is per-CPU and always DISASSOCIATED. 81 */ 82 POOL_BH = 1 << 0, /* is a BH pool */ 83 POOL_MANAGER_ACTIVE = 1 << 1, /* being managed */ 84 POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */ 85 POOL_BH_DRAINING = 1 << 3, /* draining after CPU offline */ 86 }; 87 88 enum worker_flags { 89 /* worker flags */ 90 WORKER_DIE = 1 << 1, /* die die die */ 91 WORKER_IDLE = 1 << 2, /* is idle */ 92 WORKER_PREP = 1 << 3, /* preparing to run works */ 93 WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */ 94 WORKER_UNBOUND = 1 << 7, /* worker is unbound */ 95 WORKER_REBOUND = 1 << 8, /* worker was rebound */ 96 97 WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE | 98 WORKER_UNBOUND | WORKER_REBOUND, 99 }; 100 101 enum work_cancel_flags { 102 WORK_CANCEL_DELAYED = 1 << 0, /* canceling a delayed_work */ 103 WORK_CANCEL_DISABLE = 1 << 1, /* canceling to disable */ 104 }; 105 106 enum wq_internal_consts { 107 NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */ 108 109 UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */ 110 BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */ 111 112 MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */ 113 IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */ 114 115 MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2, 116 /* call for help after 10ms 117 (min two ticks) */ 118 MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */ 119 CREATE_COOLDOWN = HZ, /* time to breath after fail */ 120 121 RESCUER_BATCH = 16, /* process items per turn */ 122 123 /* 124 * Rescue workers are used only on emergencies and shared by 125 * all cpus. Give MIN_NICE. 126 */ 127 RESCUER_NICE_LEVEL = MIN_NICE, 128 HIGHPRI_NICE_LEVEL = MIN_NICE, 129 130 WQ_NAME_LEN = 32, 131 WORKER_ID_LEN = 10 + WQ_NAME_LEN, /* "kworker/R-" + WQ_NAME_LEN */ 132 }; 133 134 /* Layout of shards within one LLC pod */ 135 struct llc_shard_layout { 136 int nr_large_shards; /* number of large shards (cores_per_shard + 1) */ 137 int cores_per_shard; /* base number of cores per default shard */ 138 int nr_shards; /* total number of shards */ 139 /* nr_default shards = (nr_shards - nr_large_shards) */ 140 }; 141 142 /* 143 * We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and 144 * MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because 145 * msecs_to_jiffies() can't be an initializer. 146 */ 147 #define BH_WORKER_JIFFIES msecs_to_jiffies(2) 148 #define BH_WORKER_RESTARTS 10 149 150 /* 151 * Structure fields follow one of the following exclusion rules. 152 * 153 * I: Modifiable by initialization/destruction paths and read-only for 154 * everyone else. 155 * 156 * P: Preemption protected. Disabling preemption is enough and should 157 * only be modified and accessed from the local cpu. 158 * 159 * L: pool->lock protected. Access with pool->lock held. 160 * 161 * LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for 162 * reads. 163 * 164 * K: Only modified by worker while holding pool->lock. Can be safely read by 165 * self, while holding pool->lock or from IRQ context if %current is the 166 * kworker. 167 * 168 * S: Only modified by worker self. 169 * 170 * A: wq_pool_attach_mutex protected. 171 * 172 * PL: wq_pool_mutex protected. 173 * 174 * PR: wq_pool_mutex protected for writes. RCU protected for reads. 175 * 176 * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads. 177 * 178 * PWR: wq_pool_mutex and wq->mutex protected for writes. Either or 179 * RCU for reads. 180 * 181 * WQ: wq->mutex protected. 182 * 183 * WR: wq->mutex protected for writes. RCU protected for reads. 184 * 185 * WO: wq->mutex protected for writes. Updated with WRITE_ONCE() and can be read 186 * with READ_ONCE() without locking. 187 * 188 * MD: wq_mayday_lock protected. 189 * 190 * WD: Used internally by the watchdog. 191 */ 192 193 /* struct worker is defined in workqueue_internal.h */ 194 195 struct worker_pool { 196 raw_spinlock_t lock; /* the pool lock */ 197 int cpu; /* I: the associated cpu */ 198 int node; /* I: the associated node ID */ 199 int id; /* I: pool ID */ 200 unsigned int flags; /* L: flags */ 201 202 unsigned long last_progress_ts; /* L: last forward progress timestamp */ 203 bool cpu_stall; /* WD: stalled cpu bound pool */ 204 205 /* 206 * The counter is incremented in a process context on the associated CPU 207 * w/ preemption disabled, and decremented or reset in the same context 208 * but w/ pool->lock held. The readers grab pool->lock and are 209 * guaranteed to see if the counter reached zero. 210 */ 211 int nr_running; 212 213 struct list_head worklist; /* L: list of pending works */ 214 215 int nr_workers; /* L: total number of workers */ 216 int nr_idle; /* L: currently idle workers */ 217 218 struct list_head idle_list; /* L: list of idle workers */ 219 struct timer_list idle_timer; /* L: worker idle timeout */ 220 struct work_struct idle_cull_work; /* L: worker idle cleanup */ 221 222 struct timer_list mayday_timer; /* L: SOS timer for workers */ 223 224 /* a workers is either on busy_hash or idle_list, or the manager */ 225 DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER); 226 /* L: hash of busy workers */ 227 228 struct worker *manager; /* L: purely informational */ 229 struct list_head workers; /* A: attached workers */ 230 231 struct ida worker_ida; /* worker IDs for task name */ 232 233 struct workqueue_attrs *attrs; /* I: worker attributes */ 234 struct hlist_node hash_node; /* PL: unbound_pool_hash node */ 235 int refcnt; /* PL: refcnt for unbound pools */ 236 #ifdef CONFIG_PREEMPT_RT 237 spinlock_t cb_lock; /* BH worker cancel lock */ 238 #endif 239 /* 240 * Destruction of pool is RCU protected to allow dereferences 241 * from get_work_pool(). 242 */ 243 struct rcu_head rcu; 244 }; 245 246 /* 247 * Per-pool_workqueue statistics. These can be monitored using 248 * tools/workqueue/wq_monitor.py. 249 */ 250 enum pool_workqueue_stats { 251 PWQ_STAT_STARTED, /* work items started execution */ 252 PWQ_STAT_COMPLETED, /* work items completed execution */ 253 PWQ_STAT_CPU_TIME, /* total CPU time consumed */ 254 PWQ_STAT_CPU_INTENSIVE, /* wq_cpu_intensive_thresh_us violations */ 255 PWQ_STAT_CM_WAKEUP, /* concurrency-management worker wakeups */ 256 PWQ_STAT_REPATRIATED, /* unbound workers brought back into scope */ 257 PWQ_STAT_MAYDAY, /* maydays to rescuer */ 258 PWQ_STAT_RESCUED, /* linked work items executed by rescuer */ 259 260 PWQ_NR_STATS, 261 }; 262 263 /* 264 * The per-pool workqueue. While queued, bits below WORK_PWQ_SHIFT 265 * of work_struct->data are used for flags and the remaining high bits 266 * point to the pwq; thus, pwqs need to be aligned at two's power of the 267 * number of flag bits. 268 */ 269 struct pool_workqueue { 270 struct worker_pool *pool; /* I: the associated pool */ 271 struct workqueue_struct *wq; /* I: the owning workqueue */ 272 int work_color; /* L: current color */ 273 int flush_color; /* L: flushing color */ 274 int refcnt; /* L: reference count */ 275 int nr_in_flight[WORK_NR_COLORS]; 276 /* L: nr of in_flight works */ 277 bool plugged; /* L: execution suspended */ 278 279 /* 280 * nr_active management and WORK_STRUCT_INACTIVE: 281 * 282 * When pwq->nr_active >= max_active, new work item is queued to 283 * pwq->inactive_works instead of pool->worklist and marked with 284 * WORK_STRUCT_INACTIVE. 285 * 286 * All work items marked with WORK_STRUCT_INACTIVE do not participate in 287 * nr_active and all work items in pwq->inactive_works are marked with 288 * WORK_STRUCT_INACTIVE. But not all WORK_STRUCT_INACTIVE work items are 289 * in pwq->inactive_works. Some of them are ready to run in 290 * pool->worklist or worker->scheduled. Those work itmes are only struct 291 * wq_barrier which is used for flush_work() and should not participate 292 * in nr_active. For non-barrier work item, it is marked with 293 * WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works. 294 */ 295 int nr_active; /* L: nr of active works */ 296 struct list_head inactive_works; /* L: inactive works */ 297 struct list_head pending_node; /* LN: node on wq_node_nr_active->pending_pwqs */ 298 struct list_head pwqs_node; /* WR: node on wq->pwqs */ 299 struct list_head mayday_node; /* MD: node on wq->maydays */ 300 struct work_struct mayday_cursor; /* L: cursor on pool->worklist */ 301 302 u64 stats[PWQ_NR_STATS]; 303 304 /* 305 * Release of unbound pwq is punted to a kthread_worker. See put_pwq() 306 * and pwq_release_workfn() for details. pool_workqueue itself is also 307 * RCU protected so that the first pwq can be determined without 308 * grabbing wq->mutex. 309 */ 310 struct kthread_work release_work; 311 struct rcu_head rcu; 312 } __aligned(1 << WORK_STRUCT_PWQ_SHIFT); 313 314 /* 315 * Structure used to wait for workqueue flush. 316 */ 317 struct wq_flusher { 318 struct list_head list; /* WQ: list of flushers */ 319 int flush_color; /* WQ: flush color waiting for */ 320 struct completion done; /* flush completion */ 321 }; 322 323 struct wq_device; 324 325 /* 326 * Unlike in a per-cpu workqueue where max_active limits its concurrency level 327 * on each CPU, in an unbound workqueue, max_active applies to the whole system. 328 * As sharing a single nr_active across multiple sockets can be very expensive, 329 * the counting and enforcement is per NUMA node. 330 * 331 * The following struct is used to enforce per-node max_active. When a pwq wants 332 * to start executing a work item, it should increment ->nr using 333 * tryinc_node_nr_active(). If acquisition fails due to ->nr already being over 334 * ->max, the pwq is queued on ->pending_pwqs. As in-flight work items finish 335 * and decrement ->nr, node_activate_pending_pwq() activates the pending pwqs in 336 * round-robin order. 337 */ 338 struct wq_node_nr_active { 339 int max; /* per-node max_active */ 340 atomic_t nr; /* per-node nr_active */ 341 raw_spinlock_t lock; /* nests inside pool locks */ 342 struct list_head pending_pwqs; /* LN: pwqs with inactive works */ 343 }; 344 345 /* 346 * The externally visible workqueue. It relays the issued work items to 347 * the appropriate worker_pool through its pool_workqueues. 348 */ 349 struct workqueue_struct { 350 struct list_head pwqs; /* WR: all pwqs of this wq */ 351 struct list_head list; /* PR: list of all workqueues */ 352 353 struct mutex mutex; /* protects this wq */ 354 int work_color; /* WQ: current work color */ 355 int flush_color; /* WQ: current flush color */ 356 atomic_t nr_pwqs_to_flush; /* flush in progress */ 357 struct wq_flusher *first_flusher; /* WQ: first flusher */ 358 struct list_head flusher_queue; /* WQ: flush waiters */ 359 struct list_head flusher_overflow; /* WQ: flush overflow list */ 360 361 struct list_head maydays; /* MD: pwqs requesting rescue */ 362 struct worker *rescuer; /* MD: rescue worker */ 363 364 int nr_drainers; /* WQ: drain in progress */ 365 366 /* See alloc_workqueue() function comment for info on min/max_active */ 367 int max_active; /* WO: max active works */ 368 int min_active; /* WO: min active works */ 369 int saved_max_active; /* WQ: saved max_active */ 370 int saved_min_active; /* WQ: saved min_active */ 371 372 struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */ 373 struct pool_workqueue __rcu *dfl_pwq; /* PW: only for unbound wqs */ 374 375 #ifdef CONFIG_SYSFS 376 struct wq_device *wq_dev; /* I: for sysfs interface */ 377 #endif 378 #ifdef CONFIG_LOCKDEP 379 char *lock_name; 380 struct lock_class_key key; 381 struct lockdep_map __lockdep_map; 382 struct lockdep_map *lockdep_map; 383 #endif 384 char name[WQ_NAME_LEN]; /* I: workqueue name */ 385 386 /* 387 * Destruction of workqueue_struct is RCU protected to allow walking 388 * the workqueues list without grabbing wq_pool_mutex. 389 * This is used to dump all workqueues from sysrq. 390 */ 391 struct rcu_head rcu; 392 393 /* hot fields used during command issue, aligned to cacheline */ 394 unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */ 395 struct pool_workqueue __rcu * __percpu *cpu_pwq; /* I: per-cpu pwqs */ 396 struct wq_node_nr_active *node_nr_active[]; /* I: per-node nr_active */ 397 }; 398 399 /* 400 * Each pod type describes how CPUs should be grouped for unbound workqueues. 401 * See the comment above workqueue_attrs->affn_scope. 402 */ 403 struct wq_pod_type { 404 int nr_pods; /* number of pods */ 405 cpumask_var_t *pod_cpus; /* pod -> cpus */ 406 int *pod_node; /* pod -> node */ 407 int *cpu_pod; /* cpu -> pod */ 408 }; 409 410 struct work_offq_data { 411 u32 pool_id; 412 u32 disable; 413 u32 flags; 414 }; 415 416 static const char * const wq_affn_names[WQ_AFFN_NR_TYPES] = { 417 [WQ_AFFN_DFL] = "default", 418 [WQ_AFFN_CPU] = "cpu", 419 [WQ_AFFN_SMT] = "smt", 420 [WQ_AFFN_CACHE] = "cache", 421 [WQ_AFFN_CACHE_SHARD] = "cache_shard", 422 [WQ_AFFN_NUMA] = "numa", 423 [WQ_AFFN_SYSTEM] = "system", 424 }; 425 426 /* 427 * Per-cpu work items which run for longer than the following threshold are 428 * automatically considered CPU intensive and excluded from concurrency 429 * management to prevent them from noticeably delaying other per-cpu work items. 430 * ULONG_MAX indicates that the user hasn't overridden it with a boot parameter. 431 * The actual value is initialized in wq_cpu_intensive_thresh_init(). 432 */ 433 static unsigned long wq_cpu_intensive_thresh_us = ULONG_MAX; 434 module_param_named(cpu_intensive_thresh_us, wq_cpu_intensive_thresh_us, ulong, 0644); 435 #ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT 436 static unsigned int wq_cpu_intensive_warning_thresh = 4; 437 module_param_named(cpu_intensive_warning_thresh, wq_cpu_intensive_warning_thresh, uint, 0644); 438 #endif 439 440 /* see the comment above the definition of WQ_POWER_EFFICIENT */ 441 static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT); 442 module_param_named(power_efficient, wq_power_efficient, bool, 0444); 443 444 static unsigned int wq_cache_shard_size = 8; 445 module_param_named(cache_shard_size, wq_cache_shard_size, uint, 0444); 446 447 static bool wq_online; /* can kworkers be created yet? */ 448 static bool wq_topo_initialized __read_mostly = false; 449 450 static struct kmem_cache *pwq_cache; 451 452 static struct wq_pod_type wq_pod_types[WQ_AFFN_NR_TYPES]; 453 static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE_SHARD; 454 455 /* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */ 456 static struct workqueue_attrs *unbound_wq_update_pwq_attrs_buf; 457 458 static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */ 459 static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */ 460 static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */ 461 /* wait for manager to go away */ 462 static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait); 463 464 static LIST_HEAD(workqueues); /* PR: list of all workqueues */ 465 static bool workqueue_freezing; /* PL: have wqs started freezing? */ 466 467 /* PL: mirror the cpu_online_mask excluding the CPU in the midst of hotplugging */ 468 static cpumask_var_t wq_online_cpumask; 469 470 /* PL&A: allowable cpus for unbound wqs and work items */ 471 static cpumask_var_t wq_unbound_cpumask; 472 473 /* PL: user requested unbound cpumask via sysfs */ 474 static cpumask_var_t wq_requested_unbound_cpumask; 475 476 /* PL: isolated cpumask to be excluded from unbound cpumask */ 477 static cpumask_var_t wq_isolated_cpumask; 478 479 /* for further constrain wq_unbound_cpumask by cmdline parameter*/ 480 static struct cpumask wq_cmdline_cpumask __initdata; 481 482 /* CPU where unbound work was last round robin scheduled from this CPU */ 483 static DEFINE_PER_CPU(int, wq_rr_cpu_last); 484 485 /* 486 * Local execution of unbound work items is no longer guaranteed. The 487 * following always forces round-robin CPU selection on unbound work items 488 * to uncover usages which depend on it. 489 */ 490 #ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU 491 static bool wq_debug_force_rr_cpu = true; 492 #else 493 static bool wq_debug_force_rr_cpu = false; 494 #endif 495 module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644); 496 497 /* to raise softirq for the BH worker pools on other CPUs */ 498 static DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_work [NR_STD_WORKER_POOLS], bh_pool_irq_works); 499 500 /* the BH worker pools */ 501 static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], bh_worker_pools); 502 503 /* the per-cpu worker pools */ 504 static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools); 505 506 static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */ 507 508 /* PL: hash of all unbound pools keyed by pool->attrs */ 509 static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER); 510 511 /* I: attributes used when instantiating standard unbound pools on demand */ 512 static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS]; 513 514 /* I: attributes used when instantiating ordered pools on demand */ 515 static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS]; 516 517 /* 518 * I: kthread_worker to release pwq's. pwq release needs to be bounced to a 519 * process context while holding a pool lock. Bounce to a dedicated kthread 520 * worker to avoid A-A deadlocks. 521 */ 522 static struct kthread_worker *pwq_release_worker __ro_after_init; 523 524 struct workqueue_struct *system_wq __ro_after_init; 525 EXPORT_SYMBOL(system_wq); 526 struct workqueue_struct *system_percpu_wq __ro_after_init; 527 EXPORT_SYMBOL(system_percpu_wq); 528 struct workqueue_struct *system_highpri_wq __ro_after_init; 529 EXPORT_SYMBOL_GPL(system_highpri_wq); 530 struct workqueue_struct *system_long_wq __ro_after_init; 531 EXPORT_SYMBOL_GPL(system_long_wq); 532 struct workqueue_struct *system_unbound_wq __ro_after_init; 533 EXPORT_SYMBOL_GPL(system_unbound_wq); 534 struct workqueue_struct *system_dfl_wq __ro_after_init; 535 EXPORT_SYMBOL_GPL(system_dfl_wq); 536 struct workqueue_struct *system_freezable_wq __ro_after_init; 537 EXPORT_SYMBOL_GPL(system_freezable_wq); 538 struct workqueue_struct *system_power_efficient_wq __ro_after_init; 539 EXPORT_SYMBOL_GPL(system_power_efficient_wq); 540 struct workqueue_struct *system_freezable_power_efficient_wq __ro_after_init; 541 EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq); 542 struct workqueue_struct *system_bh_wq; 543 EXPORT_SYMBOL_GPL(system_bh_wq); 544 struct workqueue_struct *system_bh_highpri_wq; 545 EXPORT_SYMBOL_GPL(system_bh_highpri_wq); 546 struct workqueue_struct *system_dfl_long_wq __ro_after_init; 547 EXPORT_SYMBOL_GPL(system_dfl_long_wq); 548 549 static int worker_thread(void *__worker); 550 static void workqueue_sysfs_unregister(struct workqueue_struct *wq); 551 static void show_pwq(struct pool_workqueue *pwq); 552 static void show_one_worker_pool(struct worker_pool *pool); 553 554 #define CREATE_TRACE_POINTS 555 #include <trace/events/workqueue.h> 556 557 #define assert_rcu_or_pool_mutex() \ 558 RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() && \ 559 !lockdep_is_held(&wq_pool_mutex), \ 560 "RCU or wq_pool_mutex should be held") 561 562 #define for_each_bh_worker_pool(pool, cpu) \ 563 for ((pool) = &per_cpu(bh_worker_pools, cpu)[0]; \ 564 (pool) < &per_cpu(bh_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \ 565 (pool)++) 566 567 #define for_each_cpu_worker_pool(pool, cpu) \ 568 for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \ 569 (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \ 570 (pool)++) 571 572 /** 573 * for_each_pool - iterate through all worker_pools in the system 574 * @pool: iteration cursor 575 * @pi: integer used for iteration 576 * 577 * This must be called either with wq_pool_mutex held or RCU read 578 * locked. If the pool needs to be used beyond the locking in effect, the 579 * caller is responsible for guaranteeing that the pool stays online. 580 * 581 * The if/else clause exists only for the lockdep assertion and can be 582 * ignored. 583 */ 584 #define for_each_pool(pool, pi) \ 585 idr_for_each_entry(&worker_pool_idr, pool, pi) \ 586 if (({ assert_rcu_or_pool_mutex(); false; })) { } \ 587 else 588 589 /** 590 * for_each_pool_worker - iterate through all workers of a worker_pool 591 * @worker: iteration cursor 592 * @pool: worker_pool to iterate workers of 593 * 594 * This must be called with wq_pool_attach_mutex. 595 * 596 * The if/else clause exists only for the lockdep assertion and can be 597 * ignored. 598 */ 599 #define for_each_pool_worker(worker, pool) \ 600 list_for_each_entry((worker), &(pool)->workers, node) \ 601 if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \ 602 else 603 604 /** 605 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue 606 * @pwq: iteration cursor 607 * @wq: the target workqueue 608 * 609 * This must be called either with wq->mutex held or RCU read locked. 610 * If the pwq needs to be used beyond the locking in effect, the caller is 611 * responsible for guaranteeing that the pwq stays online. 612 * 613 * The if/else clause exists only for the lockdep assertion and can be 614 * ignored. 615 */ 616 #define for_each_pwq(pwq, wq) \ 617 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \ 618 lockdep_is_held(&(wq->mutex))) 619 620 #ifdef CONFIG_DEBUG_OBJECTS_WORK 621 622 static const struct debug_obj_descr work_debug_descr; 623 624 static void *work_debug_hint(void *addr) 625 { 626 return ((struct work_struct *) addr)->func; 627 } 628 629 static bool work_is_static_object(void *addr) 630 { 631 struct work_struct *work = addr; 632 633 return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work)); 634 } 635 636 /* 637 * fixup_init is called when: 638 * - an active object is initialized 639 */ 640 static bool work_fixup_init(void *addr, enum debug_obj_state state) 641 { 642 struct work_struct *work = addr; 643 644 switch (state) { 645 case ODEBUG_STATE_ACTIVE: 646 cancel_work_sync(work); 647 debug_object_init(work, &work_debug_descr); 648 return true; 649 default: 650 return false; 651 } 652 } 653 654 /* 655 * fixup_free is called when: 656 * - an active object is freed 657 */ 658 static bool work_fixup_free(void *addr, enum debug_obj_state state) 659 { 660 struct work_struct *work = addr; 661 662 switch (state) { 663 case ODEBUG_STATE_ACTIVE: 664 cancel_work_sync(work); 665 debug_object_free(work, &work_debug_descr); 666 return true; 667 default: 668 return false; 669 } 670 } 671 672 static const struct debug_obj_descr work_debug_descr = { 673 .name = "work_struct", 674 .debug_hint = work_debug_hint, 675 .is_static_object = work_is_static_object, 676 .fixup_init = work_fixup_init, 677 .fixup_free = work_fixup_free, 678 }; 679 680 static inline void debug_work_activate(struct work_struct *work) 681 { 682 debug_object_activate(work, &work_debug_descr); 683 } 684 685 static inline void debug_work_deactivate(struct work_struct *work) 686 { 687 debug_object_deactivate(work, &work_debug_descr); 688 } 689 690 void __init_work(struct work_struct *work, int onstack) 691 { 692 if (onstack) 693 debug_object_init_on_stack(work, &work_debug_descr); 694 else 695 debug_object_init(work, &work_debug_descr); 696 } 697 EXPORT_SYMBOL_GPL(__init_work); 698 699 void destroy_work_on_stack(struct work_struct *work) 700 { 701 debug_object_free(work, &work_debug_descr); 702 } 703 EXPORT_SYMBOL_GPL(destroy_work_on_stack); 704 705 void destroy_delayed_work_on_stack(struct delayed_work *work) 706 { 707 timer_destroy_on_stack(&work->timer); 708 debug_object_free(&work->work, &work_debug_descr); 709 } 710 EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack); 711 712 #else 713 static inline void debug_work_activate(struct work_struct *work) { } 714 static inline void debug_work_deactivate(struct work_struct *work) { } 715 #endif 716 717 /** 718 * worker_pool_assign_id - allocate ID and assign it to @pool 719 * @pool: the pool pointer of interest 720 * 721 * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned 722 * successfully, -errno on failure. 723 */ 724 static int worker_pool_assign_id(struct worker_pool *pool) 725 { 726 int ret; 727 728 lockdep_assert_held(&wq_pool_mutex); 729 730 ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE, 731 GFP_KERNEL); 732 if (ret >= 0) { 733 pool->id = ret; 734 return 0; 735 } 736 return ret; 737 } 738 739 static struct pool_workqueue __rcu ** 740 unbound_pwq_slot(struct workqueue_struct *wq, int cpu) 741 { 742 if (cpu >= 0) 743 return per_cpu_ptr(wq->cpu_pwq, cpu); 744 else 745 return &wq->dfl_pwq; 746 } 747 748 /* @cpu < 0 for dfl_pwq */ 749 static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu) 750 { 751 return rcu_dereference_check(*unbound_pwq_slot(wq, cpu), 752 lockdep_is_held(&wq_pool_mutex) || 753 lockdep_is_held(&wq->mutex)); 754 } 755 756 /** 757 * unbound_effective_cpumask - effective cpumask of an unbound workqueue 758 * @wq: workqueue of interest 759 * 760 * @wq->unbound_attrs->cpumask contains the cpumask requested by the user which 761 * is masked with wq_unbound_cpumask to determine the effective cpumask. The 762 * default pwq is always mapped to the pool with the current effective cpumask. 763 */ 764 static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq) 765 { 766 return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask; 767 } 768 769 static unsigned int work_color_to_flags(int color) 770 { 771 return color << WORK_STRUCT_COLOR_SHIFT; 772 } 773 774 static int get_work_color(unsigned long work_data) 775 { 776 return (work_data >> WORK_STRUCT_COLOR_SHIFT) & 777 ((1 << WORK_STRUCT_COLOR_BITS) - 1); 778 } 779 780 static int work_next_color(int color) 781 { 782 return (color + 1) % WORK_NR_COLORS; 783 } 784 785 static unsigned long pool_offq_flags(struct worker_pool *pool) 786 { 787 return (pool->flags & POOL_BH) ? WORK_OFFQ_BH : 0; 788 } 789 790 /* 791 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data 792 * contain the pointer to the queued pwq. Once execution starts, the flag 793 * is cleared and the high bits contain OFFQ flags and pool ID. 794 * 795 * set_work_pwq(), set_work_pool_and_clear_pending() and mark_work_canceling() 796 * can be used to set the pwq, pool or clear work->data. These functions should 797 * only be called while the work is owned - ie. while the PENDING bit is set. 798 * 799 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq 800 * corresponding to a work. Pool is available once the work has been 801 * queued anywhere after initialization until it is sync canceled. pwq is 802 * available only while the work item is queued. 803 */ 804 static inline void set_work_data(struct work_struct *work, unsigned long data) 805 { 806 WARN_ON_ONCE(!work_pending(work)); 807 atomic_long_set(&work->data, data | work_static(work)); 808 } 809 810 static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq, 811 unsigned long flags) 812 { 813 set_work_data(work, (unsigned long)pwq | WORK_STRUCT_PENDING | 814 WORK_STRUCT_PWQ | flags); 815 } 816 817 static void set_work_pool_and_keep_pending(struct work_struct *work, 818 int pool_id, unsigned long flags) 819 { 820 set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) | 821 WORK_STRUCT_PENDING | flags); 822 } 823 824 static void set_work_pool_and_clear_pending(struct work_struct *work, 825 int pool_id, unsigned long flags) 826 { 827 /* 828 * The following wmb is paired with the implied mb in 829 * test_and_set_bit(PENDING) and ensures all updates to @work made 830 * here are visible to and precede any updates by the next PENDING 831 * owner. 832 */ 833 smp_wmb(); 834 set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) | 835 flags); 836 /* 837 * The following mb guarantees that previous clear of a PENDING bit 838 * will not be reordered with any speculative LOADS or STORES from 839 * work->current_func, which is executed afterwards. This possible 840 * reordering can lead to a missed execution on attempt to queue 841 * the same @work. E.g. consider this case: 842 * 843 * CPU#0 CPU#1 844 * ---------------------------- -------------------------------- 845 * 846 * 1 STORE event_indicated 847 * 2 queue_work_on() { 848 * 3 test_and_set_bit(PENDING) 849 * 4 } set_..._and_clear_pending() { 850 * 5 set_work_data() # clear bit 851 * 6 smp_mb() 852 * 7 work->current_func() { 853 * 8 LOAD event_indicated 854 * } 855 * 856 * Without an explicit full barrier speculative LOAD on line 8 can 857 * be executed before CPU#0 does STORE on line 1. If that happens, 858 * CPU#0 observes the PENDING bit is still set and new execution of 859 * a @work is not queued in a hope, that CPU#1 will eventually 860 * finish the queued @work. Meanwhile CPU#1 does not see 861 * event_indicated is set, because speculative LOAD was executed 862 * before actual STORE. 863 */ 864 smp_mb(); 865 } 866 867 static inline struct pool_workqueue *work_struct_pwq(unsigned long data) 868 { 869 return (struct pool_workqueue *)(data & WORK_STRUCT_PWQ_MASK); 870 } 871 872 static struct pool_workqueue *get_work_pwq(struct work_struct *work) 873 { 874 unsigned long data = atomic_long_read(&work->data); 875 876 if (data & WORK_STRUCT_PWQ) 877 return work_struct_pwq(data); 878 else 879 return NULL; 880 } 881 882 /** 883 * get_work_pool - return the worker_pool a given work was associated with 884 * @work: the work item of interest 885 * 886 * Pools are created and destroyed under wq_pool_mutex, and allows read 887 * access under RCU read lock. As such, this function should be 888 * called under wq_pool_mutex or inside of a rcu_read_lock() region. 889 * 890 * All fields of the returned pool are accessible as long as the above 891 * mentioned locking is in effect. If the returned pool needs to be used 892 * beyond the critical section, the caller is responsible for ensuring the 893 * returned pool is and stays online. 894 * 895 * Return: The worker_pool @work was last associated with. %NULL if none. 896 */ 897 static struct worker_pool *get_work_pool(struct work_struct *work) 898 { 899 unsigned long data = atomic_long_read(&work->data); 900 int pool_id; 901 902 assert_rcu_or_pool_mutex(); 903 904 if (data & WORK_STRUCT_PWQ) 905 return work_struct_pwq(data)->pool; 906 907 pool_id = data >> WORK_OFFQ_POOL_SHIFT; 908 if (pool_id == WORK_OFFQ_POOL_NONE) 909 return NULL; 910 911 return idr_find(&worker_pool_idr, pool_id); 912 } 913 914 static unsigned long shift_and_mask(unsigned long v, u32 shift, u32 bits) 915 { 916 return (v >> shift) & ((1U << bits) - 1); 917 } 918 919 static void work_offqd_unpack(struct work_offq_data *offqd, unsigned long data) 920 { 921 WARN_ON_ONCE(data & WORK_STRUCT_PWQ); 922 923 offqd->pool_id = shift_and_mask(data, WORK_OFFQ_POOL_SHIFT, 924 WORK_OFFQ_POOL_BITS); 925 offqd->disable = shift_and_mask(data, WORK_OFFQ_DISABLE_SHIFT, 926 WORK_OFFQ_DISABLE_BITS); 927 offqd->flags = data & WORK_OFFQ_FLAG_MASK; 928 } 929 930 static unsigned long work_offqd_pack_flags(struct work_offq_data *offqd) 931 { 932 return ((unsigned long)offqd->disable << WORK_OFFQ_DISABLE_SHIFT) | 933 ((unsigned long)offqd->flags); 934 } 935 936 /* 937 * Policy functions. These define the policies on how the global worker 938 * pools are managed. Unless noted otherwise, these functions assume that 939 * they're being called with pool->lock held. 940 */ 941 942 /* 943 * Need to wake up a worker? Called from anything but currently 944 * running workers. 945 * 946 * Note that, because unbound workers never contribute to nr_running, this 947 * function will always return %true for unbound pools as long as the 948 * worklist isn't empty. 949 */ 950 static bool need_more_worker(struct worker_pool *pool) 951 { 952 return !list_empty(&pool->worklist) && !pool->nr_running; 953 } 954 955 /* Can I start working? Called from busy but !running workers. */ 956 static bool may_start_working(struct worker_pool *pool) 957 { 958 return pool->nr_idle; 959 } 960 961 /* Do I need to keep working? Called from currently running workers. */ 962 static bool keep_working(struct worker_pool *pool) 963 { 964 return !list_empty(&pool->worklist) && (pool->nr_running <= 1); 965 } 966 967 /* Do we need a new worker? Called from manager. */ 968 static bool need_to_create_worker(struct worker_pool *pool) 969 { 970 return need_more_worker(pool) && !may_start_working(pool); 971 } 972 973 /* Do we have too many workers and should some go away? */ 974 static bool too_many_workers(struct worker_pool *pool) 975 { 976 bool managing = pool->flags & POOL_MANAGER_ACTIVE; 977 int nr_idle = pool->nr_idle + managing; /* manager is considered idle */ 978 int nr_busy = pool->nr_workers - nr_idle; 979 980 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy; 981 } 982 983 /** 984 * worker_set_flags - set worker flags and adjust nr_running accordingly 985 * @worker: self 986 * @flags: flags to set 987 * 988 * Set @flags in @worker->flags and adjust nr_running accordingly. 989 */ 990 static inline void worker_set_flags(struct worker *worker, unsigned int flags) 991 { 992 struct worker_pool *pool = worker->pool; 993 994 lockdep_assert_held(&pool->lock); 995 996 /* If transitioning into NOT_RUNNING, adjust nr_running. */ 997 if ((flags & WORKER_NOT_RUNNING) && 998 !(worker->flags & WORKER_NOT_RUNNING)) { 999 pool->nr_running--; 1000 } 1001 1002 worker->flags |= flags; 1003 } 1004 1005 /** 1006 * worker_clr_flags - clear worker flags and adjust nr_running accordingly 1007 * @worker: self 1008 * @flags: flags to clear 1009 * 1010 * Clear @flags in @worker->flags and adjust nr_running accordingly. 1011 */ 1012 static inline void worker_clr_flags(struct worker *worker, unsigned int flags) 1013 { 1014 struct worker_pool *pool = worker->pool; 1015 unsigned int oflags = worker->flags; 1016 1017 lockdep_assert_held(&pool->lock); 1018 1019 worker->flags &= ~flags; 1020 1021 /* 1022 * If transitioning out of NOT_RUNNING, increment nr_running. Note 1023 * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask 1024 * of multiple flags, not a single flag. 1025 */ 1026 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING)) 1027 if (!(worker->flags & WORKER_NOT_RUNNING)) 1028 pool->nr_running++; 1029 } 1030 1031 /* Return the first idle worker. Called with pool->lock held. */ 1032 static struct worker *first_idle_worker(struct worker_pool *pool) 1033 { 1034 if (unlikely(list_empty(&pool->idle_list))) 1035 return NULL; 1036 1037 return list_first_entry(&pool->idle_list, struct worker, entry); 1038 } 1039 1040 /** 1041 * worker_enter_idle - enter idle state 1042 * @worker: worker which is entering idle state 1043 * 1044 * @worker is entering idle state. Update stats and idle timer if 1045 * necessary. 1046 * 1047 * LOCKING: 1048 * raw_spin_lock_irq(pool->lock). 1049 */ 1050 static void worker_enter_idle(struct worker *worker) 1051 { 1052 struct worker_pool *pool = worker->pool; 1053 1054 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) || 1055 WARN_ON_ONCE(!list_empty(&worker->entry) && 1056 (worker->hentry.next || worker->hentry.pprev))) 1057 return; 1058 1059 /* can't use worker_set_flags(), also called from create_worker() */ 1060 worker->flags |= WORKER_IDLE; 1061 pool->nr_idle++; 1062 worker->last_active = jiffies; 1063 1064 /* idle_list is LIFO */ 1065 list_add(&worker->entry, &pool->idle_list); 1066 1067 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) 1068 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); 1069 1070 /* Sanity check nr_running. */ 1071 WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running); 1072 } 1073 1074 /** 1075 * worker_leave_idle - leave idle state 1076 * @worker: worker which is leaving idle state 1077 * 1078 * @worker is leaving idle state. Update stats. 1079 * 1080 * LOCKING: 1081 * raw_spin_lock_irq(pool->lock). 1082 */ 1083 static void worker_leave_idle(struct worker *worker) 1084 { 1085 struct worker_pool *pool = worker->pool; 1086 1087 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE))) 1088 return; 1089 worker_clr_flags(worker, WORKER_IDLE); 1090 pool->nr_idle--; 1091 list_del_init(&worker->entry); 1092 } 1093 1094 /** 1095 * find_worker_executing_work - find worker which is executing a work 1096 * @pool: pool of interest 1097 * @work: work to find worker for 1098 * 1099 * Find a worker which is executing @work on @pool by searching 1100 * @pool->busy_hash which is keyed by the address of @work. For a worker 1101 * to match, its current execution should match the address of @work and 1102 * its work function. This is to avoid unwanted dependency between 1103 * unrelated work executions through a work item being recycled while still 1104 * being executed. 1105 * 1106 * This is a bit tricky. A work item may be freed once its execution 1107 * starts and nothing prevents the freed area from being recycled for 1108 * another work item. If the same work item address ends up being reused 1109 * before the original execution finishes, workqueue will identify the 1110 * recycled work item as currently executing and make it wait until the 1111 * current execution finishes, introducing an unwanted dependency. 1112 * 1113 * This function checks the work item address and work function to avoid 1114 * false positives. Note that this isn't complete as one may construct a 1115 * work function which can introduce dependency onto itself through a 1116 * recycled work item. Well, if somebody wants to shoot oneself in the 1117 * foot that badly, there's only so much we can do, and if such deadlock 1118 * actually occurs, it should be easy to locate the culprit work function. 1119 * 1120 * CONTEXT: 1121 * raw_spin_lock_irq(pool->lock). 1122 * 1123 * Return: 1124 * Pointer to worker which is executing @work if found, %NULL 1125 * otherwise. 1126 */ 1127 static struct worker *find_worker_executing_work(struct worker_pool *pool, 1128 struct work_struct *work) 1129 { 1130 struct worker *worker; 1131 1132 hash_for_each_possible(pool->busy_hash, worker, hentry, 1133 (unsigned long)work) 1134 if (worker->current_work == work && 1135 worker->current_func == work->func) 1136 return worker; 1137 1138 return NULL; 1139 } 1140 1141 static void mayday_cursor_func(struct work_struct *work) 1142 { 1143 /* should not be processed, only for marking position */ 1144 BUG(); 1145 } 1146 1147 /** 1148 * move_linked_works - move linked works to a list 1149 * @work: start of series of works to be scheduled 1150 * @head: target list to append @work to 1151 * @nextp: out parameter for nested worklist walking 1152 * 1153 * Schedule linked works starting from @work to @head. Work series to be 1154 * scheduled starts at @work and includes any consecutive work with 1155 * WORK_STRUCT_LINKED set in its predecessor. See assign_work() for details on 1156 * @nextp. 1157 * 1158 * CONTEXT: 1159 * raw_spin_lock_irq(pool->lock). 1160 */ 1161 static void move_linked_works(struct work_struct *work, struct list_head *head, 1162 struct work_struct **nextp) 1163 { 1164 struct work_struct *n; 1165 1166 /* 1167 * Linked worklist will always end before the end of the list, 1168 * use NULL for list head. 1169 */ 1170 list_for_each_entry_safe_from(work, n, NULL, entry) { 1171 list_move_tail(&work->entry, head); 1172 if (!(*work_data_bits(work) & WORK_STRUCT_LINKED)) 1173 break; 1174 } 1175 1176 /* 1177 * If we're already inside safe list traversal and have moved 1178 * multiple works to the scheduled queue, the next position 1179 * needs to be updated. 1180 */ 1181 if (nextp) 1182 *nextp = n; 1183 } 1184 1185 /** 1186 * assign_work - assign a work item and its linked work items to a worker 1187 * @work: work to assign 1188 * @worker: worker to assign to 1189 * @nextp: out parameter for nested worklist walking 1190 * 1191 * Assign @work and its linked work items to @worker. If @work is already being 1192 * executed by another worker in the same pool, it'll be punted there. 1193 * 1194 * If @nextp is not NULL, it's updated to point to the next work of the last 1195 * scheduled work. This allows assign_work() to be nested inside 1196 * list_for_each_entry_safe(). 1197 * 1198 * Returns %true if @work was successfully assigned to @worker. %false if @work 1199 * was punted to another worker already executing it. 1200 */ 1201 static bool assign_work(struct work_struct *work, struct worker *worker, 1202 struct work_struct **nextp) 1203 { 1204 struct worker_pool *pool = worker->pool; 1205 struct worker *collision; 1206 1207 lockdep_assert_held(&pool->lock); 1208 1209 /* The cursor work should not be processed */ 1210 if (unlikely(work->func == mayday_cursor_func)) { 1211 /* only worker_thread() can possibly take this branch */ 1212 WARN_ON_ONCE(worker->rescue_wq); 1213 if (nextp) 1214 *nextp = list_next_entry(work, entry); 1215 list_del_init(&work->entry); 1216 return false; 1217 } 1218 1219 /* 1220 * A single work shouldn't be executed concurrently by multiple workers. 1221 * __queue_work() ensures that @work doesn't jump to a different pool 1222 * while still running in the previous pool. Here, we should ensure that 1223 * @work is not executed concurrently by multiple workers from the same 1224 * pool. Check whether anyone is already processing the work. If so, 1225 * defer the work to the currently executing one. 1226 */ 1227 collision = find_worker_executing_work(pool, work); 1228 if (unlikely(collision)) { 1229 move_linked_works(work, &collision->scheduled, nextp); 1230 return false; 1231 } 1232 1233 move_linked_works(work, &worker->scheduled, nextp); 1234 return true; 1235 } 1236 1237 static struct irq_work *bh_pool_irq_work(struct worker_pool *pool) 1238 { 1239 int high = pool->attrs->nice == HIGHPRI_NICE_LEVEL ? 1 : 0; 1240 1241 return &per_cpu(bh_pool_irq_works, pool->cpu)[high]; 1242 } 1243 1244 static void kick_bh_pool(struct worker_pool *pool) 1245 { 1246 #ifdef CONFIG_SMP 1247 /* see drain_dead_softirq_workfn() for BH_DRAINING */ 1248 if (unlikely(pool->cpu != smp_processor_id() && 1249 !(pool->flags & POOL_BH_DRAINING))) { 1250 irq_work_queue_on(bh_pool_irq_work(pool), pool->cpu); 1251 return; 1252 } 1253 #endif 1254 if (pool->attrs->nice == HIGHPRI_NICE_LEVEL) 1255 raise_softirq_irqoff(HI_SOFTIRQ); 1256 else 1257 raise_softirq_irqoff(TASKLET_SOFTIRQ); 1258 } 1259 1260 /** 1261 * kick_pool - wake up an idle worker if necessary 1262 * @pool: pool to kick 1263 * 1264 * @pool may have pending work items. Wake up worker if necessary. Returns 1265 * whether a worker was woken up. 1266 */ 1267 static bool kick_pool(struct worker_pool *pool) 1268 { 1269 struct worker *worker = first_idle_worker(pool); 1270 struct task_struct *p; 1271 1272 lockdep_assert_held(&pool->lock); 1273 1274 if (!need_more_worker(pool) || !worker) 1275 return false; 1276 1277 if (pool->flags & POOL_BH) { 1278 kick_bh_pool(pool); 1279 return true; 1280 } 1281 1282 p = worker->task; 1283 1284 #ifdef CONFIG_SMP 1285 /* 1286 * Idle @worker is about to execute @work and waking up provides an 1287 * opportunity to migrate @worker at a lower cost by setting the task's 1288 * wake_cpu field. Let's see if we want to move @worker to improve 1289 * execution locality. 1290 * 1291 * We're waking the worker that went idle the latest and there's some 1292 * chance that @worker is marked idle but hasn't gone off CPU yet. If 1293 * so, setting the wake_cpu won't do anything. As this is a best-effort 1294 * optimization and the race window is narrow, let's leave as-is for 1295 * now. If this becomes pronounced, we can skip over workers which are 1296 * still on cpu when picking an idle worker. 1297 * 1298 * If @pool has non-strict affinity, @worker might have ended up outside 1299 * its affinity scope. Repatriate. 1300 */ 1301 if (!pool->attrs->affn_strict && 1302 !cpumask_test_cpu(p->wake_cpu, pool->attrs->__pod_cpumask)) { 1303 struct work_struct *work = list_first_entry(&pool->worklist, 1304 struct work_struct, entry); 1305 int wake_cpu = cpumask_any_and_distribute(pool->attrs->__pod_cpumask, 1306 cpu_online_mask); 1307 if (wake_cpu < nr_cpu_ids) { 1308 p->wake_cpu = wake_cpu; 1309 get_work_pwq(work)->stats[PWQ_STAT_REPATRIATED]++; 1310 } 1311 } 1312 #endif 1313 wake_up_process(p); 1314 return true; 1315 } 1316 1317 #ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT 1318 1319 /* 1320 * Concurrency-managed per-cpu work items that hog CPU for longer than 1321 * wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism, 1322 * which prevents them from stalling other concurrency-managed work items. If a 1323 * work function keeps triggering this mechanism, it's likely that the work item 1324 * should be using an unbound workqueue instead. 1325 * 1326 * wq_cpu_intensive_report() tracks work functions which trigger such conditions 1327 * and report them so that they can be examined and converted to use unbound 1328 * workqueues as appropriate. To avoid flooding the console, each violating work 1329 * function is tracked and reported with exponential backoff. 1330 */ 1331 #define WCI_MAX_ENTS 128 1332 1333 struct wci_ent { 1334 work_func_t func; 1335 atomic64_t cnt; 1336 struct hlist_node hash_node; 1337 }; 1338 1339 static struct wci_ent wci_ents[WCI_MAX_ENTS]; 1340 static int wci_nr_ents; 1341 static DEFINE_RAW_SPINLOCK(wci_lock); 1342 static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS)); 1343 1344 static struct wci_ent *wci_find_ent(work_func_t func) 1345 { 1346 struct wci_ent *ent; 1347 1348 hash_for_each_possible_rcu(wci_hash, ent, hash_node, 1349 (unsigned long)func) { 1350 if (ent->func == func) 1351 return ent; 1352 } 1353 return NULL; 1354 } 1355 1356 static void wq_cpu_intensive_report(work_func_t func) 1357 { 1358 struct wci_ent *ent; 1359 1360 restart: 1361 ent = wci_find_ent(func); 1362 if (ent) { 1363 u64 cnt; 1364 1365 /* 1366 * Start reporting from the warning_thresh and back off 1367 * exponentially. 1368 */ 1369 cnt = atomic64_inc_return_relaxed(&ent->cnt); 1370 if (wq_cpu_intensive_warning_thresh && 1371 cnt >= wq_cpu_intensive_warning_thresh && 1372 is_power_of_2(cnt + 1 - wq_cpu_intensive_warning_thresh)) 1373 printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n", 1374 ent->func, wq_cpu_intensive_thresh_us, 1375 atomic64_read(&ent->cnt)); 1376 return; 1377 } 1378 1379 /* 1380 * @func is a new violation. Allocate a new entry for it. If wcn_ents[] 1381 * is exhausted, something went really wrong and we probably made enough 1382 * noise already. 1383 */ 1384 if (wci_nr_ents >= WCI_MAX_ENTS) 1385 return; 1386 1387 raw_spin_lock(&wci_lock); 1388 1389 if (wci_nr_ents >= WCI_MAX_ENTS) { 1390 raw_spin_unlock(&wci_lock); 1391 return; 1392 } 1393 1394 if (wci_find_ent(func)) { 1395 raw_spin_unlock(&wci_lock); 1396 goto restart; 1397 } 1398 1399 ent = &wci_ents[wci_nr_ents++]; 1400 ent->func = func; 1401 atomic64_set(&ent->cnt, 0); 1402 hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func); 1403 1404 raw_spin_unlock(&wci_lock); 1405 1406 goto restart; 1407 } 1408 1409 #else /* CONFIG_WQ_CPU_INTENSIVE_REPORT */ 1410 static void wq_cpu_intensive_report(work_func_t func) {} 1411 #endif /* CONFIG_WQ_CPU_INTENSIVE_REPORT */ 1412 1413 /** 1414 * wq_worker_running - a worker is running again 1415 * @task: task waking up 1416 * 1417 * This function is called when a worker returns from schedule() 1418 */ 1419 void wq_worker_running(struct task_struct *task) 1420 { 1421 struct worker *worker = kthread_data(task); 1422 1423 if (!READ_ONCE(worker->sleeping)) 1424 return; 1425 1426 /* 1427 * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check 1428 * and the nr_running increment below, we may ruin the nr_running reset 1429 * and leave with an unexpected pool->nr_running == 1 on the newly unbound 1430 * pool. Protect against such race. 1431 */ 1432 preempt_disable(); 1433 if (!(worker->flags & WORKER_NOT_RUNNING)) 1434 worker->pool->nr_running++; 1435 preempt_enable(); 1436 1437 /* 1438 * CPU intensive auto-detection cares about how long a work item hogged 1439 * CPU without sleeping. Reset the starting timestamp on wakeup. 1440 */ 1441 worker->current_at = worker->task->se.sum_exec_runtime; 1442 1443 WRITE_ONCE(worker->sleeping, 0); 1444 } 1445 1446 /** 1447 * wq_worker_sleeping - a worker is going to sleep 1448 * @task: task going to sleep 1449 * 1450 * This function is called from schedule() when a busy worker is 1451 * going to sleep. 1452 */ 1453 void wq_worker_sleeping(struct task_struct *task) 1454 { 1455 struct worker *worker = kthread_data(task); 1456 struct worker_pool *pool; 1457 1458 /* 1459 * Rescuers, which may not have all the fields set up like normal 1460 * workers, also reach here, let's not access anything before 1461 * checking NOT_RUNNING. 1462 */ 1463 if (worker->flags & WORKER_NOT_RUNNING) 1464 return; 1465 1466 pool = worker->pool; 1467 1468 /* Return if preempted before wq_worker_running() was reached */ 1469 if (READ_ONCE(worker->sleeping)) 1470 return; 1471 1472 WRITE_ONCE(worker->sleeping, 1); 1473 raw_spin_lock_irq(&pool->lock); 1474 1475 /* 1476 * Recheck in case unbind_workers() preempted us. We don't 1477 * want to decrement nr_running after the worker is unbound 1478 * and nr_running has been reset. 1479 */ 1480 if (worker->flags & WORKER_NOT_RUNNING) { 1481 raw_spin_unlock_irq(&pool->lock); 1482 return; 1483 } 1484 1485 pool->nr_running--; 1486 if (kick_pool(pool)) 1487 worker->current_pwq->stats[PWQ_STAT_CM_WAKEUP]++; 1488 1489 raw_spin_unlock_irq(&pool->lock); 1490 } 1491 1492 /** 1493 * wq_worker_tick - a scheduler tick occurred while a kworker is running 1494 * @task: task currently running 1495 * 1496 * Called from sched_tick(). We're in the IRQ context and the current 1497 * worker's fields which follow the 'K' locking rule can be accessed safely. 1498 */ 1499 void wq_worker_tick(struct task_struct *task) 1500 { 1501 struct worker *worker = kthread_data(task); 1502 struct pool_workqueue *pwq = worker->current_pwq; 1503 struct worker_pool *pool = worker->pool; 1504 1505 if (!pwq) 1506 return; 1507 1508 pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC; 1509 1510 if (!wq_cpu_intensive_thresh_us) 1511 return; 1512 1513 /* 1514 * If the current worker is concurrency managed and hogged the CPU for 1515 * longer than wq_cpu_intensive_thresh_us, it's automatically marked 1516 * CPU_INTENSIVE to avoid stalling other concurrency-managed work items. 1517 * 1518 * Set @worker->sleeping means that @worker is in the process of 1519 * switching out voluntarily and won't be contributing to 1520 * @pool->nr_running until it wakes up. As wq_worker_sleeping() also 1521 * decrements ->nr_running, setting CPU_INTENSIVE here can lead to 1522 * double decrements. The task is releasing the CPU anyway. Let's skip. 1523 * We probably want to make this prettier in the future. 1524 */ 1525 if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) || 1526 worker->task->se.sum_exec_runtime - worker->current_at < 1527 wq_cpu_intensive_thresh_us * NSEC_PER_USEC) 1528 return; 1529 1530 raw_spin_lock(&pool->lock); 1531 1532 worker_set_flags(worker, WORKER_CPU_INTENSIVE); 1533 wq_cpu_intensive_report(worker->current_func); 1534 pwq->stats[PWQ_STAT_CPU_INTENSIVE]++; 1535 1536 if (kick_pool(pool)) 1537 pwq->stats[PWQ_STAT_CM_WAKEUP]++; 1538 1539 raw_spin_unlock(&pool->lock); 1540 } 1541 1542 /** 1543 * wq_worker_last_func - retrieve worker's last work function 1544 * @task: Task to retrieve last work function of. 1545 * 1546 * Determine the last function a worker executed. This is called from 1547 * the scheduler to get a worker's last known identity. 1548 * 1549 * CONTEXT: 1550 * raw_spin_lock_irq(rq->lock) 1551 * 1552 * This function is called during schedule() when a kworker is going 1553 * to sleep. It's used by psi to identify aggregation workers during 1554 * dequeuing, to allow periodic aggregation to shut-off when that 1555 * worker is the last task in the system or cgroup to go to sleep. 1556 * 1557 * As this function doesn't involve any workqueue-related locking, it 1558 * only returns stable values when called from inside the scheduler's 1559 * queuing and dequeuing paths, when @task, which must be a kworker, 1560 * is guaranteed to not be processing any works. 1561 * 1562 * Return: 1563 * The last work function %current executed as a worker, NULL if it 1564 * hasn't executed any work yet. 1565 */ 1566 work_func_t wq_worker_last_func(struct task_struct *task) 1567 { 1568 struct worker *worker = kthread_data(task); 1569 1570 return worker->last_func; 1571 } 1572 1573 /** 1574 * wq_node_nr_active - Determine wq_node_nr_active to use 1575 * @wq: workqueue of interest 1576 * @node: NUMA node, can be %NUMA_NO_NODE 1577 * 1578 * Determine wq_node_nr_active to use for @wq on @node. Returns: 1579 * 1580 * - %NULL for per-cpu workqueues as they don't need to use shared nr_active. 1581 * 1582 * - node_nr_active[nr_node_ids] if @node is %NUMA_NO_NODE. 1583 * 1584 * - Otherwise, node_nr_active[@node]. 1585 */ 1586 static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq, 1587 int node) 1588 { 1589 if (!(wq->flags & WQ_UNBOUND)) 1590 return NULL; 1591 1592 if (node == NUMA_NO_NODE) 1593 node = nr_node_ids; 1594 1595 return wq->node_nr_active[node]; 1596 } 1597 1598 /** 1599 * wq_update_node_max_active - Update per-node max_actives to use 1600 * @wq: workqueue to update 1601 * @off_cpu: CPU that's going down, -1 if a CPU is not going down 1602 * 1603 * Update @wq->node_nr_active[]->max. @wq must be unbound. max_active is 1604 * distributed among nodes according to the proportions of numbers of online 1605 * cpus. The result is always between @wq->min_active and max_active. 1606 */ 1607 static void wq_update_node_max_active(struct workqueue_struct *wq, int off_cpu) 1608 { 1609 struct cpumask *effective = unbound_effective_cpumask(wq); 1610 int min_active = READ_ONCE(wq->min_active); 1611 int max_active = READ_ONCE(wq->max_active); 1612 int total_cpus, node; 1613 1614 lockdep_assert_held(&wq->mutex); 1615 1616 if (!wq_topo_initialized) 1617 return; 1618 1619 if (off_cpu >= 0 && !cpumask_test_cpu(off_cpu, effective)) 1620 off_cpu = -1; 1621 1622 total_cpus = cpumask_weight_and(effective, cpu_online_mask); 1623 if (off_cpu >= 0) 1624 total_cpus--; 1625 1626 /* If all CPUs of the wq get offline, use the default values */ 1627 if (unlikely(!total_cpus)) { 1628 for_each_node(node) 1629 wq_node_nr_active(wq, node)->max = min_active; 1630 1631 wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active; 1632 return; 1633 } 1634 1635 for_each_node(node) { 1636 int node_cpus; 1637 1638 node_cpus = cpumask_weight_and(effective, cpumask_of_node(node)); 1639 if (off_cpu >= 0 && cpu_to_node(off_cpu) == node) 1640 node_cpus--; 1641 1642 wq_node_nr_active(wq, node)->max = 1643 clamp(DIV_ROUND_UP(max_active * node_cpus, total_cpus), 1644 min_active, max_active); 1645 } 1646 1647 wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active; 1648 } 1649 1650 /** 1651 * get_pwq - get an extra reference on the specified pool_workqueue 1652 * @pwq: pool_workqueue to get 1653 * 1654 * Obtain an extra reference on @pwq. The caller should guarantee that 1655 * @pwq has positive refcnt and be holding the matching pool->lock. 1656 */ 1657 static void get_pwq(struct pool_workqueue *pwq) 1658 { 1659 lockdep_assert_held(&pwq->pool->lock); 1660 WARN_ON_ONCE(pwq->refcnt <= 0); 1661 pwq->refcnt++; 1662 } 1663 1664 /** 1665 * put_pwq - put a pool_workqueue reference 1666 * @pwq: pool_workqueue to put 1667 * 1668 * Drop a reference of @pwq. If its refcnt reaches zero, schedule its 1669 * destruction. The caller should be holding the matching pool->lock. 1670 */ 1671 static void put_pwq(struct pool_workqueue *pwq) 1672 { 1673 lockdep_assert_held(&pwq->pool->lock); 1674 if (likely(--pwq->refcnt)) 1675 return; 1676 /* 1677 * @pwq can't be released under pool->lock, bounce to a dedicated 1678 * kthread_worker to avoid A-A deadlocks. 1679 */ 1680 kthread_queue_work(pwq_release_worker, &pwq->release_work); 1681 } 1682 1683 /** 1684 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock 1685 * @pwq: pool_workqueue to put (can be %NULL) 1686 * 1687 * put_pwq() with locking. This function also allows %NULL @pwq. 1688 */ 1689 static void put_pwq_unlocked(struct pool_workqueue *pwq) 1690 { 1691 if (pwq) { 1692 /* 1693 * As both pwqs and pools are RCU protected, the 1694 * following lock operations are safe. 1695 */ 1696 raw_spin_lock_irq(&pwq->pool->lock); 1697 put_pwq(pwq); 1698 raw_spin_unlock_irq(&pwq->pool->lock); 1699 } 1700 } 1701 1702 static bool pwq_is_empty(struct pool_workqueue *pwq) 1703 { 1704 return !pwq->nr_active && list_empty(&pwq->inactive_works); 1705 } 1706 1707 static void __pwq_activate_work(struct pool_workqueue *pwq, 1708 struct work_struct *work) 1709 { 1710 unsigned long *wdb = work_data_bits(work); 1711 1712 WARN_ON_ONCE(!(*wdb & WORK_STRUCT_INACTIVE)); 1713 trace_workqueue_activate_work(work); 1714 if (list_empty(&pwq->pool->worklist)) 1715 pwq->pool->last_progress_ts = jiffies; 1716 move_linked_works(work, &pwq->pool->worklist, NULL); 1717 __clear_bit(WORK_STRUCT_INACTIVE_BIT, wdb); 1718 } 1719 1720 static bool tryinc_node_nr_active(struct wq_node_nr_active *nna) 1721 { 1722 int max = READ_ONCE(nna->max); 1723 int old = atomic_read(&nna->nr); 1724 1725 do { 1726 if (old >= max) 1727 return false; 1728 } while (!atomic_try_cmpxchg_relaxed(&nna->nr, &old, old + 1)); 1729 1730 return true; 1731 } 1732 1733 /** 1734 * pwq_tryinc_nr_active - Try to increment nr_active for a pwq 1735 * @pwq: pool_workqueue of interest 1736 * @fill: max_active may have increased, try to increase concurrency level 1737 * 1738 * Try to increment nr_active for @pwq. Returns %true if an nr_active count is 1739 * successfully obtained. %false otherwise. 1740 */ 1741 static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill) 1742 { 1743 struct workqueue_struct *wq = pwq->wq; 1744 struct worker_pool *pool = pwq->pool; 1745 struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node); 1746 bool obtained = false; 1747 1748 lockdep_assert_held(&pool->lock); 1749 1750 if (!nna) { 1751 /* BH or per-cpu workqueue, pwq->nr_active is sufficient */ 1752 obtained = pwq->nr_active < READ_ONCE(wq->max_active); 1753 goto out; 1754 } 1755 1756 if (unlikely(pwq->plugged)) 1757 return false; 1758 1759 /* 1760 * Unbound workqueue uses per-node shared nr_active $nna. If @pwq is 1761 * already waiting on $nna, pwq_dec_nr_active() will maintain the 1762 * concurrency level. Don't jump the line. 1763 * 1764 * We need to ignore the pending test after max_active has increased as 1765 * pwq_dec_nr_active() can only maintain the concurrency level but not 1766 * increase it. This is indicated by @fill. 1767 */ 1768 if (!list_empty(&pwq->pending_node) && likely(!fill)) 1769 goto out; 1770 1771 obtained = tryinc_node_nr_active(nna); 1772 if (obtained) 1773 goto out; 1774 1775 /* 1776 * Lockless acquisition failed. Lock, add ourself to $nna->pending_pwqs 1777 * and try again. The smp_mb() is paired with the implied memory barrier 1778 * of atomic_dec_return() in pwq_dec_nr_active() to ensure that either 1779 * we see the decremented $nna->nr or they see non-empty 1780 * $nna->pending_pwqs. 1781 */ 1782 raw_spin_lock(&nna->lock); 1783 1784 if (list_empty(&pwq->pending_node)) 1785 list_add_tail(&pwq->pending_node, &nna->pending_pwqs); 1786 else if (likely(!fill)) 1787 goto out_unlock; 1788 1789 smp_mb(); 1790 1791 obtained = tryinc_node_nr_active(nna); 1792 1793 /* 1794 * If @fill, @pwq might have already been pending. Being spuriously 1795 * pending in cold paths doesn't affect anything. Let's leave it be. 1796 */ 1797 if (obtained && likely(!fill)) 1798 list_del_init(&pwq->pending_node); 1799 1800 out_unlock: 1801 raw_spin_unlock(&nna->lock); 1802 out: 1803 if (obtained) 1804 pwq->nr_active++; 1805 return obtained; 1806 } 1807 1808 /** 1809 * pwq_activate_first_inactive - Activate the first inactive work item on a pwq 1810 * @pwq: pool_workqueue of interest 1811 * @fill: max_active may have increased, try to increase concurrency level 1812 * 1813 * Activate the first inactive work item of @pwq if available and allowed by 1814 * max_active limit. 1815 * 1816 * Returns %true if an inactive work item has been activated. %false if no 1817 * inactive work item is found or max_active limit is reached. 1818 */ 1819 static bool pwq_activate_first_inactive(struct pool_workqueue *pwq, bool fill) 1820 { 1821 struct work_struct *work = 1822 list_first_entry_or_null(&pwq->inactive_works, 1823 struct work_struct, entry); 1824 1825 if (work && pwq_tryinc_nr_active(pwq, fill)) { 1826 __pwq_activate_work(pwq, work); 1827 return true; 1828 } else { 1829 return false; 1830 } 1831 } 1832 1833 /** 1834 * unplug_oldest_pwq - unplug the oldest pool_workqueue 1835 * @wq: workqueue_struct where its oldest pwq is to be unplugged 1836 * 1837 * This function should only be called for ordered workqueues where only the 1838 * oldest pwq is unplugged, the others are plugged to suspend execution to 1839 * ensure proper work item ordering:: 1840 * 1841 * dfl_pwq --------------+ [P] - plugged 1842 * | 1843 * v 1844 * pwqs -> A -> B [P] -> C [P] (newest) 1845 * | | | 1846 * 1 3 5 1847 * | | | 1848 * 2 4 6 1849 * 1850 * When the oldest pwq is drained and removed, this function should be called 1851 * to unplug the next oldest one to start its work item execution. Note that 1852 * pwq's are linked into wq->pwqs with the oldest first, so the first one in 1853 * the list is the oldest. 1854 */ 1855 static void unplug_oldest_pwq(struct workqueue_struct *wq) 1856 { 1857 struct pool_workqueue *pwq; 1858 1859 lockdep_assert_held(&wq->mutex); 1860 1861 /* Caller should make sure that pwqs isn't empty before calling */ 1862 pwq = list_first_entry_or_null(&wq->pwqs, struct pool_workqueue, 1863 pwqs_node); 1864 raw_spin_lock_irq(&pwq->pool->lock); 1865 if (pwq->plugged) { 1866 pwq->plugged = false; 1867 if (pwq_activate_first_inactive(pwq, true)) { 1868 /* 1869 * While plugged, queueing skips activation which 1870 * includes bumping the nr_active count and adding the 1871 * pwq to nna->pending_pwqs if the count can't be 1872 * obtained. We need to restore both for the pwq being 1873 * unplugged. The first call activates the first 1874 * inactive work item and the second, if there are more 1875 * inactive, puts the pwq on pending_pwqs. 1876 */ 1877 pwq_activate_first_inactive(pwq, false); 1878 1879 kick_pool(pwq->pool); 1880 } 1881 } 1882 raw_spin_unlock_irq(&pwq->pool->lock); 1883 } 1884 1885 /** 1886 * node_activate_pending_pwq - Activate a pending pwq on a wq_node_nr_active 1887 * @nna: wq_node_nr_active to activate a pending pwq for 1888 * @caller_pool: worker_pool the caller is locking 1889 * 1890 * Activate a pwq in @nna->pending_pwqs. Called with @caller_pool locked. 1891 * @caller_pool may be unlocked and relocked to lock other worker_pools. 1892 */ 1893 static void node_activate_pending_pwq(struct wq_node_nr_active *nna, 1894 struct worker_pool *caller_pool) 1895 { 1896 struct worker_pool *locked_pool = caller_pool; 1897 struct pool_workqueue *pwq; 1898 struct work_struct *work; 1899 1900 lockdep_assert_held(&caller_pool->lock); 1901 1902 raw_spin_lock(&nna->lock); 1903 retry: 1904 pwq = list_first_entry_or_null(&nna->pending_pwqs, 1905 struct pool_workqueue, pending_node); 1906 if (!pwq) 1907 goto out_unlock; 1908 1909 /* 1910 * If @pwq is for a different pool than @locked_pool, we need to lock 1911 * @pwq->pool->lock. Let's trylock first. If unsuccessful, do the unlock 1912 * / lock dance. For that, we also need to release @nna->lock as it's 1913 * nested inside pool locks. 1914 */ 1915 if (pwq->pool != locked_pool) { 1916 raw_spin_unlock(&locked_pool->lock); 1917 locked_pool = pwq->pool; 1918 if (!raw_spin_trylock(&locked_pool->lock)) { 1919 raw_spin_unlock(&nna->lock); 1920 raw_spin_lock(&locked_pool->lock); 1921 raw_spin_lock(&nna->lock); 1922 goto retry; 1923 } 1924 } 1925 1926 /* 1927 * $pwq may not have any inactive work items due to e.g. cancellations. 1928 * Drop it from pending_pwqs and see if there's another one. 1929 */ 1930 work = list_first_entry_or_null(&pwq->inactive_works, 1931 struct work_struct, entry); 1932 if (!work) { 1933 list_del_init(&pwq->pending_node); 1934 goto retry; 1935 } 1936 1937 /* 1938 * Acquire an nr_active count and activate the inactive work item. If 1939 * $pwq still has inactive work items, rotate it to the end of the 1940 * pending_pwqs so that we round-robin through them. This means that 1941 * inactive work items are not activated in queueing order which is fine 1942 * given that there has never been any ordering across different pwqs. 1943 */ 1944 if (likely(tryinc_node_nr_active(nna))) { 1945 pwq->nr_active++; 1946 __pwq_activate_work(pwq, work); 1947 1948 if (list_empty(&pwq->inactive_works)) 1949 list_del_init(&pwq->pending_node); 1950 else 1951 list_move_tail(&pwq->pending_node, &nna->pending_pwqs); 1952 1953 /* if activating a foreign pool, make sure it's running */ 1954 if (pwq->pool != caller_pool) 1955 kick_pool(pwq->pool); 1956 } 1957 1958 out_unlock: 1959 raw_spin_unlock(&nna->lock); 1960 if (locked_pool != caller_pool) { 1961 raw_spin_unlock(&locked_pool->lock); 1962 raw_spin_lock(&caller_pool->lock); 1963 } 1964 } 1965 1966 /** 1967 * pwq_dec_nr_active - Retire an active count 1968 * @pwq: pool_workqueue of interest 1969 * 1970 * Decrement @pwq's nr_active and try to activate the first inactive work item. 1971 * For unbound workqueues, this function may temporarily drop @pwq->pool->lock. 1972 */ 1973 static void pwq_dec_nr_active(struct pool_workqueue *pwq) 1974 { 1975 struct worker_pool *pool = pwq->pool; 1976 struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node); 1977 1978 lockdep_assert_held(&pool->lock); 1979 1980 /* 1981 * @pwq->nr_active should be decremented for both percpu and unbound 1982 * workqueues. 1983 */ 1984 pwq->nr_active--; 1985 1986 /* 1987 * For a percpu workqueue, it's simple. Just need to kick the first 1988 * inactive work item on @pwq itself. 1989 */ 1990 if (!nna) { 1991 pwq_activate_first_inactive(pwq, false); 1992 return; 1993 } 1994 1995 /* 1996 * If @pwq is for an unbound workqueue, it's more complicated because 1997 * multiple pwqs and pools may be sharing the nr_active count. When a 1998 * pwq needs to wait for an nr_active count, it puts itself on 1999 * $nna->pending_pwqs. The following atomic_dec_return()'s implied 2000 * memory barrier is paired with smp_mb() in pwq_tryinc_nr_active() to 2001 * guarantee that either we see non-empty pending_pwqs or they see 2002 * decremented $nna->nr. 2003 * 2004 * $nna->max may change as CPUs come online/offline and @pwq->wq's 2005 * max_active gets updated. However, it is guaranteed to be equal to or 2006 * larger than @pwq->wq->min_active which is above zero unless freezing. 2007 * This maintains the forward progress guarantee. 2008 */ 2009 if (atomic_dec_return(&nna->nr) >= READ_ONCE(nna->max)) 2010 return; 2011 2012 if (!list_empty(&nna->pending_pwqs)) 2013 node_activate_pending_pwq(nna, pool); 2014 } 2015 2016 /** 2017 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight 2018 * @pwq: pwq of interest 2019 * @work_data: work_data of work which left the queue 2020 * 2021 * A work either has completed or is removed from pending queue, 2022 * decrement nr_in_flight of its pwq and handle workqueue flushing. 2023 * 2024 * NOTE: 2025 * For unbound workqueues, this function may temporarily drop @pwq->pool->lock 2026 * and thus should be called after all other state updates for the in-flight 2027 * work item is complete. 2028 * 2029 * CONTEXT: 2030 * raw_spin_lock_irq(pool->lock). 2031 */ 2032 static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data) 2033 { 2034 int color = get_work_color(work_data); 2035 2036 if (!(work_data & WORK_STRUCT_INACTIVE)) 2037 pwq_dec_nr_active(pwq); 2038 2039 pwq->nr_in_flight[color]--; 2040 2041 /* is flush in progress and are we at the flushing tip? */ 2042 if (likely(pwq->flush_color != color)) 2043 goto out_put; 2044 2045 /* are there still in-flight works? */ 2046 if (pwq->nr_in_flight[color]) 2047 goto out_put; 2048 2049 /* this pwq is done, clear flush_color */ 2050 pwq->flush_color = -1; 2051 2052 /* 2053 * If this was the last pwq, wake up the first flusher. It 2054 * will handle the rest. 2055 */ 2056 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush)) 2057 complete(&pwq->wq->first_flusher->done); 2058 out_put: 2059 put_pwq(pwq); 2060 } 2061 2062 /** 2063 * try_to_grab_pending - steal work item from worklist and disable irq 2064 * @work: work item to steal 2065 * @cflags: %WORK_CANCEL_ flags 2066 * @irq_flags: place to store irq state 2067 * 2068 * Try to grab PENDING bit of @work. This function can handle @work in any 2069 * stable state - idle, on timer or on worklist. 2070 * 2071 * Return: 2072 * 2073 * ======== ================================================================ 2074 * 1 if @work was pending and we successfully stole PENDING 2075 * 0 if @work was idle and we claimed PENDING 2076 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry 2077 * ======== ================================================================ 2078 * 2079 * Note: 2080 * On >= 0 return, the caller owns @work's PENDING bit. To avoid getting 2081 * interrupted while holding PENDING and @work off queue, irq must be 2082 * disabled on entry. This, combined with delayed_work->timer being 2083 * irqsafe, ensures that we return -EAGAIN for finite short period of time. 2084 * 2085 * On successful return, >= 0, irq is disabled and the caller is 2086 * responsible for releasing it using local_irq_restore(*@irq_flags). 2087 * 2088 * This function is safe to call from any context including IRQ handler. 2089 */ 2090 static int try_to_grab_pending(struct work_struct *work, u32 cflags, 2091 unsigned long *irq_flags) 2092 { 2093 struct worker_pool *pool; 2094 struct pool_workqueue *pwq; 2095 2096 local_irq_save(*irq_flags); 2097 2098 /* try to steal the timer if it exists */ 2099 if (cflags & WORK_CANCEL_DELAYED) { 2100 struct delayed_work *dwork = to_delayed_work(work); 2101 2102 /* 2103 * dwork->timer is irqsafe. If timer_delete() fails, it's 2104 * guaranteed that the timer is not queued anywhere and not 2105 * running on the local CPU. 2106 */ 2107 if (likely(timer_delete(&dwork->timer))) 2108 return 1; 2109 } 2110 2111 /* try to claim PENDING the normal way */ 2112 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) 2113 return 0; 2114 2115 rcu_read_lock(); 2116 /* 2117 * The queueing is in progress, or it is already queued. Try to 2118 * steal it from ->worklist without clearing WORK_STRUCT_PENDING. 2119 */ 2120 pool = get_work_pool(work); 2121 if (!pool) 2122 goto fail; 2123 2124 raw_spin_lock(&pool->lock); 2125 /* 2126 * work->data is guaranteed to point to pwq only while the work 2127 * item is queued on pwq->wq, and both updating work->data to point 2128 * to pwq on queueing and to pool on dequeueing are done under 2129 * pwq->pool->lock. This in turn guarantees that, if work->data 2130 * points to pwq which is associated with a locked pool, the work 2131 * item is currently queued on that pool. 2132 */ 2133 pwq = get_work_pwq(work); 2134 if (pwq && pwq->pool == pool) { 2135 unsigned long work_data = *work_data_bits(work); 2136 2137 debug_work_deactivate(work); 2138 2139 /* 2140 * A cancelable inactive work item must be in the 2141 * pwq->inactive_works since a queued barrier can't be 2142 * canceled (see the comments in insert_wq_barrier()). 2143 * 2144 * An inactive work item cannot be deleted directly because 2145 * it might have linked barrier work items which, if left 2146 * on the inactive_works list, will confuse pwq->nr_active 2147 * management later on and cause stall. Move the linked 2148 * barrier work items to the worklist when deleting the grabbed 2149 * item. Also keep WORK_STRUCT_INACTIVE in work_data, so that 2150 * it doesn't participate in nr_active management in later 2151 * pwq_dec_nr_in_flight(). 2152 */ 2153 if (work_data & WORK_STRUCT_INACTIVE) 2154 move_linked_works(work, &pwq->pool->worklist, NULL); 2155 2156 list_del_init(&work->entry); 2157 2158 /* 2159 * work->data points to pwq iff queued. Let's point to pool. As 2160 * this destroys work->data needed by the next step, stash it. 2161 */ 2162 set_work_pool_and_keep_pending(work, pool->id, 2163 pool_offq_flags(pool)); 2164 2165 /* must be the last step, see the function comment */ 2166 pwq_dec_nr_in_flight(pwq, work_data); 2167 2168 raw_spin_unlock(&pool->lock); 2169 rcu_read_unlock(); 2170 return 1; 2171 } 2172 raw_spin_unlock(&pool->lock); 2173 fail: 2174 rcu_read_unlock(); 2175 local_irq_restore(*irq_flags); 2176 return -EAGAIN; 2177 } 2178 2179 /** 2180 * work_grab_pending - steal work item from worklist and disable irq 2181 * @work: work item to steal 2182 * @cflags: %WORK_CANCEL_ flags 2183 * @irq_flags: place to store IRQ state 2184 * 2185 * Grab PENDING bit of @work. @work can be in any stable state - idle, on timer 2186 * or on worklist. 2187 * 2188 * Can be called from any context. IRQ is disabled on return with IRQ state 2189 * stored in *@irq_flags. The caller is responsible for re-enabling it using 2190 * local_irq_restore(). 2191 * 2192 * Returns %true if @work was pending. %false if idle. 2193 */ 2194 static bool work_grab_pending(struct work_struct *work, u32 cflags, 2195 unsigned long *irq_flags) 2196 { 2197 int ret; 2198 2199 while (true) { 2200 ret = try_to_grab_pending(work, cflags, irq_flags); 2201 if (ret >= 0) 2202 return ret; 2203 cpu_relax(); 2204 } 2205 } 2206 2207 /** 2208 * insert_work - insert a work into a pool 2209 * @pwq: pwq @work belongs to 2210 * @work: work to insert 2211 * @head: insertion point 2212 * @extra_flags: extra WORK_STRUCT_* flags to set 2213 * 2214 * Insert @work which belongs to @pwq after @head. @extra_flags is or'd to 2215 * work_struct flags. 2216 * 2217 * CONTEXT: 2218 * raw_spin_lock_irq(pool->lock). 2219 */ 2220 static void insert_work(struct pool_workqueue *pwq, struct work_struct *work, 2221 struct list_head *head, unsigned int extra_flags) 2222 { 2223 debug_work_activate(work); 2224 2225 /* record the work call stack in order to print it in KASAN reports */ 2226 kasan_record_aux_stack(work); 2227 2228 /* we own @work, set data and link */ 2229 set_work_pwq(work, pwq, extra_flags); 2230 list_add_tail(&work->entry, head); 2231 get_pwq(pwq); 2232 } 2233 2234 /* 2235 * Test whether @work is being queued from another work executing on the 2236 * same workqueue. 2237 */ 2238 static bool is_chained_work(struct workqueue_struct *wq) 2239 { 2240 struct worker *worker; 2241 2242 worker = current_wq_worker(); 2243 /* 2244 * Return %true iff I'm a worker executing a work item on @wq. If 2245 * I'm @worker, it's safe to dereference it without locking. 2246 */ 2247 return worker && worker->current_pwq->wq == wq; 2248 } 2249 2250 /* 2251 * When queueing an unbound work item to a wq, prefer local CPU if allowed 2252 * by wq_unbound_cpumask. Otherwise, round robin among the allowed ones to 2253 * avoid perturbing sensitive tasks. 2254 */ 2255 static int wq_select_unbound_cpu(int cpu) 2256 { 2257 int new_cpu; 2258 2259 if (likely(!wq_debug_force_rr_cpu)) { 2260 if (cpumask_test_cpu(cpu, wq_unbound_cpumask)) 2261 return cpu; 2262 } else { 2263 pr_warn_once("workqueue: round-robin CPU selection forced, expect performance impact\n"); 2264 } 2265 2266 new_cpu = __this_cpu_read(wq_rr_cpu_last); 2267 new_cpu = cpumask_next_and_wrap(new_cpu, wq_unbound_cpumask, cpu_online_mask); 2268 if (unlikely(new_cpu >= nr_cpu_ids)) 2269 return cpu; 2270 __this_cpu_write(wq_rr_cpu_last, new_cpu); 2271 2272 return new_cpu; 2273 } 2274 2275 static void __queue_work(int cpu, struct workqueue_struct *wq, 2276 struct work_struct *work) 2277 { 2278 struct pool_workqueue *pwq; 2279 struct worker_pool *last_pool, *pool; 2280 unsigned int work_flags; 2281 unsigned int req_cpu = cpu; 2282 2283 /* 2284 * While a work item is PENDING && off queue, a task trying to 2285 * steal the PENDING will busy-loop waiting for it to either get 2286 * queued or lose PENDING. Grabbing PENDING and queueing should 2287 * happen with IRQ disabled. 2288 */ 2289 lockdep_assert_irqs_disabled(); 2290 2291 /* 2292 * For a draining wq, only works from the same workqueue are 2293 * allowed. The __WQ_DESTROYING helps to spot the issue that 2294 * queues a new work item to a wq after destroy_workqueue(wq). 2295 */ 2296 if (unlikely(wq->flags & (__WQ_DESTROYING | __WQ_DRAINING) && 2297 WARN_ONCE(!is_chained_work(wq), "workqueue: cannot queue %ps on wq %s\n", 2298 work->func, wq->name))) { 2299 struct work_offq_data offqd; 2300 2301 /* 2302 * State on entry: PENDING is set, work is off-queue (no 2303 * insert_work() has run). 2304 * 2305 * Returning without clearing PENDING would leave the work 2306 * in a weird state (PENDING=1, PWQ=0, entry empty) 2307 */ 2308 work_offqd_unpack(&offqd, *work_data_bits(work)); 2309 set_work_pool_and_clear_pending(work, offqd.pool_id, 2310 work_offqd_pack_flags(&offqd)); 2311 return; 2312 } 2313 rcu_read_lock(); 2314 retry: 2315 /* pwq which will be used unless @work is executing elsewhere */ 2316 if (req_cpu == WORK_CPU_UNBOUND) { 2317 if (wq->flags & WQ_UNBOUND) 2318 cpu = wq_select_unbound_cpu(raw_smp_processor_id()); 2319 else 2320 cpu = raw_smp_processor_id(); 2321 } 2322 2323 pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu)); 2324 pool = pwq->pool; 2325 2326 /* 2327 * If @work was previously on a different pool, it might still be 2328 * running there, in which case the work needs to be queued on that 2329 * pool to guarantee non-reentrancy. 2330 * 2331 * For ordered workqueue, work items must be queued on the newest pwq 2332 * for accurate order management. Guaranteed order also guarantees 2333 * non-reentrancy. See the comments above unplug_oldest_pwq(). 2334 */ 2335 last_pool = get_work_pool(work); 2336 if (last_pool && last_pool != pool && !(wq->flags & __WQ_ORDERED)) { 2337 struct worker *worker; 2338 2339 raw_spin_lock(&last_pool->lock); 2340 2341 worker = find_worker_executing_work(last_pool, work); 2342 2343 if (worker && worker->current_pwq->wq == wq) { 2344 pwq = worker->current_pwq; 2345 pool = pwq->pool; 2346 WARN_ON_ONCE(pool != last_pool); 2347 } else { 2348 /* meh... not running there, queue here */ 2349 raw_spin_unlock(&last_pool->lock); 2350 raw_spin_lock(&pool->lock); 2351 } 2352 } else { 2353 raw_spin_lock(&pool->lock); 2354 } 2355 2356 /* 2357 * pwq is determined and locked. For unbound pools, we could have raced 2358 * with pwq release and it could already be dead. If its refcnt is zero, 2359 * repeat pwq selection. Note that unbound pwqs never die without 2360 * another pwq replacing it in cpu_pwq or while work items are executing 2361 * on it, so the retrying is guaranteed to make forward-progress. 2362 */ 2363 if (unlikely(!pwq->refcnt)) { 2364 if (wq->flags & WQ_UNBOUND) { 2365 raw_spin_unlock(&pool->lock); 2366 cpu_relax(); 2367 goto retry; 2368 } 2369 /* oops */ 2370 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt", 2371 wq->name, cpu); 2372 } 2373 2374 /* pwq determined, queue */ 2375 trace_workqueue_queue_work(req_cpu, pwq, work); 2376 2377 if (WARN_ON(!list_empty(&work->entry))) 2378 goto out; 2379 2380 pwq->nr_in_flight[pwq->work_color]++; 2381 work_flags = work_color_to_flags(pwq->work_color); 2382 2383 /* 2384 * Limit the number of concurrently active work items to max_active. 2385 * @work must also queue behind existing inactive work items to maintain 2386 * ordering when max_active changes. See wq_adjust_max_active(). 2387 */ 2388 if (list_empty(&pwq->inactive_works) && pwq_tryinc_nr_active(pwq, false)) { 2389 if (list_empty(&pool->worklist)) 2390 pool->last_progress_ts = jiffies; 2391 2392 trace_workqueue_activate_work(work); 2393 insert_work(pwq, work, &pool->worklist, work_flags); 2394 kick_pool(pool); 2395 } else { 2396 work_flags |= WORK_STRUCT_INACTIVE; 2397 insert_work(pwq, work, &pwq->inactive_works, work_flags); 2398 } 2399 2400 out: 2401 raw_spin_unlock(&pool->lock); 2402 rcu_read_unlock(); 2403 } 2404 2405 static bool clear_pending_if_disabled(struct work_struct *work) 2406 { 2407 unsigned long data = *work_data_bits(work); 2408 struct work_offq_data offqd; 2409 2410 if (likely((data & WORK_STRUCT_PWQ) || 2411 !(data & WORK_OFFQ_DISABLE_MASK))) 2412 return false; 2413 2414 work_offqd_unpack(&offqd, data); 2415 set_work_pool_and_clear_pending(work, offqd.pool_id, 2416 work_offqd_pack_flags(&offqd)); 2417 return true; 2418 } 2419 2420 /** 2421 * queue_work_on - queue work on specific cpu 2422 * @cpu: CPU number to execute work on 2423 * @wq: workqueue to use 2424 * @work: work to queue 2425 * 2426 * We queue the work to a specific CPU, the caller must ensure it 2427 * can't go away. Callers that fail to ensure that the specified 2428 * CPU cannot go away will execute on a randomly chosen CPU. 2429 * But note well that callers specifying a CPU that never has been 2430 * online will get a splat. 2431 * 2432 * Return: %false if @work was already on a queue, %true otherwise. 2433 */ 2434 bool queue_work_on(int cpu, struct workqueue_struct *wq, 2435 struct work_struct *work) 2436 { 2437 bool ret = false; 2438 unsigned long irq_flags; 2439 2440 local_irq_save(irq_flags); 2441 2442 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) && 2443 !clear_pending_if_disabled(work)) { 2444 __queue_work(cpu, wq, work); 2445 ret = true; 2446 } 2447 2448 local_irq_restore(irq_flags); 2449 return ret; 2450 } 2451 EXPORT_SYMBOL(queue_work_on); 2452 2453 /** 2454 * select_numa_node_cpu - Select a CPU based on NUMA node 2455 * @node: NUMA node ID that we want to select a CPU from 2456 * 2457 * This function will attempt to find a "random" cpu available on a given 2458 * node. If there are no CPUs available on the given node it will return 2459 * WORK_CPU_UNBOUND indicating that we should just schedule to any 2460 * available CPU if we need to schedule this work. 2461 */ 2462 static int select_numa_node_cpu(int node) 2463 { 2464 int cpu; 2465 2466 /* Delay binding to CPU if node is not valid or online */ 2467 if (node < 0 || node >= MAX_NUMNODES || !node_online(node)) 2468 return WORK_CPU_UNBOUND; 2469 2470 /* Use local node/cpu if we are already there */ 2471 cpu = raw_smp_processor_id(); 2472 if (node == cpu_to_node(cpu)) 2473 return cpu; 2474 2475 /* Use "random" otherwise know as "first" online CPU of node */ 2476 cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask); 2477 2478 /* If CPU is valid return that, otherwise just defer */ 2479 return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND; 2480 } 2481 2482 /** 2483 * queue_work_node - queue work on a "random" cpu for a given NUMA node 2484 * @node: NUMA node that we are targeting the work for 2485 * @wq: workqueue to use 2486 * @work: work to queue 2487 * 2488 * We queue the work to a "random" CPU within a given NUMA node. The basic 2489 * idea here is to provide a way to somehow associate work with a given 2490 * NUMA node. 2491 * 2492 * This function will only make a best effort attempt at getting this onto 2493 * the right NUMA node. If no node is requested or the requested node is 2494 * offline then we just fall back to standard queue_work behavior. 2495 * 2496 * Currently the "random" CPU ends up being the first available CPU in the 2497 * intersection of cpu_online_mask and the cpumask of the node, unless we 2498 * are running on the node. In that case we just use the current CPU. 2499 * 2500 * Return: %false if @work was already on a queue, %true otherwise. 2501 */ 2502 bool queue_work_node(int node, struct workqueue_struct *wq, 2503 struct work_struct *work) 2504 { 2505 unsigned long irq_flags; 2506 bool ret = false; 2507 2508 /* 2509 * This current implementation is specific to unbound workqueues. 2510 * Specifically we only return the first available CPU for a given 2511 * node instead of cycling through individual CPUs within the node. 2512 * 2513 * If this is used with a per-cpu workqueue then the logic in 2514 * workqueue_select_cpu_near would need to be updated to allow for 2515 * some round robin type logic. 2516 */ 2517 WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)); 2518 2519 local_irq_save(irq_flags); 2520 2521 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) && 2522 !clear_pending_if_disabled(work)) { 2523 int cpu = select_numa_node_cpu(node); 2524 2525 __queue_work(cpu, wq, work); 2526 ret = true; 2527 } 2528 2529 local_irq_restore(irq_flags); 2530 return ret; 2531 } 2532 EXPORT_SYMBOL_GPL(queue_work_node); 2533 2534 void delayed_work_timer_fn(struct timer_list *t) 2535 { 2536 struct delayed_work *dwork = timer_container_of(dwork, t, timer); 2537 2538 /* should have been called from irqsafe timer with irq already off */ 2539 __queue_work(dwork->cpu, dwork->wq, &dwork->work); 2540 } 2541 EXPORT_SYMBOL(delayed_work_timer_fn); 2542 2543 static void __queue_delayed_work(int cpu, struct workqueue_struct *wq, 2544 struct delayed_work *dwork, unsigned long delay) 2545 { 2546 struct timer_list *timer = &dwork->timer; 2547 struct work_struct *work = &dwork->work; 2548 2549 WARN_ON_ONCE(timer->function != delayed_work_timer_fn); 2550 WARN_ON_ONCE(timer_pending(timer)); 2551 WARN_ON_ONCE(!list_empty(&work->entry)); 2552 2553 /* 2554 * If @delay is 0, queue @dwork->work immediately. This is for 2555 * both optimization and correctness. The earliest @timer can 2556 * expire is on the closest next tick and delayed_work users depend 2557 * on that there's no such delay when @delay is 0. 2558 */ 2559 if (!delay) { 2560 __queue_work(cpu, wq, &dwork->work); 2561 return; 2562 } 2563 2564 WARN_ON_ONCE(cpu != WORK_CPU_UNBOUND && !cpu_online(cpu)); 2565 dwork->wq = wq; 2566 dwork->cpu = cpu; 2567 timer->expires = jiffies + delay; 2568 2569 if (housekeeping_enabled(HK_TYPE_TIMER)) { 2570 /* If the current cpu is a housekeeping cpu, use it. */ 2571 cpu = smp_processor_id(); 2572 if (!housekeeping_test_cpu(cpu, HK_TYPE_TIMER)) 2573 cpu = housekeeping_any_cpu(HK_TYPE_TIMER); 2574 add_timer_on(timer, cpu); 2575 } else { 2576 if (likely(cpu == WORK_CPU_UNBOUND)) 2577 add_timer_global(timer); 2578 else 2579 add_timer_on(timer, cpu); 2580 } 2581 } 2582 2583 /** 2584 * queue_delayed_work_on - queue work on specific CPU after delay 2585 * @cpu: CPU number to execute work on 2586 * @wq: workqueue to use 2587 * @dwork: work to queue 2588 * @delay: number of jiffies to wait before queueing 2589 * 2590 * We queue the delayed_work to a specific CPU, for non-zero delays the 2591 * caller must ensure it is online and can't go away. Callers that fail 2592 * to ensure this, may get @dwork->timer queued to an offlined CPU and 2593 * this will prevent queueing of @dwork->work unless the offlined CPU 2594 * becomes online again. 2595 * 2596 * Return: %false if @work was already on a queue, %true otherwise. If 2597 * @delay is zero and @dwork is idle, it will be scheduled for immediate 2598 * execution. 2599 */ 2600 bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq, 2601 struct delayed_work *dwork, unsigned long delay) 2602 { 2603 struct work_struct *work = &dwork->work; 2604 bool ret = false; 2605 unsigned long irq_flags; 2606 2607 /* read the comment in __queue_work() */ 2608 local_irq_save(irq_flags); 2609 2610 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) && 2611 !clear_pending_if_disabled(work)) { 2612 __queue_delayed_work(cpu, wq, dwork, delay); 2613 ret = true; 2614 } 2615 2616 local_irq_restore(irq_flags); 2617 return ret; 2618 } 2619 EXPORT_SYMBOL(queue_delayed_work_on); 2620 2621 /** 2622 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU 2623 * @cpu: CPU number to execute work on 2624 * @wq: workqueue to use 2625 * @dwork: work to queue 2626 * @delay: number of jiffies to wait before queueing 2627 * 2628 * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise, 2629 * modify @dwork's timer so that it expires after @delay. If @delay is 2630 * zero, @work is guaranteed to be scheduled immediately regardless of its 2631 * current state. 2632 * 2633 * Return: %false if @dwork was idle and queued, %true if @dwork was 2634 * pending and its timer was modified. 2635 * 2636 * This function is safe to call from any context including IRQ handler. 2637 * See try_to_grab_pending() for details. 2638 */ 2639 bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq, 2640 struct delayed_work *dwork, unsigned long delay) 2641 { 2642 unsigned long irq_flags; 2643 bool ret; 2644 2645 ret = work_grab_pending(&dwork->work, WORK_CANCEL_DELAYED, &irq_flags); 2646 2647 if (!clear_pending_if_disabled(&dwork->work)) 2648 __queue_delayed_work(cpu, wq, dwork, delay); 2649 2650 local_irq_restore(irq_flags); 2651 return ret; 2652 } 2653 EXPORT_SYMBOL_GPL(mod_delayed_work_on); 2654 2655 static void rcu_work_rcufn(struct rcu_head *rcu) 2656 { 2657 struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu); 2658 2659 /* read the comment in __queue_work() */ 2660 local_irq_disable(); 2661 __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work); 2662 local_irq_enable(); 2663 } 2664 2665 /** 2666 * queue_rcu_work - queue work after a RCU grace period 2667 * @wq: workqueue to use 2668 * @rwork: work to queue 2669 * 2670 * Return: %false if @rwork was already pending, %true otherwise. Note 2671 * that a full RCU grace period is guaranteed only after a %true return. 2672 * While @rwork is guaranteed to be executed after a %false return, the 2673 * execution may happen before a full RCU grace period has passed. 2674 */ 2675 bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) 2676 { 2677 struct work_struct *work = &rwork->work; 2678 2679 /* 2680 * rcu_work can't be canceled or disabled. Warn if the user reached 2681 * inside @rwork and disabled the inner work. 2682 */ 2683 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) && 2684 !WARN_ON_ONCE(clear_pending_if_disabled(work))) { 2685 rwork->wq = wq; 2686 call_rcu_hurry(&rwork->rcu, rcu_work_rcufn); 2687 return true; 2688 } 2689 2690 return false; 2691 } 2692 EXPORT_SYMBOL(queue_rcu_work); 2693 2694 static struct worker *alloc_worker(int node) 2695 { 2696 struct worker *worker; 2697 2698 worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node); 2699 if (worker) { 2700 INIT_LIST_HEAD(&worker->entry); 2701 INIT_LIST_HEAD(&worker->scheduled); 2702 INIT_LIST_HEAD(&worker->node); 2703 /* on creation a worker is in !idle && prep state */ 2704 worker->flags = WORKER_PREP; 2705 } 2706 return worker; 2707 } 2708 2709 static cpumask_t *pool_allowed_cpus(struct worker_pool *pool) 2710 { 2711 if (pool->cpu < 0 && pool->attrs->affn_strict) 2712 return pool->attrs->__pod_cpumask; 2713 else 2714 return pool->attrs->cpumask; 2715 } 2716 2717 /** 2718 * worker_attach_to_pool() - attach a worker to a pool 2719 * @worker: worker to be attached 2720 * @pool: the target pool 2721 * 2722 * Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and 2723 * cpu-binding of @worker are kept coordinated with the pool across 2724 * cpu-[un]hotplugs. 2725 */ 2726 static void worker_attach_to_pool(struct worker *worker, 2727 struct worker_pool *pool) 2728 { 2729 mutex_lock(&wq_pool_attach_mutex); 2730 2731 /* 2732 * The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains stable 2733 * across this function. See the comments above the flag definition for 2734 * details. BH workers are, while per-CPU, always DISASSOCIATED. 2735 */ 2736 if (pool->flags & POOL_DISASSOCIATED) { 2737 worker->flags |= WORKER_UNBOUND; 2738 } else { 2739 WARN_ON_ONCE(pool->flags & POOL_BH); 2740 kthread_set_per_cpu(worker->task, pool->cpu); 2741 } 2742 2743 if (worker->rescue_wq) 2744 set_cpus_allowed_ptr(worker->task, pool_allowed_cpus(pool)); 2745 2746 list_add_tail(&worker->node, &pool->workers); 2747 worker->pool = pool; 2748 2749 mutex_unlock(&wq_pool_attach_mutex); 2750 } 2751 2752 static void unbind_worker(struct worker *worker) 2753 { 2754 lockdep_assert_held(&wq_pool_attach_mutex); 2755 2756 kthread_set_per_cpu(worker->task, -1); 2757 if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask)) 2758 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0); 2759 else 2760 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0); 2761 } 2762 2763 2764 static void detach_worker(struct worker *worker) 2765 { 2766 lockdep_assert_held(&wq_pool_attach_mutex); 2767 2768 unbind_worker(worker); 2769 list_del(&worker->node); 2770 } 2771 2772 /** 2773 * worker_detach_from_pool() - detach a worker from its pool 2774 * @worker: worker which is attached to its pool 2775 * 2776 * Undo the attaching which had been done in worker_attach_to_pool(). The 2777 * caller worker shouldn't access to the pool after detached except it has 2778 * other reference to the pool. 2779 */ 2780 static void worker_detach_from_pool(struct worker *worker) 2781 { 2782 struct worker_pool *pool = worker->pool; 2783 2784 /* there is one permanent BH worker per CPU which should never detach */ 2785 WARN_ON_ONCE(pool->flags & POOL_BH); 2786 2787 mutex_lock(&wq_pool_attach_mutex); 2788 detach_worker(worker); 2789 worker->pool = NULL; 2790 mutex_unlock(&wq_pool_attach_mutex); 2791 2792 /* clear leftover flags without pool->lock after it is detached */ 2793 worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND); 2794 } 2795 2796 static int format_worker_id(char *buf, size_t size, struct worker *worker, 2797 struct worker_pool *pool) 2798 { 2799 if (worker->rescue_wq) 2800 return scnprintf(buf, size, "kworker/R-%s", 2801 worker->rescue_wq->name); 2802 2803 if (pool) { 2804 if (pool->cpu >= 0) 2805 return scnprintf(buf, size, "kworker/%d:%d%s", 2806 pool->cpu, worker->id, 2807 pool->attrs->nice < 0 ? "H" : ""); 2808 else 2809 return scnprintf(buf, size, "kworker/u%d:%d", 2810 pool->id, worker->id); 2811 } else { 2812 return scnprintf(buf, size, "kworker/dying"); 2813 } 2814 } 2815 2816 /** 2817 * create_worker - create a new workqueue worker 2818 * @pool: pool the new worker will belong to 2819 * 2820 * Create and start a new worker which is attached to @pool. 2821 * 2822 * CONTEXT: 2823 * Might sleep. Does GFP_KERNEL allocations. 2824 * 2825 * Return: 2826 * Pointer to the newly created worker. 2827 */ 2828 static struct worker *create_worker(struct worker_pool *pool) 2829 { 2830 struct worker *worker; 2831 int id; 2832 2833 /* ID is needed to determine kthread name */ 2834 id = ida_alloc(&pool->worker_ida, GFP_KERNEL); 2835 if (id < 0) { 2836 pr_err_once("workqueue: Failed to allocate a worker ID: %pe\n", 2837 ERR_PTR(id)); 2838 return NULL; 2839 } 2840 2841 worker = alloc_worker(pool->node); 2842 if (!worker) { 2843 pr_err_once("workqueue: Failed to allocate a worker\n"); 2844 goto fail; 2845 } 2846 2847 worker->id = id; 2848 2849 if (!(pool->flags & POOL_BH)) { 2850 char id_buf[WORKER_ID_LEN]; 2851 2852 format_worker_id(id_buf, sizeof(id_buf), worker, pool); 2853 worker->task = kthread_create_on_node(worker_thread, worker, 2854 pool->node, "%s", id_buf); 2855 if (IS_ERR(worker->task)) { 2856 if (PTR_ERR(worker->task) == -EINTR) { 2857 pr_err("workqueue: Interrupted when creating a worker thread \"%s\"\n", 2858 id_buf); 2859 } else { 2860 pr_err_once("workqueue: Failed to create a worker thread: %pe", 2861 worker->task); 2862 } 2863 goto fail; 2864 } 2865 2866 set_user_nice(worker->task, pool->attrs->nice); 2867 kthread_bind_mask(worker->task, pool_allowed_cpus(pool)); 2868 } 2869 2870 /* successful, attach the worker to the pool */ 2871 worker_attach_to_pool(worker, pool); 2872 2873 /* start the newly created worker */ 2874 raw_spin_lock_irq(&pool->lock); 2875 2876 worker->pool->nr_workers++; 2877 worker_enter_idle(worker); 2878 2879 /* 2880 * @worker is waiting on a completion in kthread() and will trigger hung 2881 * check if not woken up soon. As kick_pool() is noop if @pool is empty, 2882 * wake it up explicitly. 2883 */ 2884 if (worker->task) 2885 wake_up_process(worker->task); 2886 2887 raw_spin_unlock_irq(&pool->lock); 2888 2889 return worker; 2890 2891 fail: 2892 ida_free(&pool->worker_ida, id); 2893 kfree(worker); 2894 return NULL; 2895 } 2896 2897 static void detach_dying_workers(struct list_head *cull_list) 2898 { 2899 struct worker *worker; 2900 2901 list_for_each_entry(worker, cull_list, entry) 2902 detach_worker(worker); 2903 } 2904 2905 static void reap_dying_workers(struct list_head *cull_list) 2906 { 2907 struct worker *worker, *tmp; 2908 2909 list_for_each_entry_safe(worker, tmp, cull_list, entry) { 2910 list_del_init(&worker->entry); 2911 kthread_stop_put(worker->task); 2912 kfree(worker); 2913 } 2914 } 2915 2916 /** 2917 * set_worker_dying - Tag a worker for destruction 2918 * @worker: worker to be destroyed 2919 * @list: transfer worker away from its pool->idle_list and into list 2920 * 2921 * Tag @worker for destruction and adjust @pool stats accordingly. The worker 2922 * should be idle. 2923 * 2924 * CONTEXT: 2925 * raw_spin_lock_irq(pool->lock). 2926 */ 2927 static void set_worker_dying(struct worker *worker, struct list_head *list) 2928 { 2929 struct worker_pool *pool = worker->pool; 2930 2931 lockdep_assert_held(&pool->lock); 2932 lockdep_assert_held(&wq_pool_attach_mutex); 2933 2934 /* sanity check frenzy */ 2935 if (WARN_ON(worker->current_work) || 2936 WARN_ON(!list_empty(&worker->scheduled)) || 2937 WARN_ON(!(worker->flags & WORKER_IDLE))) 2938 return; 2939 2940 pool->nr_workers--; 2941 pool->nr_idle--; 2942 2943 worker->flags |= WORKER_DIE; 2944 2945 list_move(&worker->entry, list); 2946 2947 /* get an extra task struct reference for later kthread_stop_put() */ 2948 get_task_struct(worker->task); 2949 } 2950 2951 /** 2952 * idle_worker_timeout - check if some idle workers can now be deleted. 2953 * @t: The pool's idle_timer that just expired 2954 * 2955 * The timer is armed in worker_enter_idle(). Note that it isn't disarmed in 2956 * worker_leave_idle(), as a worker flicking between idle and active while its 2957 * pool is at the too_many_workers() tipping point would cause too much timer 2958 * housekeeping overhead. Since IDLE_WORKER_TIMEOUT is long enough, we just let 2959 * it expire and re-evaluate things from there. 2960 */ 2961 static void idle_worker_timeout(struct timer_list *t) 2962 { 2963 struct worker_pool *pool = timer_container_of(pool, t, idle_timer); 2964 bool do_cull = false; 2965 2966 if (work_pending(&pool->idle_cull_work)) 2967 return; 2968 2969 raw_spin_lock_irq(&pool->lock); 2970 2971 if (too_many_workers(pool)) { 2972 struct worker *worker; 2973 unsigned long expires; 2974 2975 /* idle_list is kept in LIFO order, check the last one */ 2976 worker = list_last_entry(&pool->idle_list, struct worker, entry); 2977 expires = worker->last_active + IDLE_WORKER_TIMEOUT; 2978 do_cull = !time_before(jiffies, expires); 2979 2980 if (!do_cull) 2981 mod_timer(&pool->idle_timer, expires); 2982 } 2983 raw_spin_unlock_irq(&pool->lock); 2984 2985 if (do_cull) 2986 queue_work(system_dfl_wq, &pool->idle_cull_work); 2987 } 2988 2989 /** 2990 * idle_cull_fn - cull workers that have been idle for too long. 2991 * @work: the pool's work for handling these idle workers 2992 * 2993 * This goes through a pool's idle workers and gets rid of those that have been 2994 * idle for at least IDLE_WORKER_TIMEOUT seconds. 2995 * 2996 * We don't want to disturb isolated CPUs because of a pcpu kworker being 2997 * culled, so this also resets worker affinity. This requires a sleepable 2998 * context, hence the split between timer callback and work item. 2999 */ 3000 static void idle_cull_fn(struct work_struct *work) 3001 { 3002 struct worker_pool *pool = container_of(work, struct worker_pool, idle_cull_work); 3003 LIST_HEAD(cull_list); 3004 3005 /* 3006 * Grabbing wq_pool_attach_mutex here ensures an already-running worker 3007 * cannot proceed beyong set_pf_worker() in its self-destruct path. 3008 * This is required as a previously-preempted worker could run after 3009 * set_worker_dying() has happened but before detach_dying_workers() did. 3010 */ 3011 mutex_lock(&wq_pool_attach_mutex); 3012 raw_spin_lock_irq(&pool->lock); 3013 3014 while (too_many_workers(pool)) { 3015 struct worker *worker; 3016 unsigned long expires; 3017 3018 worker = list_last_entry(&pool->idle_list, struct worker, entry); 3019 expires = worker->last_active + IDLE_WORKER_TIMEOUT; 3020 3021 if (time_before(jiffies, expires)) { 3022 mod_timer(&pool->idle_timer, expires); 3023 break; 3024 } 3025 3026 set_worker_dying(worker, &cull_list); 3027 } 3028 3029 raw_spin_unlock_irq(&pool->lock); 3030 detach_dying_workers(&cull_list); 3031 mutex_unlock(&wq_pool_attach_mutex); 3032 3033 reap_dying_workers(&cull_list); 3034 } 3035 3036 static void send_mayday(struct pool_workqueue *pwq) 3037 { 3038 struct workqueue_struct *wq = pwq->wq; 3039 3040 lockdep_assert_held(&wq_mayday_lock); 3041 3042 if (!wq->rescuer) 3043 return; 3044 3045 /* mayday mayday mayday */ 3046 if (list_empty(&pwq->mayday_node)) { 3047 /* 3048 * If @pwq is for an unbound wq, its base ref may be put at 3049 * any time due to an attribute change. Pin @pwq until the 3050 * rescuer is done with it. 3051 */ 3052 get_pwq(pwq); 3053 list_add_tail(&pwq->mayday_node, &wq->maydays); 3054 wake_up_process(wq->rescuer->task); 3055 pwq->stats[PWQ_STAT_MAYDAY]++; 3056 } 3057 } 3058 3059 static void pool_mayday_timeout(struct timer_list *t) 3060 { 3061 struct worker_pool *pool = timer_container_of(pool, t, mayday_timer); 3062 struct work_struct *work; 3063 3064 raw_spin_lock_irq(&pool->lock); 3065 raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */ 3066 3067 if (need_to_create_worker(pool)) { 3068 /* 3069 * We've been trying to create a new worker but 3070 * haven't been successful. We might be hitting an 3071 * allocation deadlock. Send distress signals to 3072 * rescuers. 3073 */ 3074 list_for_each_entry(work, &pool->worklist, entry) 3075 send_mayday(get_work_pwq(work)); 3076 } 3077 3078 raw_spin_unlock(&wq_mayday_lock); 3079 raw_spin_unlock_irq(&pool->lock); 3080 3081 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL); 3082 } 3083 3084 /** 3085 * maybe_create_worker - create a new worker if necessary 3086 * @pool: pool to create a new worker for 3087 * 3088 * Create a new worker for @pool if necessary. @pool is guaranteed to 3089 * have at least one idle worker on return from this function. If 3090 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is 3091 * sent to all rescuers with works scheduled on @pool to resolve 3092 * possible allocation deadlock. 3093 * 3094 * On return, need_to_create_worker() is guaranteed to be %false and 3095 * may_start_working() %true. 3096 * 3097 * LOCKING: 3098 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed 3099 * multiple times. Does GFP_KERNEL allocations. Called only from 3100 * manager. 3101 */ 3102 static void maybe_create_worker(struct worker_pool *pool) 3103 __releases(&pool->lock) 3104 __acquires(&pool->lock) 3105 { 3106 restart: 3107 raw_spin_unlock_irq(&pool->lock); 3108 3109 /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */ 3110 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); 3111 3112 while (true) { 3113 if (create_worker(pool) || !need_to_create_worker(pool)) 3114 break; 3115 3116 schedule_timeout_interruptible(CREATE_COOLDOWN); 3117 3118 if (!need_to_create_worker(pool)) 3119 break; 3120 } 3121 3122 timer_delete_sync(&pool->mayday_timer); 3123 raw_spin_lock_irq(&pool->lock); 3124 /* 3125 * This is necessary even after a new worker was just successfully 3126 * created as @pool->lock was dropped and the new worker might have 3127 * already become busy. 3128 */ 3129 if (need_to_create_worker(pool)) 3130 goto restart; 3131 } 3132 3133 #ifdef CONFIG_PREEMPT_RT 3134 static void worker_lock_callback(struct worker_pool *pool) 3135 { 3136 spin_lock(&pool->cb_lock); 3137 } 3138 3139 static void worker_unlock_callback(struct worker_pool *pool) 3140 { 3141 spin_unlock(&pool->cb_lock); 3142 } 3143 3144 static void workqueue_callback_cancel_wait_running(struct worker_pool *pool) 3145 { 3146 spin_lock(&pool->cb_lock); 3147 spin_unlock(&pool->cb_lock); 3148 } 3149 3150 #else 3151 3152 static void worker_lock_callback(struct worker_pool *pool) { } 3153 static void worker_unlock_callback(struct worker_pool *pool) { } 3154 static void workqueue_callback_cancel_wait_running(struct worker_pool *pool) { } 3155 3156 #endif 3157 3158 /** 3159 * manage_workers - manage worker pool 3160 * @worker: self 3161 * 3162 * Assume the manager role and manage the worker pool @worker belongs 3163 * to. At any given time, there can be only zero or one manager per 3164 * pool. The exclusion is handled automatically by this function. 3165 * 3166 * The caller can safely start processing works on false return. On 3167 * true return, it's guaranteed that need_to_create_worker() is false 3168 * and may_start_working() is true. 3169 * 3170 * CONTEXT: 3171 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed 3172 * multiple times. Does GFP_KERNEL allocations. 3173 * 3174 * Return: 3175 * %false if the pool doesn't need management and the caller can safely 3176 * start processing works, %true if management function was performed and 3177 * the conditions that the caller verified before calling the function may 3178 * no longer be true. 3179 */ 3180 static bool manage_workers(struct worker *worker) 3181 { 3182 struct worker_pool *pool = worker->pool; 3183 3184 if (pool->flags & POOL_MANAGER_ACTIVE) 3185 return false; 3186 3187 pool->flags |= POOL_MANAGER_ACTIVE; 3188 pool->manager = worker; 3189 3190 maybe_create_worker(pool); 3191 3192 pool->manager = NULL; 3193 pool->flags &= ~POOL_MANAGER_ACTIVE; 3194 rcuwait_wake_up(&manager_wait); 3195 return true; 3196 } 3197 3198 /** 3199 * process_one_work - process single work 3200 * @worker: self 3201 * @work: work to process 3202 * 3203 * Process @work. This function contains all the logics necessary to 3204 * process a single work including synchronization against and 3205 * interaction with other workers on the same cpu, queueing and 3206 * flushing. As long as context requirement is met, any worker can 3207 * call this function to process a work. 3208 * 3209 * CONTEXT: 3210 * raw_spin_lock_irq(pool->lock) which is released and regrabbed. 3211 */ 3212 static void process_one_work(struct worker *worker, struct work_struct *work) 3213 __releases(&pool->lock) 3214 __acquires(&pool->lock) 3215 { 3216 struct pool_workqueue *pwq = get_work_pwq(work); 3217 struct worker_pool *pool = worker->pool; 3218 unsigned long work_data; 3219 int lockdep_start_depth, rcu_start_depth; 3220 bool bh_draining = pool->flags & POOL_BH_DRAINING; 3221 #ifdef CONFIG_LOCKDEP 3222 /* 3223 * It is permissible to free the struct work_struct from 3224 * inside the function that is called from it, this we need to 3225 * take into account for lockdep too. To avoid bogus "held 3226 * lock freed" warnings as well as problems when looking into 3227 * work->lockdep_map, make a copy and use that here. 3228 */ 3229 struct lockdep_map lockdep_map; 3230 3231 lockdep_copy_map(&lockdep_map, &work->lockdep_map); 3232 #endif 3233 /* ensure we're on the correct CPU */ 3234 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) && 3235 raw_smp_processor_id() != pool->cpu); 3236 3237 /* claim and dequeue */ 3238 debug_work_deactivate(work); 3239 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work); 3240 worker->current_work = work; 3241 worker->current_func = work->func; 3242 worker->current_pwq = pwq; 3243 if (worker->task) 3244 worker->current_at = worker->task->se.sum_exec_runtime; 3245 worker->current_start = jiffies; 3246 work_data = *work_data_bits(work); 3247 worker->current_color = get_work_color(work_data); 3248 3249 /* 3250 * Record wq name for cmdline and debug reporting, may get 3251 * overridden through set_worker_desc(). 3252 */ 3253 strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN); 3254 3255 list_del_init(&work->entry); 3256 3257 /* 3258 * CPU intensive works don't participate in concurrency management. 3259 * They're the scheduler's responsibility. This takes @worker out 3260 * of concurrency management and the next code block will chain 3261 * execution of the pending work items. 3262 */ 3263 if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE)) 3264 worker_set_flags(worker, WORKER_CPU_INTENSIVE); 3265 3266 /* 3267 * Kick @pool if necessary. It's always noop for per-cpu worker pools 3268 * since nr_running would always be >= 1 at this point. This is used to 3269 * chain execution of the pending work items for WORKER_NOT_RUNNING 3270 * workers such as the UNBOUND and CPU_INTENSIVE ones. 3271 */ 3272 kick_pool(pool); 3273 3274 /* 3275 * Record the last pool and clear PENDING which should be the last 3276 * update to @work. Also, do this inside @pool->lock so that 3277 * PENDING and queued state changes happen together while IRQ is 3278 * disabled. 3279 */ 3280 set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool)); 3281 3282 pwq->stats[PWQ_STAT_STARTED]++; 3283 raw_spin_unlock_irq(&pool->lock); 3284 3285 rcu_start_depth = rcu_preempt_depth(); 3286 lockdep_start_depth = lockdep_depth(current); 3287 /* see drain_dead_softirq_workfn() */ 3288 if (!bh_draining) 3289 lock_map_acquire(pwq->wq->lockdep_map); 3290 lock_map_acquire(&lockdep_map); 3291 /* 3292 * Strictly speaking we should mark the invariant state without holding 3293 * any locks, that is, before these two lock_map_acquire()'s. 3294 * 3295 * However, that would result in: 3296 * 3297 * A(W1) 3298 * WFC(C) 3299 * A(W1) 3300 * C(C) 3301 * 3302 * Which would create W1->C->W1 dependencies, even though there is no 3303 * actual deadlock possible. There are two solutions, using a 3304 * read-recursive acquire on the work(queue) 'locks', but this will then 3305 * hit the lockdep limitation on recursive locks, or simply discard 3306 * these locks. 3307 * 3308 * AFAICT there is no possible deadlock scenario between the 3309 * flush_work() and complete() primitives (except for single-threaded 3310 * workqueues), so hiding them isn't a problem. 3311 */ 3312 lockdep_invariant_state(true); 3313 trace_workqueue_execute_start(work); 3314 worker->current_func(work); 3315 /* 3316 * While we must be careful to not use "work" after this, the trace 3317 * point will only record its address. 3318 */ 3319 trace_workqueue_execute_end(work, worker->current_func); 3320 3321 lock_map_release(&lockdep_map); 3322 if (!bh_draining) 3323 lock_map_release(pwq->wq->lockdep_map); 3324 3325 if (unlikely((worker->task && in_atomic()) || 3326 lockdep_depth(current) != lockdep_start_depth || 3327 rcu_preempt_depth() != rcu_start_depth)) { 3328 pr_err("BUG: workqueue leaked atomic, lock or RCU: %s[%d]\n" 3329 " preempt=0x%08x lock=%d->%d RCU=%d->%d workfn=%ps\n", 3330 current->comm, task_pid_nr(current), preempt_count(), 3331 lockdep_start_depth, lockdep_depth(current), 3332 rcu_start_depth, rcu_preempt_depth(), 3333 worker->current_func); 3334 debug_show_held_locks(current); 3335 dump_stack(); 3336 } 3337 3338 /* 3339 * The following prevents a kworker from hogging CPU on !PREEMPTION 3340 * kernels, where a requeueing work item waiting for something to 3341 * happen could deadlock with stop_machine as such work item could 3342 * indefinitely requeue itself while all other CPUs are trapped in 3343 * stop_machine. At the same time, report a quiescent RCU state so 3344 * the same condition doesn't freeze RCU. 3345 */ 3346 if (worker->task) 3347 cond_resched(); 3348 3349 raw_spin_lock_irq(&pool->lock); 3350 3351 pwq->stats[PWQ_STAT_COMPLETED]++; 3352 3353 /* 3354 * In addition to %WQ_CPU_INTENSIVE, @worker may also have been marked 3355 * CPU intensive by wq_worker_tick() if @work hogged CPU longer than 3356 * wq_cpu_intensive_thresh_us. Clear it. 3357 */ 3358 worker_clr_flags(worker, WORKER_CPU_INTENSIVE); 3359 3360 /* tag the worker for identification in schedule() */ 3361 worker->last_func = worker->current_func; 3362 3363 /* we're done with it, release */ 3364 hash_del(&worker->hentry); 3365 worker->current_work = NULL; 3366 worker->current_func = NULL; 3367 worker->current_pwq = NULL; 3368 worker->current_color = INT_MAX; 3369 3370 /* must be the last step, see the function comment */ 3371 pwq_dec_nr_in_flight(pwq, work_data); 3372 } 3373 3374 /** 3375 * process_scheduled_works - process scheduled works 3376 * @worker: self 3377 * 3378 * Process all scheduled works. Please note that the scheduled list 3379 * may change while processing a work, so this function repeatedly 3380 * fetches a work from the top and executes it. 3381 * 3382 * CONTEXT: 3383 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed 3384 * multiple times. 3385 */ 3386 static void process_scheduled_works(struct worker *worker) 3387 { 3388 struct work_struct *work; 3389 bool first = true; 3390 3391 while ((work = list_first_entry_or_null(&worker->scheduled, 3392 struct work_struct, entry))) { 3393 if (first) { 3394 worker->pool->last_progress_ts = jiffies; 3395 first = false; 3396 } 3397 process_one_work(worker, work); 3398 } 3399 } 3400 3401 static void set_pf_worker(bool val) 3402 { 3403 mutex_lock(&wq_pool_attach_mutex); 3404 if (val) 3405 current->flags |= PF_WQ_WORKER; 3406 else 3407 current->flags &= ~PF_WQ_WORKER; 3408 mutex_unlock(&wq_pool_attach_mutex); 3409 } 3410 3411 /** 3412 * worker_thread - the worker thread function 3413 * @__worker: self 3414 * 3415 * The worker thread function. All workers belong to a worker_pool - 3416 * either a per-cpu one or dynamic unbound one. These workers process all 3417 * work items regardless of their specific target workqueue. The only 3418 * exception is work items which belong to workqueues with a rescuer which 3419 * will be explained in rescuer_thread(). 3420 * 3421 * Return: 0 3422 */ 3423 static int worker_thread(void *__worker) 3424 { 3425 struct worker *worker = __worker; 3426 struct worker_pool *pool = worker->pool; 3427 3428 /* tell the scheduler that this is a workqueue worker */ 3429 set_pf_worker(true); 3430 woke_up: 3431 raw_spin_lock_irq(&pool->lock); 3432 3433 /* am I supposed to die? */ 3434 if (unlikely(worker->flags & WORKER_DIE)) { 3435 raw_spin_unlock_irq(&pool->lock); 3436 set_pf_worker(false); 3437 /* 3438 * The worker is dead and PF_WQ_WORKER is cleared, worker->pool 3439 * shouldn't be accessed, reset it to NULL in case otherwise. 3440 */ 3441 worker->pool = NULL; 3442 ida_free(&pool->worker_ida, worker->id); 3443 return 0; 3444 } 3445 3446 worker_leave_idle(worker); 3447 recheck: 3448 /* no more worker necessary? */ 3449 if (!need_more_worker(pool)) 3450 goto sleep; 3451 3452 /* do we need to manage? */ 3453 if (unlikely(!may_start_working(pool)) && manage_workers(worker)) 3454 goto recheck; 3455 3456 /* 3457 * ->scheduled list can only be filled while a worker is 3458 * preparing to process a work or actually processing it. 3459 * Make sure nobody diddled with it while I was sleeping. 3460 */ 3461 WARN_ON_ONCE(!list_empty(&worker->scheduled)); 3462 3463 /* 3464 * Finish PREP stage. We're guaranteed to have at least one idle 3465 * worker or that someone else has already assumed the manager 3466 * role. This is where @worker starts participating in concurrency 3467 * management if applicable and concurrency management is restored 3468 * after being rebound. See rebind_workers() for details. 3469 */ 3470 worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND); 3471 3472 do { 3473 struct work_struct *work = 3474 list_first_entry(&pool->worklist, 3475 struct work_struct, entry); 3476 3477 if (assign_work(work, worker, NULL)) 3478 process_scheduled_works(worker); 3479 } while (keep_working(pool)); 3480 3481 worker_set_flags(worker, WORKER_PREP); 3482 sleep: 3483 /* 3484 * pool->lock is held and there's no work to process and no need to 3485 * manage, sleep. Workers are woken up only while holding 3486 * pool->lock or from local cpu, so setting the current state 3487 * before releasing pool->lock is enough to prevent losing any 3488 * event. 3489 */ 3490 worker_enter_idle(worker); 3491 __set_current_state(TASK_IDLE); 3492 raw_spin_unlock_irq(&pool->lock); 3493 schedule(); 3494 goto woke_up; 3495 } 3496 3497 static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescuer) 3498 { 3499 struct worker_pool *pool = pwq->pool; 3500 struct work_struct *cursor = &pwq->mayday_cursor; 3501 struct work_struct *work, *n; 3502 3503 /* have work items to rescue? */ 3504 if (!pwq->nr_active) 3505 return false; 3506 3507 /* need rescue? */ 3508 if (!need_to_create_worker(pool)) { 3509 /* 3510 * The pool has idle workers and doesn't need the rescuer, so it 3511 * could simply return false here. 3512 * 3513 * However, the memory pressure might not be fully relieved. 3514 * In PERCPU pool with concurrency enabled, having idle workers 3515 * does not necessarily mean memory pressure is gone; it may 3516 * simply mean regular workers have woken up, completed their 3517 * work, and gone idle again due to concurrency limits. 3518 * 3519 * In this case, those working workers may later sleep again, 3520 * the pool may run out of idle workers, and it will have to 3521 * allocate new ones and wait for the timer to send mayday, 3522 * causing unnecessary delay - especially if memory pressure 3523 * was never resolved throughout. 3524 * 3525 * Do more work if memory pressure is still on to reduce 3526 * relapse, using (pool->flags & POOL_MANAGER_ACTIVE), though 3527 * not precisely, unless there are other PWQs needing help. 3528 */ 3529 if (!(pool->flags & POOL_MANAGER_ACTIVE) || 3530 !list_empty(&pwq->wq->maydays)) 3531 return false; 3532 } 3533 3534 /* search from the start or cursor if available */ 3535 if (list_empty(&cursor->entry)) 3536 work = list_first_entry(&pool->worklist, struct work_struct, entry); 3537 else 3538 work = list_next_entry(cursor, entry); 3539 3540 /* find the next work item to rescue */ 3541 list_for_each_entry_safe_from(work, n, &pool->worklist, entry) { 3542 if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) { 3543 pwq->stats[PWQ_STAT_RESCUED]++; 3544 /* put the cursor for next search */ 3545 list_move_tail(&cursor->entry, &n->entry); 3546 return true; 3547 } 3548 } 3549 3550 return false; 3551 } 3552 3553 /** 3554 * rescuer_thread - the rescuer thread function 3555 * @__rescuer: self 3556 * 3557 * Workqueue rescuer thread function. There's one rescuer for each 3558 * workqueue which has WQ_MEM_RECLAIM set. 3559 * 3560 * Regular work processing on a pool may block trying to create a new 3561 * worker which uses GFP_KERNEL allocation which has slight chance of 3562 * developing into deadlock if some works currently on the same queue 3563 * need to be processed to satisfy the GFP_KERNEL allocation. This is 3564 * the problem rescuer solves. 3565 * 3566 * When such condition is possible, the pool summons rescuers of all 3567 * workqueues which have works queued on the pool and let them process 3568 * those works so that forward progress can be guaranteed. 3569 * 3570 * This should happen rarely. 3571 * 3572 * Return: 0 3573 */ 3574 static int rescuer_thread(void *__rescuer) 3575 { 3576 struct worker *rescuer = __rescuer; 3577 struct workqueue_struct *wq = rescuer->rescue_wq; 3578 bool should_stop; 3579 3580 set_user_nice(current, RESCUER_NICE_LEVEL); 3581 3582 /* 3583 * Mark rescuer as worker too. As WORKER_PREP is never cleared, it 3584 * doesn't participate in concurrency management. 3585 */ 3586 set_pf_worker(true); 3587 repeat: 3588 set_current_state(TASK_IDLE); 3589 3590 /* 3591 * By the time the rescuer is requested to stop, the workqueue 3592 * shouldn't have any work pending, but @wq->maydays may still have 3593 * pwq(s) queued. This can happen by non-rescuer workers consuming 3594 * all the work items before the rescuer got to them. Go through 3595 * @wq->maydays processing before acting on should_stop so that the 3596 * list is always empty on exit. 3597 */ 3598 should_stop = kthread_should_stop(); 3599 3600 /* see whether any pwq is asking for help */ 3601 raw_spin_lock_irq(&wq_mayday_lock); 3602 3603 while (!list_empty(&wq->maydays)) { 3604 struct pool_workqueue *pwq = list_first_entry(&wq->maydays, 3605 struct pool_workqueue, mayday_node); 3606 struct worker_pool *pool = pwq->pool; 3607 unsigned int count = 0; 3608 3609 __set_current_state(TASK_RUNNING); 3610 list_del_init(&pwq->mayday_node); 3611 3612 raw_spin_unlock_irq(&wq_mayday_lock); 3613 3614 worker_attach_to_pool(rescuer, pool); 3615 3616 raw_spin_lock_irq(&pool->lock); 3617 3618 WARN_ON_ONCE(!list_empty(&rescuer->scheduled)); 3619 3620 while (assign_rescuer_work(pwq, rescuer)) { 3621 process_scheduled_works(rescuer); 3622 3623 /* 3624 * If the per-turn work item limit is reached and other 3625 * PWQs are in mayday, requeue mayday for this PWQ and 3626 * let the rescuer handle the other PWQs first. 3627 */ 3628 if (++count > RESCUER_BATCH && !list_empty(&pwq->wq->maydays) && 3629 pwq->nr_active && need_to_create_worker(pool)) { 3630 raw_spin_lock(&wq_mayday_lock); 3631 send_mayday(pwq); 3632 raw_spin_unlock(&wq_mayday_lock); 3633 break; 3634 } 3635 } 3636 3637 /* The cursor can not be left behind without the rescuer watching it. */ 3638 if (!list_empty(&pwq->mayday_cursor.entry) && list_empty(&pwq->mayday_node)) 3639 list_del_init(&pwq->mayday_cursor.entry); 3640 3641 /* 3642 * Leave this pool. Notify regular workers; otherwise, we end up 3643 * with 0 concurrency and stalling the execution. 3644 */ 3645 kick_pool(pool); 3646 3647 raw_spin_unlock_irq(&pool->lock); 3648 3649 worker_detach_from_pool(rescuer); 3650 3651 /* 3652 * Put the reference grabbed by send_mayday(). @pool might 3653 * go away any time after it. 3654 */ 3655 put_pwq_unlocked(pwq); 3656 3657 raw_spin_lock_irq(&wq_mayday_lock); 3658 } 3659 3660 raw_spin_unlock_irq(&wq_mayday_lock); 3661 3662 if (should_stop) { 3663 __set_current_state(TASK_RUNNING); 3664 set_pf_worker(false); 3665 return 0; 3666 } 3667 3668 /* rescuers should never participate in concurrency management */ 3669 WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING)); 3670 schedule(); 3671 goto repeat; 3672 } 3673 3674 static void bh_worker(struct worker *worker) 3675 { 3676 struct worker_pool *pool = worker->pool; 3677 int nr_restarts = BH_WORKER_RESTARTS; 3678 unsigned long end = jiffies + BH_WORKER_JIFFIES; 3679 3680 worker_lock_callback(pool); 3681 raw_spin_lock_irq(&pool->lock); 3682 worker_leave_idle(worker); 3683 3684 /* 3685 * This function follows the structure of worker_thread(). See there for 3686 * explanations on each step. 3687 */ 3688 if (!need_more_worker(pool)) 3689 goto done; 3690 3691 WARN_ON_ONCE(!list_empty(&worker->scheduled)); 3692 worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND); 3693 3694 do { 3695 struct work_struct *work = 3696 list_first_entry(&pool->worklist, 3697 struct work_struct, entry); 3698 3699 if (assign_work(work, worker, NULL)) 3700 process_scheduled_works(worker); 3701 } while (keep_working(pool) && 3702 --nr_restarts && time_before(jiffies, end)); 3703 3704 worker_set_flags(worker, WORKER_PREP); 3705 done: 3706 worker_enter_idle(worker); 3707 kick_pool(pool); 3708 raw_spin_unlock_irq(&pool->lock); 3709 worker_unlock_callback(pool); 3710 } 3711 3712 /* 3713 * TODO: Convert all tasklet users to workqueue and use softirq directly. 3714 * 3715 * This is currently called from tasklet[_hi]action() and thus is also called 3716 * whenever there are tasklets to run. Let's do an early exit if there's nothing 3717 * queued. Once conversion from tasklet is complete, the need_more_worker() test 3718 * can be dropped. 3719 * 3720 * After full conversion, we'll add worker->softirq_action, directly use the 3721 * softirq action and obtain the worker pointer from the softirq_action pointer. 3722 */ 3723 void workqueue_softirq_action(bool highpri) 3724 { 3725 struct worker_pool *pool = 3726 &per_cpu(bh_worker_pools, smp_processor_id())[highpri]; 3727 if (need_more_worker(pool)) 3728 bh_worker(list_first_entry(&pool->workers, struct worker, node)); 3729 } 3730 3731 struct wq_drain_dead_softirq_work { 3732 struct work_struct work; 3733 struct worker_pool *pool; 3734 struct completion done; 3735 }; 3736 3737 static void drain_dead_softirq_workfn(struct work_struct *work) 3738 { 3739 struct wq_drain_dead_softirq_work *dead_work = 3740 container_of(work, struct wq_drain_dead_softirq_work, work); 3741 struct worker_pool *pool = dead_work->pool; 3742 bool repeat; 3743 3744 /* 3745 * @pool's CPU is dead and we want to execute its still pending work 3746 * items from this BH work item which is running on a different CPU. As 3747 * its CPU is dead, @pool can't be kicked and, as work execution path 3748 * will be nested, a lockdep annotation needs to be suppressed. Mark 3749 * @pool with %POOL_BH_DRAINING for the special treatments. 3750 */ 3751 raw_spin_lock_irq(&pool->lock); 3752 pool->flags |= POOL_BH_DRAINING; 3753 raw_spin_unlock_irq(&pool->lock); 3754 3755 bh_worker(list_first_entry(&pool->workers, struct worker, node)); 3756 3757 raw_spin_lock_irq(&pool->lock); 3758 pool->flags &= ~POOL_BH_DRAINING; 3759 repeat = need_more_worker(pool); 3760 raw_spin_unlock_irq(&pool->lock); 3761 3762 /* 3763 * bh_worker() might hit consecutive execution limit and bail. If there 3764 * still are pending work items, reschedule self and return so that we 3765 * don't hog this CPU's BH. 3766 */ 3767 if (repeat) { 3768 if (pool->attrs->nice == HIGHPRI_NICE_LEVEL) 3769 queue_work(system_bh_highpri_wq, work); 3770 else 3771 queue_work(system_bh_wq, work); 3772 } else { 3773 complete(&dead_work->done); 3774 } 3775 } 3776 3777 /* 3778 * @cpu is dead. Drain the remaining BH work items on the current CPU. It's 3779 * possible to allocate dead_work per CPU and avoid flushing. However, then we 3780 * have to worry about draining overlapping with CPU coming back online or 3781 * nesting (one CPU's dead_work queued on another CPU which is also dead and so 3782 * on). Let's keep it simple and drain them synchronously. These are BH work 3783 * items which shouldn't be requeued on the same pool. Shouldn't take long. 3784 */ 3785 void workqueue_softirq_dead(unsigned int cpu) 3786 { 3787 int i; 3788 3789 for (i = 0; i < NR_STD_WORKER_POOLS; i++) { 3790 struct worker_pool *pool = &per_cpu(bh_worker_pools, cpu)[i]; 3791 struct wq_drain_dead_softirq_work dead_work; 3792 3793 if (!need_more_worker(pool)) 3794 continue; 3795 3796 INIT_WORK_ONSTACK(&dead_work.work, drain_dead_softirq_workfn); 3797 dead_work.pool = pool; 3798 init_completion(&dead_work.done); 3799 3800 if (pool->attrs->nice == HIGHPRI_NICE_LEVEL) 3801 queue_work(system_bh_highpri_wq, &dead_work.work); 3802 else 3803 queue_work(system_bh_wq, &dead_work.work); 3804 3805 wait_for_completion(&dead_work.done); 3806 destroy_work_on_stack(&dead_work.work); 3807 } 3808 } 3809 3810 /** 3811 * check_flush_dependency - check for flush dependency sanity 3812 * @target_wq: workqueue being flushed 3813 * @target_work: work item being flushed (NULL for workqueue flushes) 3814 * @from_cancel: are we called from the work cancel path 3815 * 3816 * %current is trying to flush the whole @target_wq or @target_work on it. 3817 * If this is not the cancel path (which implies work being flushed is either 3818 * already running, or will not be at all), check if @target_wq doesn't have 3819 * %WQ_MEM_RECLAIM and verify that %current is not reclaiming memory or running 3820 * on a workqueue which doesn't have %WQ_MEM_RECLAIM as that can break forward- 3821 * progress guarantee leading to a deadlock. 3822 */ 3823 static void check_flush_dependency(struct workqueue_struct *target_wq, 3824 struct work_struct *target_work, 3825 bool from_cancel) 3826 { 3827 work_func_t target_func; 3828 struct worker *worker; 3829 3830 if (from_cancel || target_wq->flags & WQ_MEM_RECLAIM) 3831 return; 3832 3833 worker = current_wq_worker(); 3834 target_func = target_work ? target_work->func : NULL; 3835 3836 WARN_ONCE(current->flags & PF_MEMALLOC, 3837 "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps", 3838 current->pid, current->comm, target_wq->name, target_func); 3839 WARN_ONCE(worker && ((worker->current_pwq->wq->flags & 3840 (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM), 3841 "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps", 3842 worker->current_pwq->wq->name, worker->current_func, 3843 target_wq->name, target_func); 3844 } 3845 3846 struct wq_barrier { 3847 struct work_struct work; 3848 struct completion done; 3849 struct task_struct *task; /* purely informational */ 3850 }; 3851 3852 static void wq_barrier_func(struct work_struct *work) 3853 { 3854 struct wq_barrier *barr = container_of(work, struct wq_barrier, work); 3855 complete(&barr->done); 3856 } 3857 3858 /** 3859 * insert_wq_barrier - insert a barrier work 3860 * @pwq: pwq to insert barrier into 3861 * @barr: wq_barrier to insert 3862 * @target: target work to attach @barr to 3863 * @worker: worker currently executing @target, NULL if @target is not executing 3864 * 3865 * @barr is linked to @target such that @barr is completed only after 3866 * @target finishes execution. Please note that the ordering 3867 * guarantee is observed only with respect to @target and on the local 3868 * cpu. 3869 * 3870 * Currently, a queued barrier can't be canceled. This is because 3871 * try_to_grab_pending() can't determine whether the work to be 3872 * grabbed is at the head of the queue and thus can't clear LINKED 3873 * flag of the previous work while there must be a valid next work 3874 * after a work with LINKED flag set. 3875 * 3876 * Note that when @worker is non-NULL, @target may be modified 3877 * underneath us, so we can't reliably determine pwq from @target. 3878 * 3879 * CONTEXT: 3880 * raw_spin_lock_irq(pool->lock). 3881 */ 3882 static void insert_wq_barrier(struct pool_workqueue *pwq, 3883 struct wq_barrier *barr, 3884 struct work_struct *target, struct worker *worker) 3885 { 3886 static __maybe_unused struct lock_class_key bh_key, thr_key; 3887 unsigned int work_flags = 0; 3888 unsigned int work_color; 3889 struct list_head *head; 3890 3891 /* 3892 * debugobject calls are safe here even with pool->lock locked 3893 * as we know for sure that this will not trigger any of the 3894 * checks and call back into the fixup functions where we 3895 * might deadlock. 3896 * 3897 * BH and threaded workqueues need separate lockdep keys to avoid 3898 * spuriously triggering "inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W} 3899 * usage". 3900 */ 3901 INIT_WORK_ONSTACK_KEY(&barr->work, wq_barrier_func, 3902 (pwq->wq->flags & WQ_BH) ? &bh_key : &thr_key); 3903 __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work)); 3904 3905 init_completion_map(&barr->done, &target->lockdep_map); 3906 3907 barr->task = current; 3908 3909 /* The barrier work item does not participate in nr_active. */ 3910 work_flags |= WORK_STRUCT_INACTIVE; 3911 3912 /* 3913 * If @target is currently being executed, schedule the 3914 * barrier to the worker; otherwise, put it after @target. 3915 */ 3916 if (worker) { 3917 head = worker->scheduled.next; 3918 work_color = worker->current_color; 3919 } else { 3920 unsigned long *bits = work_data_bits(target); 3921 3922 head = target->entry.next; 3923 /* there can already be other linked works, inherit and set */ 3924 work_flags |= *bits & WORK_STRUCT_LINKED; 3925 work_color = get_work_color(*bits); 3926 __set_bit(WORK_STRUCT_LINKED_BIT, bits); 3927 } 3928 3929 pwq->nr_in_flight[work_color]++; 3930 work_flags |= work_color_to_flags(work_color); 3931 3932 insert_work(pwq, &barr->work, head, work_flags); 3933 } 3934 3935 /** 3936 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing 3937 * @wq: workqueue being flushed 3938 * @flush_color: new flush color, < 0 for no-op 3939 * @work_color: new work color, < 0 for no-op 3940 * 3941 * Prepare pwqs for workqueue flushing. 3942 * 3943 * If @flush_color is non-negative, flush_color on all pwqs should be 3944 * -1. If no pwq has in-flight commands at the specified color, all 3945 * pwq->flush_color's stay at -1 and %false is returned. If any pwq 3946 * has in flight commands, its pwq->flush_color is set to 3947 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq 3948 * wakeup logic is armed and %true is returned. 3949 * 3950 * The caller should have initialized @wq->first_flusher prior to 3951 * calling this function with non-negative @flush_color. If 3952 * @flush_color is negative, no flush color update is done and %false 3953 * is returned. 3954 * 3955 * If @work_color is non-negative, all pwqs should have the same 3956 * work_color which is previous to @work_color and all will be 3957 * advanced to @work_color. 3958 * 3959 * CONTEXT: 3960 * mutex_lock(wq->mutex). 3961 * 3962 * Return: 3963 * %true if @flush_color >= 0 and there's something to flush. %false 3964 * otherwise. 3965 */ 3966 static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq, 3967 int flush_color, int work_color) 3968 { 3969 bool wait = false; 3970 struct pool_workqueue *pwq; 3971 struct worker_pool *current_pool = NULL; 3972 3973 if (flush_color >= 0) { 3974 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush)); 3975 atomic_set(&wq->nr_pwqs_to_flush, 1); 3976 } 3977 3978 /* 3979 * For unbound workqueue, pwqs will map to only a few pools. 3980 * Most of the time, pwqs within the same pool will be linked 3981 * sequentially to wq->pwqs by cpu index. So in the majority 3982 * of pwq iters, the pool is the same, only doing lock/unlock 3983 * if the pool has changed. This can largely reduce expensive 3984 * lock operations. 3985 */ 3986 for_each_pwq(pwq, wq) { 3987 if (current_pool != pwq->pool) { 3988 if (likely(current_pool)) 3989 raw_spin_unlock_irq(¤t_pool->lock); 3990 current_pool = pwq->pool; 3991 raw_spin_lock_irq(¤t_pool->lock); 3992 } 3993 3994 if (flush_color >= 0) { 3995 WARN_ON_ONCE(pwq->flush_color != -1); 3996 3997 if (pwq->nr_in_flight[flush_color]) { 3998 pwq->flush_color = flush_color; 3999 atomic_inc(&wq->nr_pwqs_to_flush); 4000 wait = true; 4001 } 4002 } 4003 4004 if (work_color >= 0) { 4005 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); 4006 pwq->work_color = work_color; 4007 } 4008 4009 } 4010 4011 if (current_pool) 4012 raw_spin_unlock_irq(¤t_pool->lock); 4013 4014 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush)) 4015 complete(&wq->first_flusher->done); 4016 4017 return wait; 4018 } 4019 4020 static void touch_wq_lockdep_map(struct workqueue_struct *wq) 4021 { 4022 #ifdef CONFIG_LOCKDEP 4023 if (unlikely(!wq->lockdep_map)) 4024 return; 4025 4026 if (wq->flags & WQ_BH) 4027 local_bh_disable(); 4028 4029 lock_map_acquire(wq->lockdep_map); 4030 lock_map_release(wq->lockdep_map); 4031 4032 if (wq->flags & WQ_BH) 4033 local_bh_enable(); 4034 #endif 4035 } 4036 4037 static void touch_work_lockdep_map(struct work_struct *work, 4038 struct workqueue_struct *wq) 4039 { 4040 #ifdef CONFIG_LOCKDEP 4041 if (wq->flags & WQ_BH) 4042 local_bh_disable(); 4043 4044 lock_map_acquire(&work->lockdep_map); 4045 lock_map_release(&work->lockdep_map); 4046 4047 if (wq->flags & WQ_BH) 4048 local_bh_enable(); 4049 #endif 4050 } 4051 4052 /** 4053 * __flush_workqueue - ensure that any scheduled work has run to completion. 4054 * @wq: workqueue to flush 4055 * 4056 * This function sleeps until all work items which were queued on entry 4057 * have finished execution, but it is not livelocked by new incoming ones. 4058 */ 4059 void __flush_workqueue(struct workqueue_struct *wq) 4060 { 4061 struct wq_flusher this_flusher = { 4062 .list = LIST_HEAD_INIT(this_flusher.list), 4063 .flush_color = -1, 4064 .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, (*wq->lockdep_map)), 4065 }; 4066 int next_color; 4067 4068 if (WARN_ON(!wq_online)) 4069 return; 4070 4071 touch_wq_lockdep_map(wq); 4072 4073 mutex_lock(&wq->mutex); 4074 4075 /* 4076 * Start-to-wait phase 4077 */ 4078 next_color = work_next_color(wq->work_color); 4079 4080 if (next_color != wq->flush_color) { 4081 /* 4082 * Color space is not full. The current work_color 4083 * becomes our flush_color and work_color is advanced 4084 * by one. 4085 */ 4086 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow)); 4087 this_flusher.flush_color = wq->work_color; 4088 wq->work_color = next_color; 4089 4090 if (!wq->first_flusher) { 4091 /* no flush in progress, become the first flusher */ 4092 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); 4093 4094 wq->first_flusher = &this_flusher; 4095 4096 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color, 4097 wq->work_color)) { 4098 /* nothing to flush, done */ 4099 wq->flush_color = next_color; 4100 wq->first_flusher = NULL; 4101 goto out_unlock; 4102 } 4103 } else { 4104 /* wait in queue */ 4105 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color); 4106 list_add_tail(&this_flusher.list, &wq->flusher_queue); 4107 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); 4108 } 4109 } else { 4110 /* 4111 * Oops, color space is full, wait on overflow queue. 4112 * The next flush completion will assign us 4113 * flush_color and transfer to flusher_queue. 4114 */ 4115 list_add_tail(&this_flusher.list, &wq->flusher_overflow); 4116 } 4117 4118 check_flush_dependency(wq, NULL, false); 4119 4120 mutex_unlock(&wq->mutex); 4121 4122 wait_for_completion(&this_flusher.done); 4123 4124 /* 4125 * Wake-up-and-cascade phase 4126 * 4127 * First flushers are responsible for cascading flushes and 4128 * handling overflow. Non-first flushers can simply return. 4129 */ 4130 if (READ_ONCE(wq->first_flusher) != &this_flusher) 4131 return; 4132 4133 mutex_lock(&wq->mutex); 4134 4135 /* we might have raced, check again with mutex held */ 4136 if (wq->first_flusher != &this_flusher) 4137 goto out_unlock; 4138 4139 WRITE_ONCE(wq->first_flusher, NULL); 4140 4141 WARN_ON_ONCE(!list_empty(&this_flusher.list)); 4142 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); 4143 4144 while (true) { 4145 struct wq_flusher *next, *tmp; 4146 4147 /* complete all the flushers sharing the current flush color */ 4148 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) { 4149 if (next->flush_color != wq->flush_color) 4150 break; 4151 list_del_init(&next->list); 4152 complete(&next->done); 4153 } 4154 4155 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) && 4156 wq->flush_color != work_next_color(wq->work_color)); 4157 4158 /* this flush_color is finished, advance by one */ 4159 wq->flush_color = work_next_color(wq->flush_color); 4160 4161 /* one color has been freed, handle overflow queue */ 4162 if (!list_empty(&wq->flusher_overflow)) { 4163 /* 4164 * Assign the same color to all overflowed 4165 * flushers, advance work_color and append to 4166 * flusher_queue. This is the start-to-wait 4167 * phase for these overflowed flushers. 4168 */ 4169 list_for_each_entry(tmp, &wq->flusher_overflow, list) 4170 tmp->flush_color = wq->work_color; 4171 4172 wq->work_color = work_next_color(wq->work_color); 4173 4174 list_splice_tail_init(&wq->flusher_overflow, 4175 &wq->flusher_queue); 4176 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); 4177 } 4178 4179 if (list_empty(&wq->flusher_queue)) { 4180 WARN_ON_ONCE(wq->flush_color != wq->work_color); 4181 break; 4182 } 4183 4184 /* 4185 * Need to flush more colors. Make the next flusher 4186 * the new first flusher and arm pwqs. 4187 */ 4188 WARN_ON_ONCE(wq->flush_color == wq->work_color); 4189 WARN_ON_ONCE(wq->flush_color != next->flush_color); 4190 4191 list_del_init(&next->list); 4192 wq->first_flusher = next; 4193 4194 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1)) 4195 break; 4196 4197 /* 4198 * Meh... this color is already done, clear first 4199 * flusher and repeat cascading. 4200 */ 4201 wq->first_flusher = NULL; 4202 } 4203 4204 out_unlock: 4205 mutex_unlock(&wq->mutex); 4206 } 4207 EXPORT_SYMBOL(__flush_workqueue); 4208 4209 /** 4210 * drain_workqueue - drain a workqueue 4211 * @wq: workqueue to drain 4212 * 4213 * Wait until the workqueue becomes empty. While draining is in progress, 4214 * only chain queueing is allowed. IOW, only currently pending or running 4215 * work items on @wq can queue further work items on it. @wq is flushed 4216 * repeatedly until it becomes empty. The number of flushing is determined 4217 * by the depth of chaining and should be relatively short. Whine if it 4218 * takes too long. 4219 */ 4220 void drain_workqueue(struct workqueue_struct *wq) 4221 { 4222 unsigned int flush_cnt = 0; 4223 struct pool_workqueue *pwq; 4224 4225 /* 4226 * __queue_work() needs to test whether there are drainers, is much 4227 * hotter than drain_workqueue() and already looks at @wq->flags. 4228 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers. 4229 */ 4230 mutex_lock(&wq->mutex); 4231 if (!wq->nr_drainers++) 4232 wq->flags |= __WQ_DRAINING; 4233 mutex_unlock(&wq->mutex); 4234 reflush: 4235 __flush_workqueue(wq); 4236 4237 mutex_lock(&wq->mutex); 4238 4239 for_each_pwq(pwq, wq) { 4240 bool drained; 4241 4242 raw_spin_lock_irq(&pwq->pool->lock); 4243 drained = pwq_is_empty(pwq); 4244 raw_spin_unlock_irq(&pwq->pool->lock); 4245 4246 if (drained) 4247 continue; 4248 4249 if (++flush_cnt == 10 || 4250 (flush_cnt % 100 == 0 && flush_cnt <= 1000)) 4251 pr_warn("workqueue %s: %s() isn't complete after %u tries\n", 4252 wq->name, __func__, flush_cnt); 4253 4254 mutex_unlock(&wq->mutex); 4255 goto reflush; 4256 } 4257 4258 if (!--wq->nr_drainers) 4259 wq->flags &= ~__WQ_DRAINING; 4260 mutex_unlock(&wq->mutex); 4261 } 4262 EXPORT_SYMBOL_GPL(drain_workqueue); 4263 4264 static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, 4265 bool from_cancel) 4266 { 4267 struct worker *worker = NULL; 4268 struct worker_pool *pool; 4269 struct pool_workqueue *pwq; 4270 struct workqueue_struct *wq; 4271 4272 rcu_read_lock(); 4273 pool = get_work_pool(work); 4274 if (!pool) { 4275 rcu_read_unlock(); 4276 return false; 4277 } 4278 4279 raw_spin_lock_irq(&pool->lock); 4280 /* see the comment in try_to_grab_pending() with the same code */ 4281 pwq = get_work_pwq(work); 4282 if (pwq) { 4283 if (unlikely(pwq->pool != pool)) 4284 goto already_gone; 4285 } else { 4286 worker = find_worker_executing_work(pool, work); 4287 if (!worker) 4288 goto already_gone; 4289 pwq = worker->current_pwq; 4290 } 4291 4292 wq = pwq->wq; 4293 check_flush_dependency(wq, work, from_cancel); 4294 4295 insert_wq_barrier(pwq, barr, work, worker); 4296 raw_spin_unlock_irq(&pool->lock); 4297 4298 touch_work_lockdep_map(work, wq); 4299 4300 /* 4301 * Force a lock recursion deadlock when using flush_work() inside a 4302 * single-threaded or rescuer equipped workqueue. 4303 * 4304 * For single threaded workqueues the deadlock happens when the work 4305 * is after the work issuing the flush_work(). For rescuer equipped 4306 * workqueues the deadlock happens when the rescuer stalls, blocking 4307 * forward progress. 4308 */ 4309 if (!from_cancel && (wq->saved_max_active == 1 || wq->rescuer)) 4310 touch_wq_lockdep_map(wq); 4311 4312 rcu_read_unlock(); 4313 return true; 4314 already_gone: 4315 raw_spin_unlock_irq(&pool->lock); 4316 rcu_read_unlock(); 4317 return false; 4318 } 4319 4320 static bool __flush_work(struct work_struct *work, bool from_cancel) 4321 { 4322 struct wq_barrier barr; 4323 4324 if (WARN_ON(!wq_online)) 4325 return false; 4326 4327 if (WARN_ON(!work->func)) 4328 return false; 4329 4330 if (!start_flush_work(work, &barr, from_cancel)) 4331 return false; 4332 4333 /* 4334 * start_flush_work() returned %true. If @from_cancel is set, we know 4335 * that @work must have been executing during start_flush_work() and 4336 * can't currently be queued. Its data must contain OFFQ bits. If @work 4337 * was queued on a BH workqueue, we also know that it was running in the 4338 * BH context and thus can be busy-waited. 4339 */ 4340 if (from_cancel) { 4341 unsigned long data = *work_data_bits(work); 4342 4343 if (!WARN_ON_ONCE(data & WORK_STRUCT_PWQ) && 4344 (data & WORK_OFFQ_BH)) { 4345 /* 4346 * On RT, prevent a live lock when %current preempted 4347 * soft interrupt processing by blocking on lock which 4348 * is owned by the thread invoking the callback. 4349 */ 4350 while (!try_wait_for_completion(&barr.done)) { 4351 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 4352 struct worker_pool *pool; 4353 4354 guard(rcu)(); 4355 pool = get_work_pool(work); 4356 if (pool) 4357 workqueue_callback_cancel_wait_running(pool); 4358 } else { 4359 cpu_relax(); 4360 } 4361 } 4362 goto out_destroy; 4363 } 4364 } 4365 4366 wait_for_completion(&barr.done); 4367 4368 out_destroy: 4369 destroy_work_on_stack(&barr.work); 4370 return true; 4371 } 4372 4373 /** 4374 * flush_work - wait for a work to finish executing the last queueing instance 4375 * @work: the work to flush 4376 * 4377 * Wait until @work has finished execution. @work is guaranteed to be idle 4378 * on return if it hasn't been requeued since flush started. 4379 * 4380 * Return: 4381 * %true if flush_work() waited for the work to finish execution, 4382 * %false if it was already idle. 4383 */ 4384 bool flush_work(struct work_struct *work) 4385 { 4386 might_sleep(); 4387 return __flush_work(work, false); 4388 } 4389 EXPORT_SYMBOL_GPL(flush_work); 4390 4391 /** 4392 * flush_delayed_work - wait for a dwork to finish executing the last queueing 4393 * @dwork: the delayed work to flush 4394 * 4395 * Delayed timer is cancelled and the pending work is queued for 4396 * immediate execution. Like flush_work(), this function only 4397 * considers the last queueing instance of @dwork. 4398 * 4399 * Return: 4400 * %true if flush_work() waited for the work to finish execution, 4401 * %false if it was already idle. 4402 */ 4403 bool flush_delayed_work(struct delayed_work *dwork) 4404 { 4405 local_irq_disable(); 4406 if (timer_delete_sync(&dwork->timer)) 4407 __queue_work(dwork->cpu, dwork->wq, &dwork->work); 4408 local_irq_enable(); 4409 return flush_work(&dwork->work); 4410 } 4411 EXPORT_SYMBOL(flush_delayed_work); 4412 4413 /** 4414 * flush_rcu_work - wait for a rwork to finish executing the last queueing 4415 * @rwork: the rcu work to flush 4416 * 4417 * Return: 4418 * %true if flush_rcu_work() waited for the work to finish execution, 4419 * %false if it was already idle. 4420 */ 4421 bool flush_rcu_work(struct rcu_work *rwork) 4422 { 4423 if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) { 4424 rcu_barrier(); 4425 flush_work(&rwork->work); 4426 return true; 4427 } else { 4428 return flush_work(&rwork->work); 4429 } 4430 } 4431 EXPORT_SYMBOL(flush_rcu_work); 4432 4433 static void work_offqd_disable(struct work_offq_data *offqd) 4434 { 4435 const unsigned long max = (1lu << WORK_OFFQ_DISABLE_BITS) - 1; 4436 4437 if (likely(offqd->disable < max)) 4438 offqd->disable++; 4439 else 4440 WARN_ONCE(true, "workqueue: work disable count overflowed\n"); 4441 } 4442 4443 static void work_offqd_enable(struct work_offq_data *offqd) 4444 { 4445 if (likely(offqd->disable > 0)) 4446 offqd->disable--; 4447 else 4448 WARN_ONCE(true, "workqueue: work disable count underflowed\n"); 4449 } 4450 4451 static bool __cancel_work(struct work_struct *work, u32 cflags) 4452 { 4453 struct work_offq_data offqd; 4454 unsigned long irq_flags; 4455 int ret; 4456 4457 ret = work_grab_pending(work, cflags, &irq_flags); 4458 4459 work_offqd_unpack(&offqd, *work_data_bits(work)); 4460 4461 if (cflags & WORK_CANCEL_DISABLE) 4462 work_offqd_disable(&offqd); 4463 4464 set_work_pool_and_clear_pending(work, offqd.pool_id, 4465 work_offqd_pack_flags(&offqd)); 4466 local_irq_restore(irq_flags); 4467 return ret; 4468 } 4469 4470 static bool __cancel_work_sync(struct work_struct *work, u32 cflags) 4471 { 4472 bool ret; 4473 4474 ret = __cancel_work(work, cflags | WORK_CANCEL_DISABLE); 4475 4476 if (*work_data_bits(work) & WORK_OFFQ_BH) 4477 WARN_ON_ONCE(in_hardirq()); 4478 else 4479 might_sleep(); 4480 4481 /* 4482 * Skip __flush_work() during early boot when we know that @work isn't 4483 * executing. This allows canceling during early boot. 4484 */ 4485 if (wq_online) 4486 __flush_work(work, true); 4487 4488 if (!(cflags & WORK_CANCEL_DISABLE)) 4489 enable_work(work); 4490 4491 return ret; 4492 } 4493 4494 /* 4495 * See cancel_delayed_work() 4496 */ 4497 bool cancel_work(struct work_struct *work) 4498 { 4499 return __cancel_work(work, 0); 4500 } 4501 EXPORT_SYMBOL(cancel_work); 4502 4503 /** 4504 * cancel_work_sync - cancel a work and wait for it to finish 4505 * @work: the work to cancel 4506 * 4507 * Cancel @work and wait for its execution to finish. This function can be used 4508 * even if the work re-queues itself or migrates to another workqueue. On return 4509 * from this function, @work is guaranteed to be not pending or executing on any 4510 * CPU as long as there aren't racing enqueues. 4511 * 4512 * cancel_work_sync(&delayed_work->work) must not be used for delayed_work's. 4513 * Use cancel_delayed_work_sync() instead. 4514 * 4515 * Must be called from a sleepable context if @work was last queued on a non-BH 4516 * workqueue. Can also be called from non-hardirq atomic contexts including BH 4517 * if @work was last queued on a BH workqueue. 4518 * 4519 * Returns %true if @work was pending, %false otherwise. 4520 */ 4521 bool cancel_work_sync(struct work_struct *work) 4522 { 4523 return __cancel_work_sync(work, 0); 4524 } 4525 EXPORT_SYMBOL_GPL(cancel_work_sync); 4526 4527 /** 4528 * cancel_delayed_work - cancel a delayed work 4529 * @dwork: delayed_work to cancel 4530 * 4531 * Kill off a pending delayed_work. 4532 * 4533 * Return: %true if @dwork was pending and canceled; %false if it wasn't 4534 * pending. 4535 * 4536 * Note: 4537 * The work callback function may still be running on return, unless 4538 * it returns %true and the work doesn't re-arm itself. Explicitly flush or 4539 * use cancel_delayed_work_sync() to wait on it. 4540 * 4541 * This function is safe to call from any context including IRQ handler. 4542 */ 4543 bool cancel_delayed_work(struct delayed_work *dwork) 4544 { 4545 return __cancel_work(&dwork->work, WORK_CANCEL_DELAYED); 4546 } 4547 EXPORT_SYMBOL(cancel_delayed_work); 4548 4549 /** 4550 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish 4551 * @dwork: the delayed work cancel 4552 * 4553 * This is cancel_work_sync() for delayed works. 4554 * 4555 * Return: 4556 * %true if @dwork was pending, %false otherwise. 4557 */ 4558 bool cancel_delayed_work_sync(struct delayed_work *dwork) 4559 { 4560 return __cancel_work_sync(&dwork->work, WORK_CANCEL_DELAYED); 4561 } 4562 EXPORT_SYMBOL(cancel_delayed_work_sync); 4563 4564 /** 4565 * disable_work - Disable and cancel a work item 4566 * @work: work item to disable 4567 * 4568 * Disable @work by incrementing its disable count and cancel it if currently 4569 * pending. As long as the disable count is non-zero, any attempt to queue @work 4570 * will fail and return %false. The maximum supported disable depth is 2 to the 4571 * power of %WORK_OFFQ_DISABLE_BITS, currently 65536. 4572 * 4573 * Can be called from any context. Returns %true if @work was pending, %false 4574 * otherwise. 4575 */ 4576 bool disable_work(struct work_struct *work) 4577 { 4578 return __cancel_work(work, WORK_CANCEL_DISABLE); 4579 } 4580 EXPORT_SYMBOL_GPL(disable_work); 4581 4582 /** 4583 * disable_work_sync - Disable, cancel and drain a work item 4584 * @work: work item to disable 4585 * 4586 * Similar to disable_work() but also wait for @work to finish if currently 4587 * executing. 4588 * 4589 * Must be called from a sleepable context if @work was last queued on a non-BH 4590 * workqueue. Can also be called from non-hardirq atomic contexts including BH 4591 * if @work was last queued on a BH workqueue. 4592 * 4593 * Returns %true if @work was pending, %false otherwise. 4594 */ 4595 bool disable_work_sync(struct work_struct *work) 4596 { 4597 return __cancel_work_sync(work, WORK_CANCEL_DISABLE); 4598 } 4599 EXPORT_SYMBOL_GPL(disable_work_sync); 4600 4601 /** 4602 * enable_work - Enable a work item 4603 * @work: work item to enable 4604 * 4605 * Undo disable_work[_sync]() by decrementing @work's disable count. @work can 4606 * only be queued if its disable count is 0. 4607 * 4608 * Can be called from any context. Returns %true if the disable count reached 0. 4609 * Otherwise, %false. 4610 */ 4611 bool enable_work(struct work_struct *work) 4612 { 4613 struct work_offq_data offqd; 4614 unsigned long irq_flags; 4615 4616 work_grab_pending(work, 0, &irq_flags); 4617 4618 work_offqd_unpack(&offqd, *work_data_bits(work)); 4619 work_offqd_enable(&offqd); 4620 set_work_pool_and_clear_pending(work, offqd.pool_id, 4621 work_offqd_pack_flags(&offqd)); 4622 local_irq_restore(irq_flags); 4623 4624 return !offqd.disable; 4625 } 4626 EXPORT_SYMBOL_GPL(enable_work); 4627 4628 /** 4629 * disable_delayed_work - Disable and cancel a delayed work item 4630 * @dwork: delayed work item to disable 4631 * 4632 * disable_work() for delayed work items. 4633 */ 4634 bool disable_delayed_work(struct delayed_work *dwork) 4635 { 4636 return __cancel_work(&dwork->work, 4637 WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE); 4638 } 4639 EXPORT_SYMBOL_GPL(disable_delayed_work); 4640 4641 /** 4642 * disable_delayed_work_sync - Disable, cancel and drain a delayed work item 4643 * @dwork: delayed work item to disable 4644 * 4645 * disable_work_sync() for delayed work items. 4646 */ 4647 bool disable_delayed_work_sync(struct delayed_work *dwork) 4648 { 4649 return __cancel_work_sync(&dwork->work, 4650 WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE); 4651 } 4652 EXPORT_SYMBOL_GPL(disable_delayed_work_sync); 4653 4654 /** 4655 * enable_delayed_work - Enable a delayed work item 4656 * @dwork: delayed work item to enable 4657 * 4658 * enable_work() for delayed work items. 4659 */ 4660 bool enable_delayed_work(struct delayed_work *dwork) 4661 { 4662 return enable_work(&dwork->work); 4663 } 4664 EXPORT_SYMBOL_GPL(enable_delayed_work); 4665 4666 /** 4667 * schedule_on_each_cpu - execute a function synchronously on each online CPU 4668 * @func: the function to call 4669 * 4670 * schedule_on_each_cpu() executes @func on each online CPU using the 4671 * system workqueue and blocks until all CPUs have completed. 4672 * schedule_on_each_cpu() is very slow. 4673 * 4674 * Return: 4675 * 0 on success, -errno on failure. 4676 */ 4677 int schedule_on_each_cpu(work_func_t func) 4678 { 4679 int cpu; 4680 struct work_struct __percpu *works; 4681 4682 works = alloc_percpu(struct work_struct); 4683 if (!works) 4684 return -ENOMEM; 4685 4686 cpus_read_lock(); 4687 4688 for_each_online_cpu(cpu) { 4689 struct work_struct *work = per_cpu_ptr(works, cpu); 4690 4691 INIT_WORK(work, func); 4692 schedule_work_on(cpu, work); 4693 } 4694 4695 for_each_online_cpu(cpu) 4696 flush_work(per_cpu_ptr(works, cpu)); 4697 4698 cpus_read_unlock(); 4699 free_percpu(works); 4700 return 0; 4701 } 4702 4703 /** 4704 * execute_in_process_context - reliably execute the routine with user context 4705 * @fn: the function to execute 4706 * @ew: guaranteed storage for the execute work structure (must 4707 * be available when the work executes) 4708 * 4709 * Executes the function immediately if process context is available, 4710 * otherwise schedules the function for delayed execution. 4711 * 4712 * Return: 0 - function was executed 4713 * 1 - function was scheduled for execution 4714 */ 4715 int execute_in_process_context(work_func_t fn, struct execute_work *ew) 4716 { 4717 if (!in_interrupt()) { 4718 fn(&ew->work); 4719 return 0; 4720 } 4721 4722 INIT_WORK(&ew->work, fn); 4723 schedule_work(&ew->work); 4724 4725 return 1; 4726 } 4727 EXPORT_SYMBOL_GPL(execute_in_process_context); 4728 4729 /** 4730 * free_workqueue_attrs - free a workqueue_attrs 4731 * @attrs: workqueue_attrs to free 4732 * 4733 * Undo alloc_workqueue_attrs(). 4734 */ 4735 void free_workqueue_attrs(struct workqueue_attrs *attrs) 4736 { 4737 if (attrs) { 4738 free_cpumask_var(attrs->cpumask); 4739 free_cpumask_var(attrs->__pod_cpumask); 4740 kfree(attrs); 4741 } 4742 } 4743 4744 /** 4745 * alloc_workqueue_attrs - allocate a workqueue_attrs 4746 * 4747 * Allocate a new workqueue_attrs, initialize with default settings and 4748 * return it. 4749 * 4750 * Return: The allocated new workqueue_attr on success. %NULL on failure. 4751 */ 4752 struct workqueue_attrs *alloc_workqueue_attrs_noprof(void) 4753 { 4754 struct workqueue_attrs *attrs; 4755 4756 attrs = kzalloc_obj(*attrs); 4757 if (!attrs) 4758 goto fail; 4759 if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL)) 4760 goto fail; 4761 if (!alloc_cpumask_var(&attrs->__pod_cpumask, GFP_KERNEL)) 4762 goto fail; 4763 4764 cpumask_copy(attrs->cpumask, cpu_possible_mask); 4765 attrs->affn_scope = WQ_AFFN_DFL; 4766 return attrs; 4767 fail: 4768 free_workqueue_attrs(attrs); 4769 return NULL; 4770 } 4771 4772 static void copy_workqueue_attrs(struct workqueue_attrs *to, 4773 const struct workqueue_attrs *from) 4774 { 4775 to->nice = from->nice; 4776 cpumask_copy(to->cpumask, from->cpumask); 4777 cpumask_copy(to->__pod_cpumask, from->__pod_cpumask); 4778 to->affn_strict = from->affn_strict; 4779 4780 /* 4781 * Unlike hash and equality test, copying shouldn't ignore wq-only 4782 * fields as copying is used for both pool and wq attrs. Instead, 4783 * get_unbound_pool() explicitly clears the fields. 4784 */ 4785 to->affn_scope = from->affn_scope; 4786 to->ordered = from->ordered; 4787 } 4788 4789 /* 4790 * Some attrs fields are workqueue-only. Clear them for worker_pool's. See the 4791 * comments in 'struct workqueue_attrs' definition. 4792 */ 4793 static void wqattrs_clear_for_pool(struct workqueue_attrs *attrs) 4794 { 4795 attrs->affn_scope = WQ_AFFN_NR_TYPES; 4796 attrs->ordered = false; 4797 if (attrs->affn_strict) 4798 cpumask_copy(attrs->cpumask, cpu_possible_mask); 4799 } 4800 4801 /* hash value of the content of @attr */ 4802 static u32 wqattrs_hash(const struct workqueue_attrs *attrs) 4803 { 4804 u32 hash = 0; 4805 4806 hash = jhash_1word(attrs->nice, hash); 4807 hash = jhash_1word(attrs->affn_strict, hash); 4808 hash = jhash(cpumask_bits(attrs->__pod_cpumask), 4809 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash); 4810 if (!attrs->affn_strict) 4811 hash = jhash(cpumask_bits(attrs->cpumask), 4812 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash); 4813 return hash; 4814 } 4815 4816 /* content equality test */ 4817 static bool wqattrs_equal(const struct workqueue_attrs *a, 4818 const struct workqueue_attrs *b) 4819 { 4820 if (a->nice != b->nice) 4821 return false; 4822 if (a->affn_strict != b->affn_strict) 4823 return false; 4824 if (!cpumask_equal(a->__pod_cpumask, b->__pod_cpumask)) 4825 return false; 4826 if (!a->affn_strict && !cpumask_equal(a->cpumask, b->cpumask)) 4827 return false; 4828 return true; 4829 } 4830 4831 /* Update @attrs with actually available CPUs */ 4832 static void wqattrs_actualize_cpumask(struct workqueue_attrs *attrs, 4833 const cpumask_t *unbound_cpumask) 4834 { 4835 /* 4836 * Calculate the effective CPU mask of @attrs given @unbound_cpumask. If 4837 * @attrs->cpumask doesn't overlap with @unbound_cpumask, we fallback to 4838 * @unbound_cpumask. 4839 */ 4840 cpumask_and(attrs->cpumask, attrs->cpumask, unbound_cpumask); 4841 if (unlikely(cpumask_empty(attrs->cpumask))) 4842 cpumask_copy(attrs->cpumask, unbound_cpumask); 4843 } 4844 4845 /* find wq_pod_type to use for @attrs */ 4846 static const struct wq_pod_type * 4847 wqattrs_pod_type(const struct workqueue_attrs *attrs) 4848 { 4849 enum wq_affn_scope scope; 4850 struct wq_pod_type *pt; 4851 4852 /* to synchronize access to wq_affn_dfl */ 4853 lockdep_assert_held(&wq_pool_mutex); 4854 4855 if (attrs->affn_scope == WQ_AFFN_DFL) 4856 scope = wq_affn_dfl; 4857 else 4858 scope = attrs->affn_scope; 4859 4860 pt = &wq_pod_types[scope]; 4861 4862 if (!WARN_ON_ONCE(attrs->affn_scope == WQ_AFFN_NR_TYPES) && 4863 likely(pt->nr_pods)) 4864 return pt; 4865 4866 /* 4867 * Before workqueue_init_topology(), only SYSTEM is available which is 4868 * initialized in workqueue_init_early(). 4869 */ 4870 pt = &wq_pod_types[WQ_AFFN_SYSTEM]; 4871 BUG_ON(!pt->nr_pods); 4872 return pt; 4873 } 4874 4875 /** 4876 * init_worker_pool - initialize a newly zalloc'd worker_pool 4877 * @pool: worker_pool to initialize 4878 * 4879 * Initialize a newly zalloc'd @pool. It also allocates @pool->attrs. 4880 * 4881 * Return: 0 on success, -errno on failure. Even on failure, all fields 4882 * inside @pool proper are initialized and put_unbound_pool() can be called 4883 * on @pool safely to release it. 4884 */ 4885 static int init_worker_pool(struct worker_pool *pool) 4886 { 4887 raw_spin_lock_init(&pool->lock); 4888 pool->id = -1; 4889 pool->cpu = -1; 4890 pool->node = NUMA_NO_NODE; 4891 pool->flags |= POOL_DISASSOCIATED; 4892 pool->last_progress_ts = jiffies; 4893 INIT_LIST_HEAD(&pool->worklist); 4894 INIT_LIST_HEAD(&pool->idle_list); 4895 hash_init(pool->busy_hash); 4896 4897 timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE); 4898 INIT_WORK(&pool->idle_cull_work, idle_cull_fn); 4899 4900 timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0); 4901 4902 INIT_LIST_HEAD(&pool->workers); 4903 4904 ida_init(&pool->worker_ida); 4905 INIT_HLIST_NODE(&pool->hash_node); 4906 pool->refcnt = 1; 4907 #ifdef CONFIG_PREEMPT_RT 4908 spin_lock_init(&pool->cb_lock); 4909 #endif 4910 4911 /* shouldn't fail above this point */ 4912 pool->attrs = alloc_workqueue_attrs(); 4913 if (!pool->attrs) 4914 return -ENOMEM; 4915 4916 wqattrs_clear_for_pool(pool->attrs); 4917 4918 return 0; 4919 } 4920 4921 #ifdef CONFIG_LOCKDEP 4922 static void wq_init_lockdep(struct workqueue_struct *wq) 4923 { 4924 char *lock_name; 4925 4926 lockdep_register_key(&wq->key); 4927 lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name); 4928 if (!lock_name) 4929 lock_name = wq->name; 4930 4931 wq->lock_name = lock_name; 4932 wq->lockdep_map = &wq->__lockdep_map; 4933 lockdep_init_map(wq->lockdep_map, lock_name, &wq->key, 0); 4934 } 4935 4936 static void wq_unregister_lockdep(struct workqueue_struct *wq) 4937 { 4938 if (wq->lockdep_map != &wq->__lockdep_map) 4939 return; 4940 4941 lockdep_unregister_key(&wq->key); 4942 } 4943 4944 static void wq_free_lockdep(struct workqueue_struct *wq) 4945 { 4946 if (wq->lockdep_map != &wq->__lockdep_map) 4947 return; 4948 4949 if (wq->lock_name != wq->name) 4950 kfree(wq->lock_name); 4951 } 4952 #else 4953 static void wq_init_lockdep(struct workqueue_struct *wq) 4954 { 4955 } 4956 4957 static void wq_unregister_lockdep(struct workqueue_struct *wq) 4958 { 4959 } 4960 4961 static void wq_free_lockdep(struct workqueue_struct *wq) 4962 { 4963 } 4964 #endif 4965 4966 static void free_node_nr_active(struct wq_node_nr_active **nna_ar) 4967 { 4968 int node; 4969 4970 for_each_node(node) { 4971 kfree(nna_ar[node]); 4972 nna_ar[node] = NULL; 4973 } 4974 4975 kfree(nna_ar[nr_node_ids]); 4976 nna_ar[nr_node_ids] = NULL; 4977 } 4978 4979 static void init_node_nr_active(struct wq_node_nr_active *nna) 4980 { 4981 nna->max = WQ_DFL_MIN_ACTIVE; 4982 atomic_set(&nna->nr, 0); 4983 raw_spin_lock_init(&nna->lock); 4984 INIT_LIST_HEAD(&nna->pending_pwqs); 4985 } 4986 4987 /* 4988 * Each node's nr_active counter will be accessed mostly from its own node and 4989 * should be allocated in the node. 4990 */ 4991 static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar) 4992 { 4993 struct wq_node_nr_active *nna; 4994 int node; 4995 4996 for_each_node(node) { 4997 nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node); 4998 if (!nna) 4999 goto err_free; 5000 init_node_nr_active(nna); 5001 nna_ar[node] = nna; 5002 } 5003 5004 /* [nr_node_ids] is used as the fallback */ 5005 nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE); 5006 if (!nna) 5007 goto err_free; 5008 init_node_nr_active(nna); 5009 nna_ar[nr_node_ids] = nna; 5010 5011 return 0; 5012 5013 err_free: 5014 free_node_nr_active(nna_ar); 5015 return -ENOMEM; 5016 } 5017 5018 static void rcu_free_wq(struct rcu_head *rcu) 5019 { 5020 struct workqueue_struct *wq = 5021 container_of(rcu, struct workqueue_struct, rcu); 5022 5023 if (wq->flags & WQ_UNBOUND) 5024 free_node_nr_active(wq->node_nr_active); 5025 5026 wq_free_lockdep(wq); 5027 free_percpu(wq->cpu_pwq); 5028 free_workqueue_attrs(wq->unbound_attrs); 5029 kfree(wq); 5030 } 5031 5032 static void rcu_free_pool(struct rcu_head *rcu) 5033 { 5034 struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu); 5035 5036 ida_destroy(&pool->worker_ida); 5037 free_workqueue_attrs(pool->attrs); 5038 kfree(pool); 5039 } 5040 5041 /** 5042 * put_unbound_pool - put a worker_pool 5043 * @pool: worker_pool to put 5044 * 5045 * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU 5046 * safe manner. get_unbound_pool() calls this function on its failure path 5047 * and this function should be able to release pools which went through, 5048 * successfully or not, init_worker_pool(). 5049 * 5050 * Should be called with wq_pool_mutex held. 5051 */ 5052 static void put_unbound_pool(struct worker_pool *pool) 5053 { 5054 struct worker *worker; 5055 LIST_HEAD(cull_list); 5056 5057 lockdep_assert_held(&wq_pool_mutex); 5058 5059 if (--pool->refcnt) 5060 return; 5061 5062 /* sanity checks */ 5063 if (WARN_ON(!(pool->cpu < 0)) || 5064 WARN_ON(!list_empty(&pool->worklist))) 5065 return; 5066 5067 /* release id and unhash */ 5068 if (pool->id >= 0) 5069 idr_remove(&worker_pool_idr, pool->id); 5070 hash_del(&pool->hash_node); 5071 5072 /* 5073 * Become the manager and destroy all workers. This prevents 5074 * @pool's workers from blocking on attach_mutex. We're the last 5075 * manager and @pool gets freed with the flag set. 5076 * 5077 * Having a concurrent manager is quite unlikely to happen as we can 5078 * only get here with 5079 * pwq->refcnt == pool->refcnt == 0 5080 * which implies no work queued to the pool, which implies no worker can 5081 * become the manager. However a worker could have taken the role of 5082 * manager before the refcnts dropped to 0, since maybe_create_worker() 5083 * drops pool->lock 5084 */ 5085 while (true) { 5086 rcuwait_wait_event(&manager_wait, 5087 !(pool->flags & POOL_MANAGER_ACTIVE), 5088 TASK_UNINTERRUPTIBLE); 5089 5090 mutex_lock(&wq_pool_attach_mutex); 5091 raw_spin_lock_irq(&pool->lock); 5092 if (!(pool->flags & POOL_MANAGER_ACTIVE)) { 5093 pool->flags |= POOL_MANAGER_ACTIVE; 5094 break; 5095 } 5096 raw_spin_unlock_irq(&pool->lock); 5097 mutex_unlock(&wq_pool_attach_mutex); 5098 } 5099 5100 while ((worker = first_idle_worker(pool))) 5101 set_worker_dying(worker, &cull_list); 5102 WARN_ON(pool->nr_workers || pool->nr_idle); 5103 raw_spin_unlock_irq(&pool->lock); 5104 5105 detach_dying_workers(&cull_list); 5106 5107 mutex_unlock(&wq_pool_attach_mutex); 5108 5109 reap_dying_workers(&cull_list); 5110 5111 /* shut down the timers */ 5112 timer_delete_sync(&pool->idle_timer); 5113 cancel_work_sync(&pool->idle_cull_work); 5114 timer_delete_sync(&pool->mayday_timer); 5115 5116 /* RCU protected to allow dereferences from get_work_pool() */ 5117 call_rcu(&pool->rcu, rcu_free_pool); 5118 } 5119 5120 /** 5121 * get_unbound_pool - get a worker_pool with the specified attributes 5122 * @attrs: the attributes of the worker_pool to get 5123 * 5124 * Obtain a worker_pool which has the same attributes as @attrs, bump the 5125 * reference count and return it. If there already is a matching 5126 * worker_pool, it will be used; otherwise, this function attempts to 5127 * create a new one. 5128 * 5129 * Should be called with wq_pool_mutex held. 5130 * 5131 * Return: On success, a worker_pool with the same attributes as @attrs. 5132 * On failure, %NULL. 5133 */ 5134 static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs) 5135 { 5136 struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_NUMA]; 5137 u32 hash = wqattrs_hash(attrs); 5138 struct worker_pool *pool; 5139 int pod, node = NUMA_NO_NODE; 5140 5141 lockdep_assert_held(&wq_pool_mutex); 5142 5143 /* do we already have a matching pool? */ 5144 hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) { 5145 if (wqattrs_equal(pool->attrs, attrs)) { 5146 pool->refcnt++; 5147 return pool; 5148 } 5149 } 5150 5151 /* If __pod_cpumask is contained inside a NUMA pod, that's our node */ 5152 for (pod = 0; pod < pt->nr_pods; pod++) { 5153 if (cpumask_subset(attrs->__pod_cpumask, pt->pod_cpus[pod])) { 5154 node = pt->pod_node[pod]; 5155 break; 5156 } 5157 } 5158 5159 /* nope, create a new one */ 5160 pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, node); 5161 if (!pool || init_worker_pool(pool) < 0) 5162 goto fail; 5163 5164 pool->node = node; 5165 copy_workqueue_attrs(pool->attrs, attrs); 5166 wqattrs_clear_for_pool(pool->attrs); 5167 5168 if (worker_pool_assign_id(pool) < 0) 5169 goto fail; 5170 5171 /* create and start the initial worker */ 5172 if (wq_online && !create_worker(pool)) 5173 goto fail; 5174 5175 /* install */ 5176 hash_add(unbound_pool_hash, &pool->hash_node, hash); 5177 5178 return pool; 5179 fail: 5180 if (pool) 5181 put_unbound_pool(pool); 5182 return NULL; 5183 } 5184 5185 /* 5186 * Scheduled on pwq_release_worker by put_pwq() when an unbound pwq hits zero 5187 * refcnt and needs to be destroyed. 5188 */ 5189 static void pwq_release_workfn(struct kthread_work *work) 5190 { 5191 struct pool_workqueue *pwq = container_of(work, struct pool_workqueue, 5192 release_work); 5193 struct workqueue_struct *wq = pwq->wq; 5194 struct worker_pool *pool = pwq->pool; 5195 bool is_last = false; 5196 5197 /* 5198 * When @pwq is not linked, it doesn't hold any reference to the 5199 * @wq, and @wq is invalid to access. 5200 */ 5201 if (!list_empty(&pwq->pwqs_node)) { 5202 mutex_lock(&wq->mutex); 5203 list_del_rcu(&pwq->pwqs_node); 5204 is_last = list_empty(&wq->pwqs); 5205 5206 /* 5207 * For ordered workqueue with a plugged dfl_pwq, restart it now. 5208 */ 5209 if (!is_last && (wq->flags & __WQ_ORDERED)) 5210 unplug_oldest_pwq(wq); 5211 5212 mutex_unlock(&wq->mutex); 5213 } 5214 5215 if (wq->flags & WQ_UNBOUND) { 5216 mutex_lock(&wq_pool_mutex); 5217 put_unbound_pool(pool); 5218 mutex_unlock(&wq_pool_mutex); 5219 } 5220 5221 if (!list_empty(&pwq->pending_node)) { 5222 struct wq_node_nr_active *nna = 5223 wq_node_nr_active(pwq->wq, pwq->pool->node); 5224 5225 raw_spin_lock_irq(&nna->lock); 5226 list_del_init(&pwq->pending_node); 5227 raw_spin_unlock_irq(&nna->lock); 5228 } 5229 5230 kfree_rcu(pwq, rcu); 5231 5232 /* 5233 * If we're the last pwq going away, @wq is already dead and no one 5234 * is gonna access it anymore. Schedule RCU free. 5235 */ 5236 if (is_last) { 5237 wq_unregister_lockdep(wq); 5238 call_rcu(&wq->rcu, rcu_free_wq); 5239 } 5240 } 5241 5242 /* initialize newly allocated @pwq which is associated with @wq and @pool */ 5243 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq, 5244 struct worker_pool *pool) 5245 { 5246 BUG_ON((unsigned long)pwq & ~WORK_STRUCT_PWQ_MASK); 5247 5248 memset(pwq, 0, sizeof(*pwq)); 5249 5250 pwq->pool = pool; 5251 pwq->wq = wq; 5252 pwq->flush_color = -1; 5253 pwq->refcnt = 1; 5254 INIT_LIST_HEAD(&pwq->inactive_works); 5255 INIT_LIST_HEAD(&pwq->pending_node); 5256 INIT_LIST_HEAD(&pwq->pwqs_node); 5257 INIT_LIST_HEAD(&pwq->mayday_node); 5258 kthread_init_work(&pwq->release_work, pwq_release_workfn); 5259 5260 /* 5261 * Set the dummy cursor work with valid function and get_work_pwq(). 5262 * 5263 * The cursor work should only be in the pwq->pool->worklist, and 5264 * should not be treated as a processable work item. 5265 * 5266 * WORK_STRUCT_PENDING and WORK_STRUCT_INACTIVE just make it less 5267 * surprise for kernel debugging tools and reviewers. 5268 */ 5269 INIT_WORK(&pwq->mayday_cursor, mayday_cursor_func); 5270 atomic_long_set(&pwq->mayday_cursor.data, (unsigned long)pwq | 5271 WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | WORK_STRUCT_INACTIVE); 5272 } 5273 5274 /* sync @pwq with the current state of its associated wq and link it */ 5275 static void link_pwq(struct pool_workqueue *pwq) 5276 { 5277 struct workqueue_struct *wq = pwq->wq; 5278 5279 lockdep_assert_held(&wq->mutex); 5280 5281 /* may be called multiple times, ignore if already linked */ 5282 if (!list_empty(&pwq->pwqs_node)) 5283 return; 5284 5285 /* set the matching work_color */ 5286 pwq->work_color = wq->work_color; 5287 5288 /* link in @pwq */ 5289 list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs); 5290 } 5291 5292 /* obtain a pool matching @attr and create a pwq associating the pool and @wq */ 5293 static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq, 5294 const struct workqueue_attrs *attrs) 5295 { 5296 struct worker_pool *pool; 5297 struct pool_workqueue *pwq; 5298 5299 lockdep_assert_held(&wq_pool_mutex); 5300 5301 pool = get_unbound_pool(attrs); 5302 if (!pool) 5303 return NULL; 5304 5305 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); 5306 if (!pwq) { 5307 put_unbound_pool(pool); 5308 return NULL; 5309 } 5310 5311 init_pwq(pwq, wq, pool); 5312 return pwq; 5313 } 5314 5315 static void apply_wqattrs_lock(void) 5316 { 5317 mutex_lock(&wq_pool_mutex); 5318 } 5319 5320 static void apply_wqattrs_unlock(void) 5321 { 5322 mutex_unlock(&wq_pool_mutex); 5323 } 5324 5325 /** 5326 * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod 5327 * @attrs: the wq_attrs of the default pwq of the target workqueue 5328 * @cpu: the target CPU 5329 * 5330 * Calculate the cpumask a workqueue with @attrs should use on @pod. 5331 * The result is stored in @attrs->__pod_cpumask. 5332 * 5333 * If pod affinity is not enabled, @attrs->cpumask is always used. If enabled 5334 * and @pod has online CPUs requested by @attrs, the returned cpumask is the 5335 * intersection of the possible CPUs of @pod and @attrs->cpumask. 5336 * 5337 * The caller is responsible for ensuring that the cpumask of @pod stays stable. 5338 */ 5339 static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu) 5340 { 5341 const struct wq_pod_type *pt = wqattrs_pod_type(attrs); 5342 int pod = pt->cpu_pod[cpu]; 5343 5344 /* calculate possible CPUs in @pod that @attrs wants */ 5345 cpumask_and(attrs->__pod_cpumask, pt->pod_cpus[pod], attrs->cpumask); 5346 /* does @pod have any online CPUs @attrs wants? */ 5347 if (!cpumask_intersects(attrs->__pod_cpumask, wq_online_cpumask)) { 5348 cpumask_copy(attrs->__pod_cpumask, attrs->cpumask); 5349 return; 5350 } 5351 } 5352 5353 /* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */ 5354 static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq, 5355 int cpu, struct pool_workqueue *pwq) 5356 { 5357 struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu); 5358 struct pool_workqueue *old_pwq; 5359 5360 lockdep_assert_held(&wq_pool_mutex); 5361 lockdep_assert_held(&wq->mutex); 5362 5363 /* link_pwq() can handle duplicate calls */ 5364 link_pwq(pwq); 5365 5366 old_pwq = rcu_access_pointer(*slot); 5367 rcu_assign_pointer(*slot, pwq); 5368 return old_pwq; 5369 } 5370 5371 /* context to store the prepared attrs & pwqs before applying */ 5372 struct apply_wqattrs_ctx { 5373 struct workqueue_struct *wq; /* target workqueue */ 5374 struct workqueue_attrs *attrs; /* attrs to apply */ 5375 struct list_head list; /* queued for batching commit */ 5376 struct pool_workqueue *dfl_pwq; 5377 struct pool_workqueue *pwq_tbl[]; 5378 }; 5379 5380 /* free the resources after success or abort */ 5381 static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx) 5382 { 5383 if (ctx) { 5384 int cpu; 5385 5386 for_each_possible_cpu(cpu) 5387 put_pwq_unlocked(ctx->pwq_tbl[cpu]); 5388 put_pwq_unlocked(ctx->dfl_pwq); 5389 5390 free_workqueue_attrs(ctx->attrs); 5391 5392 kfree(ctx); 5393 } 5394 } 5395 5396 /* allocate the attrs and pwqs for later installation */ 5397 static struct apply_wqattrs_ctx * 5398 apply_wqattrs_prepare(struct workqueue_struct *wq, 5399 const struct workqueue_attrs *attrs, 5400 const cpumask_var_t unbound_cpumask) 5401 { 5402 struct apply_wqattrs_ctx *ctx; 5403 struct workqueue_attrs *new_attrs; 5404 int cpu; 5405 5406 lockdep_assert_held(&wq_pool_mutex); 5407 5408 if (WARN_ON(attrs->affn_scope < 0 || 5409 attrs->affn_scope >= WQ_AFFN_NR_TYPES)) 5410 return ERR_PTR(-EINVAL); 5411 5412 ctx = kzalloc_flex(*ctx, pwq_tbl, nr_cpu_ids); 5413 5414 new_attrs = alloc_workqueue_attrs(); 5415 if (!ctx || !new_attrs) 5416 goto out_free; 5417 5418 /* 5419 * If something goes wrong during CPU up/down, we'll fall back to 5420 * the default pwq covering whole @attrs->cpumask. Always create 5421 * it even if we don't use it immediately. 5422 */ 5423 copy_workqueue_attrs(new_attrs, attrs); 5424 wqattrs_actualize_cpumask(new_attrs, unbound_cpumask); 5425 cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask); 5426 ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); 5427 if (!ctx->dfl_pwq) 5428 goto out_free; 5429 5430 for_each_possible_cpu(cpu) { 5431 if (new_attrs->ordered) { 5432 ctx->dfl_pwq->refcnt++; 5433 ctx->pwq_tbl[cpu] = ctx->dfl_pwq; 5434 } else { 5435 wq_calc_pod_cpumask(new_attrs, cpu); 5436 ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs); 5437 if (!ctx->pwq_tbl[cpu]) 5438 goto out_free; 5439 } 5440 } 5441 5442 /* save the user configured attrs and sanitize it. */ 5443 copy_workqueue_attrs(new_attrs, attrs); 5444 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask); 5445 cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask); 5446 ctx->attrs = new_attrs; 5447 5448 /* 5449 * For initialized ordered workqueues, there should only be one pwq 5450 * (dfl_pwq). Set the plugged flag of ctx->dfl_pwq to suspend execution 5451 * of newly queued work items until execution of older work items in 5452 * the old pwq's have completed. 5453 */ 5454 if ((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)) 5455 ctx->dfl_pwq->plugged = true; 5456 5457 ctx->wq = wq; 5458 return ctx; 5459 5460 out_free: 5461 free_workqueue_attrs(new_attrs); 5462 apply_wqattrs_cleanup(ctx); 5463 return ERR_PTR(-ENOMEM); 5464 } 5465 5466 /* set attrs and install prepared pwqs, @ctx points to old pwqs on return */ 5467 static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx) 5468 { 5469 int cpu; 5470 5471 /* all pwqs have been created successfully, let's install'em */ 5472 mutex_lock(&ctx->wq->mutex); 5473 5474 copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs); 5475 5476 /* save the previous pwqs and install the new ones */ 5477 for_each_possible_cpu(cpu) 5478 ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu, 5479 ctx->pwq_tbl[cpu]); 5480 ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq); 5481 5482 /* update node_nr_active->max */ 5483 wq_update_node_max_active(ctx->wq, -1); 5484 5485 mutex_unlock(&ctx->wq->mutex); 5486 } 5487 5488 static int apply_workqueue_attrs_locked(struct workqueue_struct *wq, 5489 const struct workqueue_attrs *attrs) 5490 { 5491 struct apply_wqattrs_ctx *ctx; 5492 5493 /* only unbound workqueues can change attributes */ 5494 if (WARN_ON(!(wq->flags & WQ_UNBOUND))) 5495 return -EINVAL; 5496 5497 ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask); 5498 if (IS_ERR(ctx)) 5499 return PTR_ERR(ctx); 5500 5501 /* the ctx has been prepared successfully, let's commit it */ 5502 apply_wqattrs_commit(ctx); 5503 apply_wqattrs_cleanup(ctx); 5504 5505 return 0; 5506 } 5507 5508 /** 5509 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue 5510 * @wq: the target workqueue 5511 * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs() 5512 * 5513 * Apply @attrs to an unbound workqueue @wq. Unless disabled, this function maps 5514 * a separate pwq to each CPU pod with possibles CPUs in @attrs->cpumask so that 5515 * work items are affine to the pod it was issued on. Older pwqs are released as 5516 * in-flight work items finish. Note that a work item which repeatedly requeues 5517 * itself back-to-back will stay on its current pwq. 5518 * 5519 * Performs GFP_KERNEL allocations. 5520 * 5521 * Return: 0 on success and -errno on failure. 5522 */ 5523 int apply_workqueue_attrs(struct workqueue_struct *wq, 5524 const struct workqueue_attrs *attrs) 5525 { 5526 int ret; 5527 5528 mutex_lock(&wq_pool_mutex); 5529 ret = apply_workqueue_attrs_locked(wq, attrs); 5530 mutex_unlock(&wq_pool_mutex); 5531 5532 return ret; 5533 } 5534 5535 /** 5536 * unbound_wq_update_pwq - update a pwq slot for CPU hot[un]plug 5537 * @wq: the target workqueue 5538 * @cpu: the CPU to update the pwq slot for 5539 * 5540 * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and 5541 * %CPU_DOWN_FAILED. @cpu is in the same pod of the CPU being hot[un]plugged. 5542 * 5543 * 5544 * If pod affinity can't be adjusted due to memory allocation failure, it falls 5545 * back to @wq->dfl_pwq which may not be optimal but is always correct. 5546 * 5547 * Note that when the last allowed CPU of a pod goes offline for a workqueue 5548 * with a cpumask spanning multiple pods, the workers which were already 5549 * executing the work items for the workqueue will lose their CPU affinity and 5550 * may execute on any CPU. This is similar to how per-cpu workqueues behave on 5551 * CPU_DOWN. If a workqueue user wants strict affinity, it's the user's 5552 * responsibility to flush the work item from CPU_DOWN_PREPARE. 5553 */ 5554 static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu) 5555 { 5556 struct pool_workqueue *old_pwq = NULL, *pwq; 5557 struct workqueue_attrs *target_attrs; 5558 5559 lockdep_assert_held(&wq_pool_mutex); 5560 5561 if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered) 5562 return; 5563 5564 /* 5565 * We don't wanna alloc/free wq_attrs for each wq for each CPU. 5566 * Let's use a preallocated one. The following buf is protected by 5567 * CPU hotplug exclusion. 5568 */ 5569 target_attrs = unbound_wq_update_pwq_attrs_buf; 5570 5571 copy_workqueue_attrs(target_attrs, wq->unbound_attrs); 5572 wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask); 5573 5574 /* nothing to do if the target cpumask matches the current pwq */ 5575 wq_calc_pod_cpumask(target_attrs, cpu); 5576 if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs)) 5577 return; 5578 5579 /* create a new pwq */ 5580 pwq = alloc_unbound_pwq(wq, target_attrs); 5581 if (!pwq) { 5582 pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n", 5583 wq->name); 5584 goto use_dfl_pwq; 5585 } 5586 5587 /* Install the new pwq. */ 5588 mutex_lock(&wq->mutex); 5589 old_pwq = install_unbound_pwq(wq, cpu, pwq); 5590 goto out_unlock; 5591 5592 use_dfl_pwq: 5593 mutex_lock(&wq->mutex); 5594 pwq = unbound_pwq(wq, -1); 5595 raw_spin_lock_irq(&pwq->pool->lock); 5596 get_pwq(pwq); 5597 raw_spin_unlock_irq(&pwq->pool->lock); 5598 old_pwq = install_unbound_pwq(wq, cpu, pwq); 5599 out_unlock: 5600 mutex_unlock(&wq->mutex); 5601 put_pwq_unlocked(old_pwq); 5602 } 5603 5604 static int alloc_and_link_pwqs(struct workqueue_struct *wq) 5605 { 5606 bool highpri = wq->flags & WQ_HIGHPRI; 5607 int cpu, ret; 5608 5609 lockdep_assert_held(&wq_pool_mutex); 5610 5611 wq->cpu_pwq = alloc_percpu(struct pool_workqueue *); 5612 if (!wq->cpu_pwq) 5613 goto enomem; 5614 5615 if (!(wq->flags & WQ_UNBOUND)) { 5616 struct worker_pool __percpu *pools; 5617 5618 if (wq->flags & WQ_BH) 5619 pools = bh_worker_pools; 5620 else 5621 pools = cpu_worker_pools; 5622 5623 for_each_possible_cpu(cpu) { 5624 struct pool_workqueue **pwq_p; 5625 struct worker_pool *pool; 5626 5627 pool = &(per_cpu_ptr(pools, cpu)[highpri]); 5628 pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu); 5629 5630 *pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, 5631 pool->node); 5632 if (!*pwq_p) 5633 goto enomem; 5634 5635 init_pwq(*pwq_p, wq, pool); 5636 5637 mutex_lock(&wq->mutex); 5638 link_pwq(*pwq_p); 5639 mutex_unlock(&wq->mutex); 5640 } 5641 return 0; 5642 } 5643 5644 if (wq->flags & __WQ_ORDERED) { 5645 struct pool_workqueue *dfl_pwq; 5646 5647 ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]); 5648 /* there should only be single pwq for ordering guarantee */ 5649 dfl_pwq = rcu_access_pointer(wq->dfl_pwq); 5650 WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node || 5651 wq->pwqs.prev != &dfl_pwq->pwqs_node), 5652 "ordering guarantee broken for workqueue %s\n", wq->name); 5653 } else { 5654 ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]); 5655 } 5656 5657 if (ret) 5658 goto enomem; 5659 return 0; 5660 5661 enomem: 5662 if (wq->cpu_pwq) { 5663 for_each_possible_cpu(cpu) { 5664 struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); 5665 5666 if (pwq) { 5667 /* 5668 * Unlink pwq from wq->pwqs since link_pwq() 5669 * may have already added it. wq->mutex is not 5670 * needed as the wq has not been published yet. 5671 */ 5672 if (!list_empty(&pwq->pwqs_node)) 5673 list_del_rcu(&pwq->pwqs_node); 5674 kmem_cache_free(pwq_cache, pwq); 5675 } 5676 } 5677 free_percpu(wq->cpu_pwq); 5678 wq->cpu_pwq = NULL; 5679 } 5680 return -ENOMEM; 5681 } 5682 5683 static int wq_clamp_max_active(int max_active, unsigned int flags, 5684 const char *name) 5685 { 5686 if (max_active < 1 || max_active > WQ_MAX_ACTIVE) 5687 pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n", 5688 max_active, name, 1, WQ_MAX_ACTIVE); 5689 5690 return clamp_val(max_active, 1, WQ_MAX_ACTIVE); 5691 } 5692 5693 /* 5694 * Workqueues which may be used during memory reclaim should have a rescuer 5695 * to guarantee forward progress. 5696 */ 5697 static int init_rescuer(struct workqueue_struct *wq) 5698 { 5699 struct worker *rescuer; 5700 char id_buf[WORKER_ID_LEN]; 5701 int ret; 5702 5703 lockdep_assert_held(&wq_pool_mutex); 5704 5705 if (!(wq->flags & WQ_MEM_RECLAIM)) 5706 return 0; 5707 5708 rescuer = alloc_worker(NUMA_NO_NODE); 5709 if (!rescuer) { 5710 pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n", 5711 wq->name); 5712 return -ENOMEM; 5713 } 5714 5715 rescuer->rescue_wq = wq; 5716 format_worker_id(id_buf, sizeof(id_buf), rescuer, NULL); 5717 5718 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", id_buf); 5719 if (IS_ERR(rescuer->task)) { 5720 ret = PTR_ERR(rescuer->task); 5721 pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe", 5722 wq->name, ERR_PTR(ret)); 5723 kfree(rescuer); 5724 return ret; 5725 } 5726 5727 wq->rescuer = rescuer; 5728 5729 /* initial cpumask is consistent with the detached rescuer and unbind_worker() */ 5730 if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask)) 5731 kthread_bind_mask(rescuer->task, wq_unbound_cpumask); 5732 else 5733 kthread_bind_mask(rescuer->task, cpu_possible_mask); 5734 5735 wake_up_process(rescuer->task); 5736 5737 return 0; 5738 } 5739 5740 /** 5741 * wq_adjust_max_active - update a wq's max_active to the current setting 5742 * @wq: target workqueue 5743 * 5744 * If @wq isn't freezing, set @wq->max_active to the saved_max_active and 5745 * activate inactive work items accordingly. If @wq is freezing, clear 5746 * @wq->max_active to zero. 5747 */ 5748 static void wq_adjust_max_active(struct workqueue_struct *wq) 5749 { 5750 bool activated; 5751 int new_max, new_min; 5752 5753 lockdep_assert_held(&wq->mutex); 5754 5755 if ((wq->flags & WQ_FREEZABLE) && workqueue_freezing) { 5756 new_max = 0; 5757 new_min = 0; 5758 } else { 5759 new_max = wq->saved_max_active; 5760 new_min = wq->saved_min_active; 5761 } 5762 5763 if (wq->max_active == new_max && wq->min_active == new_min) 5764 return; 5765 5766 /* 5767 * Update @wq->max/min_active and then kick inactive work items if more 5768 * active work items are allowed. This doesn't break work item ordering 5769 * because new work items are always queued behind existing inactive 5770 * work items if there are any. 5771 */ 5772 WRITE_ONCE(wq->max_active, new_max); 5773 WRITE_ONCE(wq->min_active, new_min); 5774 5775 if (wq->flags & WQ_UNBOUND) 5776 wq_update_node_max_active(wq, -1); 5777 5778 if (new_max == 0) 5779 return; 5780 5781 /* 5782 * Round-robin through pwq's activating the first inactive work item 5783 * until max_active is filled. 5784 */ 5785 do { 5786 struct pool_workqueue *pwq; 5787 5788 activated = false; 5789 for_each_pwq(pwq, wq) { 5790 unsigned long irq_flags; 5791 5792 /* can be called during early boot w/ irq disabled */ 5793 raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags); 5794 if (pwq_activate_first_inactive(pwq, true)) { 5795 activated = true; 5796 kick_pool(pwq->pool); 5797 } 5798 raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags); 5799 } 5800 } while (activated); 5801 } 5802 5803 __printf(1, 0) 5804 static struct workqueue_struct *__alloc_workqueue(const char *fmt, 5805 unsigned int flags, 5806 int max_active, va_list args) 5807 { 5808 struct workqueue_struct *wq; 5809 size_t wq_size; 5810 int name_len; 5811 5812 if (flags & WQ_BH) { 5813 if (WARN_ON_ONCE(flags & ~__WQ_BH_ALLOWS)) 5814 return NULL; 5815 if (WARN_ON_ONCE(max_active)) 5816 return NULL; 5817 } 5818 5819 /* see the comment above the definition of WQ_POWER_EFFICIENT */ 5820 if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient) 5821 flags |= WQ_UNBOUND; 5822 5823 /* allocate wq and format name */ 5824 if (flags & WQ_UNBOUND) 5825 wq_size = struct_size(wq, node_nr_active, nr_node_ids + 1); 5826 else 5827 wq_size = sizeof(*wq); 5828 5829 wq = kzalloc_noprof(wq_size, GFP_KERNEL); 5830 if (!wq) 5831 return NULL; 5832 5833 if (flags & WQ_UNBOUND) { 5834 wq->unbound_attrs = alloc_workqueue_attrs_noprof(); 5835 if (!wq->unbound_attrs) 5836 goto err_free_wq; 5837 } 5838 5839 name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args); 5840 5841 if (name_len >= WQ_NAME_LEN) 5842 pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n", 5843 wq->name); 5844 5845 if (flags & WQ_BH) { 5846 /* 5847 * BH workqueues always share a single execution context per CPU 5848 * and don't impose any max_active limit. 5849 */ 5850 max_active = INT_MAX; 5851 } else { 5852 max_active = max_active ?: WQ_DFL_ACTIVE; 5853 max_active = wq_clamp_max_active(max_active, flags, wq->name); 5854 } 5855 5856 /* init wq */ 5857 wq->flags = flags; 5858 wq->max_active = max_active; 5859 wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE); 5860 wq->saved_max_active = wq->max_active; 5861 wq->saved_min_active = wq->min_active; 5862 mutex_init(&wq->mutex); 5863 atomic_set(&wq->nr_pwqs_to_flush, 0); 5864 INIT_LIST_HEAD(&wq->pwqs); 5865 INIT_LIST_HEAD(&wq->flusher_queue); 5866 INIT_LIST_HEAD(&wq->flusher_overflow); 5867 INIT_LIST_HEAD(&wq->maydays); 5868 5869 INIT_LIST_HEAD(&wq->list); 5870 5871 if (flags & WQ_UNBOUND) { 5872 if (alloc_node_nr_active(wq->node_nr_active) < 0) 5873 goto err_free_wq; 5874 } 5875 5876 /* 5877 * wq_pool_mutex protects the workqueues list, allocations of PWQs, 5878 * and the global freeze state. 5879 */ 5880 apply_wqattrs_lock(); 5881 5882 if (alloc_and_link_pwqs(wq) < 0) 5883 goto err_unlock_free_node_nr_active; 5884 5885 mutex_lock(&wq->mutex); 5886 wq_adjust_max_active(wq); 5887 mutex_unlock(&wq->mutex); 5888 5889 list_add_tail_rcu(&wq->list, &workqueues); 5890 5891 if (wq_online && init_rescuer(wq) < 0) 5892 goto err_unlock_destroy; 5893 5894 apply_wqattrs_unlock(); 5895 5896 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq)) 5897 goto err_destroy; 5898 5899 return wq; 5900 5901 err_unlock_free_node_nr_active: 5902 apply_wqattrs_unlock(); 5903 /* 5904 * Failed alloc_and_link_pwqs() may leave pending pwq->release_work, 5905 * flushing the pwq_release_worker ensures that the pwq_release_workfn() 5906 * completes before calling kfree(wq). 5907 */ 5908 if (wq->flags & WQ_UNBOUND) { 5909 kthread_flush_worker(pwq_release_worker); 5910 free_node_nr_active(wq->node_nr_active); 5911 } 5912 err_free_wq: 5913 free_workqueue_attrs(wq->unbound_attrs); 5914 kfree(wq); 5915 return NULL; 5916 err_unlock_destroy: 5917 apply_wqattrs_unlock(); 5918 err_destroy: 5919 destroy_workqueue(wq); 5920 return NULL; 5921 } 5922 5923 __printf(1, 0) 5924 static struct workqueue_struct *alloc_workqueue_va(const char *fmt, 5925 unsigned int flags, 5926 int max_active, 5927 va_list args) 5928 { 5929 struct workqueue_struct *wq; 5930 5931 wq = __alloc_workqueue(fmt, flags, max_active, args); 5932 if (wq) 5933 wq_init_lockdep(wq); 5934 5935 return wq; 5936 } 5937 5938 __printf(1, 4) 5939 struct workqueue_struct *alloc_workqueue_noprof(const char *fmt, 5940 unsigned int flags, 5941 int max_active, ...) 5942 { 5943 struct workqueue_struct *wq; 5944 va_list args; 5945 5946 va_start(args, max_active); 5947 wq = alloc_workqueue_va(fmt, flags, max_active, args); 5948 va_end(args); 5949 5950 return wq; 5951 } 5952 EXPORT_SYMBOL_GPL(alloc_workqueue_noprof); 5953 5954 static void devm_workqueue_release(void *res) 5955 { 5956 destroy_workqueue(res); 5957 } 5958 5959 __printf(2, 5) struct workqueue_struct * 5960 devm_alloc_workqueue_noprof(struct device *dev, const char *fmt, 5961 unsigned int flags, int max_active, ...) 5962 { 5963 struct workqueue_struct *wq; 5964 va_list args; 5965 int ret; 5966 5967 va_start(args, max_active); 5968 wq = alloc_workqueue_va(fmt, flags, max_active, args); 5969 va_end(args); 5970 if (!wq) 5971 return NULL; 5972 5973 ret = devm_add_action_or_reset(dev, devm_workqueue_release, wq); 5974 if (ret) 5975 return NULL; 5976 5977 return wq; 5978 } 5979 EXPORT_SYMBOL_GPL(devm_alloc_workqueue_noprof); 5980 5981 #ifdef CONFIG_LOCKDEP 5982 __printf(1, 5) 5983 struct workqueue_struct * 5984 alloc_workqueue_lockdep_map(const char *fmt, unsigned int flags, 5985 int max_active, struct lockdep_map *lockdep_map, ...) 5986 { 5987 struct workqueue_struct *wq; 5988 va_list args; 5989 5990 va_start(args, lockdep_map); 5991 wq = __alloc_workqueue(fmt, flags, max_active, args); 5992 va_end(args); 5993 if (!wq) 5994 return NULL; 5995 5996 wq->lockdep_map = lockdep_map; 5997 5998 return wq; 5999 } 6000 EXPORT_SYMBOL_GPL(alloc_workqueue_lockdep_map); 6001 #endif 6002 6003 static bool pwq_busy(struct pool_workqueue *pwq) 6004 { 6005 int i; 6006 6007 for (i = 0; i < WORK_NR_COLORS; i++) 6008 if (pwq->nr_in_flight[i]) 6009 return true; 6010 6011 if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1)) 6012 return true; 6013 if (!pwq_is_empty(pwq)) 6014 return true; 6015 6016 return false; 6017 } 6018 6019 /** 6020 * destroy_workqueue - safely terminate a workqueue 6021 * @wq: target workqueue 6022 * 6023 * Safely destroy a workqueue. All work currently pending will be done first. 6024 * 6025 * This function does NOT guarantee that non-pending work that has been 6026 * submitted with queue_delayed_work() and similar functions will be done 6027 * before destroying the workqueue. The fundamental problem is that, currently, 6028 * the workqueue has no way of accessing non-pending delayed_work. delayed_work 6029 * is only linked on the timer-side. All delayed_work must, therefore, be 6030 * canceled before calling this function. 6031 * 6032 * TODO: It would be better if the problem described above wouldn't exist and 6033 * destroy_workqueue() would cleanly cancel all pending and non-pending 6034 * delayed_work. 6035 */ 6036 void destroy_workqueue(struct workqueue_struct *wq) 6037 { 6038 struct pool_workqueue *pwq; 6039 int cpu; 6040 6041 /* 6042 * Remove it from sysfs first so that sanity check failure doesn't 6043 * lead to sysfs name conflicts. 6044 */ 6045 workqueue_sysfs_unregister(wq); 6046 6047 /* mark the workqueue destruction is in progress */ 6048 mutex_lock(&wq->mutex); 6049 wq->flags |= __WQ_DESTROYING; 6050 mutex_unlock(&wq->mutex); 6051 6052 /* drain it before proceeding with destruction */ 6053 drain_workqueue(wq); 6054 6055 /* kill rescuer, if sanity checks fail, leave it w/o rescuer */ 6056 if (wq->rescuer) { 6057 /* rescuer will empty maydays list before exiting */ 6058 kthread_stop(wq->rescuer->task); 6059 kfree(wq->rescuer); 6060 wq->rescuer = NULL; 6061 } 6062 6063 /* 6064 * Sanity checks - grab all the locks so that we wait for all 6065 * in-flight operations which may do put_pwq(). 6066 */ 6067 mutex_lock(&wq_pool_mutex); 6068 mutex_lock(&wq->mutex); 6069 for_each_pwq(pwq, wq) { 6070 raw_spin_lock_irq(&pwq->pool->lock); 6071 if (WARN_ON(pwq_busy(pwq))) { 6072 pr_warn("%s: %s has the following busy pwq\n", 6073 __func__, wq->name); 6074 show_pwq(pwq); 6075 raw_spin_unlock_irq(&pwq->pool->lock); 6076 mutex_unlock(&wq->mutex); 6077 mutex_unlock(&wq_pool_mutex); 6078 show_one_workqueue(wq); 6079 return; 6080 } 6081 raw_spin_unlock_irq(&pwq->pool->lock); 6082 } 6083 mutex_unlock(&wq->mutex); 6084 6085 /* 6086 * wq list is used to freeze wq, remove from list after 6087 * flushing is complete in case freeze races us. 6088 */ 6089 list_del_rcu(&wq->list); 6090 mutex_unlock(&wq_pool_mutex); 6091 6092 /* 6093 * We're the sole accessor of @wq. Directly access cpu_pwq and dfl_pwq 6094 * to put the base refs. @wq will be auto-destroyed from the last 6095 * pwq_put. RCU read lock prevents @wq from going away from under us. 6096 */ 6097 rcu_read_lock(); 6098 6099 for_each_possible_cpu(cpu) { 6100 put_pwq_unlocked(unbound_pwq(wq, cpu)); 6101 RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL); 6102 } 6103 6104 put_pwq_unlocked(unbound_pwq(wq, -1)); 6105 RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL); 6106 6107 rcu_read_unlock(); 6108 } 6109 EXPORT_SYMBOL_GPL(destroy_workqueue); 6110 6111 /** 6112 * workqueue_set_max_active - adjust max_active of a workqueue 6113 * @wq: target workqueue 6114 * @max_active: new max_active value. 6115 * 6116 * Set max_active of @wq to @max_active. See the alloc_workqueue() function 6117 * comment. 6118 * 6119 * CONTEXT: 6120 * Don't call from IRQ context. 6121 */ 6122 void workqueue_set_max_active(struct workqueue_struct *wq, int max_active) 6123 { 6124 /* max_active doesn't mean anything for BH workqueues */ 6125 if (WARN_ON(wq->flags & WQ_BH)) 6126 return; 6127 /* disallow meddling with max_active for ordered workqueues */ 6128 if (WARN_ON(wq->flags & __WQ_ORDERED)) 6129 return; 6130 6131 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name); 6132 6133 mutex_lock(&wq->mutex); 6134 6135 wq->saved_max_active = max_active; 6136 if (wq->flags & WQ_UNBOUND) 6137 wq->saved_min_active = min(wq->saved_min_active, max_active); 6138 6139 wq_adjust_max_active(wq); 6140 6141 mutex_unlock(&wq->mutex); 6142 } 6143 EXPORT_SYMBOL_GPL(workqueue_set_max_active); 6144 6145 /** 6146 * workqueue_set_min_active - adjust min_active of an unbound workqueue 6147 * @wq: target unbound workqueue 6148 * @min_active: new min_active value 6149 * 6150 * Set min_active of an unbound workqueue. Unlike other types of workqueues, an 6151 * unbound workqueue is not guaranteed to be able to process max_active 6152 * interdependent work items. Instead, an unbound workqueue is guaranteed to be 6153 * able to process min_active number of interdependent work items which is 6154 * %WQ_DFL_MIN_ACTIVE by default. 6155 * 6156 * Use this function to adjust the min_active value between 0 and the current 6157 * max_active. 6158 */ 6159 void workqueue_set_min_active(struct workqueue_struct *wq, int min_active) 6160 { 6161 /* min_active is only meaningful for non-ordered unbound workqueues */ 6162 if (WARN_ON((wq->flags & (WQ_BH | WQ_UNBOUND | __WQ_ORDERED)) != 6163 WQ_UNBOUND)) 6164 return; 6165 6166 mutex_lock(&wq->mutex); 6167 wq->saved_min_active = clamp(min_active, 0, wq->saved_max_active); 6168 wq_adjust_max_active(wq); 6169 mutex_unlock(&wq->mutex); 6170 } 6171 6172 /** 6173 * current_work - retrieve %current task's work struct 6174 * 6175 * Determine if %current task is a workqueue worker and what it's working on. 6176 * Useful to find out the context that the %current task is running in. 6177 * 6178 * Return: work struct if %current task is a workqueue worker, %NULL otherwise. 6179 */ 6180 struct work_struct *current_work(void) 6181 { 6182 struct worker *worker = current_wq_worker(); 6183 6184 return worker ? worker->current_work : NULL; 6185 } 6186 EXPORT_SYMBOL(current_work); 6187 6188 /** 6189 * current_is_workqueue_rescuer - is %current workqueue rescuer? 6190 * 6191 * Determine whether %current is a workqueue rescuer. Can be used from 6192 * work functions to determine whether it's being run off the rescuer task. 6193 * 6194 * Return: %true if %current is a workqueue rescuer. %false otherwise. 6195 */ 6196 bool current_is_workqueue_rescuer(void) 6197 { 6198 struct worker *worker = current_wq_worker(); 6199 6200 return worker && worker->rescue_wq; 6201 } 6202 6203 /** 6204 * workqueue_congested - test whether a workqueue is congested 6205 * @cpu: CPU in question 6206 * @wq: target workqueue 6207 * 6208 * Test whether @wq's cpu workqueue for @cpu is congested. There is 6209 * no synchronization around this function and the test result is 6210 * unreliable and only useful as advisory hints or for debugging. 6211 * 6212 * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU. 6213 * 6214 * With the exception of ordered workqueues, all workqueues have per-cpu 6215 * pool_workqueues, each with its own congested state. A workqueue being 6216 * congested on one CPU doesn't mean that the workqueue is contested on any 6217 * other CPUs. 6218 * 6219 * Return: 6220 * %true if congested, %false otherwise. 6221 */ 6222 bool workqueue_congested(int cpu, struct workqueue_struct *wq) 6223 { 6224 struct pool_workqueue *pwq; 6225 bool ret; 6226 6227 preempt_disable(); 6228 6229 if (cpu == WORK_CPU_UNBOUND) 6230 cpu = smp_processor_id(); 6231 6232 pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); 6233 ret = !list_empty(&pwq->inactive_works); 6234 6235 preempt_enable(); 6236 6237 return ret; 6238 } 6239 EXPORT_SYMBOL_GPL(workqueue_congested); 6240 6241 /** 6242 * work_busy - test whether a work is currently pending or running 6243 * @work: the work to be tested 6244 * 6245 * Test whether @work is currently pending or running. There is no 6246 * synchronization around this function and the test result is 6247 * unreliable and only useful as advisory hints or for debugging. 6248 * 6249 * Return: 6250 * OR'd bitmask of WORK_BUSY_* bits. 6251 */ 6252 unsigned int work_busy(struct work_struct *work) 6253 { 6254 struct worker_pool *pool; 6255 unsigned long irq_flags; 6256 unsigned int ret = 0; 6257 6258 if (work_pending(work)) 6259 ret |= WORK_BUSY_PENDING; 6260 6261 rcu_read_lock(); 6262 pool = get_work_pool(work); 6263 if (pool) { 6264 raw_spin_lock_irqsave(&pool->lock, irq_flags); 6265 if (find_worker_executing_work(pool, work)) 6266 ret |= WORK_BUSY_RUNNING; 6267 raw_spin_unlock_irqrestore(&pool->lock, irq_flags); 6268 } 6269 rcu_read_unlock(); 6270 6271 return ret; 6272 } 6273 EXPORT_SYMBOL_GPL(work_busy); 6274 6275 /** 6276 * set_worker_desc - set description for the current work item 6277 * @fmt: printf-style format string 6278 * @...: arguments for the format string 6279 * 6280 * This function can be called by a running work function to describe what 6281 * the work item is about. If the worker task gets dumped, this 6282 * information will be printed out together to help debugging. The 6283 * description can be at most WORKER_DESC_LEN including the trailing '\0'. 6284 */ 6285 void set_worker_desc(const char *fmt, ...) 6286 { 6287 struct worker *worker = current_wq_worker(); 6288 va_list args; 6289 6290 if (worker) { 6291 va_start(args, fmt); 6292 vsnprintf(worker->desc, sizeof(worker->desc), fmt, args); 6293 va_end(args); 6294 } 6295 } 6296 EXPORT_SYMBOL_GPL(set_worker_desc); 6297 6298 /** 6299 * print_worker_info - print out worker information and description 6300 * @log_lvl: the log level to use when printing 6301 * @task: target task 6302 * 6303 * If @task is a worker and currently executing a work item, print out the 6304 * name of the workqueue being serviced and worker description set with 6305 * set_worker_desc() by the currently executing work item. 6306 * 6307 * This function can be safely called on any task as long as the 6308 * task_struct itself is accessible. While safe, this function isn't 6309 * synchronized and may print out mixups or garbages of limited length. 6310 */ 6311 void print_worker_info(const char *log_lvl, struct task_struct *task) 6312 { 6313 work_func_t *fn = NULL; 6314 char name[WQ_NAME_LEN] = { }; 6315 char desc[WORKER_DESC_LEN] = { }; 6316 struct pool_workqueue *pwq = NULL; 6317 struct workqueue_struct *wq = NULL; 6318 struct worker *worker; 6319 6320 if (!(task->flags & PF_WQ_WORKER)) 6321 return; 6322 6323 /* 6324 * This function is called without any synchronization and @task 6325 * could be in any state. Be careful with dereferences. 6326 */ 6327 worker = kthread_probe_data(task); 6328 6329 /* 6330 * Carefully copy the associated workqueue's workfn, name and desc. 6331 * Keep the original last '\0' in case the original is garbage. 6332 */ 6333 copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn)); 6334 copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq)); 6335 copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq)); 6336 copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1); 6337 copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1); 6338 6339 if (fn || name[0] || desc[0]) { 6340 printk("%sWorkqueue: %s %ps", log_lvl, name, fn); 6341 if (strcmp(name, desc)) 6342 pr_cont(" (%s)", desc); 6343 pr_cont("\n"); 6344 } 6345 } 6346 6347 static void pr_cont_pool_info(struct worker_pool *pool) 6348 { 6349 pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask); 6350 if (pool->node != NUMA_NO_NODE) 6351 pr_cont(" node=%d", pool->node); 6352 pr_cont(" flags=0x%x", pool->flags); 6353 if (pool->flags & POOL_BH) 6354 pr_cont(" bh%s", 6355 pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : ""); 6356 else 6357 pr_cont(" nice=%d", pool->attrs->nice); 6358 } 6359 6360 static void pr_cont_worker_id(struct worker *worker) 6361 { 6362 struct worker_pool *pool = worker->pool; 6363 6364 if (pool->flags & POOL_BH) 6365 pr_cont("bh%s", 6366 pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : ""); 6367 else 6368 pr_cont("%d%s", task_pid_nr(worker->task), 6369 worker->rescue_wq ? "(RESCUER)" : ""); 6370 } 6371 6372 struct pr_cont_work_struct { 6373 bool comma; 6374 work_func_t func; 6375 long ctr; 6376 }; 6377 6378 static void pr_cont_work_flush(bool comma, work_func_t func, struct pr_cont_work_struct *pcwsp) 6379 { 6380 if (!pcwsp->ctr) 6381 goto out_record; 6382 if (func == pcwsp->func) { 6383 pcwsp->ctr++; 6384 return; 6385 } 6386 if (pcwsp->ctr == 1) 6387 pr_cont("%s %ps", pcwsp->comma ? "," : "", pcwsp->func); 6388 else 6389 pr_cont("%s %ld*%ps", pcwsp->comma ? "," : "", pcwsp->ctr, pcwsp->func); 6390 pcwsp->ctr = 0; 6391 out_record: 6392 if ((long)func == -1L) 6393 return; 6394 pcwsp->comma = comma; 6395 pcwsp->func = func; 6396 pcwsp->ctr = 1; 6397 } 6398 6399 static void pr_cont_work(bool comma, struct work_struct *work, struct pr_cont_work_struct *pcwsp) 6400 { 6401 if (work->func == wq_barrier_func) { 6402 struct wq_barrier *barr; 6403 6404 barr = container_of(work, struct wq_barrier, work); 6405 6406 pr_cont_work_flush(comma, (work_func_t)-1, pcwsp); 6407 pr_cont("%s BAR(%d)", comma ? "," : "", 6408 task_pid_nr(barr->task)); 6409 } else { 6410 if (!comma) 6411 pr_cont_work_flush(comma, (work_func_t)-1, pcwsp); 6412 pr_cont_work_flush(comma, work->func, pcwsp); 6413 } 6414 } 6415 6416 static void show_pwq(struct pool_workqueue *pwq) 6417 { 6418 struct pr_cont_work_struct pcws = { .ctr = 0, }; 6419 struct worker_pool *pool = pwq->pool; 6420 struct work_struct *work; 6421 struct worker *worker; 6422 bool has_in_flight = false, has_pending = false; 6423 int bkt; 6424 6425 pr_info(" pwq %d:", pool->id); 6426 pr_cont_pool_info(pool); 6427 6428 pr_cont(" active=%d refcnt=%d%s\n", 6429 pwq->nr_active, pwq->refcnt, 6430 !list_empty(&pwq->mayday_node) ? " MAYDAY" : ""); 6431 6432 hash_for_each(pool->busy_hash, bkt, worker, hentry) { 6433 if (worker->current_pwq == pwq) { 6434 has_in_flight = true; 6435 break; 6436 } 6437 } 6438 if (has_in_flight) { 6439 bool comma = false; 6440 6441 pr_info(" in-flight:"); 6442 hash_for_each(pool->busy_hash, bkt, worker, hentry) { 6443 if (worker->current_pwq != pwq) 6444 continue; 6445 6446 pr_cont(" %s", comma ? "," : ""); 6447 pr_cont_worker_id(worker); 6448 pr_cont(":%ps", worker->current_func); 6449 pr_cont(" for %us", 6450 jiffies_to_msecs(jiffies - worker->current_start) / 1000); 6451 list_for_each_entry(work, &worker->scheduled, entry) 6452 pr_cont_work(false, work, &pcws); 6453 pr_cont_work_flush(comma, (work_func_t)-1L, &pcws); 6454 comma = true; 6455 } 6456 pr_cont("\n"); 6457 } 6458 6459 list_for_each_entry(work, &pool->worklist, entry) { 6460 if (get_work_pwq(work) == pwq) { 6461 has_pending = true; 6462 break; 6463 } 6464 } 6465 if (has_pending) { 6466 bool comma = false; 6467 6468 pr_info(" pending:"); 6469 list_for_each_entry(work, &pool->worklist, entry) { 6470 if (get_work_pwq(work) != pwq) 6471 continue; 6472 6473 pr_cont_work(comma, work, &pcws); 6474 comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED); 6475 } 6476 pr_cont_work_flush(comma, (work_func_t)-1L, &pcws); 6477 pr_cont("\n"); 6478 } 6479 6480 if (!list_empty(&pwq->inactive_works)) { 6481 bool comma = false; 6482 6483 pr_info(" inactive:"); 6484 list_for_each_entry(work, &pwq->inactive_works, entry) { 6485 pr_cont_work(comma, work, &pcws); 6486 comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED); 6487 } 6488 pr_cont_work_flush(comma, (work_func_t)-1L, &pcws); 6489 pr_cont("\n"); 6490 } 6491 } 6492 6493 /** 6494 * show_one_workqueue - dump state of specified workqueue 6495 * @wq: workqueue whose state will be printed 6496 */ 6497 void show_one_workqueue(struct workqueue_struct *wq) 6498 { 6499 struct pool_workqueue *pwq; 6500 bool idle = true; 6501 unsigned long irq_flags; 6502 6503 for_each_pwq(pwq, wq) { 6504 if (!pwq_is_empty(pwq)) { 6505 idle = false; 6506 break; 6507 } 6508 } 6509 if (idle) /* Nothing to print for idle workqueue */ 6510 return; 6511 6512 pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags); 6513 6514 for_each_pwq(pwq, wq) { 6515 raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags); 6516 if (!pwq_is_empty(pwq)) { 6517 /* 6518 * Defer printing to avoid deadlocks in console 6519 * drivers that queue work while holding locks 6520 * also taken in their write paths. 6521 */ 6522 printk_deferred_enter(); 6523 show_pwq(pwq); 6524 printk_deferred_exit(); 6525 } 6526 raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags); 6527 /* 6528 * We could be printing a lot from atomic context, e.g. 6529 * sysrq-t -> show_all_workqueues(). Avoid triggering 6530 * hard lockup. 6531 */ 6532 touch_nmi_watchdog(); 6533 } 6534 6535 } 6536 6537 /** 6538 * show_one_worker_pool - dump state of specified worker pool 6539 * @pool: worker pool whose state will be printed 6540 */ 6541 static void show_one_worker_pool(struct worker_pool *pool) 6542 { 6543 struct worker *worker; 6544 bool first = true; 6545 unsigned long irq_flags; 6546 unsigned long hung = 0; 6547 6548 raw_spin_lock_irqsave(&pool->lock, irq_flags); 6549 if (pool->nr_workers == pool->nr_idle) 6550 goto next_pool; 6551 6552 /* How long the first pending work is waiting for a worker. */ 6553 if (!list_empty(&pool->worklist)) 6554 hung = jiffies_to_msecs(jiffies - pool->last_progress_ts) / 1000; 6555 6556 /* 6557 * Defer printing to avoid deadlocks in console drivers that 6558 * queue work while holding locks also taken in their write 6559 * paths. 6560 */ 6561 printk_deferred_enter(); 6562 pr_info("pool %d:", pool->id); 6563 pr_cont_pool_info(pool); 6564 pr_cont(" hung=%lus workers=%d", hung, pool->nr_workers); 6565 if (pool->manager) 6566 pr_cont(" manager: %d", 6567 task_pid_nr(pool->manager->task)); 6568 list_for_each_entry(worker, &pool->idle_list, entry) { 6569 pr_cont(" %s", first ? "idle: " : ""); 6570 pr_cont_worker_id(worker); 6571 first = false; 6572 } 6573 pr_cont("\n"); 6574 printk_deferred_exit(); 6575 next_pool: 6576 raw_spin_unlock_irqrestore(&pool->lock, irq_flags); 6577 /* 6578 * We could be printing a lot from atomic context, e.g. 6579 * sysrq-t -> show_all_workqueues(). Avoid triggering 6580 * hard lockup. 6581 */ 6582 touch_nmi_watchdog(); 6583 6584 } 6585 6586 /** 6587 * show_all_workqueues - dump workqueue state 6588 * 6589 * Called from a sysrq handler and prints out all busy workqueues and pools. 6590 */ 6591 void show_all_workqueues(void) 6592 { 6593 struct workqueue_struct *wq; 6594 struct worker_pool *pool; 6595 int pi; 6596 6597 rcu_read_lock(); 6598 6599 pr_info("Showing busy workqueues and worker pools:\n"); 6600 6601 list_for_each_entry_rcu(wq, &workqueues, list) 6602 show_one_workqueue(wq); 6603 6604 for_each_pool(pool, pi) 6605 show_one_worker_pool(pool); 6606 6607 rcu_read_unlock(); 6608 } 6609 6610 /** 6611 * show_freezable_workqueues - dump freezable workqueue state 6612 * 6613 * Called from try_to_freeze_tasks() and prints out all freezable workqueues 6614 * still busy. 6615 */ 6616 void show_freezable_workqueues(void) 6617 { 6618 struct workqueue_struct *wq; 6619 6620 rcu_read_lock(); 6621 6622 pr_info("Showing freezable workqueues that are still busy:\n"); 6623 6624 list_for_each_entry_rcu(wq, &workqueues, list) { 6625 if (!(wq->flags & WQ_FREEZABLE)) 6626 continue; 6627 show_one_workqueue(wq); 6628 } 6629 6630 rcu_read_unlock(); 6631 } 6632 6633 /* used to show worker information through /proc/PID/{comm,stat,status} */ 6634 void wq_worker_comm(char *buf, size_t size, struct task_struct *task) 6635 { 6636 /* stabilize PF_WQ_WORKER and worker pool association */ 6637 mutex_lock(&wq_pool_attach_mutex); 6638 6639 if (task->flags & PF_WQ_WORKER) { 6640 struct worker *worker = kthread_data(task); 6641 struct worker_pool *pool = worker->pool; 6642 int off; 6643 6644 off = format_worker_id(buf, size, worker, pool); 6645 6646 if (pool) { 6647 raw_spin_lock_irq(&pool->lock); 6648 /* 6649 * ->desc tracks information (wq name or 6650 * set_worker_desc()) for the latest execution. If 6651 * current, prepend '+', otherwise '-'. 6652 */ 6653 if (worker->desc[0] != '\0') { 6654 if (worker->current_work) 6655 scnprintf(buf + off, size - off, "+%s", 6656 worker->desc); 6657 else 6658 scnprintf(buf + off, size - off, "-%s", 6659 worker->desc); 6660 } 6661 raw_spin_unlock_irq(&pool->lock); 6662 } 6663 } else { 6664 strscpy(buf, task->comm, size); 6665 } 6666 6667 mutex_unlock(&wq_pool_attach_mutex); 6668 } 6669 6670 #ifdef CONFIG_SMP 6671 6672 /* 6673 * CPU hotplug. 6674 * 6675 * There are two challenges in supporting CPU hotplug. Firstly, there 6676 * are a lot of assumptions on strong associations among work, pwq and 6677 * pool which make migrating pending and scheduled works very 6678 * difficult to implement without impacting hot paths. Secondly, 6679 * worker pools serve mix of short, long and very long running works making 6680 * blocked draining impractical. 6681 * 6682 * This is solved by allowing the pools to be disassociated from the CPU 6683 * running as an unbound one and allowing it to be reattached later if the 6684 * cpu comes back online. 6685 */ 6686 6687 static void unbind_workers(int cpu) 6688 { 6689 struct worker_pool *pool; 6690 struct worker *worker; 6691 6692 for_each_cpu_worker_pool(pool, cpu) { 6693 mutex_lock(&wq_pool_attach_mutex); 6694 raw_spin_lock_irq(&pool->lock); 6695 6696 /* 6697 * We've blocked all attach/detach operations. Make all workers 6698 * unbound and set DISASSOCIATED. Before this, all workers 6699 * must be on the cpu. After this, they may become diasporas. 6700 * And the preemption disabled section in their sched callbacks 6701 * are guaranteed to see WORKER_UNBOUND since the code here 6702 * is on the same cpu. 6703 */ 6704 for_each_pool_worker(worker, pool) 6705 worker->flags |= WORKER_UNBOUND; 6706 6707 pool->flags |= POOL_DISASSOCIATED; 6708 6709 /* 6710 * The handling of nr_running in sched callbacks are disabled 6711 * now. Zap nr_running. After this, nr_running stays zero and 6712 * need_more_worker() and keep_working() are always true as 6713 * long as the worklist is not empty. This pool now behaves as 6714 * an unbound (in terms of concurrency management) pool which 6715 * are served by workers tied to the pool. 6716 */ 6717 pool->nr_running = 0; 6718 6719 /* 6720 * With concurrency management just turned off, a busy 6721 * worker blocking could lead to lengthy stalls. Kick off 6722 * unbound chain execution of currently pending work items. 6723 */ 6724 kick_pool(pool); 6725 6726 raw_spin_unlock_irq(&pool->lock); 6727 6728 for_each_pool_worker(worker, pool) 6729 unbind_worker(worker); 6730 6731 mutex_unlock(&wq_pool_attach_mutex); 6732 } 6733 } 6734 6735 /** 6736 * rebind_workers - rebind all workers of a pool to the associated CPU 6737 * @pool: pool of interest 6738 * 6739 * @pool->cpu is coming online. Rebind all workers to the CPU. 6740 */ 6741 static void rebind_workers(struct worker_pool *pool) 6742 { 6743 struct worker *worker; 6744 6745 lockdep_assert_held(&wq_pool_attach_mutex); 6746 6747 /* 6748 * Restore CPU affinity of all workers. As all idle workers should 6749 * be on the run-queue of the associated CPU before any local 6750 * wake-ups for concurrency management happen, restore CPU affinity 6751 * of all workers first and then clear UNBOUND. As we're called 6752 * from CPU_ONLINE, the following shouldn't fail. 6753 */ 6754 for_each_pool_worker(worker, pool) { 6755 kthread_set_per_cpu(worker->task, pool->cpu); 6756 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, 6757 pool_allowed_cpus(pool)) < 0); 6758 } 6759 6760 raw_spin_lock_irq(&pool->lock); 6761 6762 pool->flags &= ~POOL_DISASSOCIATED; 6763 6764 for_each_pool_worker(worker, pool) { 6765 unsigned int worker_flags = worker->flags; 6766 6767 /* 6768 * We want to clear UNBOUND but can't directly call 6769 * worker_clr_flags() or adjust nr_running. Atomically 6770 * replace UNBOUND with another NOT_RUNNING flag REBOUND. 6771 * @worker will clear REBOUND using worker_clr_flags() when 6772 * it initiates the next execution cycle thus restoring 6773 * concurrency management. Note that when or whether 6774 * @worker clears REBOUND doesn't affect correctness. 6775 * 6776 * WRITE_ONCE() is necessary because @worker->flags may be 6777 * tested without holding any lock in 6778 * wq_worker_running(). Without it, NOT_RUNNING test may 6779 * fail incorrectly leading to premature concurrency 6780 * management operations. 6781 */ 6782 WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND)); 6783 worker_flags |= WORKER_REBOUND; 6784 worker_flags &= ~WORKER_UNBOUND; 6785 WRITE_ONCE(worker->flags, worker_flags); 6786 } 6787 6788 raw_spin_unlock_irq(&pool->lock); 6789 } 6790 6791 /** 6792 * restore_unbound_workers_cpumask - restore cpumask of unbound workers 6793 * @pool: unbound pool of interest 6794 * @cpu: the CPU which is coming up 6795 * 6796 * An unbound pool may end up with a cpumask which doesn't have any online 6797 * CPUs. When a worker of such pool get scheduled, the scheduler resets 6798 * its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any 6799 * online CPU before, cpus_allowed of all its workers should be restored. 6800 */ 6801 static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu) 6802 { 6803 static cpumask_t cpumask; 6804 struct worker *worker; 6805 6806 lockdep_assert_held(&wq_pool_attach_mutex); 6807 6808 /* is @cpu allowed for @pool? */ 6809 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask)) 6810 return; 6811 6812 cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask); 6813 6814 /* as we're called from CPU_ONLINE, the following shouldn't fail */ 6815 for_each_pool_worker(worker, pool) 6816 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0); 6817 } 6818 6819 int workqueue_prepare_cpu(unsigned int cpu) 6820 { 6821 struct worker_pool *pool; 6822 6823 for_each_cpu_worker_pool(pool, cpu) { 6824 if (pool->nr_workers) 6825 continue; 6826 if (!create_worker(pool)) 6827 return -ENOMEM; 6828 } 6829 return 0; 6830 } 6831 6832 int workqueue_online_cpu(unsigned int cpu) 6833 { 6834 struct worker_pool *pool; 6835 struct workqueue_struct *wq; 6836 int pi; 6837 6838 mutex_lock(&wq_pool_mutex); 6839 6840 cpumask_set_cpu(cpu, wq_online_cpumask); 6841 6842 for_each_pool(pool, pi) { 6843 /* BH pools aren't affected by hotplug */ 6844 if (pool->flags & POOL_BH) 6845 continue; 6846 6847 mutex_lock(&wq_pool_attach_mutex); 6848 if (pool->cpu == cpu) 6849 rebind_workers(pool); 6850 else if (pool->cpu < 0) 6851 restore_unbound_workers_cpumask(pool, cpu); 6852 mutex_unlock(&wq_pool_attach_mutex); 6853 } 6854 6855 /* update pod affinity of unbound workqueues */ 6856 list_for_each_entry(wq, &workqueues, list) { 6857 struct workqueue_attrs *attrs = wq->unbound_attrs; 6858 6859 if (attrs) { 6860 const struct wq_pod_type *pt = wqattrs_pod_type(attrs); 6861 int tcpu; 6862 6863 for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]]) 6864 unbound_wq_update_pwq(wq, tcpu); 6865 6866 mutex_lock(&wq->mutex); 6867 wq_update_node_max_active(wq, -1); 6868 mutex_unlock(&wq->mutex); 6869 } 6870 } 6871 6872 mutex_unlock(&wq_pool_mutex); 6873 return 0; 6874 } 6875 6876 int workqueue_offline_cpu(unsigned int cpu) 6877 { 6878 struct workqueue_struct *wq; 6879 6880 /* unbinding per-cpu workers should happen on the local CPU */ 6881 if (WARN_ON(cpu != smp_processor_id())) 6882 return -1; 6883 6884 unbind_workers(cpu); 6885 6886 /* update pod affinity of unbound workqueues */ 6887 mutex_lock(&wq_pool_mutex); 6888 6889 cpumask_clear_cpu(cpu, wq_online_cpumask); 6890 6891 list_for_each_entry(wq, &workqueues, list) { 6892 struct workqueue_attrs *attrs = wq->unbound_attrs; 6893 6894 if (attrs) { 6895 const struct wq_pod_type *pt = wqattrs_pod_type(attrs); 6896 int tcpu; 6897 6898 for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]]) 6899 unbound_wq_update_pwq(wq, tcpu); 6900 6901 mutex_lock(&wq->mutex); 6902 wq_update_node_max_active(wq, cpu); 6903 mutex_unlock(&wq->mutex); 6904 } 6905 } 6906 mutex_unlock(&wq_pool_mutex); 6907 6908 return 0; 6909 } 6910 6911 struct work_for_cpu { 6912 struct work_struct work; 6913 long (*fn)(void *); 6914 void *arg; 6915 long ret; 6916 }; 6917 6918 static void work_for_cpu_fn(struct work_struct *work) 6919 { 6920 struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work); 6921 6922 wfc->ret = wfc->fn(wfc->arg); 6923 } 6924 6925 /** 6926 * work_on_cpu_key - run a function in thread context on a particular cpu 6927 * @cpu: the cpu to run on 6928 * @fn: the function to run 6929 * @arg: the function arg 6930 * @key: The lock class key for lock debugging purposes 6931 * 6932 * It is up to the caller to ensure that the cpu doesn't go offline. 6933 * The caller must not hold any locks which would prevent @fn from completing. 6934 * 6935 * Return: The value @fn returns. 6936 */ 6937 long work_on_cpu_key(int cpu, long (*fn)(void *), 6938 void *arg, struct lock_class_key *key) 6939 { 6940 struct work_for_cpu wfc = { .fn = fn, .arg = arg }; 6941 6942 INIT_WORK_ONSTACK_KEY(&wfc.work, work_for_cpu_fn, key); 6943 schedule_work_on(cpu, &wfc.work); 6944 flush_work(&wfc.work); 6945 destroy_work_on_stack(&wfc.work); 6946 return wfc.ret; 6947 } 6948 EXPORT_SYMBOL_GPL(work_on_cpu_key); 6949 #endif /* CONFIG_SMP */ 6950 6951 #ifdef CONFIG_FREEZER 6952 6953 /** 6954 * freeze_workqueues_begin - begin freezing workqueues 6955 * 6956 * Start freezing workqueues. After this function returns, all freezable 6957 * workqueues will queue new works to their inactive_works list instead of 6958 * pool->worklist. 6959 * 6960 * CONTEXT: 6961 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's. 6962 */ 6963 void freeze_workqueues_begin(void) 6964 { 6965 struct workqueue_struct *wq; 6966 6967 mutex_lock(&wq_pool_mutex); 6968 6969 WARN_ON_ONCE(workqueue_freezing); 6970 workqueue_freezing = true; 6971 6972 list_for_each_entry(wq, &workqueues, list) { 6973 mutex_lock(&wq->mutex); 6974 wq_adjust_max_active(wq); 6975 mutex_unlock(&wq->mutex); 6976 } 6977 6978 mutex_unlock(&wq_pool_mutex); 6979 } 6980 6981 /** 6982 * freeze_workqueues_busy - are freezable workqueues still busy? 6983 * 6984 * Check whether freezing is complete. This function must be called 6985 * between freeze_workqueues_begin() and thaw_workqueues(). 6986 * 6987 * CONTEXT: 6988 * Grabs and releases wq_pool_mutex. 6989 * 6990 * Return: 6991 * %true if some freezable workqueues are still busy. %false if freezing 6992 * is complete. 6993 */ 6994 bool freeze_workqueues_busy(void) 6995 { 6996 bool busy = false; 6997 struct workqueue_struct *wq; 6998 struct pool_workqueue *pwq; 6999 7000 mutex_lock(&wq_pool_mutex); 7001 7002 WARN_ON_ONCE(!workqueue_freezing); 7003 7004 list_for_each_entry(wq, &workqueues, list) { 7005 if (!(wq->flags & WQ_FREEZABLE)) 7006 continue; 7007 /* 7008 * nr_active is monotonically decreasing. It's safe 7009 * to peek without lock. 7010 */ 7011 rcu_read_lock(); 7012 for_each_pwq(pwq, wq) { 7013 WARN_ON_ONCE(pwq->nr_active < 0); 7014 if (pwq->nr_active) { 7015 busy = true; 7016 rcu_read_unlock(); 7017 goto out_unlock; 7018 } 7019 } 7020 rcu_read_unlock(); 7021 } 7022 out_unlock: 7023 mutex_unlock(&wq_pool_mutex); 7024 return busy; 7025 } 7026 7027 /** 7028 * thaw_workqueues - thaw workqueues 7029 * 7030 * Thaw workqueues. Normal queueing is restored and all collected 7031 * frozen works are transferred to their respective pool worklists. 7032 * 7033 * CONTEXT: 7034 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's. 7035 */ 7036 void thaw_workqueues(void) 7037 { 7038 struct workqueue_struct *wq; 7039 7040 mutex_lock(&wq_pool_mutex); 7041 7042 if (!workqueue_freezing) 7043 goto out_unlock; 7044 7045 workqueue_freezing = false; 7046 7047 /* restore max_active and repopulate worklist */ 7048 list_for_each_entry(wq, &workqueues, list) { 7049 mutex_lock(&wq->mutex); 7050 wq_adjust_max_active(wq); 7051 mutex_unlock(&wq->mutex); 7052 } 7053 7054 out_unlock: 7055 mutex_unlock(&wq_pool_mutex); 7056 } 7057 #endif /* CONFIG_FREEZER */ 7058 7059 static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask) 7060 { 7061 LIST_HEAD(ctxs); 7062 int ret = 0; 7063 struct workqueue_struct *wq; 7064 struct apply_wqattrs_ctx *ctx, *n; 7065 7066 lockdep_assert_held(&wq_pool_mutex); 7067 7068 list_for_each_entry(wq, &workqueues, list) { 7069 if (!(wq->flags & WQ_UNBOUND) || (wq->flags & __WQ_DESTROYING)) 7070 continue; 7071 7072 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask); 7073 if (IS_ERR(ctx)) { 7074 ret = PTR_ERR(ctx); 7075 break; 7076 } 7077 7078 list_add_tail(&ctx->list, &ctxs); 7079 } 7080 7081 list_for_each_entry_safe(ctx, n, &ctxs, list) { 7082 if (!ret) 7083 apply_wqattrs_commit(ctx); 7084 apply_wqattrs_cleanup(ctx); 7085 } 7086 7087 if (!ret) { 7088 int cpu; 7089 struct worker_pool *pool; 7090 struct worker *worker; 7091 7092 mutex_lock(&wq_pool_attach_mutex); 7093 cpumask_copy(wq_unbound_cpumask, unbound_cpumask); 7094 /* rescuer needs to respect cpumask changes when it is not attached */ 7095 list_for_each_entry(wq, &workqueues, list) { 7096 if (wq->rescuer && !wq->rescuer->pool) 7097 unbind_worker(wq->rescuer); 7098 } 7099 /* DISASSOCIATED worker needs to respect wq_unbound_cpumask */ 7100 for_each_possible_cpu(cpu) { 7101 for_each_cpu_worker_pool(pool, cpu) { 7102 if (!(pool->flags & POOL_DISASSOCIATED)) 7103 continue; 7104 for_each_pool_worker(worker, pool) 7105 unbind_worker(worker); 7106 } 7107 } 7108 mutex_unlock(&wq_pool_attach_mutex); 7109 } 7110 return ret; 7111 } 7112 7113 /** 7114 * workqueue_unbound_housekeeping_update - Propagate housekeeping cpumask update 7115 * @hk: the new housekeeping cpumask 7116 * 7117 * Update the unbound workqueue cpumask on top of the new housekeeping cpumask such 7118 * that the effective unbound affinity is the intersection of the new housekeeping 7119 * with the requested affinity set via nohz_full=/isolcpus= or sysfs. 7120 * 7121 * Return: 0 on success and -errno on failure. 7122 */ 7123 int workqueue_unbound_housekeeping_update(const struct cpumask *hk) 7124 { 7125 cpumask_var_t cpumask; 7126 int ret = 0; 7127 7128 if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL)) 7129 return -ENOMEM; 7130 7131 mutex_lock(&wq_pool_mutex); 7132 7133 /* 7134 * If the operation fails, it will fall back to 7135 * wq_requested_unbound_cpumask which is initially set to 7136 * HK_TYPE_DOMAIN house keeping mask and rewritten 7137 * by any subsequent write to workqueue/cpumask sysfs file. 7138 */ 7139 if (!cpumask_and(cpumask, wq_requested_unbound_cpumask, hk)) 7140 cpumask_copy(cpumask, wq_requested_unbound_cpumask); 7141 if (!cpumask_equal(cpumask, wq_unbound_cpumask)) 7142 ret = workqueue_apply_unbound_cpumask(cpumask); 7143 7144 /* Save the current isolated cpumask & export it via sysfs */ 7145 if (!ret) 7146 cpumask_andnot(wq_isolated_cpumask, cpu_possible_mask, hk); 7147 7148 mutex_unlock(&wq_pool_mutex); 7149 free_cpumask_var(cpumask); 7150 return ret; 7151 } 7152 7153 static int parse_affn_scope(const char *val) 7154 { 7155 return sysfs_match_string(wq_affn_names, val); 7156 } 7157 7158 static int wq_affn_dfl_set(const char *val, const struct kernel_param *kp) 7159 { 7160 struct workqueue_struct *wq; 7161 int affn, cpu; 7162 7163 affn = parse_affn_scope(val); 7164 if (affn < 0) 7165 return affn; 7166 if (affn == WQ_AFFN_DFL) 7167 return -EINVAL; 7168 7169 cpus_read_lock(); 7170 mutex_lock(&wq_pool_mutex); 7171 7172 wq_affn_dfl = affn; 7173 7174 list_for_each_entry(wq, &workqueues, list) { 7175 for_each_online_cpu(cpu) 7176 unbound_wq_update_pwq(wq, cpu); 7177 } 7178 7179 mutex_unlock(&wq_pool_mutex); 7180 cpus_read_unlock(); 7181 7182 return 0; 7183 } 7184 7185 static int wq_affn_dfl_get(char *buffer, const struct kernel_param *kp) 7186 { 7187 return scnprintf(buffer, PAGE_SIZE, "%s\n", wq_affn_names[wq_affn_dfl]); 7188 } 7189 7190 static const struct kernel_param_ops wq_affn_dfl_ops = { 7191 .set = wq_affn_dfl_set, 7192 .get = wq_affn_dfl_get, 7193 }; 7194 7195 module_param_cb(default_affinity_scope, &wq_affn_dfl_ops, NULL, 0644); 7196 7197 #ifdef CONFIG_SYSFS 7198 /* 7199 * Workqueues with WQ_SYSFS flag set is visible to userland via 7200 * /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the 7201 * following attributes. 7202 * 7203 * per_cpu RO bool : whether the workqueue is per-cpu or unbound 7204 * max_active RW int : maximum number of in-flight work items 7205 * 7206 * Unbound workqueues have the following extra attributes. 7207 * 7208 * nice RW int : nice value of the workers 7209 * cpumask RW mask : bitmask of allowed CPUs for the workers 7210 * affinity_scope RW str : worker CPU affinity scope (cache, numa, none) 7211 * affinity_strict RW bool : worker CPU affinity is strict 7212 */ 7213 struct wq_device { 7214 struct workqueue_struct *wq; 7215 struct device dev; 7216 }; 7217 7218 static struct workqueue_struct *dev_to_wq(struct device *dev) 7219 { 7220 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev); 7221 7222 return wq_dev->wq; 7223 } 7224 7225 static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr, 7226 char *buf) 7227 { 7228 struct workqueue_struct *wq = dev_to_wq(dev); 7229 7230 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND)); 7231 } 7232 static DEVICE_ATTR_RO(per_cpu); 7233 7234 static ssize_t max_active_show(struct device *dev, 7235 struct device_attribute *attr, char *buf) 7236 { 7237 struct workqueue_struct *wq = dev_to_wq(dev); 7238 7239 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active); 7240 } 7241 7242 static ssize_t max_active_store(struct device *dev, 7243 struct device_attribute *attr, const char *buf, 7244 size_t count) 7245 { 7246 struct workqueue_struct *wq = dev_to_wq(dev); 7247 int val; 7248 7249 if (sscanf(buf, "%d", &val) != 1 || val <= 0) 7250 return -EINVAL; 7251 7252 workqueue_set_max_active(wq, val); 7253 return count; 7254 } 7255 static DEVICE_ATTR_RW(max_active); 7256 7257 static struct attribute *wq_sysfs_attrs[] = { 7258 &dev_attr_per_cpu.attr, 7259 &dev_attr_max_active.attr, 7260 NULL, 7261 }; 7262 7263 static umode_t wq_sysfs_is_visible(struct kobject *kobj, struct attribute *a, int n) 7264 { 7265 struct device *dev = kobj_to_dev(kobj); 7266 struct workqueue_struct *wq = dev_to_wq(dev); 7267 7268 /* 7269 * Adjusting max_active breaks ordering guarantee. Changing it has no 7270 * effect on BH worker. Limit max_active to RO in such case. 7271 */ 7272 if (wq->flags & (WQ_BH | __WQ_ORDERED)) 7273 return 0444; 7274 return a->mode; 7275 } 7276 7277 static const struct attribute_group wq_sysfs_group = { 7278 .is_visible = wq_sysfs_is_visible, 7279 .attrs = wq_sysfs_attrs, 7280 }; 7281 __ATTRIBUTE_GROUPS(wq_sysfs); 7282 7283 static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr, 7284 char *buf) 7285 { 7286 struct workqueue_struct *wq = dev_to_wq(dev); 7287 int written; 7288 7289 mutex_lock(&wq->mutex); 7290 written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice); 7291 mutex_unlock(&wq->mutex); 7292 7293 return written; 7294 } 7295 7296 /* prepare workqueue_attrs for sysfs store operations */ 7297 static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq) 7298 { 7299 struct workqueue_attrs *attrs; 7300 7301 lockdep_assert_held(&wq_pool_mutex); 7302 7303 attrs = alloc_workqueue_attrs(); 7304 if (!attrs) 7305 return NULL; 7306 7307 copy_workqueue_attrs(attrs, wq->unbound_attrs); 7308 return attrs; 7309 } 7310 7311 static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr, 7312 const char *buf, size_t count) 7313 { 7314 struct workqueue_struct *wq = dev_to_wq(dev); 7315 struct workqueue_attrs *attrs; 7316 int ret = -ENOMEM; 7317 7318 apply_wqattrs_lock(); 7319 7320 attrs = wq_sysfs_prep_attrs(wq); 7321 if (!attrs) 7322 goto out_unlock; 7323 7324 if (sscanf(buf, "%d", &attrs->nice) == 1 && 7325 attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE) 7326 ret = apply_workqueue_attrs_locked(wq, attrs); 7327 else 7328 ret = -EINVAL; 7329 7330 out_unlock: 7331 apply_wqattrs_unlock(); 7332 free_workqueue_attrs(attrs); 7333 return ret ?: count; 7334 } 7335 7336 static ssize_t wq_cpumask_show(struct device *dev, 7337 struct device_attribute *attr, char *buf) 7338 { 7339 struct workqueue_struct *wq = dev_to_wq(dev); 7340 int written; 7341 7342 mutex_lock(&wq->mutex); 7343 written = scnprintf(buf, PAGE_SIZE, "%*pb\n", 7344 cpumask_pr_args(wq->unbound_attrs->cpumask)); 7345 mutex_unlock(&wq->mutex); 7346 return written; 7347 } 7348 7349 static ssize_t wq_cpumask_store(struct device *dev, 7350 struct device_attribute *attr, 7351 const char *buf, size_t count) 7352 { 7353 struct workqueue_struct *wq = dev_to_wq(dev); 7354 struct workqueue_attrs *attrs; 7355 int ret = -ENOMEM; 7356 7357 apply_wqattrs_lock(); 7358 7359 attrs = wq_sysfs_prep_attrs(wq); 7360 if (!attrs) 7361 goto out_unlock; 7362 7363 ret = cpumask_parse(buf, attrs->cpumask); 7364 if (!ret) 7365 ret = apply_workqueue_attrs_locked(wq, attrs); 7366 7367 out_unlock: 7368 apply_wqattrs_unlock(); 7369 free_workqueue_attrs(attrs); 7370 return ret ?: count; 7371 } 7372 7373 static ssize_t wq_affn_scope_show(struct device *dev, 7374 struct device_attribute *attr, char *buf) 7375 { 7376 struct workqueue_struct *wq = dev_to_wq(dev); 7377 int written; 7378 7379 mutex_lock(&wq->mutex); 7380 if (wq->unbound_attrs->affn_scope == WQ_AFFN_DFL) 7381 written = scnprintf(buf, PAGE_SIZE, "%s (%s)\n", 7382 wq_affn_names[WQ_AFFN_DFL], 7383 wq_affn_names[wq_affn_dfl]); 7384 else 7385 written = scnprintf(buf, PAGE_SIZE, "%s\n", 7386 wq_affn_names[wq->unbound_attrs->affn_scope]); 7387 mutex_unlock(&wq->mutex); 7388 7389 return written; 7390 } 7391 7392 static ssize_t wq_affn_scope_store(struct device *dev, 7393 struct device_attribute *attr, 7394 const char *buf, size_t count) 7395 { 7396 struct workqueue_struct *wq = dev_to_wq(dev); 7397 struct workqueue_attrs *attrs; 7398 int affn, ret = -ENOMEM; 7399 7400 affn = parse_affn_scope(buf); 7401 if (affn < 0) 7402 return affn; 7403 7404 apply_wqattrs_lock(); 7405 attrs = wq_sysfs_prep_attrs(wq); 7406 if (attrs) { 7407 attrs->affn_scope = affn; 7408 ret = apply_workqueue_attrs_locked(wq, attrs); 7409 } 7410 apply_wqattrs_unlock(); 7411 free_workqueue_attrs(attrs); 7412 return ret ?: count; 7413 } 7414 7415 static ssize_t wq_affinity_strict_show(struct device *dev, 7416 struct device_attribute *attr, char *buf) 7417 { 7418 struct workqueue_struct *wq = dev_to_wq(dev); 7419 7420 return scnprintf(buf, PAGE_SIZE, "%d\n", 7421 wq->unbound_attrs->affn_strict); 7422 } 7423 7424 static ssize_t wq_affinity_strict_store(struct device *dev, 7425 struct device_attribute *attr, 7426 const char *buf, size_t count) 7427 { 7428 struct workqueue_struct *wq = dev_to_wq(dev); 7429 struct workqueue_attrs *attrs; 7430 int v, ret = -ENOMEM; 7431 7432 if (sscanf(buf, "%d", &v) != 1) 7433 return -EINVAL; 7434 7435 apply_wqattrs_lock(); 7436 attrs = wq_sysfs_prep_attrs(wq); 7437 if (attrs) { 7438 attrs->affn_strict = (bool)v; 7439 ret = apply_workqueue_attrs_locked(wq, attrs); 7440 } 7441 apply_wqattrs_unlock(); 7442 free_workqueue_attrs(attrs); 7443 return ret ?: count; 7444 } 7445 7446 static struct device_attribute wq_sysfs_unbound_attrs[] = { 7447 __ATTR(nice, 0644, wq_nice_show, wq_nice_store), 7448 __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store), 7449 __ATTR(affinity_scope, 0644, wq_affn_scope_show, wq_affn_scope_store), 7450 __ATTR(affinity_strict, 0644, wq_affinity_strict_show, wq_affinity_strict_store), 7451 __ATTR_NULL, 7452 }; 7453 7454 static const struct bus_type wq_subsys = { 7455 .name = "workqueue", 7456 .dev_groups = wq_sysfs_groups, 7457 }; 7458 7459 /** 7460 * workqueue_set_unbound_cpumask - Set the low-level unbound cpumask 7461 * @cpumask: the cpumask to set 7462 * 7463 * The low-level workqueues cpumask is a global cpumask that limits 7464 * the affinity of all unbound workqueues. This function check the @cpumask 7465 * and apply it to all unbound workqueues and updates all pwqs of them. 7466 * 7467 * Return: 0 - Success 7468 * -EINVAL - Invalid @cpumask 7469 * -ENOMEM - Failed to allocate memory for attrs or pwqs. 7470 */ 7471 static int workqueue_set_unbound_cpumask(cpumask_var_t cpumask) 7472 { 7473 int ret = -EINVAL; 7474 7475 /* 7476 * Not excluding isolated cpus on purpose. 7477 * If the user wishes to include them, we allow that. 7478 */ 7479 cpumask_and(cpumask, cpumask, cpu_possible_mask); 7480 if (!cpumask_empty(cpumask)) { 7481 ret = 0; 7482 apply_wqattrs_lock(); 7483 if (!cpumask_equal(cpumask, wq_unbound_cpumask)) 7484 ret = workqueue_apply_unbound_cpumask(cpumask); 7485 if (!ret) 7486 cpumask_copy(wq_requested_unbound_cpumask, cpumask); 7487 apply_wqattrs_unlock(); 7488 } 7489 7490 return ret; 7491 } 7492 7493 static ssize_t __wq_cpumask_show(struct device *dev, 7494 struct device_attribute *attr, char *buf, cpumask_var_t mask) 7495 { 7496 int written; 7497 7498 mutex_lock(&wq_pool_mutex); 7499 written = scnprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(mask)); 7500 mutex_unlock(&wq_pool_mutex); 7501 7502 return written; 7503 } 7504 7505 static ssize_t cpumask_requested_show(struct device *dev, 7506 struct device_attribute *attr, char *buf) 7507 { 7508 return __wq_cpumask_show(dev, attr, buf, wq_requested_unbound_cpumask); 7509 } 7510 static DEVICE_ATTR_RO(cpumask_requested); 7511 7512 static ssize_t cpumask_isolated_show(struct device *dev, 7513 struct device_attribute *attr, char *buf) 7514 { 7515 return __wq_cpumask_show(dev, attr, buf, wq_isolated_cpumask); 7516 } 7517 static DEVICE_ATTR_RO(cpumask_isolated); 7518 7519 static ssize_t cpumask_show(struct device *dev, 7520 struct device_attribute *attr, char *buf) 7521 { 7522 return __wq_cpumask_show(dev, attr, buf, wq_unbound_cpumask); 7523 } 7524 7525 static ssize_t cpumask_store(struct device *dev, 7526 struct device_attribute *attr, const char *buf, size_t count) 7527 { 7528 cpumask_var_t cpumask; 7529 int ret; 7530 7531 if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL)) 7532 return -ENOMEM; 7533 7534 ret = cpumask_parse(buf, cpumask); 7535 if (!ret) 7536 ret = workqueue_set_unbound_cpumask(cpumask); 7537 7538 free_cpumask_var(cpumask); 7539 return ret ? ret : count; 7540 } 7541 static DEVICE_ATTR_RW(cpumask); 7542 7543 static struct attribute *wq_sysfs_cpumask_attrs[] = { 7544 &dev_attr_cpumask.attr, 7545 &dev_attr_cpumask_requested.attr, 7546 &dev_attr_cpumask_isolated.attr, 7547 NULL, 7548 }; 7549 ATTRIBUTE_GROUPS(wq_sysfs_cpumask); 7550 7551 static int __init wq_sysfs_init(void) 7552 { 7553 return subsys_virtual_register(&wq_subsys, wq_sysfs_cpumask_groups); 7554 } 7555 core_initcall(wq_sysfs_init); 7556 7557 static void wq_device_release(struct device *dev) 7558 { 7559 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev); 7560 7561 kfree(wq_dev); 7562 } 7563 7564 /** 7565 * workqueue_sysfs_register - make a workqueue visible in sysfs 7566 * @wq: the workqueue to register 7567 * 7568 * Expose @wq in sysfs under /sys/bus/workqueue/devices. 7569 * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set 7570 * which is the preferred method. 7571 * 7572 * Workqueue user should use this function directly iff it wants to apply 7573 * workqueue_attrs before making the workqueue visible in sysfs; otherwise, 7574 * apply_workqueue_attrs() may race against userland updating the 7575 * attributes. 7576 * 7577 * Return: 0 on success, -errno on failure. 7578 */ 7579 int workqueue_sysfs_register(struct workqueue_struct *wq) 7580 { 7581 struct wq_device *wq_dev; 7582 int ret; 7583 7584 wq->wq_dev = wq_dev = kzalloc_obj(*wq_dev); 7585 if (!wq_dev) 7586 return -ENOMEM; 7587 7588 wq_dev->wq = wq; 7589 wq_dev->dev.bus = &wq_subsys; 7590 wq_dev->dev.release = wq_device_release; 7591 dev_set_name(&wq_dev->dev, "%s", wq->name); 7592 7593 /* 7594 * unbound_attrs are created separately. Suppress uevent until 7595 * everything is ready. 7596 */ 7597 dev_set_uevent_suppress(&wq_dev->dev, true); 7598 7599 ret = device_register(&wq_dev->dev); 7600 if (ret) { 7601 put_device(&wq_dev->dev); 7602 wq->wq_dev = NULL; 7603 return ret; 7604 } 7605 7606 if (wq->flags & WQ_UNBOUND) { 7607 struct device_attribute *attr; 7608 7609 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) { 7610 ret = device_create_file(&wq_dev->dev, attr); 7611 if (ret) { 7612 device_unregister(&wq_dev->dev); 7613 wq->wq_dev = NULL; 7614 return ret; 7615 } 7616 } 7617 } 7618 7619 dev_set_uevent_suppress(&wq_dev->dev, false); 7620 kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD); 7621 return 0; 7622 } 7623 7624 /** 7625 * workqueue_sysfs_unregister - undo workqueue_sysfs_register() 7626 * @wq: the workqueue to unregister 7627 * 7628 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister. 7629 */ 7630 static void workqueue_sysfs_unregister(struct workqueue_struct *wq) 7631 { 7632 struct wq_device *wq_dev = wq->wq_dev; 7633 7634 if (!wq->wq_dev) 7635 return; 7636 7637 wq->wq_dev = NULL; 7638 device_unregister(&wq_dev->dev); 7639 } 7640 #else /* CONFIG_SYSFS */ 7641 static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { } 7642 #endif /* CONFIG_SYSFS */ 7643 7644 /* 7645 * Workqueue watchdog. 7646 * 7647 * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal 7648 * flush dependency, a concurrency managed work item which stays RUNNING 7649 * indefinitely. Workqueue stalls can be very difficult to debug as the 7650 * usual warning mechanisms don't trigger and internal workqueue state is 7651 * largely opaque. 7652 * 7653 * Workqueue watchdog monitors all worker pools periodically and dumps 7654 * state if some pools failed to make forward progress for a while where 7655 * forward progress is defined as the first item on ->worklist changing. 7656 * 7657 * This mechanism is controlled through the kernel parameter 7658 * "workqueue.watchdog_thresh" which can be updated at runtime through the 7659 * corresponding sysfs parameter file. 7660 */ 7661 #ifdef CONFIG_WQ_WATCHDOG 7662 7663 static unsigned long wq_watchdog_thresh = 30; 7664 static struct timer_list wq_watchdog_timer; 7665 7666 static unsigned long wq_watchdog_touched = INITIAL_JIFFIES; 7667 static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES; 7668 7669 static unsigned int wq_panic_on_stall = CONFIG_BOOTPARAM_WQ_STALL_PANIC; 7670 module_param_named(panic_on_stall, wq_panic_on_stall, uint, 0644); 7671 7672 static unsigned int wq_panic_on_stall_time; 7673 module_param_named(panic_on_stall_time, wq_panic_on_stall_time, uint, 0644); 7674 MODULE_PARM_DESC(panic_on_stall_time, "Panic if stall exceeds this many seconds (0=disabled)"); 7675 7676 /* 7677 * Show workers that might prevent the processing of pending work items. 7678 * A busy worker that is not running on the CPU (e.g. sleeping in 7679 * wait_event_idle() with PF_WQ_WORKER cleared) can stall the pool just as 7680 * effectively as a CPU-bound one, so dump every in-flight worker. 7681 */ 7682 static void show_cpu_pool_busy_workers(struct worker_pool *pool) 7683 { 7684 struct worker *worker; 7685 unsigned long irq_flags; 7686 int bkt; 7687 7688 raw_spin_lock_irqsave(&pool->lock, irq_flags); 7689 7690 hash_for_each(pool->busy_hash, bkt, worker, hentry) { 7691 /* 7692 * Defer printing to avoid deadlocks in console 7693 * drivers that queue work while holding locks 7694 * also taken in their write paths. 7695 */ 7696 printk_deferred_enter(); 7697 7698 pr_info("pool %d:\n", pool->id); 7699 sched_show_task(worker->task); 7700 7701 printk_deferred_exit(); 7702 } 7703 7704 raw_spin_unlock_irqrestore(&pool->lock, irq_flags); 7705 } 7706 7707 static void show_cpu_pools_busy_workers(void) 7708 { 7709 struct worker_pool *pool; 7710 int pi; 7711 7712 pr_info("Showing backtraces of busy workers in stalled worker pools:\n"); 7713 7714 rcu_read_lock(); 7715 7716 for_each_pool(pool, pi) { 7717 if (pool->cpu_stall) 7718 show_cpu_pool_busy_workers(pool); 7719 7720 } 7721 7722 rcu_read_unlock(); 7723 } 7724 7725 /* 7726 * It triggers a panic in two scenarios: when the total number of stalls 7727 * exceeds a threshold, and when a stall lasts longer than 7728 * wq_panic_on_stall_time 7729 */ 7730 static void panic_on_wq_watchdog(unsigned int stall_time_sec) 7731 { 7732 static unsigned int wq_stall; 7733 7734 if (wq_panic_on_stall) { 7735 wq_stall++; 7736 if (wq_stall >= wq_panic_on_stall) 7737 panic("workqueue: %u stall(s) exceeded threshold %u\n", 7738 wq_stall, wq_panic_on_stall); 7739 } 7740 7741 if (wq_panic_on_stall_time && stall_time_sec >= wq_panic_on_stall_time) 7742 panic("workqueue: stall lasted %us, exceeding threshold %us\n", 7743 stall_time_sec, wq_panic_on_stall_time); 7744 } 7745 7746 static void wq_watchdog_reset_touched(void) 7747 { 7748 int cpu; 7749 7750 wq_watchdog_touched = jiffies; 7751 for_each_possible_cpu(cpu) 7752 per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies; 7753 } 7754 7755 static void wq_watchdog_timer_fn(struct timer_list *unused) 7756 { 7757 unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ; 7758 unsigned int max_stall_time = 0; 7759 bool lockup_detected = false; 7760 bool cpu_pool_stall = false; 7761 unsigned long now = jiffies; 7762 struct worker_pool *pool; 7763 unsigned int stall_time; 7764 int pi; 7765 7766 if (!thresh) 7767 return; 7768 7769 for_each_pool(pool, pi) { 7770 unsigned long pool_ts, touched, ts; 7771 7772 pool->cpu_stall = false; 7773 if (list_empty(&pool->worklist)) 7774 continue; 7775 7776 /* 7777 * If a virtual machine is stopped by the host it can look to 7778 * the watchdog like a stall. 7779 */ 7780 kvm_check_and_clear_guest_paused(); 7781 7782 /* get the latest of pool and touched timestamps */ 7783 if (pool->cpu >= 0) 7784 touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu)); 7785 else 7786 touched = READ_ONCE(wq_watchdog_touched); 7787 pool_ts = READ_ONCE(pool->last_progress_ts); 7788 7789 if (time_after(pool_ts, touched)) 7790 ts = pool_ts; 7791 else 7792 ts = touched; 7793 7794 /* 7795 * Did we stall? 7796 * 7797 * Do a lockless check first to do not disturb the system. 7798 * 7799 * Prevent false positives by double checking the timestamp 7800 * under pool->lock. The lock makes sure that the check reads 7801 * an updated pool->last_progress_ts when this CPU saw 7802 * an already updated pool->worklist above. It seems better 7803 * than adding another barrier into __queue_work() which 7804 * is a hotter path. 7805 */ 7806 if (time_after(now, ts + thresh)) { 7807 scoped_guard(raw_spinlock_irqsave, &pool->lock) { 7808 pool_ts = pool->last_progress_ts; 7809 if (time_after(pool_ts, touched)) 7810 ts = pool_ts; 7811 else 7812 ts = touched; 7813 } 7814 if (!time_after(now, ts + thresh)) 7815 continue; 7816 7817 lockup_detected = true; 7818 stall_time = jiffies_to_msecs(now - pool_ts) / 1000; 7819 max_stall_time = max(max_stall_time, stall_time); 7820 if (pool->cpu >= 0 && !(pool->flags & POOL_BH)) { 7821 pool->cpu_stall = true; 7822 cpu_pool_stall = true; 7823 } 7824 pr_emerg("BUG: workqueue lockup - pool"); 7825 pr_cont_pool_info(pool); 7826 pr_cont(" stuck for %us!\n", stall_time); 7827 } 7828 } 7829 7830 if (lockup_detected) 7831 show_all_workqueues(); 7832 7833 if (cpu_pool_stall) 7834 show_cpu_pools_busy_workers(); 7835 7836 if (lockup_detected) 7837 panic_on_wq_watchdog(max_stall_time); 7838 7839 wq_watchdog_reset_touched(); 7840 mod_timer(&wq_watchdog_timer, jiffies + thresh); 7841 } 7842 7843 notrace void wq_watchdog_touch(int cpu) 7844 { 7845 unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ; 7846 unsigned long touch_ts = READ_ONCE(wq_watchdog_touched); 7847 unsigned long now = jiffies; 7848 7849 if (cpu >= 0) 7850 per_cpu(wq_watchdog_touched_cpu, cpu) = now; 7851 else 7852 WARN_ONCE(1, "%s should be called with valid CPU", __func__); 7853 7854 /* Don't unnecessarily store to global cacheline */ 7855 if (time_after(now, touch_ts + thresh / 4)) 7856 WRITE_ONCE(wq_watchdog_touched, jiffies); 7857 } 7858 7859 static void wq_watchdog_set_thresh(unsigned long thresh) 7860 { 7861 wq_watchdog_thresh = 0; 7862 timer_delete_sync(&wq_watchdog_timer); 7863 7864 if (thresh) { 7865 wq_watchdog_thresh = thresh; 7866 wq_watchdog_reset_touched(); 7867 mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ); 7868 } 7869 } 7870 7871 static int wq_watchdog_param_set_thresh(const char *val, 7872 const struct kernel_param *kp) 7873 { 7874 unsigned long thresh; 7875 int ret; 7876 7877 ret = kstrtoul(val, 0, &thresh); 7878 if (ret) 7879 return ret; 7880 7881 if (system_percpu_wq) 7882 wq_watchdog_set_thresh(thresh); 7883 else 7884 wq_watchdog_thresh = thresh; 7885 7886 return 0; 7887 } 7888 7889 static const struct kernel_param_ops wq_watchdog_thresh_ops = { 7890 .set = wq_watchdog_param_set_thresh, 7891 .get = param_get_ulong, 7892 }; 7893 7894 module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh, 7895 0644); 7896 7897 static void wq_watchdog_init(void) 7898 { 7899 timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE); 7900 wq_watchdog_set_thresh(wq_watchdog_thresh); 7901 } 7902 7903 #else /* CONFIG_WQ_WATCHDOG */ 7904 7905 static inline void wq_watchdog_init(void) { } 7906 7907 #endif /* CONFIG_WQ_WATCHDOG */ 7908 7909 static void bh_pool_kick_normal(struct irq_work *irq_work) 7910 { 7911 raise_softirq_irqoff(TASKLET_SOFTIRQ); 7912 } 7913 7914 static void bh_pool_kick_highpri(struct irq_work *irq_work) 7915 { 7916 raise_softirq_irqoff(HI_SOFTIRQ); 7917 } 7918 7919 static void __init restrict_unbound_cpumask(const char *name, const struct cpumask *mask) 7920 { 7921 if (!cpumask_intersects(wq_unbound_cpumask, mask)) { 7922 pr_warn("workqueue: Restricting unbound_cpumask (%*pb) with %s (%*pb) leaves no CPU, ignoring\n", 7923 cpumask_pr_args(wq_unbound_cpumask), name, cpumask_pr_args(mask)); 7924 return; 7925 } 7926 7927 cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, mask); 7928 } 7929 7930 static void __init init_cpu_worker_pool(struct worker_pool *pool, int cpu, int nice) 7931 { 7932 BUG_ON(init_worker_pool(pool)); 7933 pool->cpu = cpu; 7934 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu)); 7935 cpumask_copy(pool->attrs->__pod_cpumask, cpumask_of(cpu)); 7936 pool->attrs->nice = nice; 7937 pool->attrs->affn_strict = true; 7938 pool->node = cpu_to_node(cpu); 7939 7940 /* alloc pool ID */ 7941 mutex_lock(&wq_pool_mutex); 7942 BUG_ON(worker_pool_assign_id(pool)); 7943 mutex_unlock(&wq_pool_mutex); 7944 } 7945 7946 /** 7947 * workqueue_init_early - early init for workqueue subsystem 7948 * 7949 * This is the first step of three-staged workqueue subsystem initialization and 7950 * invoked as soon as the bare basics - memory allocation, cpumasks and idr are 7951 * up. It sets up all the data structures and system workqueues and allows early 7952 * boot code to create workqueues and queue/cancel work items. Actual work item 7953 * execution starts only after kthreads can be created and scheduled right 7954 * before early initcalls. 7955 */ 7956 void __init workqueue_init_early(void) 7957 { 7958 struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM]; 7959 int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL }; 7960 void (*irq_work_fns[NR_STD_WORKER_POOLS])(struct irq_work *) = 7961 { bh_pool_kick_normal, bh_pool_kick_highpri }; 7962 int i, cpu; 7963 7964 BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long)); 7965 7966 BUG_ON(!alloc_cpumask_var(&wq_online_cpumask, GFP_KERNEL)); 7967 BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL)); 7968 BUG_ON(!alloc_cpumask_var(&wq_requested_unbound_cpumask, GFP_KERNEL)); 7969 BUG_ON(!zalloc_cpumask_var(&wq_isolated_cpumask, GFP_KERNEL)); 7970 7971 cpumask_copy(wq_online_cpumask, cpu_online_mask); 7972 cpumask_copy(wq_unbound_cpumask, cpu_possible_mask); 7973 restrict_unbound_cpumask("HK_TYPE_DOMAIN", housekeeping_cpumask(HK_TYPE_DOMAIN)); 7974 if (!cpumask_empty(&wq_cmdline_cpumask)) 7975 restrict_unbound_cpumask("workqueue.unbound_cpus", &wq_cmdline_cpumask); 7976 7977 cpumask_copy(wq_requested_unbound_cpumask, wq_unbound_cpumask); 7978 cpumask_andnot(wq_isolated_cpumask, cpu_possible_mask, 7979 housekeeping_cpumask(HK_TYPE_DOMAIN)); 7980 pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC); 7981 7982 unbound_wq_update_pwq_attrs_buf = alloc_workqueue_attrs(); 7983 BUG_ON(!unbound_wq_update_pwq_attrs_buf); 7984 7985 /* 7986 * If nohz_full is enabled, set power efficient workqueue as unbound. 7987 * This allows workqueue items to be moved to HK CPUs. 7988 */ 7989 if (housekeeping_enabled(HK_TYPE_TICK)) 7990 wq_power_efficient = true; 7991 7992 /* initialize WQ_AFFN_SYSTEM pods */ 7993 pt->pod_cpus = kzalloc_objs(pt->pod_cpus[0], 1); 7994 pt->pod_node = kzalloc_objs(pt->pod_node[0], 1); 7995 pt->cpu_pod = kzalloc_objs(pt->cpu_pod[0], nr_cpu_ids); 7996 BUG_ON(!pt->pod_cpus || !pt->pod_node || !pt->cpu_pod); 7997 7998 BUG_ON(!zalloc_cpumask_var_node(&pt->pod_cpus[0], GFP_KERNEL, NUMA_NO_NODE)); 7999 8000 pt->nr_pods = 1; 8001 cpumask_copy(pt->pod_cpus[0], cpu_possible_mask); 8002 pt->pod_node[0] = NUMA_NO_NODE; 8003 pt->cpu_pod[0] = 0; 8004 8005 /* initialize BH and CPU pools */ 8006 for_each_possible_cpu(cpu) { 8007 struct worker_pool *pool; 8008 8009 i = 0; 8010 for_each_bh_worker_pool(pool, cpu) { 8011 init_cpu_worker_pool(pool, cpu, std_nice[i]); 8012 pool->flags |= POOL_BH; 8013 init_irq_work(bh_pool_irq_work(pool), irq_work_fns[i]); 8014 i++; 8015 } 8016 8017 i = 0; 8018 for_each_cpu_worker_pool(pool, cpu) 8019 init_cpu_worker_pool(pool, cpu, std_nice[i++]); 8020 } 8021 8022 /* create default unbound and ordered wq attrs */ 8023 for (i = 0; i < NR_STD_WORKER_POOLS; i++) { 8024 struct workqueue_attrs *attrs; 8025 8026 BUG_ON(!(attrs = alloc_workqueue_attrs())); 8027 attrs->nice = std_nice[i]; 8028 unbound_std_wq_attrs[i] = attrs; 8029 8030 /* 8031 * An ordered wq should have only one pwq as ordering is 8032 * guaranteed by max_active which is enforced by pwqs. 8033 */ 8034 BUG_ON(!(attrs = alloc_workqueue_attrs())); 8035 attrs->nice = std_nice[i]; 8036 attrs->ordered = true; 8037 ordered_wq_attrs[i] = attrs; 8038 } 8039 8040 system_wq = alloc_workqueue("events", WQ_PERCPU, 0); 8041 system_percpu_wq = alloc_workqueue("events", WQ_PERCPU, 0); 8042 system_highpri_wq = alloc_workqueue("events_highpri", 8043 WQ_HIGHPRI | WQ_PERCPU, 0); 8044 system_long_wq = alloc_workqueue("events_long", WQ_PERCPU, 0); 8045 system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, WQ_MAX_ACTIVE); 8046 system_dfl_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, WQ_MAX_ACTIVE); 8047 system_freezable_wq = alloc_workqueue("events_freezable", 8048 WQ_FREEZABLE | WQ_PERCPU, 0); 8049 system_power_efficient_wq = alloc_workqueue("events_power_efficient", 8050 WQ_POWER_EFFICIENT | WQ_PERCPU, 0); 8051 system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_pwr_efficient", 8052 WQ_FREEZABLE | WQ_POWER_EFFICIENT | WQ_PERCPU, 0); 8053 system_bh_wq = alloc_workqueue("events_bh", WQ_BH | WQ_PERCPU, 0); 8054 system_bh_highpri_wq = alloc_workqueue("events_bh_highpri", 8055 WQ_BH | WQ_HIGHPRI | WQ_PERCPU, 0); 8056 system_dfl_long_wq = alloc_workqueue("events_dfl_long", WQ_UNBOUND, WQ_MAX_ACTIVE); 8057 BUG_ON(!system_wq || !system_percpu_wq|| !system_highpri_wq || !system_long_wq || 8058 !system_unbound_wq || !system_freezable_wq || !system_dfl_wq || 8059 !system_power_efficient_wq || 8060 !system_freezable_power_efficient_wq || 8061 !system_bh_wq || !system_bh_highpri_wq || !system_dfl_long_wq); 8062 } 8063 8064 static void __init wq_cpu_intensive_thresh_init(void) 8065 { 8066 unsigned long thresh; 8067 unsigned long bogo; 8068 8069 pwq_release_worker = kthread_run_worker(0, "pool_workqueue_release"); 8070 BUG_ON(IS_ERR(pwq_release_worker)); 8071 8072 /* if the user set it to a specific value, keep it */ 8073 if (wq_cpu_intensive_thresh_us != ULONG_MAX) 8074 return; 8075 8076 /* 8077 * The default of 10ms is derived from the fact that most modern (as of 8078 * 2023) processors can do a lot in 10ms and that it's just below what 8079 * most consider human-perceivable. However, the kernel also runs on a 8080 * lot slower CPUs including microcontrollers where the threshold is way 8081 * too low. 8082 * 8083 * Let's scale up the threshold upto 1 second if BogoMips is below 4000. 8084 * This is by no means accurate but it doesn't have to be. The mechanism 8085 * is still useful even when the threshold is fully scaled up. Also, as 8086 * the reports would usually be applicable to everyone, some machines 8087 * operating on longer thresholds won't significantly diminish their 8088 * usefulness. 8089 */ 8090 thresh = 10 * USEC_PER_MSEC; 8091 8092 /* see init/calibrate.c for lpj -> BogoMIPS calculation */ 8093 bogo = max_t(unsigned long, loops_per_jiffy / 500000 * HZ, 1); 8094 if (bogo < 4000) 8095 thresh = min_t(unsigned long, thresh * 4000 / bogo, USEC_PER_SEC); 8096 8097 pr_debug("wq_cpu_intensive_thresh: lpj=%lu BogoMIPS=%lu thresh_us=%lu\n", 8098 loops_per_jiffy, bogo, thresh); 8099 8100 wq_cpu_intensive_thresh_us = thresh; 8101 } 8102 8103 /** 8104 * workqueue_init - bring workqueue subsystem fully online 8105 * 8106 * This is the second step of three-staged workqueue subsystem initialization 8107 * and invoked as soon as kthreads can be created and scheduled. Workqueues have 8108 * been created and work items queued on them, but there are no kworkers 8109 * executing the work items yet. Populate the worker pools with the initial 8110 * workers and enable future kworker creations. 8111 */ 8112 void __init workqueue_init(void) 8113 { 8114 struct workqueue_struct *wq; 8115 struct worker_pool *pool; 8116 int cpu, bkt; 8117 8118 wq_cpu_intensive_thresh_init(); 8119 8120 mutex_lock(&wq_pool_mutex); 8121 8122 /* 8123 * Per-cpu pools created earlier could be missing node hint. Fix them 8124 * up. Also, create a rescuer for workqueues that requested it. 8125 */ 8126 for_each_possible_cpu(cpu) { 8127 for_each_bh_worker_pool(pool, cpu) 8128 pool->node = cpu_to_node(cpu); 8129 for_each_cpu_worker_pool(pool, cpu) 8130 pool->node = cpu_to_node(cpu); 8131 } 8132 8133 list_for_each_entry(wq, &workqueues, list) { 8134 WARN(init_rescuer(wq), 8135 "workqueue: failed to create early rescuer for %s", 8136 wq->name); 8137 } 8138 8139 mutex_unlock(&wq_pool_mutex); 8140 8141 /* 8142 * Create the initial workers. A BH pool has one pseudo worker that 8143 * represents the shared BH execution context and thus doesn't get 8144 * affected by hotplug events. Create the BH pseudo workers for all 8145 * possible CPUs here. 8146 */ 8147 for_each_possible_cpu(cpu) 8148 for_each_bh_worker_pool(pool, cpu) 8149 BUG_ON(!create_worker(pool)); 8150 8151 for_each_online_cpu(cpu) { 8152 for_each_cpu_worker_pool(pool, cpu) { 8153 pool->flags &= ~POOL_DISASSOCIATED; 8154 BUG_ON(!create_worker(pool)); 8155 } 8156 } 8157 8158 hash_for_each(unbound_pool_hash, bkt, pool, hash_node) 8159 BUG_ON(!create_worker(pool)); 8160 8161 wq_online = true; 8162 wq_watchdog_init(); 8163 } 8164 8165 /* 8166 * Initialize @pt by first initializing @pt->cpu_pod[] with pod IDs according to 8167 * @cpu_shares_pod(). Each subset of CPUs that share a pod is assigned a unique 8168 * and consecutive pod ID. The rest of @pt is initialized accordingly. 8169 */ 8170 static void __init init_pod_type(struct wq_pod_type *pt, 8171 bool (*cpus_share_pod)(int, int)) 8172 { 8173 int cur, pre, cpu, pod; 8174 8175 pt->nr_pods = 0; 8176 8177 /* init @pt->cpu_pod[] according to @cpus_share_pod() */ 8178 pt->cpu_pod = kzalloc_objs(pt->cpu_pod[0], nr_cpu_ids); 8179 BUG_ON(!pt->cpu_pod); 8180 8181 for_each_possible_cpu(cur) { 8182 for_each_possible_cpu(pre) { 8183 if (pre >= cur) { 8184 pt->cpu_pod[cur] = pt->nr_pods++; 8185 break; 8186 } 8187 if (cpus_share_pod(cur, pre)) { 8188 pt->cpu_pod[cur] = pt->cpu_pod[pre]; 8189 break; 8190 } 8191 } 8192 } 8193 8194 /* init the rest to match @pt->cpu_pod[] */ 8195 pt->pod_cpus = kzalloc_objs(pt->pod_cpus[0], pt->nr_pods); 8196 pt->pod_node = kzalloc_objs(pt->pod_node[0], pt->nr_pods); 8197 BUG_ON(!pt->pod_cpus || !pt->pod_node); 8198 8199 for (pod = 0; pod < pt->nr_pods; pod++) 8200 BUG_ON(!zalloc_cpumask_var(&pt->pod_cpus[pod], GFP_KERNEL)); 8201 8202 for_each_possible_cpu(cpu) { 8203 cpumask_set_cpu(cpu, pt->pod_cpus[pt->cpu_pod[cpu]]); 8204 pt->pod_node[pt->cpu_pod[cpu]] = cpu_to_node(cpu); 8205 } 8206 } 8207 8208 static bool __init cpus_dont_share(int cpu0, int cpu1) 8209 { 8210 return false; 8211 } 8212 8213 static bool __init cpus_share_smt(int cpu0, int cpu1) 8214 { 8215 #ifdef CONFIG_SCHED_SMT 8216 return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1)); 8217 #else 8218 return false; 8219 #endif 8220 } 8221 8222 static bool __init cpus_share_numa(int cpu0, int cpu1) 8223 { 8224 return cpu_to_node(cpu0) == cpu_to_node(cpu1); 8225 } 8226 8227 /* Maps each CPU to its shard index within the LLC pod it belongs to */ 8228 static int cpu_shard_id[NR_CPUS] __initdata; 8229 8230 /** 8231 * llc_count_cores - count distinct cores (SMT groups) within an LLC pod 8232 * @pod_cpus: the cpumask of CPUs in the LLC pod 8233 * @smt_pods: the SMT pod type, used to identify sibling groups 8234 * 8235 * A core is represented by the lowest-numbered CPU in its SMT group. Returns 8236 * the number of distinct cores found in @pod_cpus. 8237 */ 8238 static int __init llc_count_cores(const struct cpumask *pod_cpus, 8239 struct wq_pod_type *smt_pods) 8240 { 8241 const struct cpumask *sibling_cpus; 8242 int nr_cores = 0, c; 8243 8244 /* 8245 * Count distinct cores by only counting the first CPU in each 8246 * SMT sibling group. 8247 */ 8248 for_each_cpu(c, pod_cpus) { 8249 sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]]; 8250 if (cpumask_first(sibling_cpus) == c) 8251 nr_cores++; 8252 } 8253 8254 return nr_cores; 8255 } 8256 8257 /* 8258 * llc_shard_size - number of cores in a given shard 8259 * 8260 * Cores are spread as evenly as possible. The first @nr_large_shards shards are 8261 * "large shards" with (cores_per_shard + 1) cores; the rest are "default 8262 * shards" with cores_per_shard cores. 8263 */ 8264 static int __init llc_shard_size(int shard_id, int cores_per_shard, int nr_large_shards) 8265 { 8266 /* The first @nr_large_shards shards are large shards */ 8267 if (shard_id < nr_large_shards) 8268 return cores_per_shard + 1; 8269 8270 /* The remaining shards are default shards */ 8271 return cores_per_shard; 8272 } 8273 8274 /* 8275 * llc_calc_shard_layout - compute the shard layout for an LLC pod 8276 * @nr_cores: number of distinct cores in the LLC pod 8277 * 8278 * Chooses the number of shards that keeps average shard size closest to 8279 * wq_cache_shard_size. Returns a struct describing the total number of shards, 8280 * the base size of each, and how many are large shards. 8281 */ 8282 static struct llc_shard_layout __init llc_calc_shard_layout(int nr_cores) 8283 { 8284 struct llc_shard_layout layout; 8285 8286 /* Ensure at least one shard; pick the count closest to the target size */ 8287 layout.nr_shards = max(1, DIV_ROUND_CLOSEST(nr_cores, wq_cache_shard_size)); 8288 layout.cores_per_shard = nr_cores / layout.nr_shards; 8289 layout.nr_large_shards = nr_cores % layout.nr_shards; 8290 8291 return layout; 8292 } 8293 8294 /* 8295 * llc_shard_is_full - check whether a shard has reached its core capacity 8296 * @cores_in_shard: number of cores already assigned to this shard 8297 * @shard_id: index of the shard being checked 8298 * @layout: the shard layout computed by llc_calc_shard_layout() 8299 * 8300 * Returns true if @cores_in_shard equals the expected size for @shard_id. 8301 */ 8302 static bool __init llc_shard_is_full(int cores_in_shard, int shard_id, 8303 const struct llc_shard_layout *layout) 8304 { 8305 return cores_in_shard == llc_shard_size(shard_id, layout->cores_per_shard, 8306 layout->nr_large_shards); 8307 } 8308 8309 /** 8310 * llc_populate_cpu_shard_id - populate cpu_shard_id[] for each CPU in an LLC pod 8311 * @pod_cpus: the cpumask of CPUs in the LLC pod 8312 * @smt_pods: the SMT pod type, used to identify sibling groups 8313 * @nr_cores: number of distinct cores in @pod_cpus (from llc_count_cores()) 8314 * 8315 * Walks @pod_cpus in order. At each SMT group leader, advances to the next 8316 * shard once the current shard is full. Results are written to cpu_shard_id[]. 8317 */ 8318 static void __init llc_populate_cpu_shard_id(const struct cpumask *pod_cpus, 8319 struct wq_pod_type *smt_pods, 8320 int nr_cores) 8321 { 8322 struct llc_shard_layout layout = llc_calc_shard_layout(nr_cores); 8323 const struct cpumask *sibling_cpus; 8324 /* Count the number of cores in the current shard_id */ 8325 int cores_in_shard = 0; 8326 unsigned int leader; 8327 /* This is a cursor for the shards. Go from zero to nr_shards - 1*/ 8328 int shard_id = 0; 8329 int c; 8330 8331 /* Iterate at every CPU for a given LLC pod, and assign it a shard */ 8332 for_each_cpu(c, pod_cpus) { 8333 sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]]; 8334 if (cpumask_first(sibling_cpus) == c) { 8335 /* This is the CPU leader for the siblings */ 8336 if (llc_shard_is_full(cores_in_shard, shard_id, &layout)) { 8337 shard_id++; 8338 cores_in_shard = 0; 8339 } 8340 cores_in_shard++; 8341 cpu_shard_id[c] = shard_id; 8342 } else { 8343 /* 8344 * The siblings' shard MUST be the same as the leader. 8345 * never split threads in the same core. 8346 */ 8347 leader = cpumask_first(sibling_cpus); 8348 8349 /* 8350 * This check silences a Warray-bounds warning on UP 8351 * configs where NR_CPUS=1 makes cpu_shard_id[] 8352 * a single-element array, and the compiler can't 8353 * prove the index is always 0. 8354 */ 8355 if (WARN_ON_ONCE(leader >= nr_cpu_ids)) 8356 continue; 8357 cpu_shard_id[c] = cpu_shard_id[leader]; 8358 } 8359 } 8360 8361 WARN_ON_ONCE(shard_id != (layout.nr_shards - 1)); 8362 } 8363 8364 /** 8365 * precompute_cache_shard_ids - assign each CPU its shard index within its LLC 8366 * 8367 * Iterates over all LLC pods. For each pod, counts distinct cores then assigns 8368 * shard indices to all CPUs in the pod. Must be called after WQ_AFFN_CACHE and 8369 * WQ_AFFN_SMT have been initialized. 8370 */ 8371 static void __init precompute_cache_shard_ids(void) 8372 { 8373 struct wq_pod_type *llc_pods = &wq_pod_types[WQ_AFFN_CACHE]; 8374 struct wq_pod_type *smt_pods = &wq_pod_types[WQ_AFFN_SMT]; 8375 const struct cpumask *cpus_sharing_llc; 8376 int nr_cores; 8377 int pod; 8378 8379 if (!wq_cache_shard_size) { 8380 pr_warn("workqueue: cache_shard_size must be > 0, setting to 1\n"); 8381 wq_cache_shard_size = 1; 8382 } 8383 8384 for (pod = 0; pod < llc_pods->nr_pods; pod++) { 8385 cpus_sharing_llc = llc_pods->pod_cpus[pod]; 8386 8387 /* Number of cores in this given LLC */ 8388 nr_cores = llc_count_cores(cpus_sharing_llc, smt_pods); 8389 llc_populate_cpu_shard_id(cpus_sharing_llc, smt_pods, nr_cores); 8390 } 8391 } 8392 8393 /* 8394 * cpus_share_cache_shard - test whether two CPUs belong to the same cache shard 8395 * 8396 * Two CPUs share a cache shard if they are in the same LLC and have the same 8397 * shard index. Used as the pod affinity callback for WQ_AFFN_CACHE_SHARD. 8398 */ 8399 static bool __init cpus_share_cache_shard(int cpu0, int cpu1) 8400 { 8401 if (!cpus_share_cache(cpu0, cpu1)) 8402 return false; 8403 8404 return cpu_shard_id[cpu0] == cpu_shard_id[cpu1]; 8405 } 8406 8407 /** 8408 * workqueue_init_topology - initialize CPU pods for unbound workqueues 8409 * 8410 * This is the third step of three-staged workqueue subsystem initialization and 8411 * invoked after SMP and topology information are fully initialized. It 8412 * initializes the unbound CPU pods accordingly. 8413 */ 8414 void __init workqueue_init_topology(void) 8415 { 8416 struct workqueue_struct *wq; 8417 int cpu; 8418 8419 init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share); 8420 init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt); 8421 init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache); 8422 precompute_cache_shard_ids(); 8423 init_pod_type(&wq_pod_types[WQ_AFFN_CACHE_SHARD], cpus_share_cache_shard); 8424 init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa); 8425 8426 wq_topo_initialized = true; 8427 8428 mutex_lock(&wq_pool_mutex); 8429 8430 /* 8431 * Workqueues allocated earlier would have all CPUs sharing the default 8432 * worker pool. Explicitly call unbound_wq_update_pwq() on all workqueue 8433 * and CPU combinations to apply per-pod sharing. 8434 */ 8435 list_for_each_entry(wq, &workqueues, list) { 8436 for_each_online_cpu(cpu) 8437 unbound_wq_update_pwq(wq, cpu); 8438 if (wq->flags & WQ_UNBOUND) { 8439 mutex_lock(&wq->mutex); 8440 wq_update_node_max_active(wq, -1); 8441 mutex_unlock(&wq->mutex); 8442 } 8443 } 8444 8445 mutex_unlock(&wq_pool_mutex); 8446 } 8447 8448 void __warn_flushing_systemwide_wq(void) 8449 { 8450 pr_warn("WARNING: Flushing system-wide workqueues will be prohibited in near future.\n"); 8451 dump_stack(); 8452 } 8453 EXPORT_SYMBOL(__warn_flushing_systemwide_wq); 8454 8455 static int __init workqueue_unbound_cpus_setup(char *str) 8456 { 8457 if (cpulist_parse(str, &wq_cmdline_cpumask) < 0) { 8458 cpumask_clear(&wq_cmdline_cpumask); 8459 pr_warn("workqueue.unbound_cpus: incorrect CPU range, using default\n"); 8460 } 8461 8462 return 1; 8463 } 8464 __setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup); 8465