Lines Matching +full:- +full:affinity
1 // SPDX-License-Identifier: GPL-2.0-only
84 WARN_ON(!(k->flags & PF_KTHREAD)); in to_kthread()
85 return k->worker_private; in to_kthread()
92 if (!kthread || !kthread->full_name) { in get_kthread_comm()
93 strscpy(buf, tsk->comm, buf_size); in get_kthread_comm()
97 strscpy_pad(buf, kthread->full_name, buf_size); in get_kthread_comm()
111 init_completion(&kthread->exited); in set_kthread_struct()
112 init_completion(&kthread->parked); in set_kthread_struct()
113 INIT_LIST_HEAD(&kthread->affinity_node); in set_kthread_struct()
114 p->vfork_done = &kthread->exited; in set_kthread_struct()
116 kthread->task = p; in set_kthread_struct()
117 kthread->node = tsk_fork_get_node(current); in set_kthread_struct()
118 p->worker_private = kthread; in set_kthread_struct()
134 WARN_ON_ONCE(kthread->blkcg_css); in free_kthread_struct()
136 k->worker_private = NULL; in free_kthread_struct()
137 kfree(kthread->full_name); in free_kthread_struct()
142 * kthread_should_stop - should this kthread return now?
150 return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags); in kthread_should_stop()
156 return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags); in __kthread_should_park()
160 * kthread_should_park - should this kthread park now?
183 return kthread->flags & (BIT(KTHREAD_SHOULD_STOP) | BIT(KTHREAD_SHOULD_PARK)); in kthread_should_stop_or_park()
187 * kthread_freezable_should_stop - should this freezable kthread return now?
212 * kthread_func - return the function specified on kthread creation
221 return kthread->threadfn; in kthread_func()
227 * kthread_data - return data value specified on kthread creation
236 return to_kthread(task)->data; in kthread_data()
241 * kthread_probe_data - speculative version of kthread_data()
255 copy_from_kernel_nofault(&data, &kthread->data, sizeof(data)); in kthread_probe_data()
264 * possible pending wakeups to avoid store-store collisions on in __kthread_parkme()
265 * task->state. in __kthread_parkme()
272 if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags)) in __kthread_parkme()
281 complete(&self->parked); in __kthread_parkme()
296 kthread->result = result; in kthread_do_exit()
297 if (!list_empty(&kthread->affinity_node)) { in kthread_do_exit()
299 list_del(&kthread->affinity_node); in kthread_do_exit()
302 if (kthread->preferred_affinity) { in kthread_do_exit()
303 kfree(kthread->preferred_affinity); in kthread_do_exit()
304 kthread->preferred_affinity = NULL; in kthread_do_exit()
310 * kthread_complete_and_exit - Exit the current kthread.
336 if (kthread->preferred_affinity) { in kthread_fetch_affinity()
337 pref = kthread->preferred_affinity; in kthread_fetch_affinity()
339 if (kthread->node == NUMA_NO_NODE) in kthread_fetch_affinity()
342 pref = cpumask_of_node(kthread->node); in kthread_fetch_affinity()
353 cpumask_var_t affinity; in kthread_affine_node() local
358 if (!zalloc_cpumask_var(&affinity, GFP_KERNEL)) { in kthread_affine_node()
364 WARN_ON_ONCE(!list_empty(&kthread->affinity_node)); in kthread_affine_node()
365 list_add_tail(&kthread->affinity_node, &kthread_affinity_list); in kthread_affine_node()
368 * - a racing CPU going down will either fail on the subsequent in kthread_affine_node()
371 * - a racing CPU going up will be handled by kthreads_online_cpu() in kthread_affine_node()
373 kthread_fetch_affinity(kthread, affinity); in kthread_affine_node()
374 set_cpus_allowed_ptr(current, affinity); in kthread_affine_node()
377 free_cpumask_var(affinity); in kthread_affine_node()
385 int (*threadfn)(void *data) = create->threadfn; in kthread()
386 void *data = create->data; in kthread()
394 done = xchg(&create->done, NULL); in kthread()
396 kfree(create->full_name); in kthread()
398 kthread_exit(-EINTR); in kthread()
401 self->full_name = create->full_name; in kthread()
402 self->threadfn = threadfn; in kthread()
403 self->data = data; in kthread()
413 create->result = current; in kthread()
423 self->started = 1; in kthread()
426 * Apply default node affinity if no call to kthread_bind[_mask]() nor in kthread()
427 * kthread_affine_preferred() was issued before the first wake-up. in kthread()
429 if (!(current->flags & PF_NO_SETAFFINITY) && !self->preferred_affinity) in kthread()
432 ret = -EINTR; in kthread()
433 if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) { in kthread()
446 return tsk->pref_node_fork; in tsk_fork_get_node()
456 current->pref_node_fork = create->node; in create_kthread()
459 pid = kernel_thread(kthread, create, create->full_name, in create_kthread()
463 struct completion *done = xchg(&create->done, NULL); in create_kthread()
465 kfree(create->full_name); in create_kthread()
470 create->result = ERR_PTR(pid); in create_kthread()
486 return ERR_PTR(-ENOMEM); in __kthread_create_on_node()
487 create->threadfn = threadfn; in __kthread_create_on_node()
488 create->data = data; in __kthread_create_on_node()
489 create->node = node; in __kthread_create_on_node()
490 create->done = &done; in __kthread_create_on_node()
491 create->full_name = kvasprintf(GFP_KERNEL, namefmt, args); in __kthread_create_on_node()
492 if (!create->full_name) { in __kthread_create_on_node()
493 task = ERR_PTR(-ENOMEM); in __kthread_create_on_node()
498 list_add_tail(&create->list, &kthread_create_list); in __kthread_create_on_node()
513 if (xchg(&create->done, NULL)) in __kthread_create_on_node()
514 return ERR_PTR(-EINTR); in __kthread_create_on_node()
521 task = create->result; in __kthread_create_on_node()
528 * kthread_create_on_node - create a kthread.
532 * @namefmt: printf-style name for the thread.
540 * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE.
548 * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR).
573 scoped_guard (raw_spinlock_irqsave, &p->pi_lock) in __kthread_bind_mask()
577 p->flags |= PF_NO_SETAFFINITY; in __kthread_bind_mask()
589 WARN_ON_ONCE(kthread->started); in kthread_bind_mask()
593 * kthread_bind - bind a just-created kthread to a cpu.
605 WARN_ON_ONCE(kthread->started); in kthread_bind()
610 * kthread_create_on_cpu - Create a cpu bound kthread
614 * @namefmt: printf-style name for the thread. Format is restricted
631 to_kthread(p)->cpu = cpu; in kthread_create_on_cpu()
642 WARN_ON_ONCE(!(k->flags & PF_NO_SETAFFINITY)); in kthread_set_per_cpu()
645 clear_bit(KTHREAD_IS_PER_CPU, &kthread->flags); in kthread_set_per_cpu()
649 kthread->cpu = cpu; in kthread_set_per_cpu()
650 set_bit(KTHREAD_IS_PER_CPU, &kthread->flags); in kthread_set_per_cpu()
659 return test_bit(KTHREAD_IS_PER_CPU, &kthread->flags); in kthread_is_per_cpu()
663 * kthread_unpark - unpark a thread created by kthread_create().
674 if (!test_bit(KTHREAD_SHOULD_PARK, &kthread->flags)) in kthread_unpark()
680 if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags)) in kthread_unpark()
681 __kthread_bind(k, kthread->cpu, TASK_PARKED); in kthread_unpark()
683 clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags); in kthread_unpark()
692 * kthread_park - park a thread created by kthread_create().
700 * Returns 0 if the thread is parked, -ENOSYS if the thread exited.
707 if (WARN_ON(k->flags & PF_EXITING)) in kthread_park()
708 return -ENOSYS; in kthread_park()
710 if (WARN_ON_ONCE(test_bit(KTHREAD_SHOULD_PARK, &kthread->flags))) in kthread_park()
711 return -EBUSY; in kthread_park()
713 set_bit(KTHREAD_SHOULD_PARK, &kthread->flags); in kthread_park()
720 wait_for_completion(&kthread->parked); in kthread_park()
733 * kthread_stop - stop a thread created by kthread_create().
744 * Returns the result of threadfn(), or %-EINTR if wake_up_process()
756 set_bit(KTHREAD_SHOULD_STOP, &kthread->flags); in kthread_stop()
760 wait_for_completion(&kthread->exited); in kthread_stop()
761 ret = kthread->result; in kthread_stop()
770 * kthread_stop_put - stop a thread and put its task struct
797 current->flags |= PF_NOFREEZE; in kthreadd()
814 list_del_init(&create->list); in kthreadd()
828 * kthread_affine_preferred - Define a kthread's preferred affinity
833 * Similar to kthread_bind_mask() except that the affinity is not a requirement
837 * Returns 0 if the affinity has been applied.
842 cpumask_var_t affinity; in kthread_affine_preferred() local
845 if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE) || kthread->started) { in kthread_affine_preferred()
847 return -EINVAL; in kthread_affine_preferred()
850 WARN_ON_ONCE(kthread->preferred_affinity); in kthread_affine_preferred()
852 if (!zalloc_cpumask_var(&affinity, GFP_KERNEL)) in kthread_affine_preferred()
853 return -ENOMEM; in kthread_affine_preferred()
855 kthread->preferred_affinity = kzalloc_obj(struct cpumask); in kthread_affine_preferred()
856 if (!kthread->preferred_affinity) { in kthread_affine_preferred()
857 ret = -ENOMEM; in kthread_affine_preferred()
862 cpumask_copy(kthread->preferred_affinity, mask); in kthread_affine_preferred()
863 WARN_ON_ONCE(!list_empty(&kthread->affinity_node)); in kthread_affine_preferred()
864 list_add_tail(&kthread->affinity_node, &kthread_affinity_list); in kthread_affine_preferred()
865 kthread_fetch_affinity(kthread, affinity); in kthread_affine_preferred()
867 scoped_guard (raw_spinlock_irqsave, &p->pi_lock) in kthread_affine_preferred()
868 set_cpus_allowed_force(p, affinity); in kthread_affine_preferred()
872 free_cpumask_var(affinity); in kthread_affine_preferred()
880 cpumask_var_t affinity; in kthreads_update_affinity() local
889 if (!zalloc_cpumask_var(&affinity, GFP_KERNEL)) in kthreads_update_affinity()
890 return -ENOMEM; in kthreads_update_affinity()
895 if (WARN_ON_ONCE((k->task->flags & PF_NO_SETAFFINITY) || in kthreads_update_affinity()
896 kthread_is_per_cpu(k->task))) { in kthreads_update_affinity()
897 ret = -EINVAL; in kthreads_update_affinity()
902 * Unbound kthreads without preferred affinity are already affine in kthreads_update_affinity()
907 * But kthreads with a preferred affinity or node are different: in kthreads_update_affinity()
913 if (force || k->preferred_affinity || k->node != NUMA_NO_NODE) { in kthreads_update_affinity()
914 kthread_fetch_affinity(k, affinity); in kthreads_update_affinity()
915 set_cpus_allowed_ptr(k->task, affinity); in kthreads_update_affinity()
919 free_cpumask_var(affinity); in kthreads_update_affinity()
925 * kthreads_update_housekeeping - Update kthreads affinity on cpuset change
929 * affinity.
931 * Returns 0 if successful, -ENOMEM if temporary mask couldn't
932 * be allocated or -EINVAL in case of internal error.
940 * Re-affine kthreads according to their preferences
942 * by select_fallback_rq() which default re-affines to
944 * affinity doesn't apply anymore.
963 raw_spin_lock_init(&worker->lock); in __kthread_init_worker()
964 lockdep_set_class_and_name(&worker->lock, key, name); in __kthread_init_worker()
965 INIT_LIST_HEAD(&worker->work_list); in __kthread_init_worker()
966 INIT_LIST_HEAD(&worker->delayed_work_list); in __kthread_init_worker()
971 * kthread_worker_fn - kthread function to process kthread_worker
994 WARN_ON(worker->task && worker->task != current); in kthread_worker_fn()
995 worker->task = current; in kthread_worker_fn()
997 if (worker->flags & KTW_FREEZABLE) in kthread_worker_fn()
1005 raw_spin_lock_irq(&worker->lock); in kthread_worker_fn()
1006 worker->task = NULL; in kthread_worker_fn()
1007 raw_spin_unlock_irq(&worker->lock); in kthread_worker_fn()
1012 raw_spin_lock_irq(&worker->lock); in kthread_worker_fn()
1013 if (!list_empty(&worker->work_list)) { in kthread_worker_fn()
1014 work = list_first_entry(&worker->work_list, in kthread_worker_fn()
1016 list_del_init(&work->node); in kthread_worker_fn()
1018 worker->current_work = work; in kthread_worker_fn()
1019 raw_spin_unlock_irq(&worker->lock); in kthread_worker_fn()
1022 kthread_work_func_t func = work->func; in kthread_worker_fn()
1025 work->func(work); in kthread_worker_fn()
1057 return ERR_PTR(-ENOMEM); in __kthread_create_worker_on_node()
1066 worker->flags = flags; in __kthread_create_worker_on_node()
1067 worker->task = task; in __kthread_create_worker_on_node()
1077 * kthread_create_worker_on_node - create a kthread worker
1080 * @namefmt: printf-style name for the kthread worker (task).
1082 * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
1083 * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
1101 * kthread_create_worker_on_cpu - create a kthread worker and bind it
1105 * @namefmt: printf-style name for the thread. Format is restricted
1123 * - CPU affinity gets lost when it is scheduled on an offline CPU.
1125 * - The worker might not exist when the CPU was off when the user
1132 * The pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
1133 * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
1144 kthread_bind(worker->task, cpu); in kthread_create_worker_on_cpu()
1158 lockdep_assert_held(&worker->lock); in queuing_blocked()
1160 return !list_empty(&work->node) || work->canceling; in queuing_blocked()
1166 lockdep_assert_held(&worker->lock); in kthread_insert_work_sanity_check()
1167 WARN_ON_ONCE(!list_empty(&work->node)); in kthread_insert_work_sanity_check()
1169 WARN_ON_ONCE(work->worker && work->worker != worker); in kthread_insert_work_sanity_check()
1181 list_add_tail(&work->node, pos); in kthread_insert_work()
1182 work->worker = worker; in kthread_insert_work()
1183 if (!worker->current_work && likely(worker->task)) in kthread_insert_work()
1184 wake_up_process(worker->task); in kthread_insert_work()
1188 * kthread_queue_work - queue a kthread_work
1205 raw_spin_lock_irqsave(&worker->lock, flags); in kthread_queue_work()
1207 kthread_insert_work(worker, work, &worker->work_list); in kthread_queue_work()
1210 raw_spin_unlock_irqrestore(&worker->lock, flags); in kthread_queue_work()
1216 * kthread_delayed_work_timer_fn - callback that queues the associated kthread
1227 struct kthread_work *work = &dwork->work; in kthread_delayed_work_timer_fn()
1228 struct kthread_worker *worker = work->worker; in kthread_delayed_work_timer_fn()
1238 raw_spin_lock_irqsave(&worker->lock, flags); in kthread_delayed_work_timer_fn()
1240 WARN_ON_ONCE(work->worker != worker); in kthread_delayed_work_timer_fn()
1242 /* Move the work from worker->delayed_work_list. */ in kthread_delayed_work_timer_fn()
1243 WARN_ON_ONCE(list_empty(&work->node)); in kthread_delayed_work_timer_fn()
1244 list_del_init(&work->node); in kthread_delayed_work_timer_fn()
1245 if (!work->canceling) in kthread_delayed_work_timer_fn()
1246 kthread_insert_work(worker, work, &worker->work_list); in kthread_delayed_work_timer_fn()
1248 raw_spin_unlock_irqrestore(&worker->lock, flags); in kthread_delayed_work_timer_fn()
1256 struct timer_list *timer = &dwork->timer; in __kthread_queue_delayed_work()
1257 struct kthread_work *work = &dwork->work; in __kthread_queue_delayed_work()
1259 WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn); in __kthread_queue_delayed_work()
1262 * If @delay is 0, queue @dwork->work immediately. This is for in __kthread_queue_delayed_work()
1268 kthread_insert_work(worker, work, &worker->work_list); in __kthread_queue_delayed_work()
1275 list_add(&work->node, &worker->delayed_work_list); in __kthread_queue_delayed_work()
1276 work->worker = worker; in __kthread_queue_delayed_work()
1277 timer->expires = jiffies + delay; in __kthread_queue_delayed_work()
1282 * kthread_queue_delayed_work - queue the associated kthread work
1300 struct kthread_work *work = &dwork->work; in kthread_queue_delayed_work()
1304 raw_spin_lock_irqsave(&worker->lock, flags); in kthread_queue_delayed_work()
1311 raw_spin_unlock_irqrestore(&worker->lock, flags); in kthread_queue_delayed_work()
1325 complete(&fwork->done); in kthread_flush_work_fn()
1329 * kthread_flush_work - flush a kthread_work
1343 worker = work->worker; in kthread_flush_work()
1347 raw_spin_lock_irq(&worker->lock); in kthread_flush_work()
1349 WARN_ON_ONCE(work->worker != worker); in kthread_flush_work()
1351 if (!list_empty(&work->node)) in kthread_flush_work()
1352 kthread_insert_work(worker, &fwork.work, work->node.next); in kthread_flush_work()
1353 else if (worker->current_work == work) in kthread_flush_work()
1355 worker->work_list.next); in kthread_flush_work()
1359 raw_spin_unlock_irq(&worker->lock); in kthread_flush_work()
1370 * The function is called under worker->lock. The lock is temporary
1378 struct kthread_worker *worker = work->worker; in kthread_cancel_delayed_work_timer()
1386 work->canceling++; in kthread_cancel_delayed_work_timer()
1387 raw_spin_unlock_irqrestore(&worker->lock, *flags); in kthread_cancel_delayed_work_timer()
1388 timer_delete_sync(&dwork->timer); in kthread_cancel_delayed_work_timer()
1389 raw_spin_lock_irqsave(&worker->lock, *flags); in kthread_cancel_delayed_work_timer()
1390 work->canceling--; in kthread_cancel_delayed_work_timer()
1396 * It is called under worker->lock. The caller must make sure that
1410 * be from worker->work_list or from worker->delayed_work_list. in __kthread_cancel_work()
1412 if (!list_empty(&work->node)) { in __kthread_cancel_work()
1413 list_del_init(&work->node); in __kthread_cancel_work()
1421 * kthread_mod_delayed_work - modify delay of or queue a kthread delayed work
1447 struct kthread_work *work = &dwork->work; in kthread_mod_delayed_work()
1451 raw_spin_lock_irqsave(&worker->lock, flags); in kthread_mod_delayed_work()
1454 if (!work->worker) { in kthread_mod_delayed_work()
1460 WARN_ON_ONCE(work->worker != worker); in kthread_mod_delayed_work()
1469 * The timer must be canceled first because worker->lock is released in kthread_mod_delayed_work()
1475 if (work->canceling) { in kthread_mod_delayed_work()
1485 raw_spin_unlock_irqrestore(&worker->lock, flags); in kthread_mod_delayed_work()
1492 struct kthread_worker *worker = work->worker; in __kthread_cancel_work_sync()
1499 raw_spin_lock_irqsave(&worker->lock, flags); in __kthread_cancel_work_sync()
1501 WARN_ON_ONCE(work->worker != worker); in __kthread_cancel_work_sync()
1508 if (worker->current_work != work) in __kthread_cancel_work_sync()
1515 work->canceling++; in __kthread_cancel_work_sync()
1516 raw_spin_unlock_irqrestore(&worker->lock, flags); in __kthread_cancel_work_sync()
1518 raw_spin_lock_irqsave(&worker->lock, flags); in __kthread_cancel_work_sync()
1519 work->canceling--; in __kthread_cancel_work_sync()
1522 raw_spin_unlock_irqrestore(&worker->lock, flags); in __kthread_cancel_work_sync()
1528 * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish
1532 * can be used even if the work re-queues itself. On return from this
1535 * kthread_cancel_work_sync(&delayed_work->work) must not be used for
1550 * kthread_cancel_delayed_work_sync - cancel a kthread delayed work and
1560 return __kthread_cancel_work_sync(&dwork->work, true); in kthread_cancel_delayed_work_sync()
1565 * kthread_flush_worker - flush all current works on a kthread_worker
1584 * kthread_destroy_worker - destroy a kthread worker
1588 * worker API is used only in trivial scenarios. There are no multi-step state
1599 task = worker->task; in kthread_destroy_worker()
1605 WARN_ON(!list_empty(&worker->delayed_work_list)); in kthread_destroy_worker()
1606 WARN_ON(!list_empty(&worker->work_list)); in kthread_destroy_worker()
1612 * kthread_use_mm - make the calling kthread operate on an address space
1620 WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD)); in kthread_use_mm()
1621 WARN_ON_ONCE(tsk->mm); in kthread_use_mm()
1624 * It is possible for mm to be the same as tsk->active_mm, but in kthread_use_mm()
1633 active_mm = tsk->active_mm; in kthread_use_mm()
1634 tsk->active_mm = mm; in kthread_use_mm()
1635 tsk->mm = mm; in kthread_use_mm()
1647 * that tsk->mm, and not issue an IPI. Membarrier requires a in kthread_use_mm()
1648 * memory barrier after storing to tsk->mm, before accessing in kthread_use_mm()
1649 * user-space memory. A full memory barrier for membarrier in kthread_use_mm()
1658 * kthread_unuse_mm - reverse the effect of kthread_use_mm()
1665 WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD)); in kthread_unuse_mm()
1666 WARN_ON_ONCE(!tsk->mm); in kthread_unuse_mm()
1672 * that tsk->mm, and not issue an IPI. Membarrier requires a in kthread_unuse_mm()
1673 * memory barrier after accessing user-space memory, before in kthread_unuse_mm()
1674 * clearing tsk->mm. in kthread_unuse_mm()
1678 tsk->mm = NULL; in kthread_unuse_mm()
1692 * kthread_associate_blkcg - associate blkcg to current kthread
1705 if (!(current->flags & PF_KTHREAD)) in kthread_associate_blkcg()
1711 if (kthread->blkcg_css) { in kthread_associate_blkcg()
1712 css_put(kthread->blkcg_css); in kthread_associate_blkcg()
1713 kthread->blkcg_css = NULL; in kthread_associate_blkcg()
1717 kthread->blkcg_css = css; in kthread_associate_blkcg()
1723 * kthread_blkcg - get associated blkcg css of current kthread
1731 if (current->flags & PF_KTHREAD) { in kthread_blkcg()
1734 return kthread->blkcg_css; in kthread_blkcg()