Lines Matching +full:fine +full:- +full:tune
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* sched.c - SPU scheduler.
7 * 2006-03-31 NUMA domains added.
55 * Priority of a normal, non-rt, non-niced'd process (aka nice level 0).
75 max(x * (MAX_PRIO - prio) / (NICE_WIDTH / 2), MIN_SPU_TIMESLICE)
78 * scale user-nice values [ -20 ... 0 ... 19 ] to time slice values:
87 if (ctx->prio < NORMAL_PRIO) in spu_set_timeslice()
88 ctx->time_slice = SCALE_PRIO(DEF_SPU_TIMESLICE * 4, ctx->prio); in spu_set_timeslice()
90 ctx->time_slice = SCALE_PRIO(DEF_SPU_TIMESLICE, ctx->prio); in spu_set_timeslice()
102 BUG_ON(!list_empty(&ctx->rq)); in __spu_update_sched_info()
105 * 32-Bit assignments are atomic on powerpc, and we don't care about in __spu_update_sched_info()
109 ctx->tid = current->pid; in __spu_update_sched_info()
113 * ->static_prio to start with. Unfortunately this field in __spu_update_sched_info()
115 * policy so we have to look at ->prio in this case. in __spu_update_sched_info()
117 if (rt_prio(current->prio)) in __spu_update_sched_info()
118 ctx->prio = current->prio; in __spu_update_sched_info()
120 ctx->prio = current->static_prio; in __spu_update_sched_info()
121 ctx->policy = current->policy; in __spu_update_sched_info()
131 cpumask_copy(&ctx->cpus_allowed, current->cpus_ptr); in __spu_update_sched_info()
134 ctx->last_ran = raw_smp_processor_id(); in __spu_update_sched_info()
141 if (ctx->state == SPU_STATE_RUNNABLE) { in spu_update_sched_info()
142 node = ctx->spu->node; in spu_update_sched_info()
160 if (cpumask_intersects(mask, &ctx->cpus_allowed)) in __node_allowed()
171 spin_lock(&spu_prio->runq_lock); in node_allowed()
173 spin_unlock(&spu_prio->runq_lock); in node_allowed()
190 if (spu->alloc_state != SPU_FREE) { in do_notify_spus_active()
191 struct spu_context *ctx = spu->ctx; in do_notify_spus_active()
193 &ctx->sched_flags); in do_notify_spus_active()
195 wake_up_all(&ctx->stop_wq); in do_notify_spus_active()
203 * spu_bind_context - bind spu context to physical spu
213 if (ctx->flags & SPU_CREATE_NOSCHED) in spu_bind_context()
214 atomic_inc(&cbe_spu_info[spu->node].reserved_spus); in spu_bind_context()
216 ctx->stats.slb_flt_base = spu->stats.slb_flt; in spu_bind_context()
217 ctx->stats.class2_intr_base = spu->stats.class2_intr; in spu_bind_context()
219 spu_associate_mm(spu, ctx->owner); in spu_bind_context()
221 spin_lock_irq(&spu->register_lock); in spu_bind_context()
222 spu->ctx = ctx; in spu_bind_context()
223 spu->flags = 0; in spu_bind_context()
224 ctx->spu = spu; in spu_bind_context()
225 ctx->ops = &spu_hw_ops; in spu_bind_context()
226 spu->pid = current->pid; in spu_bind_context()
227 spu->tgid = current->tgid; in spu_bind_context()
228 spu->ibox_callback = spufs_ibox_callback; in spu_bind_context()
229 spu->wbox_callback = spufs_wbox_callback; in spu_bind_context()
230 spu->stop_callback = spufs_stop_callback; in spu_bind_context()
231 spu->mfc_callback = spufs_mfc_callback; in spu_bind_context()
232 spin_unlock_irq(&spu->register_lock); in spu_bind_context()
237 spu_restore(&ctx->csa, spu); in spu_bind_context()
238 spu->timestamp = jiffies; in spu_bind_context()
239 ctx->state = SPU_STATE_RUNNABLE; in spu_bind_context()
249 BUG_ON(!mutex_is_locked(&cbe_spu_info[spu->node].list_mutex)); in sched_spu()
251 return (!spu->ctx || !(spu->ctx->flags & SPU_CREATE_NOSCHED)); in sched_spu()
258 list_for_each_entry(ctx, &gang->aff_list_head, aff_list) { in aff_merge_remaining_ctxs()
259 if (list_empty(&ctx->aff_list)) in aff_merge_remaining_ctxs()
260 list_add(&ctx->aff_list, &gang->aff_list_head); in aff_merge_remaining_ctxs()
262 gang->aff_flags |= AFF_MERGED; in aff_merge_remaining_ctxs()
270 offset = -1; in aff_set_offsets()
271 list_for_each_entry_reverse(ctx, &gang->aff_ref_ctx->aff_list, in aff_set_offsets()
273 if (&ctx->aff_list == &gang->aff_list_head) in aff_set_offsets()
275 ctx->aff_offset = offset--; in aff_set_offsets()
279 list_for_each_entry(ctx, gang->aff_ref_ctx->aff_list.prev, aff_list) { in aff_set_offsets()
280 if (&ctx->aff_list == &gang->aff_list_head) in aff_set_offsets()
282 ctx->aff_offset = offset++; in aff_set_offsets()
285 gang->aff_flags |= AFF_OFFSETS_SET; in aff_set_offsets()
318 if (spu->ctx && spu->ctx->gang && !spu->ctx->aff_offset in aff_ref_location()
319 && spu->ctx->gang->aff_ref_spu) in aff_ref_location()
320 available_spus -= spu->ctx->gang->contexts; in aff_ref_location()
323 if (available_spus < ctx->gang->contexts) { in aff_ref_location()
329 if ((!mem_aff || spu->has_mem_affinity) && in aff_ref_location()
345 mem_aff = gang->aff_ref_ctx->flags & SPU_CREATE_AFFINITY_MEM; in aff_set_ref_point_location()
349 list_for_each_entry(tmp, &gang->aff_list_head, aff_list) in aff_set_ref_point_location()
352 list_for_each_entry_reverse(ctx, &gang->aff_ref_ctx->aff_list, in aff_set_ref_point_location()
354 if (&ctx->aff_list == &gang->aff_list_head) in aff_set_ref_point_location()
356 lowest_offset = ctx->aff_offset; in aff_set_ref_point_location()
359 gang->aff_ref_spu = aff_ref_location(gang->aff_ref_ctx, mem_aff, gs, in aff_set_ref_point_location()
369 list_for_each_entry(spu, ref->aff_list.prev, aff_list) { in ctx_location()
370 BUG_ON(spu->node != node); in ctx_location()
374 offset--; in ctx_location()
377 list_for_each_entry_reverse(spu, ref->aff_list.next, aff_list) { in ctx_location()
378 BUG_ON(spu->node != node); in ctx_location()
395 struct spu_gang *gang = ctx->gang; in has_affinity()
397 if (list_empty(&ctx->aff_list)) in has_affinity()
400 if (atomic_read(&ctx->gang->aff_sched_count) == 0) in has_affinity()
401 ctx->gang->aff_ref_spu = NULL; in has_affinity()
403 if (!gang->aff_ref_spu) { in has_affinity()
404 if (!(gang->aff_flags & AFF_MERGED)) in has_affinity()
406 if (!(gang->aff_flags & AFF_OFFSETS_SET)) in has_affinity()
411 return gang->aff_ref_spu != NULL; in has_affinity()
415 * spu_unbind_context - unbind spu context from physical spu
427 if (spu->ctx->flags & SPU_CREATE_NOSCHED) in spu_unbind_context()
428 atomic_dec(&cbe_spu_info[spu->node].reserved_spus); in spu_unbind_context()
430 if (ctx->gang) in spu_unbind_context()
432 * If ctx->gang->aff_sched_count is positive, SPU affinity is in spu_unbind_context()
436 atomic_dec_if_positive(&ctx->gang->aff_sched_count); in spu_unbind_context()
439 spu_save(&ctx->csa, spu); in spu_unbind_context()
442 spin_lock_irq(&spu->register_lock); in spu_unbind_context()
443 spu->timestamp = jiffies; in spu_unbind_context()
444 ctx->state = SPU_STATE_SAVED; in spu_unbind_context()
445 spu->ibox_callback = NULL; in spu_unbind_context()
446 spu->wbox_callback = NULL; in spu_unbind_context()
447 spu->stop_callback = NULL; in spu_unbind_context()
448 spu->mfc_callback = NULL; in spu_unbind_context()
449 spu->pid = 0; in spu_unbind_context()
450 spu->tgid = 0; in spu_unbind_context()
451 ctx->ops = &spu_backing_ops; in spu_unbind_context()
452 spu->flags = 0; in spu_unbind_context()
453 spu->ctx = NULL; in spu_unbind_context()
454 spin_unlock_irq(&spu->register_lock); in spu_unbind_context()
458 ctx->stats.slb_flt += in spu_unbind_context()
459 (spu->stats.slb_flt - ctx->stats.slb_flt_base); in spu_unbind_context()
460 ctx->stats.class2_intr += in spu_unbind_context()
461 (spu->stats.class2_intr - ctx->stats.class2_intr_base); in spu_unbind_context()
465 ctx->spu = NULL; in spu_unbind_context()
468 wake_up_all(&ctx->stop_wq); in spu_unbind_context()
472 * spu_add_to_rq - add a context to the runqueue
486 * It's still quite hacky, and long-term we should proxy all other in __spu_add_to_rq()
490 if (list_empty(&ctx->rq)) { in __spu_add_to_rq()
491 list_add_tail(&ctx->rq, &spu_prio->runq[ctx->prio]); in __spu_add_to_rq()
492 set_bit(ctx->prio, spu_prio->bitmap); in __spu_add_to_rq()
493 if (!spu_prio->nr_waiting++) in __spu_add_to_rq()
500 spin_lock(&spu_prio->runq_lock); in spu_add_to_rq()
502 spin_unlock(&spu_prio->runq_lock); in spu_add_to_rq()
507 int prio = ctx->prio; in __spu_del_from_rq()
509 if (!list_empty(&ctx->rq)) { in __spu_del_from_rq()
510 if (!--spu_prio->nr_waiting) in __spu_del_from_rq()
512 list_del_init(&ctx->rq); in __spu_del_from_rq()
514 if (list_empty(&spu_prio->runq[prio])) in __spu_del_from_rq()
515 clear_bit(prio, spu_prio->bitmap); in __spu_del_from_rq()
521 spin_lock(&spu_prio->runq_lock); in spu_del_from_rq()
523 spin_unlock(&spu_prio->runq_lock); in spu_del_from_rq()
535 BUG_ON(!(ctx->flags & SPU_CREATE_NOSCHED)); in spu_prio_wait()
537 spin_lock(&spu_prio->runq_lock); in spu_prio_wait()
538 prepare_to_wait_exclusive(&ctx->stop_wq, &wait, TASK_INTERRUPTIBLE); in spu_prio_wait()
541 spin_unlock(&spu_prio->runq_lock); in spu_prio_wait()
542 mutex_unlock(&ctx->state_mutex); in spu_prio_wait()
544 mutex_lock(&ctx->state_mutex); in spu_prio_wait()
545 spin_lock(&spu_prio->runq_lock); in spu_prio_wait()
548 spin_unlock(&spu_prio->runq_lock); in spu_prio_wait()
550 remove_wait_queue(&ctx->stop_wq, &wait); in spu_prio_wait()
560 if (ctx->gang) { in spu_get_idle()
561 mutex_lock(&ctx->gang->aff_mutex); in spu_get_idle()
563 aff_ref_spu = ctx->gang->aff_ref_spu; in spu_get_idle()
564 atomic_inc(&ctx->gang->aff_sched_count); in spu_get_idle()
565 mutex_unlock(&ctx->gang->aff_mutex); in spu_get_idle()
566 node = aff_ref_spu->node; in spu_get_idle()
569 spu = ctx_location(aff_ref_spu, ctx->aff_offset, node); in spu_get_idle()
570 if (spu && spu->alloc_state == SPU_FREE) in spu_get_idle()
574 atomic_dec(&ctx->gang->aff_sched_count); in spu_get_idle()
577 mutex_unlock(&ctx->gang->aff_mutex); in spu_get_idle()
587 if (spu->alloc_state == SPU_FREE) in spu_get_idle()
598 spu->alloc_state = SPU_USED; in spu_get_idle()
606 * find_victim - find a lower priority context to preempt
623 * a strong node affinity. We might want to fine-tune this in in find_victim()
635 struct spu_context *tmp = spu->ctx; in find_victim()
637 if (tmp && tmp->prio > ctx->prio && in find_victim()
638 !(tmp->flags & SPU_CREATE_NOSCHED) && in find_victim()
639 (!victim || tmp->prio > victim->prio)) { in find_victim()
640 victim = spu->ctx; in find_victim()
649 * This nests ctx->state_mutex, but we always lock in find_victim()
658 if (!mutex_trylock(&victim->state_mutex)) { in find_victim()
664 spu = victim->spu; in find_victim()
665 if (!spu || victim->prio <= ctx->prio) { in find_victim()
671 mutex_unlock(&victim->state_mutex); in find_victim()
680 cbe_spu_info[node].nr_active--; in find_victim()
684 victim->stats.invol_ctx_switch++; in find_victim()
685 spu->stats.invol_ctx_switch++; in find_victim()
686 if (test_bit(SPU_SCHED_SPU_RUN, &victim->sched_flags)) in find_victim()
689 mutex_unlock(&victim->state_mutex); in find_victim()
701 int node = spu->node; in __spu_schedule()
707 if (spu->ctx == NULL) { in __spu_schedule()
710 spu->alloc_state = SPU_USED; in __spu_schedule()
716 wake_up_all(&ctx->run_wq); in __spu_schedule()
725 mutex_lock(&ctx->state_mutex); in spu_schedule()
726 if (ctx->state == SPU_STATE_SAVED) in spu_schedule()
732 * spu_unschedule - remove a context from a spu, and possibly release it.
737 * Unbinds the context @ctx from the SPU @spu. If @free_spu is non-zero, the
742 * Should be called with ctx->state_mutex held.
747 int node = spu->node; in spu_unschedule()
750 cbe_spu_info[node].nr_active--; in spu_unschedule()
752 spu->alloc_state = SPU_FREE; in spu_unschedule()
754 ctx->stats.invol_ctx_switch++; in spu_unschedule()
755 spu->stats.invol_ctx_switch++; in spu_unschedule()
760 * spu_activate - find a free spu for a context and execute it
778 if (ctx->spu) in spu_activate()
783 return -ERESTARTSYS; in spu_activate()
790 if (!spu && rt_prio(ctx->prio)) in spu_activate()
795 runcntl = ctx->ops->runcntl_read(ctx); in spu_activate()
803 if (ctx->flags & SPU_CREATE_NOSCHED) { in spu_activate()
814 * grab_runnable_context - try to find a runnable context
824 spin_lock(&spu_prio->runq_lock); in grab_runnable_context()
825 best = find_first_bit(spu_prio->bitmap, prio); in grab_runnable_context()
827 struct list_head *rq = &spu_prio->runq[best]; in grab_runnable_context()
840 spin_unlock(&spu_prio->runq_lock); in grab_runnable_context()
846 struct spu *spu = ctx->spu; in __spu_deactivate()
850 new = grab_runnable_context(max_prio, spu->node); in __spu_deactivate()
854 if (new->flags & SPU_CREATE_NOSCHED) in __spu_deactivate()
855 wake_up(&new->stop_wq); in __spu_deactivate()
861 mutex_lock(&ctx->state_mutex); in __spu_deactivate()
871 * spu_deactivate - unbind a context from its physical spu
884 * spu_yield - yield a physical spu if others are waiting
894 if (!(ctx->flags & SPU_CREATE_NOSCHED)) { in spu_yield()
895 mutex_lock(&ctx->state_mutex); in spu_yield()
897 mutex_unlock(&ctx->state_mutex); in spu_yield()
909 if (ctx->state != SPU_STATE_RUNNABLE) in spusched_tick()
911 if (ctx->flags & SPU_CREATE_NOSCHED) in spusched_tick()
913 if (ctx->policy == SCHED_FIFO) in spusched_tick()
916 if (--ctx->time_slice && test_bit(SPU_SCHED_SPU_RUN, &ctx->sched_flags)) in spusched_tick()
919 spu = ctx->spu; in spusched_tick()
923 new = grab_runnable_context(ctx->prio + 1, spu->node); in spusched_tick()
926 if (test_bit(SPU_SCHED_SPU_RUN, &ctx->sched_flags)) in spusched_tick()
930 if (!ctx->time_slice) in spusched_tick()
931 ctx->time_slice++; in spusched_tick()
941 * count_active_contexts - count nr of active tasks
955 nr_active += spu_prio->nr_waiting; in count_active_contexts()
961 * spu_calc_load - update the avenrun load estimates.
968 unsigned long active_tasks; /* fixed-point */ in spu_calc_load()
1002 struct spu_context *ctx = spu->ctx; in spusched_thread()
1029 delta = curtime - ctx->stats.tstamp; in spuctx_switch_state()
1031 WARN_ON(!mutex_is_locked(&ctx->state_mutex)); in spuctx_switch_state()
1034 spu = ctx->spu; in spuctx_switch_state()
1035 old_state = ctx->stats.util_state; in spuctx_switch_state()
1036 ctx->stats.util_state = new_state; in spuctx_switch_state()
1037 ctx->stats.tstamp = curtime; in spuctx_switch_state()
1043 ctx->stats.times[old_state] += delta; in spuctx_switch_state()
1044 spu->stats.times[old_state] += delta; in spuctx_switch_state()
1045 spu->stats.util_state = new_state; in spuctx_switch_state()
1046 spu->stats.tstamp = curtime; in spuctx_switch_state()
1047 node = spu->node; in spuctx_switch_state()
1075 idr_get_cursor(&task_active_pid_ns(current)->idr) - 1); in show_spu_loadavg()
1083 int err = -ENOMEM, i; in spu_sched_init()
1090 INIT_LIST_HEAD(&spu_prio->runq[i]); in spu_sched_init()
1091 __clear_bit(i, spu_prio->bitmap); in spu_sched_init()
1093 spin_lock_init(&spu_prio->runq_lock); in spu_sched_init()
1136 if (spu->alloc_state != SPU_FREE) in spu_sched_exit()
1137 spu->alloc_state = SPU_FREE; in spu_sched_exit()