Lines Matching full:group
43 * The scheduling happens at the scheduling group level, each group
52 * rotating the groups passed to the firmware so every group gets
68 * queue ring-buffer, and the group is scheduled for execution if it
71 * Kernel-side group scheduling is timeslice-based. When we have less
74 * groups than slots, we let each group a chance to execute stuff for
76 * to schedule. The group selection algorithm is based on
81 * group/queue state that would be based on information we wouldn't have
83 * reason we don't do 'cooperative' scheduling (encoding FW group slot
85 * drm_gpu_scheduler::prepare_job() hook, and treating group rotation as
101 * struct panthor_csg_slot - Command stream group slot
103 * This represents a FW slot for a scheduling group.
106 /** @group: Scheduling group bound to this slot. */
107 struct panthor_group *group; member
109 /** @priority: Group priority. */
113 * @idle: True if the group bound to this slot is idle.
115 * A group is idle when it has nothing waiting for execution on
123 * enum panthor_csg_priority - Group priority
126 /** @PANTHOR_CSG_PRIORITY_LOW: Low priority group. */
129 /** @PANTHOR_CSG_PRIORITY_MEDIUM: Medium priority group. */
132 /** @PANTHOR_CSG_PRIORITY_HIGH: High priority group. */
136 * @PANTHOR_CSG_PRIORITY_RT: Real-time priority group.
139 * non-real-time groups. When such a group becomes executable,
140 * it will evict the group with the lowest non-rt priority if
141 * there's no free group slot available.
160 * Used for the scheduler tick, group update or other kind of FW
232 * @runnable: Runnable group lists.
234 * When a group has queues that want to execute something,
242 * @idle: Idle group lists.
244 * When all queues of a group are idle (either because they
256 * Insert panthor_group::wait_node here when a group is waiting
265 * @csg_slots: FW command stream group slots.
269 /** @csg_slot_count: Number of command stream group slots exposed by the FW. */
272 /** @cs_slot_count: Number of command stream slot per group slot exposed by the FW. */
278 /** @used_csg_slot_count: Number of command stream group slot currently used. */
285 * @might_have_idle_groups: True if an active group might have become idle.
386 * is assigned to group_slot + 1 when the group is assigned a slot. But
388 * point, so don't have to wake up all queues in a group every time one
394 * @priority: Priority of the queue inside the group.
500 * enum panthor_group_state - Scheduling group state.
503 /** @PANTHOR_CS_GROUP_CREATED: Group was created, but not scheduled yet. */
506 /** @PANTHOR_CS_GROUP_ACTIVE: Group is currently scheduled. */
510 * @PANTHOR_CS_GROUP_SUSPENDED: Group was scheduled at least once, but is
516 * @PANTHOR_CS_GROUP_TERMINATED: Group was terminated.
523 * @PANTHOR_CS_GROUP_UNKNOWN_STATE: Group is an unknown state.
525 * The FW returned an inconsistent state. The group is flagged unusable
536 * struct panthor_group - Scheduling group object
545 /** @vm: VM bound to the group. */
566 /** @priority: Group priority (check panthor_csg_priority). */
584 /** @queue_count: Number of queues in this group. */
587 /** @queues: Queues owned by this group. */
591 * @csg_id: ID of the FW group slot.
593 * -1 when the group is not scheduled/active.
598 * @destroyed: True when the group has been destroyed.
600 * If a group is destroyed it becomes useless: no further jobs can be submitted
602 * release the group object.
608 * this group.
611 * and the group can't be suspended, this also leads to a timeout. In any case,
612 * any timeout situation is unrecoverable, and the group becomes useless. We
613 * simply wait for all references to be dropped so we can release the group
619 * @innocent: True when the group becomes unusable because the group suspension
622 * Sometimes the FW was put in a bad state by other groups, causing the group
624 * group innocent.
638 /** @data: Total sampled values for jobs in queues from this group. */
647 /** @fdinfo.kbo_sizes: Aggregate size of private kernel BO's held by the group. */
651 /** @task_info: Info of current->group_leader that created the group. */
660 /** @state: Group state. */
666 * Stores the state of the group and its queues when a group is suspended.
667 * Used at resume time to restore the group in its previous state.
676 * Stores the state of the group and its queues when a group that's in
679 * Used at resume time to restore the group in its previous state.
689 /** @tiler_oom_work: Work used to process tiler OOM events happening on this group. */
692 /** @term_work: Work used to finish the group termination procedure. */
696 * @release_work: Work used to release group resources.
698 * We need to postpone the group release to avoid a deadlock when
704 * @run_node: Node used to insert the group in the
711 * @wait_node: Node used to insert the group in the
730 * group_queue_work() - Queue a group work
731 * @group: Group to queue the work for.
739 #define group_queue_work(group, wname) \ argument
741 group_get(group); \
742 if (!queue_work((group)->ptdev->scheduler->wq, &(group)->wname ## _work)) \
743 group_put(group); \
783 * struct panthor_group_pool - Group pool
785 * Each file get assigned a group pool.
788 /** @xa: Xarray used to manage group handles. */
802 /** @group: Group of the queue this job will be pushed to. */
803 struct panthor_group *group; member
805 /** @queue_idx: Index of the queue inside @group. */
868 panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue *queue) in panthor_queue_get_syncwait_obj() argument
870 struct panthor_device *ptdev = group->ptdev; in panthor_queue_get_syncwait_obj()
878 bo = panthor_vm_get_bo_for_va(group->vm, in panthor_queue_get_syncwait_obj()
900 static void group_free_queue(struct panthor_group *group, struct panthor_queue *queue) in group_free_queue() argument
906 drm_WARN_ON(&group->ptdev->base, in group_free_queue()
931 struct panthor_group *group = container_of(work, in group_release_work() local
936 for (i = 0; i < group->queue_count; i++) in group_release_work()
937 group_free_queue(group, group->queues[i]); in group_release_work()
939 panthor_kernel_bo_destroy(group->suspend_buf); in group_release_work()
940 panthor_kernel_bo_destroy(group->protm_suspend_buf); in group_release_work()
941 panthor_kernel_bo_destroy(group->syncobjs); in group_release_work()
943 panthor_vm_put(group->vm); in group_release_work()
944 kfree(group); in group_release_work()
949 struct panthor_group *group = container_of(kref, in group_release() local
952 struct panthor_device *ptdev = group->ptdev; in group_release()
954 drm_WARN_ON(&ptdev->base, group->csg_id >= 0); in group_release()
955 drm_WARN_ON(&ptdev->base, !list_empty(&group->run_node)); in group_release()
956 drm_WARN_ON(&ptdev->base, !list_empty(&group->wait_node)); in group_release()
958 queue_work(panthor_cleanup_wq, &group->release_work); in group_release()
961 static void group_put(struct panthor_group *group) in group_put() argument
963 if (group) in group_put()
964 kref_put(&group->refcount, group_release); in group_put()
968 group_get(struct panthor_group *group) in group_get() argument
970 if (group) in group_get()
971 kref_get(&group->refcount); in group_get()
973 return group; in group_get()
977 * group_bind_locked() - Bind a group to a group slot
978 * @group: Group.
984 group_bind_locked(struct panthor_group *group, u32 csg_id) in group_bind_locked() argument
986 struct panthor_device *ptdev = group->ptdev; in group_bind_locked()
992 if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS || in group_bind_locked()
993 ptdev->scheduler->csg_slots[csg_id].group)) in group_bind_locked()
996 ret = panthor_vm_active(group->vm); in group_bind_locked()
1001 group_get(group); in group_bind_locked()
1002 group->csg_id = csg_id; in group_bind_locked()
1004 /* Dummy doorbell allocation: doorbell is assigned to the group and in group_bind_locked()
1009 * on queues belonging to the same group that are rarely updated. in group_bind_locked()
1011 for (u32 i = 0; i < group->queue_count; i++) in group_bind_locked()
1012 group->queues[i]->doorbell_id = csg_id + 1; in group_bind_locked()
1014 csg_slot->group = group; in group_bind_locked()
1020 * group_unbind_locked() - Unbind a group from a slot.
1021 * @group: Group to unbind.
1026 group_unbind_locked(struct panthor_group *group) in group_unbind_locked() argument
1028 struct panthor_device *ptdev = group->ptdev; in group_unbind_locked()
1033 if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS)) in group_unbind_locked()
1036 if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE)) in group_unbind_locked()
1039 slot = &ptdev->scheduler->csg_slots[group->csg_id]; in group_unbind_locked()
1040 panthor_vm_idle(group->vm); in group_unbind_locked()
1041 group->csg_id = -1; in group_unbind_locked()
1043 /* Tiler OOM events will be re-issued next time the group is scheduled. */ in group_unbind_locked()
1044 atomic_set(&group->tiler_oom, 0); in group_unbind_locked()
1045 cancel_work(&group->tiler_oom_work); in group_unbind_locked()
1047 for (u32 i = 0; i < group->queue_count; i++) in group_unbind_locked()
1048 group->queues[i]->doorbell_id = -1; in group_unbind_locked()
1050 slot->group = NULL; in group_unbind_locked()
1052 group_put(group); in group_unbind_locked()
1057 group_is_idle(struct panthor_group *group) in group_is_idle() argument
1059 struct panthor_device *ptdev = group->ptdev; in group_is_idle()
1062 if (group->csg_id >= 0) in group_is_idle()
1063 return ptdev->scheduler->csg_slots[group->csg_id].idle; in group_is_idle()
1065 inactive_queues = group->idle_queues | group->blocked_queues; in group_is_idle()
1066 return hweight32(inactive_queues) == group->queue_count; in group_is_idle()
1082 group_can_run(struct panthor_group *group) in group_can_run() argument
1084 return group->state != PANTHOR_CS_GROUP_TERMINATED && in group_can_run()
1085 group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE && in group_can_run()
1086 !group->destroyed && group->fatal_queues == 0 && in group_can_run()
1087 !group->timedout; in group_can_run()
1101 struct panthor_group *group; in queue_suspend_timeout_locked() local
1113 group = job ? job->group : NULL; in queue_suspend_timeout_locked()
1115 /* If the queue is blocked and the group is idle, we want the timer to in queue_suspend_timeout_locked()
1116 * keep running because the group can't be unblocked by other queues, in queue_suspend_timeout_locked()
1120 if (group && group_can_run(group) && in queue_suspend_timeout_locked()
1121 (group->blocked_queues & BIT(job->queue_idx)) && in queue_suspend_timeout_locked()
1122 group_is_idle(group)) in queue_suspend_timeout_locked()
1168 * @csg_id: Group slot ID.
1174 * The group slot must have a group bound to it already (group_bind_locked()).
1179 struct panthor_queue *queue = ptdev->scheduler->csg_slots[csg_id].group->queues[cs_id]; in cs_slot_prog_locked()
1210 * @csg_id: Group slot.
1216 * The group slot must have a group bound to it (group_bind_locked()).
1222 struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group; in cs_slot_reset_locked() local
1223 struct panthor_queue *queue = group->queues[cs_id]; in cs_slot_reset_locked()
1237 * csg_slot_sync_priority_locked() - Synchronize the group slot priority
1239 * @csg_id: Group slot ID.
1241 * Group slot priority update happens asynchronously. When we receive a
1262 * @csg_id: Group slot.
1265 * Queue state is updated on group suspend or STATUS_UPDATE event.
1270 struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group; in cs_slot_sync_queue_state_locked() local
1271 struct panthor_queue *queue = group->queues[cs_id]; in cs_slot_sync_queue_state_locked()
1273 panthor_fw_get_cs_iface(group->ptdev, csg_id, cs_id); in cs_slot_sync_queue_state_locked()
1281 group->idle_queues |= BIT(cs_id); in cs_slot_sync_queue_state_locked()
1285 if (list_empty(&group->wait_node)) { in cs_slot_sync_queue_state_locked()
1286 list_move_tail(&group->wait_node, in cs_slot_sync_queue_state_locked()
1287 &group->ptdev->scheduler->groups.waiting); in cs_slot_sync_queue_state_locked()
1294 group->blocked_queues |= BIT(cs_id); in cs_slot_sync_queue_state_locked()
1322 struct panthor_group *group = csg_slot->group; in csg_slot_sync_queues_state_locked() local
1327 group->idle_queues = 0; in csg_slot_sync_queues_state_locked()
1328 group->blocked_queues = 0; in csg_slot_sync_queues_state_locked()
1330 for (i = 0; i < group->queue_count; i++) { in csg_slot_sync_queues_state_locked()
1331 if (group->queues[i]) in csg_slot_sync_queues_state_locked()
1341 struct panthor_group *group; in csg_slot_sync_state_locked() local
1348 group = csg_slot->group; in csg_slot_sync_state_locked()
1350 if (!group) in csg_slot_sync_state_locked()
1353 old_state = group->state; in csg_slot_sync_state_locked()
1368 * which means the group metadata can't be trusted anymore, and in csg_slot_sync_state_locked()
1370 * suspend buffers. Flag the group state as unknown to make in csg_slot_sync_state_locked()
1395 * state when starting/resuming a new group on this in csg_slot_sync_state_locked()
1400 for (i = 0; i < group->queue_count; i++) { in csg_slot_sync_state_locked()
1401 if (group->queues[i]) in csg_slot_sync_state_locked()
1406 group->state = new_state; in csg_slot_sync_state_locked()
1414 struct panthor_group *group; in csg_slot_prog_locked() local
1427 group = csg_slot->group; in csg_slot_prog_locked()
1428 if (!group || group->state == PANTHOR_CS_GROUP_ACTIVE) in csg_slot_prog_locked()
1431 csg_iface = panthor_fw_get_csg_iface(group->ptdev, csg_id); in csg_slot_prog_locked()
1433 for (i = 0; i < group->queue_count; i++) { in csg_slot_prog_locked()
1434 if (group->queues[i]) { in csg_slot_prog_locked()
1440 csg_iface->input->allow_compute = group->compute_core_mask; in csg_slot_prog_locked()
1441 csg_iface->input->allow_fragment = group->fragment_core_mask; in csg_slot_prog_locked()
1442 csg_iface->input->allow_other = group->tiler_core_mask; in csg_slot_prog_locked()
1443 endpoint_req = CSG_EP_REQ_COMPUTE(group->max_compute_cores) | in csg_slot_prog_locked()
1444 CSG_EP_REQ_FRAGMENT(group->max_fragment_cores) | in csg_slot_prog_locked()
1445 CSG_EP_REQ_TILER(group->max_tiler_cores) | in csg_slot_prog_locked()
1449 csg_iface->input->config = panthor_vm_as(group->vm); in csg_slot_prog_locked()
1451 if (group->suspend_buf) in csg_slot_prog_locked()
1452 csg_iface->input->suspend_buf = panthor_kernel_bo_gpuva(group->suspend_buf); in csg_slot_prog_locked()
1456 if (group->protm_suspend_buf) { in csg_slot_prog_locked()
1458 panthor_kernel_bo_gpuva(group->protm_suspend_buf); in csg_slot_prog_locked()
1474 struct panthor_group *group = csg_slot->group; in cs_slot_process_fatal_event_locked() local
1485 if (group) { in cs_slot_process_fatal_event_locked()
1487 group->task_info.pid, group->task_info.comm); in cs_slot_process_fatal_event_locked()
1489 group->fatal_queues |= BIT(cs_id); in cs_slot_process_fatal_event_locked()
1520 struct panthor_group *group = csg_slot->group; in cs_slot_process_fault_event_locked() local
1521 struct panthor_queue *queue = group && cs_id < group->queue_count ? in cs_slot_process_fault_event_locked()
1522 group->queues[cs_id] : NULL; in cs_slot_process_fault_event_locked()
1550 if (group) { in cs_slot_process_fault_event_locked()
1552 group->task_info.pid, group->task_info.comm); in cs_slot_process_fault_event_locked()
1567 static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id) in group_process_tiler_oom() argument
1569 struct panthor_device *ptdev = group->ptdev; in group_process_tiler_oom()
1578 csg_id = group->csg_id; in group_process_tiler_oom()
1583 heaps = panthor_vm_get_heap_pool(group->vm, false); in group_process_tiler_oom()
1593 /* The group got scheduled out, we stop here. We will get a new tiler OOM event in group_process_tiler_oom()
1617 group->fatal_queues |= BIT(cs_id); in group_process_tiler_oom()
1623 csg_id = group->csg_id; in group_process_tiler_oom()
1640 * context because the group was scheduled out while we were in group_process_tiler_oom()
1655 struct panthor_group *group = in group_tiler_oom_work() local
1657 u32 tiler_oom = atomic_xchg(&group->tiler_oom, 0); in group_tiler_oom_work()
1662 group_process_tiler_oom(group, cs_id); in group_tiler_oom_work()
1666 group_put(group); in group_tiler_oom_work()
1675 struct panthor_group *group = csg_slot->group; in cs_slot_process_tiler_oom_event_locked() local
1679 if (drm_WARN_ON(&ptdev->base, !group)) in cs_slot_process_tiler_oom_event_locked()
1682 atomic_or(BIT(cs_id), &group->tiler_oom); in cs_slot_process_tiler_oom_event_locked()
1687 group_get(group); in cs_slot_process_tiler_oom_event_locked()
1688 if (!queue_work(sched->heap_alloc_wq, &group->tiler_oom_work)) in cs_slot_process_tiler_oom_event_locked()
1689 group_put(group); in cs_slot_process_tiler_oom_event_locked()
1752 struct panthor_group *group = csg_slot->group; in csg_slot_sync_update_locked() local
1756 if (group) in csg_slot_sync_update_locked()
1757 group_queue_work(group, sync_upd); in csg_slot_sync_update_locked()
1767 struct panthor_group *group = csg_slot->group; in csg_slot_process_progress_timer_event_locked() local
1771 group = csg_slot->group; in csg_slot_process_progress_timer_event_locked()
1772 if (!drm_WARN_ON(&ptdev->base, !group)) { in csg_slot_process_progress_timer_event_locked()
1774 group->task_info.pid, group->task_info.comm); in csg_slot_process_progress_timer_event_locked()
1776 group->timedout = true; in csg_slot_process_progress_timer_event_locked()
2038 struct panthor_group *group, *tmp; in tick_ctx_pick_groups_from_list() local
2043 list_for_each_entry_safe(group, tmp, queue, run_node) { in tick_ctx_pick_groups_from_list()
2046 if (!group_can_run(group)) in tick_ctx_pick_groups_from_list()
2049 if (skip_idle_groups && group_is_idle(group)) in tick_ctx_pick_groups_from_list()
2053 if (ctx->vms[i] == group->vm) in tick_ctx_pick_groups_from_list()
2061 group_get(group); in tick_ctx_pick_groups_from_list()
2063 list_move_tail(&group->run_node, &ctx->groups[group->priority]); in tick_ctx_pick_groups_from_list()
2065 if (group_is_idle(group)) in tick_ctx_pick_groups_from_list()
2069 ctx->vms[ctx->as_count++] = group->vm; in tick_ctx_pick_groups_from_list()
2071 if (ctx->min_priority > group->priority) in tick_ctx_pick_groups_from_list()
2072 ctx->min_priority = group->priority; in tick_ctx_pick_groups_from_list()
2082 struct panthor_group *group, in tick_ctx_insert_old_group() argument
2085 struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id]; in tick_ctx_insert_old_group()
2089 list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); in tick_ctx_insert_old_group()
2097 * important to make sure we don't let one group starve in tick_ctx_insert_old_group()
2098 * all other groups with the same group priority. in tick_ctx_insert_old_group()
2101 &ctx->old_groups[csg_slot->group->priority], in tick_ctx_insert_old_group()
2106 list_add_tail(&csg_slot->group->run_node, &other_group->run_node); in tick_ctx_insert_old_group()
2111 list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); in tick_ctx_insert_old_group()
2135 struct panthor_group *group = csg_slot->group; in tick_ctx_init() local
2138 if (!group) in tick_ctx_init()
2142 group_get(group); in tick_ctx_init()
2147 if (panthor_vm_has_unhandled_faults(group->vm)) { in tick_ctx_init()
2151 if (!group->fatal_queues) in tick_ctx_init()
2152 group->fatal_queues |= GENMASK(group->queue_count - 1, 0); in tick_ctx_init()
2155 tick_ctx_insert_old_group(sched, ctx, group, full_tick); in tick_ctx_init()
2169 group_term_post_processing(struct panthor_group *group) in group_term_post_processing() argument
2176 if (drm_WARN_ON(&group->ptdev->base, group_can_run(group))) in group_term_post_processing()
2180 for (i = 0; i < group->queue_count; i++) { in group_term_post_processing()
2181 struct panthor_queue *queue = group->queues[i]; in group_term_post_processing()
2185 if (group->fatal_queues & BIT(i)) in group_term_post_processing()
2187 else if (group->timedout) in group_term_post_processing()
2204 syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj)); in group_term_post_processing()
2207 sched_queue_work(group->ptdev->scheduler, sync_upd); in group_term_post_processing()
2219 struct panthor_group *group = in group_term_work() local
2222 group_term_post_processing(group); in group_term_work()
2223 group_put(group); in group_term_work()
2231 struct panthor_group *group, *tmp; in tick_ctx_cleanup() local
2235 list_for_each_entry_safe(group, tmp, &ctx->old_groups[i], run_node) { in tick_ctx_cleanup()
2240 group_can_run(group)); in tick_ctx_cleanup()
2242 if (!group_can_run(group)) { in tick_ctx_cleanup()
2243 list_del_init(&group->run_node); in tick_ctx_cleanup()
2244 list_del_init(&group->wait_node); in tick_ctx_cleanup()
2245 group_queue_work(group, term); in tick_ctx_cleanup()
2246 } else if (group->csg_id >= 0) { in tick_ctx_cleanup()
2247 list_del_init(&group->run_node); in tick_ctx_cleanup()
2249 list_move(&group->run_node, in tick_ctx_cleanup()
2250 group_is_idle(group) ? in tick_ctx_cleanup()
2251 &sched->groups.idle[group->priority] : in tick_ctx_cleanup()
2252 &sched->groups.runnable[group->priority]); in tick_ctx_cleanup()
2254 group_put(group); in tick_ctx_cleanup()
2265 list_for_each_entry_safe(group, tmp, &ctx->groups[i], run_node) { in tick_ctx_cleanup()
2266 if (group->csg_id >= 0) { in tick_ctx_cleanup()
2267 list_del_init(&group->run_node); in tick_ctx_cleanup()
2269 list_move(&group->run_node, in tick_ctx_cleanup()
2270 group_is_idle(group) ? in tick_ctx_cleanup()
2271 &sched->groups.idle[group->priority] : in tick_ctx_cleanup()
2272 &sched->groups.runnable[group->priority]); in tick_ctx_cleanup()
2274 group_put(group); in tick_ctx_cleanup()
2282 struct panthor_group *group, *tmp; in tick_ctx_apply() local
2294 list_for_each_entry(group, &ctx->old_groups[prio], run_node) { in tick_ctx_apply()
2295 bool term = !group_can_run(group); in tick_ctx_apply()
2296 int csg_id = group->csg_id; in tick_ctx_apply()
2308 list_for_each_entry(group, &ctx->groups[prio], run_node) { in tick_ctx_apply()
2310 int csg_id = group->csg_id; in tick_ctx_apply()
2343 list_for_each_entry(group, &ctx->old_groups[prio], run_node) { in tick_ctx_apply()
2344 /* This group is gone. Process interrupts to clear in tick_ctx_apply()
2346 * group. in tick_ctx_apply()
2348 if (group->csg_id >= 0) in tick_ctx_apply()
2349 sched_process_csg_irq_locked(ptdev, group->csg_id); in tick_ctx_apply()
2351 group_unbind_locked(group); in tick_ctx_apply()
2356 if (!sched->csg_slots[i].group) in tick_ctx_apply()
2365 list_for_each_entry(group, &ctx->groups[prio], run_node) { in tick_ctx_apply()
2366 int csg_id = group->csg_id; in tick_ctx_apply()
2380 group_bind_locked(group, csg_id); in tick_ctx_apply()
2383 group->state == PANTHOR_CS_GROUP_SUSPENDED ? in tick_ctx_apply()
2401 list_for_each_entry_safe(group, tmp, &ctx->groups[prio], run_node) { in tick_ctx_apply()
2402 list_del_init(&group->run_node); in tick_ctx_apply()
2404 /* If the group has been destroyed while we were in tick_ctx_apply()
2407 * this dangling group. in tick_ctx_apply()
2409 if (group->destroyed) in tick_ctx_apply()
2411 group_put(group); in tick_ctx_apply()
2419 list_for_each_entry_safe(group, tmp, &ctx->old_groups[prio], run_node) { in tick_ctx_apply()
2420 if (!group_can_run(group)) in tick_ctx_apply()
2423 if (group_is_idle(group)) in tick_ctx_apply()
2424 list_move_tail(&group->run_node, &sched->groups.idle[prio]); in tick_ctx_apply()
2426 list_move_tail(&group->run_node, &sched->groups.runnable[prio]); in tick_ctx_apply()
2427 group_put(group); in tick_ctx_apply()
2444 * event happens (group unblocked, new job submitted, ...). in tick_ctx_update_resched_target()
2574 static int panthor_queue_eval_syncwait(struct panthor_group *group, u8 queue_idx) in panthor_queue_eval_syncwait() argument
2576 struct panthor_queue *queue = group->queues[queue_idx]; in panthor_queue_eval_syncwait()
2584 syncobj = panthor_queue_get_syncwait_obj(group, queue); in panthor_queue_eval_syncwait()
2608 struct panthor_group *group, *tmp; in sync_upd_work() local
2612 list_for_each_entry_safe(group, tmp, &sched->groups.waiting, wait_node) { in sync_upd_work()
2613 u32 tested_queues = group->blocked_queues; in sync_upd_work()
2620 ret = panthor_queue_eval_syncwait(group, cs_id); in sync_upd_work()
2621 drm_WARN_ON(&group->ptdev->base, ret < 0); in sync_upd_work()
2629 group->blocked_queues &= ~unblocked_queues; in sync_upd_work()
2631 if (group->csg_id < 0) { in sync_upd_work()
2632 list_move(&group->run_node, in sync_upd_work()
2633 &sched->groups.runnable[group->priority]); in sync_upd_work()
2634 if (group->priority == PANTHOR_CSG_PRIORITY_RT) in sync_upd_work()
2639 if (!group->blocked_queues) in sync_upd_work()
2640 list_del_init(&group->wait_node); in sync_upd_work()
2648 static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) in group_schedule_locked() argument
2650 struct panthor_device *ptdev = group->ptdev; in group_schedule_locked()
2652 struct list_head *queue = &sched->groups.runnable[group->priority]; in group_schedule_locked()
2657 if (!group_can_run(group)) in group_schedule_locked()
2661 if ((queue_mask & group->blocked_queues) == queue_mask) in group_schedule_locked()
2664 was_idle = group_is_idle(group); in group_schedule_locked()
2665 group->idle_queues &= ~queue_mask; in group_schedule_locked()
2671 if (was_idle && !group_is_idle(group)) in group_schedule_locked()
2672 list_move_tail(&group->run_node, queue); in group_schedule_locked()
2675 if (group->priority == PANTHOR_CSG_PRIORITY_RT) { in group_schedule_locked()
2728 static void panthor_group_stop(struct panthor_group *group) in panthor_group_stop() argument
2730 struct panthor_scheduler *sched = group->ptdev->scheduler; in panthor_group_stop()
2734 for (u32 i = 0; i < group->queue_count; i++) in panthor_group_stop()
2735 queue_stop(group->queues[i], NULL); in panthor_group_stop()
2737 group_get(group); in panthor_group_stop()
2738 list_move_tail(&group->run_node, &sched->reset.stopped_groups); in panthor_group_stop()
2741 static void panthor_group_start(struct panthor_group *group) in panthor_group_start() argument
2743 struct panthor_scheduler *sched = group->ptdev->scheduler; in panthor_group_start()
2745 lockdep_assert_held(&group->ptdev->scheduler->reset.lock); in panthor_group_start()
2747 for (u32 i = 0; i < group->queue_count; i++) in panthor_group_start()
2748 queue_start(group->queues[i]); in panthor_group_start()
2750 if (group_can_run(group)) { in panthor_group_start()
2751 list_move_tail(&group->run_node, in panthor_group_start()
2752 group_is_idle(group) ? in panthor_group_start()
2753 &sched->groups.idle[group->priority] : in panthor_group_start()
2754 &sched->groups.runnable[group->priority]); in panthor_group_start()
2756 list_del_init(&group->run_node); in panthor_group_start()
2757 list_del_init(&group->wait_node); in panthor_group_start()
2758 group_queue_work(group, term); in panthor_group_start()
2761 group_put(group); in panthor_group_start()
2799 if (csg_slot->group) { in panthor_sched_suspend()
2801 group_can_run(csg_slot->group) ? in panthor_sched_suspend()
2821 /* If the group was still usable before that point, we consider in panthor_sched_suspend()
2824 if (group_can_run(csg_slot->group)) in panthor_sched_suspend()
2825 csg_slot->group->innocent = true; in panthor_sched_suspend()
2827 /* We consider group suspension failures as fatal and flag the in panthor_sched_suspend()
2828 * group as unusable by setting timedout=true. in panthor_sched_suspend()
2830 csg_slot->group->timedout = true; in panthor_sched_suspend()
2844 struct panthor_group *group = csg_slot->group; in panthor_sched_suspend() local
2850 if (group->state != PANTHOR_CS_GROUP_TERMINATED) { in panthor_sched_suspend()
2851 group->state = PANTHOR_CS_GROUP_TERMINATED; in panthor_sched_suspend()
2856 for (i = 0; i < group->queue_count; i++) { in panthor_sched_suspend()
2857 if (group->queues[i]) in panthor_sched_suspend()
2880 csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED; in panthor_sched_suspend()
2890 struct panthor_group *group = csg_slot->group; in panthor_sched_suspend() local
2892 if (!group) in panthor_sched_suspend()
2895 group_get(group); in panthor_sched_suspend()
2897 if (group->csg_id >= 0) in panthor_sched_suspend()
2898 sched_process_csg_irq_locked(ptdev, group->csg_id); in panthor_sched_suspend()
2900 group_unbind_locked(group); in panthor_sched_suspend()
2902 drm_WARN_ON(&group->ptdev->base, !list_empty(&group->run_node)); in panthor_sched_suspend()
2904 if (group_can_run(group)) { in panthor_sched_suspend()
2905 list_add(&group->run_node, in panthor_sched_suspend()
2906 &sched->groups.idle[group->priority]); in panthor_sched_suspend()
2908 /* We don't bother stopping the scheduler if the group is in panthor_sched_suspend()
2909 * faulty, the group termination work will finish the job. in panthor_sched_suspend()
2911 list_del_init(&group->wait_node); in panthor_sched_suspend()
2912 group_queue_work(group, term); in panthor_sched_suspend()
2914 group_put(group); in panthor_sched_suspend()
2922 struct panthor_group *group, *group_tmp; in panthor_sched_pre_reset() local
2942 list_for_each_entry_safe(group, group_tmp, &sched->groups.runnable[i], run_node) in panthor_sched_pre_reset()
2943 panthor_group_stop(group); in panthor_sched_pre_reset()
2947 list_for_each_entry_safe(group, group_tmp, &sched->groups.idle[i], run_node) in panthor_sched_pre_reset()
2948 panthor_group_stop(group); in panthor_sched_pre_reset()
2957 struct panthor_group *group, *group_tmp; in panthor_sched_post_reset() local
2961 list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) { in panthor_sched_post_reset()
2962 /* Consider all previously running group as terminated if the in panthor_sched_post_reset()
2966 group->state = PANTHOR_CS_GROUP_TERMINATED; in panthor_sched_post_reset()
2968 panthor_group_start(group); in panthor_sched_post_reset()
2986 struct panthor_group *group = job->group; in update_fdinfo_stats() local
2987 struct panthor_queue *queue = group->queues[job->queue_idx]; in update_fdinfo_stats()
2988 struct panthor_gpu_usage *fdinfo = &group->fdinfo.data; in update_fdinfo_stats()
2992 scoped_guard(spinlock, &group->fdinfo.lock) { in update_fdinfo_stats()
3003 struct panthor_group *group; in panthor_fdinfo_gather_group_samples() local
3010 xa_for_each(&gpool->xa, i, group) { in panthor_fdinfo_gather_group_samples()
3011 guard(spinlock)(&group->fdinfo.lock); in panthor_fdinfo_gather_group_samples()
3012 pfile->stats.cycles += group->fdinfo.data.cycles; in panthor_fdinfo_gather_group_samples()
3013 pfile->stats.time += group->fdinfo.data.time; in panthor_fdinfo_gather_group_samples()
3014 group->fdinfo.data.cycles = 0; in panthor_fdinfo_gather_group_samples()
3015 group->fdinfo.data.time = 0; in panthor_fdinfo_gather_group_samples()
3031 struct panthor_group *group = job->group; in queue_check_job_completion() local
3033 syncobj = group->syncobjs->kmap + in queue_check_job_completion()
3075 struct panthor_group *group = in group_sync_upd_work() local
3081 for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) { in group_sync_upd_work()
3082 struct panthor_queue *queue = group->queues[queue_idx]; in group_sync_upd_work()
3091 group_put(group); in group_sync_upd_work()
3149 struct panthor_group *group = job->group; in get_job_cs_params() local
3150 struct panthor_queue *queue = group->queues[job->queue_idx]; in get_job_cs_params()
3151 struct panthor_device *ptdev = group->ptdev; in get_job_cs_params()
3160 params->sync_addr = panthor_kernel_bo_gpuva(group->syncobjs) + in get_job_cs_params()
3278 struct panthor_group *group = job->group; in queue_run_job() local
3279 struct panthor_queue *queue = group->queues[job->queue_idx]; in queue_run_job()
3280 struct panthor_device *ptdev = group->ptdev; in queue_run_job()
3301 if (!group_can_run(group)) { in queue_run_job()
3337 if (group->csg_id < 0) { in queue_run_job()
3338 group_schedule_locked(group, BIT(job->queue_idx)); in queue_run_job()
3342 !(group->blocked_queues & BIT(job->queue_idx))) { in queue_run_job()
3368 struct panthor_group *group = job->group; in queue_timedout_job() local
3369 struct panthor_device *ptdev = group->ptdev; in queue_timedout_job()
3371 struct panthor_queue *queue = group->queues[job->queue_idx]; in queue_timedout_job()
3374 group->task_info.pid, group->task_info.comm, job->done_fence->seqno); in queue_timedout_job()
3381 group->timedout = true; in queue_timedout_job()
3382 if (group->csg_id >= 0) { in queue_timedout_job()
3386 * pick the group on the next tick. in queue_timedout_job()
3388 list_del_init(&group->run_node); in queue_timedout_job()
3389 list_del_init(&group->wait_node); in queue_timedout_job()
3391 group_queue_work(group, term); in queue_timedout_job()
3450 group_create_queue(struct panthor_group *group, in group_create_queue() argument
3456 .submit_wq = group->ptdev->scheduler->wq, in group_create_queue()
3466 .timeout_wq = group->ptdev->reset.wq, in group_create_queue()
3467 .dev = group->ptdev->base.dev, in group_create_queue()
3495 queue->ringbuf = panthor_kernel_bo_create(group->ptdev, group->vm, in group_create_queue()
3511 queue->iface.mem = panthor_fw_alloc_queue_iface_mem(group->ptdev, in group_create_queue()
3522 calc_profiling_ringbuf_num_slots(group->ptdev, args->ringbuf_size); in group_create_queue()
3525 panthor_kernel_bo_create(group->ptdev, group->vm, in group_create_queue()
3532 "Group job stats"); in group_create_queue()
3564 group_free_queue(group, queue); in group_create_queue()
3568 static void group_init_task_info(struct panthor_group *group) in group_init_task_info() argument
3572 group->task_info.pid = task->pid; in group_init_task_info()
3573 get_task_comm(group->task_info.comm, task); in group_init_task_info()
3577 struct panthor_group *group) in add_group_kbo_sizes() argument
3582 if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(group))) in add_group_kbo_sizes()
3584 if (drm_WARN_ON(&ptdev->base, ptdev != group->ptdev)) in add_group_kbo_sizes()
3587 group->fdinfo.kbo_sizes += group->suspend_buf->obj->size; in add_group_kbo_sizes()
3588 group->fdinfo.kbo_sizes += group->protm_suspend_buf->obj->size; in add_group_kbo_sizes()
3589 group->fdinfo.kbo_sizes += group->syncobjs->obj->size; in add_group_kbo_sizes()
3591 for (i = 0; i < group->queue_count; i++) { in add_group_kbo_sizes()
3592 queue = group->queues[i]; in add_group_kbo_sizes()
3593 group->fdinfo.kbo_sizes += queue->ringbuf->obj->size; in add_group_kbo_sizes()
3594 group->fdinfo.kbo_sizes += queue->iface.mem->obj->size; in add_group_kbo_sizes()
3595 group->fdinfo.kbo_sizes += queue->profiling.slots->obj->size; in add_group_kbo_sizes()
3610 struct panthor_group *group = NULL; in panthor_group_create() local
3630 group = kzalloc(sizeof(*group), GFP_KERNEL); in panthor_group_create()
3631 if (!group) in panthor_group_create()
3634 spin_lock_init(&group->fatal_lock); in panthor_group_create()
3635 kref_init(&group->refcount); in panthor_group_create()
3636 group->state = PANTHOR_CS_GROUP_CREATED; in panthor_group_create()
3637 group->csg_id = -1; in panthor_group_create()
3639 group->ptdev = ptdev; in panthor_group_create()
3640 group->max_compute_cores = group_args->max_compute_cores; in panthor_group_create()
3641 group->compute_core_mask = group_args->compute_core_mask; in panthor_group_create()
3642 group->max_fragment_cores = group_args->max_fragment_cores; in panthor_group_create()
3643 group->fragment_core_mask = group_args->fragment_core_mask; in panthor_group_create()
3644 group->max_tiler_cores = group_args->max_tiler_cores; in panthor_group_create()
3645 group->tiler_core_mask = group_args->tiler_core_mask; in panthor_group_create()
3646 group->priority = group_args->priority; in panthor_group_create()
3648 INIT_LIST_HEAD(&group->wait_node); in panthor_group_create()
3649 INIT_LIST_HEAD(&group->run_node); in panthor_group_create()
3650 INIT_WORK(&group->term_work, group_term_work); in panthor_group_create()
3651 INIT_WORK(&group->sync_upd_work, group_sync_upd_work); in panthor_group_create()
3652 INIT_WORK(&group->tiler_oom_work, group_tiler_oom_work); in panthor_group_create()
3653 INIT_WORK(&group->release_work, group_release_work); in panthor_group_create()
3655 group->vm = panthor_vm_pool_get_vm(pfile->vms, group_args->vm_id); in panthor_group_create()
3656 if (!group->vm) { in panthor_group_create()
3662 group->suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size); in panthor_group_create()
3663 if (IS_ERR(group->suspend_buf)) { in panthor_group_create()
3664 ret = PTR_ERR(group->suspend_buf); in panthor_group_create()
3665 group->suspend_buf = NULL; in panthor_group_create()
3670 group->protm_suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size); in panthor_group_create()
3671 if (IS_ERR(group->protm_suspend_buf)) { in panthor_group_create()
3672 ret = PTR_ERR(group->protm_suspend_buf); in panthor_group_create()
3673 group->protm_suspend_buf = NULL; in panthor_group_create()
3677 group->syncobjs = panthor_kernel_bo_create(ptdev, group->vm, in panthor_group_create()
3684 "Group sync objects"); in panthor_group_create()
3685 if (IS_ERR(group->syncobjs)) { in panthor_group_create()
3686 ret = PTR_ERR(group->syncobjs); in panthor_group_create()
3690 ret = panthor_kernel_bo_vmap(group->syncobjs); in panthor_group_create()
3694 memset(group->syncobjs->kmap, 0, in panthor_group_create()
3697 ret = xa_alloc(&gpool->xa, &gid, group, XA_LIMIT(1, MAX_GROUPS_PER_POOL), GFP_KERNEL); in panthor_group_create()
3702 group->queues[i] = group_create_queue(group, &queue_args[i], drm_client_id, gid, i); in panthor_group_create()
3703 if (IS_ERR(group->queues[i])) { in panthor_group_create()
3704 ret = PTR_ERR(group->queues[i]); in panthor_group_create()
3705 group->queues[i] = NULL; in panthor_group_create()
3709 group->queue_count++; in panthor_group_create()
3712 group->idle_queues = GENMASK(group->queue_count - 1, 0); in panthor_group_create()
3716 panthor_group_stop(group); in panthor_group_create()
3719 list_add_tail(&group->run_node, in panthor_group_create()
3720 &sched->groups.idle[group->priority]); in panthor_group_create()
3725 add_group_kbo_sizes(group->ptdev, group); in panthor_group_create()
3726 spin_lock_init(&group->fdinfo.lock); in panthor_group_create()
3728 group_init_task_info(group); in panthor_group_create()
3736 group_put(group); in panthor_group_create()
3745 struct panthor_group *group; in panthor_group_destroy() local
3747 group = xa_erase(&gpool->xa, group_handle); in panthor_group_destroy()
3748 if (!group) in panthor_group_destroy()
3753 group->destroyed = true; in panthor_group_destroy()
3754 if (group->csg_id >= 0) { in panthor_group_destroy()
3758 * pick the group on the next tick. in panthor_group_destroy()
3760 list_del_init(&group->run_node); in panthor_group_destroy()
3761 list_del_init(&group->wait_node); in panthor_group_destroy()
3762 group_queue_work(group, term); in panthor_group_destroy()
3767 group_put(group); in panthor_group_destroy()
3774 struct panthor_group *group; in group_from_handle() local
3777 group = group_get(xa_load(&pool->xa, group_handle)); in group_from_handle()
3780 return group; in group_from_handle()
3789 struct panthor_group *group; in panthor_group_get_state() local
3794 group = group_from_handle(gpool, get_state->group_handle); in panthor_group_get_state()
3795 if (!group) in panthor_group_get_state()
3801 if (group->timedout) in panthor_group_get_state()
3803 if (group->fatal_queues) { in panthor_group_get_state()
3805 get_state->fatal_queues = group->fatal_queues; in panthor_group_get_state()
3807 if (group->innocent) in panthor_group_get_state()
3811 group_put(group); in panthor_group_get_state()
3831 struct panthor_group *group; in panthor_group_pool_destroy() local
3837 xa_for_each(&gpool->xa, i, group) in panthor_group_pool_destroy()
3857 struct panthor_group *group; in panthor_fdinfo_gather_group_mem_info() local
3864 xa_for_each(&gpool->xa, i, group) { in panthor_fdinfo_gather_group_mem_info()
3865 stats->resident += group->fdinfo.kbo_sizes; in panthor_fdinfo_gather_group_mem_info()
3866 if (group->csg_id >= 0) in panthor_fdinfo_gather_group_mem_info()
3867 stats->active += group->fdinfo.kbo_sizes; in panthor_fdinfo_gather_group_mem_info()
3876 drm_WARN_ON(&job->group->ptdev->base, !list_empty(&job->node)); in job_release()
3886 group_put(job->group); in job_release()
3914 return job->group->vm; in panthor_job_vm()
3956 job->group = group_from_handle(gpool, group_handle); in panthor_job_create()
3957 if (!job->group) { in panthor_job_create()
3962 if (!group_can_run(job->group)) { in panthor_job_create()
3967 if (job->queue_idx >= job->group->queue_count || in panthor_job_create()
3968 !job->group->queues[job->queue_idx]) { in panthor_job_create()
3992 &job->group->queues[job->queue_idx]->entity, in panthor_job_create()
3993 credits, job->group, drm_client_id); in panthor_job_create()
4008 panthor_vm_update_resvs(job->group->vm, exec, &sched_job->s_fence->finished, in panthor_job_update_resvs()
4070 * of the resources being allocated to one group and the other part to in panthor_sched_init()
4071 * the other group, both groups waiting for the remaining resources to in panthor_sched_init()