1de854881SBoris Brezillon // SPDX-License-Identifier: GPL-2.0 or MIT
2de854881SBoris Brezillon /* Copyright 2023 Collabora ltd. */
3de854881SBoris Brezillon
4de854881SBoris Brezillon #include <drm/drm_drv.h>
5de854881SBoris Brezillon #include <drm/drm_exec.h>
6de854881SBoris Brezillon #include <drm/drm_gem_shmem_helper.h>
7de854881SBoris Brezillon #include <drm/drm_managed.h>
8de854881SBoris Brezillon #include <drm/gpu_scheduler.h>
9de854881SBoris Brezillon #include <drm/panthor_drm.h>
10de854881SBoris Brezillon
11de854881SBoris Brezillon #include <linux/build_bug.h>
12de854881SBoris Brezillon #include <linux/clk.h>
13de854881SBoris Brezillon #include <linux/delay.h>
14de854881SBoris Brezillon #include <linux/dma-mapping.h>
15de854881SBoris Brezillon #include <linux/dma-resv.h>
16de854881SBoris Brezillon #include <linux/firmware.h>
17de854881SBoris Brezillon #include <linux/interrupt.h>
18de854881SBoris Brezillon #include <linux/io.h>
19de854881SBoris Brezillon #include <linux/iopoll.h>
20de854881SBoris Brezillon #include <linux/iosys-map.h>
21de854881SBoris Brezillon #include <linux/module.h>
22de854881SBoris Brezillon #include <linux/platform_device.h>
23de854881SBoris Brezillon #include <linux/pm_runtime.h>
24de854881SBoris Brezillon
25de854881SBoris Brezillon #include "panthor_devfreq.h"
26de854881SBoris Brezillon #include "panthor_device.h"
27de854881SBoris Brezillon #include "panthor_fw.h"
28de854881SBoris Brezillon #include "panthor_gem.h"
29de854881SBoris Brezillon #include "panthor_gpu.h"
30de854881SBoris Brezillon #include "panthor_heap.h"
31de854881SBoris Brezillon #include "panthor_mmu.h"
32de854881SBoris Brezillon #include "panthor_regs.h"
33de854881SBoris Brezillon #include "panthor_sched.h"
34de854881SBoris Brezillon
35de854881SBoris Brezillon /**
36de854881SBoris Brezillon * DOC: Scheduler
37de854881SBoris Brezillon *
38de854881SBoris Brezillon * Mali CSF hardware adopts a firmware-assisted scheduling model, where
39de854881SBoris Brezillon * the firmware takes care of scheduling aspects, to some extent.
40de854881SBoris Brezillon *
41de854881SBoris Brezillon * The scheduling happens at the scheduling group level, each group
42de854881SBoris Brezillon * contains 1 to N queues (N is FW/hardware dependent, and exposed
43de854881SBoris Brezillon * through the firmware interface). Each queue is assigned a command
44de854881SBoris Brezillon * stream ring buffer, which serves as a way to get jobs submitted to
45de854881SBoris Brezillon * the GPU, among other things.
46de854881SBoris Brezillon *
47de854881SBoris Brezillon * The firmware can schedule a maximum of M groups (M is FW/hardware
48de854881SBoris Brezillon * dependent, and exposed through the firmware interface). Passed
49de854881SBoris Brezillon * this maximum number of groups, the kernel must take care of
50de854881SBoris Brezillon * rotating the groups passed to the firmware so every group gets
51de854881SBoris Brezillon * a chance to have his queues scheduled for execution.
52de854881SBoris Brezillon *
53de854881SBoris Brezillon * The current implementation only supports with kernel-mode queues.
54de854881SBoris Brezillon * In other terms, userspace doesn't have access to the ring-buffer.
55de854881SBoris Brezillon * Instead, userspace passes indirect command stream buffers that are
56de854881SBoris Brezillon * called from the queue ring-buffer by the kernel using a pre-defined
57de854881SBoris Brezillon * sequence of command stream instructions to ensure the userspace driver
58de854881SBoris Brezillon * always gets consistent results (cache maintenance,
59de854881SBoris Brezillon * synchronization, ...).
60de854881SBoris Brezillon *
61de854881SBoris Brezillon * We rely on the drm_gpu_scheduler framework to deal with job
62de854881SBoris Brezillon * dependencies and submission. As any other driver dealing with a
63de854881SBoris Brezillon * FW-scheduler, we use the 1:1 entity:scheduler mode, such that each
64de854881SBoris Brezillon * entity has its own job scheduler. When a job is ready to be executed
65de854881SBoris Brezillon * (all its dependencies are met), it is pushed to the appropriate
66de854881SBoris Brezillon * queue ring-buffer, and the group is scheduled for execution if it
67de854881SBoris Brezillon * wasn't already active.
68de854881SBoris Brezillon *
69de854881SBoris Brezillon * Kernel-side group scheduling is timeslice-based. When we have less
70de854881SBoris Brezillon * groups than there are slots, the periodic tick is disabled and we
71de854881SBoris Brezillon * just let the FW schedule the active groups. When there are more
72de854881SBoris Brezillon * groups than slots, we let each group a chance to execute stuff for
73de854881SBoris Brezillon * a given amount of time, and then re-evaluate and pick new groups
74de854881SBoris Brezillon * to schedule. The group selection algorithm is based on
75de854881SBoris Brezillon * priority+round-robin.
76de854881SBoris Brezillon *
77de854881SBoris Brezillon * Even though user-mode queues is out of the scope right now, the
78de854881SBoris Brezillon * current design takes them into account by avoiding any guess on the
79de854881SBoris Brezillon * group/queue state that would be based on information we wouldn't have
80de854881SBoris Brezillon * if userspace was in charge of the ring-buffer. That's also one of the
81de854881SBoris Brezillon * reason we don't do 'cooperative' scheduling (encoding FW group slot
82de854881SBoris Brezillon * reservation as dma_fence that would be returned from the
83de854881SBoris Brezillon * drm_gpu_scheduler::prepare_job() hook, and treating group rotation as
84de854881SBoris Brezillon * a queue of waiters, ordered by job submission order). This approach
85de854881SBoris Brezillon * would work for kernel-mode queues, but would make user-mode queues a
86de854881SBoris Brezillon * lot more complicated to retrofit.
87de854881SBoris Brezillon */
88de854881SBoris Brezillon
89de854881SBoris Brezillon #define JOB_TIMEOUT_MS 5000
90de854881SBoris Brezillon
91de854881SBoris Brezillon #define MIN_CS_PER_CSG 8
92de854881SBoris Brezillon
93de854881SBoris Brezillon #define MIN_CSGS 3
94de854881SBoris Brezillon #define MAX_CSG_PRIO 0xf
95de854881SBoris Brezillon
96de854881SBoris Brezillon struct panthor_group;
97de854881SBoris Brezillon
98de854881SBoris Brezillon /**
99de854881SBoris Brezillon * struct panthor_csg_slot - Command stream group slot
100de854881SBoris Brezillon *
101de854881SBoris Brezillon * This represents a FW slot for a scheduling group.
102de854881SBoris Brezillon */
103de854881SBoris Brezillon struct panthor_csg_slot {
104de854881SBoris Brezillon /** @group: Scheduling group bound to this slot. */
105de854881SBoris Brezillon struct panthor_group *group;
106de854881SBoris Brezillon
107de854881SBoris Brezillon /** @priority: Group priority. */
108de854881SBoris Brezillon u8 priority;
109de854881SBoris Brezillon
110de854881SBoris Brezillon /**
111de854881SBoris Brezillon * @idle: True if the group bound to this slot is idle.
112de854881SBoris Brezillon *
113de854881SBoris Brezillon * A group is idle when it has nothing waiting for execution on
114de854881SBoris Brezillon * all its queues, or when queues are blocked waiting for something
115de854881SBoris Brezillon * to happen (synchronization object).
116de854881SBoris Brezillon */
117de854881SBoris Brezillon bool idle;
118de854881SBoris Brezillon };
119de854881SBoris Brezillon
120de854881SBoris Brezillon /**
121de854881SBoris Brezillon * enum panthor_csg_priority - Group priority
122de854881SBoris Brezillon */
123de854881SBoris Brezillon enum panthor_csg_priority {
124de854881SBoris Brezillon /** @PANTHOR_CSG_PRIORITY_LOW: Low priority group. */
125de854881SBoris Brezillon PANTHOR_CSG_PRIORITY_LOW = 0,
126de854881SBoris Brezillon
127de854881SBoris Brezillon /** @PANTHOR_CSG_PRIORITY_MEDIUM: Medium priority group. */
128de854881SBoris Brezillon PANTHOR_CSG_PRIORITY_MEDIUM,
129de854881SBoris Brezillon
130de854881SBoris Brezillon /** @PANTHOR_CSG_PRIORITY_HIGH: High priority group. */
131de854881SBoris Brezillon PANTHOR_CSG_PRIORITY_HIGH,
132de854881SBoris Brezillon
133de854881SBoris Brezillon /**
134de854881SBoris Brezillon * @PANTHOR_CSG_PRIORITY_RT: Real-time priority group.
135de854881SBoris Brezillon *
136de854881SBoris Brezillon * Real-time priority allows one to preempt scheduling of other
137de854881SBoris Brezillon * non-real-time groups. When such a group becomes executable,
138de854881SBoris Brezillon * it will evict the group with the lowest non-rt priority if
139de854881SBoris Brezillon * there's no free group slot available.
140de854881SBoris Brezillon *
141de854881SBoris Brezillon * Currently not exposed to userspace.
142de854881SBoris Brezillon */
143de854881SBoris Brezillon PANTHOR_CSG_PRIORITY_RT,
144de854881SBoris Brezillon
145de854881SBoris Brezillon /** @PANTHOR_CSG_PRIORITY_COUNT: Number of priority levels. */
146de854881SBoris Brezillon PANTHOR_CSG_PRIORITY_COUNT,
147de854881SBoris Brezillon };
148de854881SBoris Brezillon
149de854881SBoris Brezillon /**
150de854881SBoris Brezillon * struct panthor_scheduler - Object used to manage the scheduler
151de854881SBoris Brezillon */
152de854881SBoris Brezillon struct panthor_scheduler {
153de854881SBoris Brezillon /** @ptdev: Device. */
154de854881SBoris Brezillon struct panthor_device *ptdev;
155de854881SBoris Brezillon
156de854881SBoris Brezillon /**
157de854881SBoris Brezillon * @wq: Workqueue used by our internal scheduler logic and
158de854881SBoris Brezillon * drm_gpu_scheduler.
159de854881SBoris Brezillon *
160de854881SBoris Brezillon * Used for the scheduler tick, group update or other kind of FW
161de854881SBoris Brezillon * event processing that can't be handled in the threaded interrupt
162de854881SBoris Brezillon * path. Also passed to the drm_gpu_scheduler instances embedded
163de854881SBoris Brezillon * in panthor_queue.
164de854881SBoris Brezillon */
165de854881SBoris Brezillon struct workqueue_struct *wq;
166de854881SBoris Brezillon
167de854881SBoris Brezillon /**
168de854881SBoris Brezillon * @heap_alloc_wq: Workqueue used to schedule tiler_oom works.
169de854881SBoris Brezillon *
170de854881SBoris Brezillon * We have a queue dedicated to heap chunk allocation works to avoid
171de854881SBoris Brezillon * blocking the rest of the scheduler if the allocation tries to
172de854881SBoris Brezillon * reclaim memory.
173de854881SBoris Brezillon */
174de854881SBoris Brezillon struct workqueue_struct *heap_alloc_wq;
175de854881SBoris Brezillon
176de854881SBoris Brezillon /** @tick_work: Work executed on a scheduling tick. */
177de854881SBoris Brezillon struct delayed_work tick_work;
178de854881SBoris Brezillon
179de854881SBoris Brezillon /**
180de854881SBoris Brezillon * @sync_upd_work: Work used to process synchronization object updates.
181de854881SBoris Brezillon *
182de854881SBoris Brezillon * We use this work to unblock queues/groups that were waiting on a
183de854881SBoris Brezillon * synchronization object.
184de854881SBoris Brezillon */
185de854881SBoris Brezillon struct work_struct sync_upd_work;
186de854881SBoris Brezillon
187de854881SBoris Brezillon /**
188de854881SBoris Brezillon * @fw_events_work: Work used to process FW events outside the interrupt path.
189de854881SBoris Brezillon *
190de854881SBoris Brezillon * Even if the interrupt is threaded, we need any event processing
191de854881SBoris Brezillon * that require taking the panthor_scheduler::lock to be processed
192de854881SBoris Brezillon * outside the interrupt path so we don't block the tick logic when
193de854881SBoris Brezillon * it calls panthor_fw_{csg,wait}_wait_acks(). Since most of the
194de854881SBoris Brezillon * event processing requires taking this lock, we just delegate all
195de854881SBoris Brezillon * FW event processing to the scheduler workqueue.
196de854881SBoris Brezillon */
197de854881SBoris Brezillon struct work_struct fw_events_work;
198de854881SBoris Brezillon
199de854881SBoris Brezillon /**
200de854881SBoris Brezillon * @fw_events: Bitmask encoding pending FW events.
201de854881SBoris Brezillon */
202de854881SBoris Brezillon atomic_t fw_events;
203de854881SBoris Brezillon
204de854881SBoris Brezillon /**
205de854881SBoris Brezillon * @resched_target: When the next tick should occur.
206de854881SBoris Brezillon *
207de854881SBoris Brezillon * Expressed in jiffies.
208de854881SBoris Brezillon */
209de854881SBoris Brezillon u64 resched_target;
210de854881SBoris Brezillon
211de854881SBoris Brezillon /**
212de854881SBoris Brezillon * @last_tick: When the last tick occurred.
213de854881SBoris Brezillon *
214de854881SBoris Brezillon * Expressed in jiffies.
215de854881SBoris Brezillon */
216de854881SBoris Brezillon u64 last_tick;
217de854881SBoris Brezillon
218de854881SBoris Brezillon /** @tick_period: Tick period in jiffies. */
219de854881SBoris Brezillon u64 tick_period;
220de854881SBoris Brezillon
221de854881SBoris Brezillon /**
222de854881SBoris Brezillon * @lock: Lock protecting access to all the scheduler fields.
223de854881SBoris Brezillon *
224de854881SBoris Brezillon * Should be taken in the tick work, the irq handler, and anywhere the @groups
225de854881SBoris Brezillon * fields are touched.
226de854881SBoris Brezillon */
227de854881SBoris Brezillon struct mutex lock;
228de854881SBoris Brezillon
229de854881SBoris Brezillon /** @groups: Various lists used to classify groups. */
230de854881SBoris Brezillon struct {
231de854881SBoris Brezillon /**
232de854881SBoris Brezillon * @runnable: Runnable group lists.
233de854881SBoris Brezillon *
234de854881SBoris Brezillon * When a group has queues that want to execute something,
235de854881SBoris Brezillon * its panthor_group::run_node should be inserted here.
236de854881SBoris Brezillon *
237de854881SBoris Brezillon * One list per-priority.
238de854881SBoris Brezillon */
239de854881SBoris Brezillon struct list_head runnable[PANTHOR_CSG_PRIORITY_COUNT];
240de854881SBoris Brezillon
241de854881SBoris Brezillon /**
242de854881SBoris Brezillon * @idle: Idle group lists.
243de854881SBoris Brezillon *
244de854881SBoris Brezillon * When all queues of a group are idle (either because they
245de854881SBoris Brezillon * have nothing to execute, or because they are blocked), the
246de854881SBoris Brezillon * panthor_group::run_node field should be inserted here.
247de854881SBoris Brezillon *
248de854881SBoris Brezillon * One list per-priority.
249de854881SBoris Brezillon */
250de854881SBoris Brezillon struct list_head idle[PANTHOR_CSG_PRIORITY_COUNT];
251de854881SBoris Brezillon
252de854881SBoris Brezillon /**
253de854881SBoris Brezillon * @waiting: List of groups whose queues are blocked on a
254de854881SBoris Brezillon * synchronization object.
255de854881SBoris Brezillon *
256de854881SBoris Brezillon * Insert panthor_group::wait_node here when a group is waiting
257de854881SBoris Brezillon * for synchronization objects to be signaled.
258de854881SBoris Brezillon *
259de854881SBoris Brezillon * This list is evaluated in the @sync_upd_work work.
260de854881SBoris Brezillon */
261de854881SBoris Brezillon struct list_head waiting;
262de854881SBoris Brezillon } groups;
263de854881SBoris Brezillon
264de854881SBoris Brezillon /**
265de854881SBoris Brezillon * @csg_slots: FW command stream group slots.
266de854881SBoris Brezillon */
267de854881SBoris Brezillon struct panthor_csg_slot csg_slots[MAX_CSGS];
268de854881SBoris Brezillon
269de854881SBoris Brezillon /** @csg_slot_count: Number of command stream group slots exposed by the FW. */
270de854881SBoris Brezillon u32 csg_slot_count;
271de854881SBoris Brezillon
272de854881SBoris Brezillon /** @cs_slot_count: Number of command stream slot per group slot exposed by the FW. */
273de854881SBoris Brezillon u32 cs_slot_count;
274de854881SBoris Brezillon
275de854881SBoris Brezillon /** @as_slot_count: Number of address space slots supported by the MMU. */
276de854881SBoris Brezillon u32 as_slot_count;
277de854881SBoris Brezillon
278de854881SBoris Brezillon /** @used_csg_slot_count: Number of command stream group slot currently used. */
279de854881SBoris Brezillon u32 used_csg_slot_count;
280de854881SBoris Brezillon
281de854881SBoris Brezillon /** @sb_slot_count: Number of scoreboard slots. */
282de854881SBoris Brezillon u32 sb_slot_count;
283de854881SBoris Brezillon
284de854881SBoris Brezillon /**
285de854881SBoris Brezillon * @might_have_idle_groups: True if an active group might have become idle.
286de854881SBoris Brezillon *
287de854881SBoris Brezillon * This will force a tick, so other runnable groups can be scheduled if one
288de854881SBoris Brezillon * or more active groups became idle.
289de854881SBoris Brezillon */
290de854881SBoris Brezillon bool might_have_idle_groups;
291de854881SBoris Brezillon
292de854881SBoris Brezillon /** @pm: Power management related fields. */
293de854881SBoris Brezillon struct {
294de854881SBoris Brezillon /** @has_ref: True if the scheduler owns a runtime PM reference. */
295de854881SBoris Brezillon bool has_ref;
296de854881SBoris Brezillon } pm;
297de854881SBoris Brezillon
298de854881SBoris Brezillon /** @reset: Reset related fields. */
299de854881SBoris Brezillon struct {
300de854881SBoris Brezillon /** @lock: Lock protecting the other reset fields. */
301de854881SBoris Brezillon struct mutex lock;
302de854881SBoris Brezillon
303de854881SBoris Brezillon /**
304de854881SBoris Brezillon * @in_progress: True if a reset is in progress.
305de854881SBoris Brezillon *
306de854881SBoris Brezillon * Set to true in panthor_sched_pre_reset() and back to false in
307de854881SBoris Brezillon * panthor_sched_post_reset().
308de854881SBoris Brezillon */
309de854881SBoris Brezillon atomic_t in_progress;
310de854881SBoris Brezillon
311de854881SBoris Brezillon /**
312de854881SBoris Brezillon * @stopped_groups: List containing all groups that were stopped
313de854881SBoris Brezillon * before a reset.
314de854881SBoris Brezillon *
315de854881SBoris Brezillon * Insert panthor_group::run_node in the pre_reset path.
316de854881SBoris Brezillon */
317de854881SBoris Brezillon struct list_head stopped_groups;
318de854881SBoris Brezillon } reset;
319de854881SBoris Brezillon };
320de854881SBoris Brezillon
321de854881SBoris Brezillon /**
322de854881SBoris Brezillon * struct panthor_syncobj_32b - 32-bit FW synchronization object
323de854881SBoris Brezillon */
324de854881SBoris Brezillon struct panthor_syncobj_32b {
325de854881SBoris Brezillon /** @seqno: Sequence number. */
326de854881SBoris Brezillon u32 seqno;
327de854881SBoris Brezillon
328de854881SBoris Brezillon /**
329de854881SBoris Brezillon * @status: Status.
330de854881SBoris Brezillon *
331de854881SBoris Brezillon * Not zero on failure.
332de854881SBoris Brezillon */
333de854881SBoris Brezillon u32 status;
334de854881SBoris Brezillon };
335de854881SBoris Brezillon
336de854881SBoris Brezillon /**
337de854881SBoris Brezillon * struct panthor_syncobj_64b - 64-bit FW synchronization object
338de854881SBoris Brezillon */
339de854881SBoris Brezillon struct panthor_syncobj_64b {
340de854881SBoris Brezillon /** @seqno: Sequence number. */
341de854881SBoris Brezillon u64 seqno;
342de854881SBoris Brezillon
343de854881SBoris Brezillon /**
344de854881SBoris Brezillon * @status: Status.
345de854881SBoris Brezillon *
346de854881SBoris Brezillon * Not zero on failure.
347de854881SBoris Brezillon */
348de854881SBoris Brezillon u32 status;
349de854881SBoris Brezillon
350de854881SBoris Brezillon /** @pad: MBZ. */
351de854881SBoris Brezillon u32 pad;
352de854881SBoris Brezillon };
353de854881SBoris Brezillon
354de854881SBoris Brezillon /**
355de854881SBoris Brezillon * struct panthor_queue - Execution queue
356de854881SBoris Brezillon */
357de854881SBoris Brezillon struct panthor_queue {
358de854881SBoris Brezillon /** @scheduler: DRM scheduler used for this queue. */
359de854881SBoris Brezillon struct drm_gpu_scheduler scheduler;
360de854881SBoris Brezillon
361de854881SBoris Brezillon /** @entity: DRM scheduling entity used for this queue. */
362de854881SBoris Brezillon struct drm_sched_entity entity;
363de854881SBoris Brezillon
364de854881SBoris Brezillon /**
365de854881SBoris Brezillon * @remaining_time: Time remaining before the job timeout expires.
366de854881SBoris Brezillon *
367de854881SBoris Brezillon * The job timeout is suspended when the queue is not scheduled by the
368de854881SBoris Brezillon * FW. Every time we suspend the timer, we need to save the remaining
369de854881SBoris Brezillon * time so we can restore it later on.
370de854881SBoris Brezillon */
371de854881SBoris Brezillon unsigned long remaining_time;
372de854881SBoris Brezillon
373de854881SBoris Brezillon /** @timeout_suspended: True if the job timeout was suspended. */
374de854881SBoris Brezillon bool timeout_suspended;
375de854881SBoris Brezillon
376de854881SBoris Brezillon /**
377de854881SBoris Brezillon * @doorbell_id: Doorbell assigned to this queue.
378de854881SBoris Brezillon *
379de854881SBoris Brezillon * Right now, all groups share the same doorbell, and the doorbell ID
380de854881SBoris Brezillon * is assigned to group_slot + 1 when the group is assigned a slot. But
381de854881SBoris Brezillon * we might decide to provide fine grained doorbell assignment at some
382de854881SBoris Brezillon * point, so don't have to wake up all queues in a group every time one
383de854881SBoris Brezillon * of them is updated.
384de854881SBoris Brezillon */
385de854881SBoris Brezillon u8 doorbell_id;
386de854881SBoris Brezillon
387de854881SBoris Brezillon /**
388de854881SBoris Brezillon * @priority: Priority of the queue inside the group.
389de854881SBoris Brezillon *
390de854881SBoris Brezillon * Must be less than 16 (Only 4 bits available).
391de854881SBoris Brezillon */
392de854881SBoris Brezillon u8 priority;
393de854881SBoris Brezillon #define CSF_MAX_QUEUE_PRIO GENMASK(3, 0)
394de854881SBoris Brezillon
395de854881SBoris Brezillon /** @ringbuf: Command stream ring-buffer. */
396de854881SBoris Brezillon struct panthor_kernel_bo *ringbuf;
397de854881SBoris Brezillon
398de854881SBoris Brezillon /** @iface: Firmware interface. */
399de854881SBoris Brezillon struct {
400de854881SBoris Brezillon /** @mem: FW memory allocated for this interface. */
401de854881SBoris Brezillon struct panthor_kernel_bo *mem;
402de854881SBoris Brezillon
403de854881SBoris Brezillon /** @input: Input interface. */
404de854881SBoris Brezillon struct panthor_fw_ringbuf_input_iface *input;
405de854881SBoris Brezillon
406de854881SBoris Brezillon /** @output: Output interface. */
407de854881SBoris Brezillon const struct panthor_fw_ringbuf_output_iface *output;
408de854881SBoris Brezillon
409de854881SBoris Brezillon /** @input_fw_va: FW virtual address of the input interface buffer. */
410de854881SBoris Brezillon u32 input_fw_va;
411de854881SBoris Brezillon
412de854881SBoris Brezillon /** @output_fw_va: FW virtual address of the output interface buffer. */
413de854881SBoris Brezillon u32 output_fw_va;
414de854881SBoris Brezillon } iface;
415de854881SBoris Brezillon
416de854881SBoris Brezillon /**
417de854881SBoris Brezillon * @syncwait: Stores information about the synchronization object this
418de854881SBoris Brezillon * queue is waiting on.
419de854881SBoris Brezillon */
420de854881SBoris Brezillon struct {
421de854881SBoris Brezillon /** @gpu_va: GPU address of the synchronization object. */
422de854881SBoris Brezillon u64 gpu_va;
423de854881SBoris Brezillon
424de854881SBoris Brezillon /** @ref: Reference value to compare against. */
425de854881SBoris Brezillon u64 ref;
426de854881SBoris Brezillon
427de854881SBoris Brezillon /** @gt: True if this is a greater-than test. */
428de854881SBoris Brezillon bool gt;
429de854881SBoris Brezillon
430de854881SBoris Brezillon /** @sync64: True if this is a 64-bit sync object. */
431de854881SBoris Brezillon bool sync64;
432de854881SBoris Brezillon
433de854881SBoris Brezillon /** @bo: Buffer object holding the synchronization object. */
434de854881SBoris Brezillon struct drm_gem_object *obj;
435de854881SBoris Brezillon
436de854881SBoris Brezillon /** @offset: Offset of the synchronization object inside @bo. */
437de854881SBoris Brezillon u64 offset;
438de854881SBoris Brezillon
439de854881SBoris Brezillon /**
440de854881SBoris Brezillon * @kmap: Kernel mapping of the buffer object holding the
441de854881SBoris Brezillon * synchronization object.
442de854881SBoris Brezillon */
443de854881SBoris Brezillon void *kmap;
444de854881SBoris Brezillon } syncwait;
445de854881SBoris Brezillon
446de854881SBoris Brezillon /** @fence_ctx: Fence context fields. */
447de854881SBoris Brezillon struct {
448de854881SBoris Brezillon /** @lock: Used to protect access to all fences allocated by this context. */
449de854881SBoris Brezillon spinlock_t lock;
450de854881SBoris Brezillon
451de854881SBoris Brezillon /**
452de854881SBoris Brezillon * @id: Fence context ID.
453de854881SBoris Brezillon *
454de854881SBoris Brezillon * Allocated with dma_fence_context_alloc().
455de854881SBoris Brezillon */
456de854881SBoris Brezillon u64 id;
457de854881SBoris Brezillon
458de854881SBoris Brezillon /** @seqno: Sequence number of the last initialized fence. */
459de854881SBoris Brezillon atomic64_t seqno;
460de854881SBoris Brezillon
461de854881SBoris Brezillon /**
4627b6f9ec6SBoris Brezillon * @last_fence: Fence of the last submitted job.
4637b6f9ec6SBoris Brezillon *
4647b6f9ec6SBoris Brezillon * We return this fence when we get an empty command stream.
4657b6f9ec6SBoris Brezillon * This way, we are guaranteed that all earlier jobs have completed
4667b6f9ec6SBoris Brezillon * when drm_sched_job::s_fence::finished without having to feed
4677b6f9ec6SBoris Brezillon * the CS ring buffer with a dummy job that only signals the fence.
4687b6f9ec6SBoris Brezillon */
4697b6f9ec6SBoris Brezillon struct dma_fence *last_fence;
4707b6f9ec6SBoris Brezillon
4717b6f9ec6SBoris Brezillon /**
472de854881SBoris Brezillon * @in_flight_jobs: List containing all in-flight jobs.
473de854881SBoris Brezillon *
474de854881SBoris Brezillon * Used to keep track and signal panthor_job::done_fence when the
475de854881SBoris Brezillon * synchronization object attached to the queue is signaled.
476de854881SBoris Brezillon */
477de854881SBoris Brezillon struct list_head in_flight_jobs;
478de854881SBoris Brezillon } fence_ctx;
479de854881SBoris Brezillon };
480de854881SBoris Brezillon
481de854881SBoris Brezillon /**
482de854881SBoris Brezillon * enum panthor_group_state - Scheduling group state.
483de854881SBoris Brezillon */
484de854881SBoris Brezillon enum panthor_group_state {
485de854881SBoris Brezillon /** @PANTHOR_CS_GROUP_CREATED: Group was created, but not scheduled yet. */
486de854881SBoris Brezillon PANTHOR_CS_GROUP_CREATED,
487de854881SBoris Brezillon
488de854881SBoris Brezillon /** @PANTHOR_CS_GROUP_ACTIVE: Group is currently scheduled. */
489de854881SBoris Brezillon PANTHOR_CS_GROUP_ACTIVE,
490de854881SBoris Brezillon
491de854881SBoris Brezillon /**
492de854881SBoris Brezillon * @PANTHOR_CS_GROUP_SUSPENDED: Group was scheduled at least once, but is
493de854881SBoris Brezillon * inactive/suspended right now.
494de854881SBoris Brezillon */
495de854881SBoris Brezillon PANTHOR_CS_GROUP_SUSPENDED,
496de854881SBoris Brezillon
497de854881SBoris Brezillon /**
498de854881SBoris Brezillon * @PANTHOR_CS_GROUP_TERMINATED: Group was terminated.
499de854881SBoris Brezillon *
500de854881SBoris Brezillon * Can no longer be scheduled. The only allowed action is a destruction.
501de854881SBoris Brezillon */
502de854881SBoris Brezillon PANTHOR_CS_GROUP_TERMINATED,
5038bdbd8b5SBoris Brezillon
5048bdbd8b5SBoris Brezillon /**
5058bdbd8b5SBoris Brezillon * @PANTHOR_CS_GROUP_UNKNOWN_STATE: Group is an unknown state.
5068bdbd8b5SBoris Brezillon *
5078bdbd8b5SBoris Brezillon * The FW returned an inconsistent state. The group is flagged unusable
5088bdbd8b5SBoris Brezillon * and can no longer be scheduled. The only allowed action is a
5098bdbd8b5SBoris Brezillon * destruction.
5108bdbd8b5SBoris Brezillon *
5118bdbd8b5SBoris Brezillon * When that happens, we also schedule a FW reset, to start from a fresh
5128bdbd8b5SBoris Brezillon * state.
5138bdbd8b5SBoris Brezillon */
5148bdbd8b5SBoris Brezillon PANTHOR_CS_GROUP_UNKNOWN_STATE,
515de854881SBoris Brezillon };
516de854881SBoris Brezillon
517de854881SBoris Brezillon /**
518de854881SBoris Brezillon * struct panthor_group - Scheduling group object
519de854881SBoris Brezillon */
520de854881SBoris Brezillon struct panthor_group {
521de854881SBoris Brezillon /** @refcount: Reference count */
522de854881SBoris Brezillon struct kref refcount;
523de854881SBoris Brezillon
524de854881SBoris Brezillon /** @ptdev: Device. */
525de854881SBoris Brezillon struct panthor_device *ptdev;
526de854881SBoris Brezillon
527de854881SBoris Brezillon /** @vm: VM bound to the group. */
528de854881SBoris Brezillon struct panthor_vm *vm;
529de854881SBoris Brezillon
530de854881SBoris Brezillon /** @compute_core_mask: Mask of shader cores that can be used for compute jobs. */
531de854881SBoris Brezillon u64 compute_core_mask;
532de854881SBoris Brezillon
533de854881SBoris Brezillon /** @fragment_core_mask: Mask of shader cores that can be used for fragment jobs. */
534de854881SBoris Brezillon u64 fragment_core_mask;
535de854881SBoris Brezillon
536de854881SBoris Brezillon /** @tiler_core_mask: Mask of tiler cores that can be used for tiler jobs. */
537de854881SBoris Brezillon u64 tiler_core_mask;
538de854881SBoris Brezillon
539de854881SBoris Brezillon /** @max_compute_cores: Maximum number of shader cores used for compute jobs. */
540de854881SBoris Brezillon u8 max_compute_cores;
541de854881SBoris Brezillon
542be7ffc82SLiviu Dudau /** @max_fragment_cores: Maximum number of shader cores used for fragment jobs. */
543de854881SBoris Brezillon u8 max_fragment_cores;
544de854881SBoris Brezillon
545de854881SBoris Brezillon /** @max_tiler_cores: Maximum number of tiler cores used for tiler jobs. */
546de854881SBoris Brezillon u8 max_tiler_cores;
547de854881SBoris Brezillon
548de854881SBoris Brezillon /** @priority: Group priority (check panthor_csg_priority). */
549de854881SBoris Brezillon u8 priority;
550de854881SBoris Brezillon
551de854881SBoris Brezillon /** @blocked_queues: Bitmask reflecting the blocked queues. */
552de854881SBoris Brezillon u32 blocked_queues;
553de854881SBoris Brezillon
554de854881SBoris Brezillon /** @idle_queues: Bitmask reflecting the idle queues. */
555de854881SBoris Brezillon u32 idle_queues;
556de854881SBoris Brezillon
557de854881SBoris Brezillon /** @fatal_lock: Lock used to protect access to fatal fields. */
558de854881SBoris Brezillon spinlock_t fatal_lock;
559de854881SBoris Brezillon
560de854881SBoris Brezillon /** @fatal_queues: Bitmask reflecting the queues that hit a fatal exception. */
561de854881SBoris Brezillon u32 fatal_queues;
562de854881SBoris Brezillon
563de854881SBoris Brezillon /** @tiler_oom: Mask of queues that have a tiler OOM event to process. */
564de854881SBoris Brezillon atomic_t tiler_oom;
565de854881SBoris Brezillon
566de854881SBoris Brezillon /** @queue_count: Number of queues in this group. */
567de854881SBoris Brezillon u32 queue_count;
568de854881SBoris Brezillon
569de854881SBoris Brezillon /** @queues: Queues owned by this group. */
570de854881SBoris Brezillon struct panthor_queue *queues[MAX_CS_PER_CSG];
571de854881SBoris Brezillon
572de854881SBoris Brezillon /**
573de854881SBoris Brezillon * @csg_id: ID of the FW group slot.
574de854881SBoris Brezillon *
575de854881SBoris Brezillon * -1 when the group is not scheduled/active.
576de854881SBoris Brezillon */
577de854881SBoris Brezillon int csg_id;
578de854881SBoris Brezillon
579de854881SBoris Brezillon /**
580de854881SBoris Brezillon * @destroyed: True when the group has been destroyed.
581de854881SBoris Brezillon *
582de854881SBoris Brezillon * If a group is destroyed it becomes useless: no further jobs can be submitted
583de854881SBoris Brezillon * to its queues. We simply wait for all references to be dropped so we can
584de854881SBoris Brezillon * release the group object.
585de854881SBoris Brezillon */
586de854881SBoris Brezillon bool destroyed;
587de854881SBoris Brezillon
588de854881SBoris Brezillon /**
589de854881SBoris Brezillon * @timedout: True when a timeout occurred on any of the queues owned by
590de854881SBoris Brezillon * this group.
591de854881SBoris Brezillon *
592de854881SBoris Brezillon * Timeouts can be reported by drm_sched or by the FW. In any case, any
593de854881SBoris Brezillon * timeout situation is unrecoverable, and the group becomes useless.
594de854881SBoris Brezillon * We simply wait for all references to be dropped so we can release the
595de854881SBoris Brezillon * group object.
596de854881SBoris Brezillon */
597de854881SBoris Brezillon bool timedout;
598de854881SBoris Brezillon
599de854881SBoris Brezillon /**
600de854881SBoris Brezillon * @syncobjs: Pool of per-queue synchronization objects.
601de854881SBoris Brezillon *
602de854881SBoris Brezillon * One sync object per queue. The position of the sync object is
603de854881SBoris Brezillon * determined by the queue index.
604de854881SBoris Brezillon */
605de854881SBoris Brezillon struct panthor_kernel_bo *syncobjs;
606de854881SBoris Brezillon
607de854881SBoris Brezillon /** @state: Group state. */
608de854881SBoris Brezillon enum panthor_group_state state;
609de854881SBoris Brezillon
610de854881SBoris Brezillon /**
611de854881SBoris Brezillon * @suspend_buf: Suspend buffer.
612de854881SBoris Brezillon *
613de854881SBoris Brezillon * Stores the state of the group and its queues when a group is suspended.
614de854881SBoris Brezillon * Used at resume time to restore the group in its previous state.
615de854881SBoris Brezillon *
616de854881SBoris Brezillon * The size of the suspend buffer is exposed through the FW interface.
617de854881SBoris Brezillon */
618de854881SBoris Brezillon struct panthor_kernel_bo *suspend_buf;
619de854881SBoris Brezillon
620de854881SBoris Brezillon /**
621de854881SBoris Brezillon * @protm_suspend_buf: Protection mode suspend buffer.
622de854881SBoris Brezillon *
623de854881SBoris Brezillon * Stores the state of the group and its queues when a group that's in
624de854881SBoris Brezillon * protection mode is suspended.
625de854881SBoris Brezillon *
626de854881SBoris Brezillon * Used at resume time to restore the group in its previous state.
627de854881SBoris Brezillon *
628de854881SBoris Brezillon * The size of the protection mode suspend buffer is exposed through the
629de854881SBoris Brezillon * FW interface.
630de854881SBoris Brezillon */
631de854881SBoris Brezillon struct panthor_kernel_bo *protm_suspend_buf;
632de854881SBoris Brezillon
633de854881SBoris Brezillon /** @sync_upd_work: Work used to check/signal job fences. */
634de854881SBoris Brezillon struct work_struct sync_upd_work;
635de854881SBoris Brezillon
636de854881SBoris Brezillon /** @tiler_oom_work: Work used to process tiler OOM events happening on this group. */
637de854881SBoris Brezillon struct work_struct tiler_oom_work;
638de854881SBoris Brezillon
639de854881SBoris Brezillon /** @term_work: Work used to finish the group termination procedure. */
640de854881SBoris Brezillon struct work_struct term_work;
641de854881SBoris Brezillon
642de854881SBoris Brezillon /**
643de854881SBoris Brezillon * @release_work: Work used to release group resources.
644de854881SBoris Brezillon *
645de854881SBoris Brezillon * We need to postpone the group release to avoid a deadlock when
646de854881SBoris Brezillon * the last ref is released in the tick work.
647de854881SBoris Brezillon */
648de854881SBoris Brezillon struct work_struct release_work;
649de854881SBoris Brezillon
650de854881SBoris Brezillon /**
651de854881SBoris Brezillon * @run_node: Node used to insert the group in the
652de854881SBoris Brezillon * panthor_group::groups::{runnable,idle} and
653de854881SBoris Brezillon * panthor_group::reset.stopped_groups lists.
654de854881SBoris Brezillon */
655de854881SBoris Brezillon struct list_head run_node;
656de854881SBoris Brezillon
657de854881SBoris Brezillon /**
658de854881SBoris Brezillon * @wait_node: Node used to insert the group in the
659de854881SBoris Brezillon * panthor_group::groups::waiting list.
660de854881SBoris Brezillon */
661de854881SBoris Brezillon struct list_head wait_node;
662de854881SBoris Brezillon };
663de854881SBoris Brezillon
664de854881SBoris Brezillon /**
665de854881SBoris Brezillon * group_queue_work() - Queue a group work
666de854881SBoris Brezillon * @group: Group to queue the work for.
667de854881SBoris Brezillon * @wname: Work name.
668de854881SBoris Brezillon *
669de854881SBoris Brezillon * Grabs a ref and queue a work item to the scheduler workqueue. If
670de854881SBoris Brezillon * the work was already queued, we release the reference we grabbed.
671de854881SBoris Brezillon *
672de854881SBoris Brezillon * Work callbacks must release the reference we grabbed here.
673de854881SBoris Brezillon */
674de854881SBoris Brezillon #define group_queue_work(group, wname) \
675de854881SBoris Brezillon do { \
676de854881SBoris Brezillon group_get(group); \
677de854881SBoris Brezillon if (!queue_work((group)->ptdev->scheduler->wq, &(group)->wname ## _work)) \
678de854881SBoris Brezillon group_put(group); \
679de854881SBoris Brezillon } while (0)
680de854881SBoris Brezillon
681de854881SBoris Brezillon /**
682de854881SBoris Brezillon * sched_queue_work() - Queue a scheduler work.
683de854881SBoris Brezillon * @sched: Scheduler object.
684de854881SBoris Brezillon * @wname: Work name.
685de854881SBoris Brezillon *
686de854881SBoris Brezillon * Conditionally queues a scheduler work if no reset is pending/in-progress.
687de854881SBoris Brezillon */
688de854881SBoris Brezillon #define sched_queue_work(sched, wname) \
689de854881SBoris Brezillon do { \
690de854881SBoris Brezillon if (!atomic_read(&(sched)->reset.in_progress) && \
691de854881SBoris Brezillon !panthor_device_reset_is_pending((sched)->ptdev)) \
692de854881SBoris Brezillon queue_work((sched)->wq, &(sched)->wname ## _work); \
693de854881SBoris Brezillon } while (0)
694de854881SBoris Brezillon
695de854881SBoris Brezillon /**
696de854881SBoris Brezillon * sched_queue_delayed_work() - Queue a scheduler delayed work.
697de854881SBoris Brezillon * @sched: Scheduler object.
698de854881SBoris Brezillon * @wname: Work name.
699de854881SBoris Brezillon * @delay: Work delay in jiffies.
700de854881SBoris Brezillon *
701de854881SBoris Brezillon * Conditionally queues a scheduler delayed work if no reset is
702de854881SBoris Brezillon * pending/in-progress.
703de854881SBoris Brezillon */
704de854881SBoris Brezillon #define sched_queue_delayed_work(sched, wname, delay) \
705de854881SBoris Brezillon do { \
706de854881SBoris Brezillon if (!atomic_read(&sched->reset.in_progress) && \
707de854881SBoris Brezillon !panthor_device_reset_is_pending((sched)->ptdev)) \
708de854881SBoris Brezillon mod_delayed_work((sched)->wq, &(sched)->wname ## _work, delay); \
709de854881SBoris Brezillon } while (0)
710de854881SBoris Brezillon
711de854881SBoris Brezillon /*
712de854881SBoris Brezillon * We currently set the maximum of groups per file to an arbitrary low value.
713de854881SBoris Brezillon * But this can be updated if we need more.
714de854881SBoris Brezillon */
715de854881SBoris Brezillon #define MAX_GROUPS_PER_POOL 128
716de854881SBoris Brezillon
717de854881SBoris Brezillon /**
718de854881SBoris Brezillon * struct panthor_group_pool - Group pool
719de854881SBoris Brezillon *
720de854881SBoris Brezillon * Each file get assigned a group pool.
721de854881SBoris Brezillon */
722de854881SBoris Brezillon struct panthor_group_pool {
723de854881SBoris Brezillon /** @xa: Xarray used to manage group handles. */
724de854881SBoris Brezillon struct xarray xa;
725de854881SBoris Brezillon };
726de854881SBoris Brezillon
727de854881SBoris Brezillon /**
728de854881SBoris Brezillon * struct panthor_job - Used to manage GPU job
729de854881SBoris Brezillon */
730de854881SBoris Brezillon struct panthor_job {
731de854881SBoris Brezillon /** @base: Inherit from drm_sched_job. */
732de854881SBoris Brezillon struct drm_sched_job base;
733de854881SBoris Brezillon
734de854881SBoris Brezillon /** @refcount: Reference count. */
735de854881SBoris Brezillon struct kref refcount;
736de854881SBoris Brezillon
737de854881SBoris Brezillon /** @group: Group of the queue this job will be pushed to. */
738de854881SBoris Brezillon struct panthor_group *group;
739de854881SBoris Brezillon
740de854881SBoris Brezillon /** @queue_idx: Index of the queue inside @group. */
741de854881SBoris Brezillon u32 queue_idx;
742de854881SBoris Brezillon
743de854881SBoris Brezillon /** @call_info: Information about the userspace command stream call. */
744de854881SBoris Brezillon struct {
745de854881SBoris Brezillon /** @start: GPU address of the userspace command stream. */
746de854881SBoris Brezillon u64 start;
747de854881SBoris Brezillon
748de854881SBoris Brezillon /** @size: Size of the userspace command stream. */
749de854881SBoris Brezillon u32 size;
750de854881SBoris Brezillon
751de854881SBoris Brezillon /**
752de854881SBoris Brezillon * @latest_flush: Flush ID at the time the userspace command
753de854881SBoris Brezillon * stream was built.
754de854881SBoris Brezillon *
755de854881SBoris Brezillon * Needed for the flush reduction mechanism.
756de854881SBoris Brezillon */
757de854881SBoris Brezillon u32 latest_flush;
758de854881SBoris Brezillon } call_info;
759de854881SBoris Brezillon
760de854881SBoris Brezillon /** @ringbuf: Position of this job is in the ring buffer. */
761de854881SBoris Brezillon struct {
762de854881SBoris Brezillon /** @start: Start offset. */
763de854881SBoris Brezillon u64 start;
764de854881SBoris Brezillon
765de854881SBoris Brezillon /** @end: End offset. */
766de854881SBoris Brezillon u64 end;
767de854881SBoris Brezillon } ringbuf;
768de854881SBoris Brezillon
769de854881SBoris Brezillon /**
770de854881SBoris Brezillon * @node: Used to insert the job in the panthor_queue::fence_ctx::in_flight_jobs
771de854881SBoris Brezillon * list.
772de854881SBoris Brezillon */
773de854881SBoris Brezillon struct list_head node;
774de854881SBoris Brezillon
775de854881SBoris Brezillon /** @done_fence: Fence signaled when the job is finished or cancelled. */
776de854881SBoris Brezillon struct dma_fence *done_fence;
777de854881SBoris Brezillon };
778de854881SBoris Brezillon
779de854881SBoris Brezillon static void
panthor_queue_put_syncwait_obj(struct panthor_queue * queue)780de854881SBoris Brezillon panthor_queue_put_syncwait_obj(struct panthor_queue *queue)
781de854881SBoris Brezillon {
782de854881SBoris Brezillon if (queue->syncwait.kmap) {
783de854881SBoris Brezillon struct iosys_map map = IOSYS_MAP_INIT_VADDR(queue->syncwait.kmap);
784de854881SBoris Brezillon
785de854881SBoris Brezillon drm_gem_vunmap_unlocked(queue->syncwait.obj, &map);
786de854881SBoris Brezillon queue->syncwait.kmap = NULL;
787de854881SBoris Brezillon }
788de854881SBoris Brezillon
789de854881SBoris Brezillon drm_gem_object_put(queue->syncwait.obj);
790de854881SBoris Brezillon queue->syncwait.obj = NULL;
791de854881SBoris Brezillon }
792de854881SBoris Brezillon
793de854881SBoris Brezillon static void *
panthor_queue_get_syncwait_obj(struct panthor_group * group,struct panthor_queue * queue)794de854881SBoris Brezillon panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue *queue)
795de854881SBoris Brezillon {
796de854881SBoris Brezillon struct panthor_device *ptdev = group->ptdev;
797de854881SBoris Brezillon struct panthor_gem_object *bo;
798de854881SBoris Brezillon struct iosys_map map;
799de854881SBoris Brezillon int ret;
800de854881SBoris Brezillon
801de854881SBoris Brezillon if (queue->syncwait.kmap)
802de854881SBoris Brezillon return queue->syncwait.kmap + queue->syncwait.offset;
803de854881SBoris Brezillon
804de854881SBoris Brezillon bo = panthor_vm_get_bo_for_va(group->vm,
805de854881SBoris Brezillon queue->syncwait.gpu_va,
806de854881SBoris Brezillon &queue->syncwait.offset);
807de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(bo)))
808de854881SBoris Brezillon goto err_put_syncwait_obj;
809de854881SBoris Brezillon
810de854881SBoris Brezillon queue->syncwait.obj = &bo->base.base;
811de854881SBoris Brezillon ret = drm_gem_vmap_unlocked(queue->syncwait.obj, &map);
812de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, ret))
813de854881SBoris Brezillon goto err_put_syncwait_obj;
814de854881SBoris Brezillon
815de854881SBoris Brezillon queue->syncwait.kmap = map.vaddr;
816de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, !queue->syncwait.kmap))
817de854881SBoris Brezillon goto err_put_syncwait_obj;
818de854881SBoris Brezillon
819de854881SBoris Brezillon return queue->syncwait.kmap + queue->syncwait.offset;
820de854881SBoris Brezillon
821de854881SBoris Brezillon err_put_syncwait_obj:
822de854881SBoris Brezillon panthor_queue_put_syncwait_obj(queue);
823de854881SBoris Brezillon return NULL;
824de854881SBoris Brezillon }
825de854881SBoris Brezillon
group_free_queue(struct panthor_group * group,struct panthor_queue * queue)826de854881SBoris Brezillon static void group_free_queue(struct panthor_group *group, struct panthor_queue *queue)
827de854881SBoris Brezillon {
828de854881SBoris Brezillon if (IS_ERR_OR_NULL(queue))
829de854881SBoris Brezillon return;
830de854881SBoris Brezillon
831de854881SBoris Brezillon if (queue->entity.fence_context)
832de854881SBoris Brezillon drm_sched_entity_destroy(&queue->entity);
833de854881SBoris Brezillon
834de854881SBoris Brezillon if (queue->scheduler.ops)
835de854881SBoris Brezillon drm_sched_fini(&queue->scheduler);
836de854881SBoris Brezillon
837de854881SBoris Brezillon panthor_queue_put_syncwait_obj(queue);
838de854881SBoris Brezillon
839ff60c8daSBoris Brezillon panthor_kernel_bo_destroy(queue->ringbuf);
840ff60c8daSBoris Brezillon panthor_kernel_bo_destroy(queue->iface.mem);
841de854881SBoris Brezillon
8427b6f9ec6SBoris Brezillon /* Release the last_fence we were holding, if any. */
8437b6f9ec6SBoris Brezillon dma_fence_put(queue->fence_ctx.last_fence);
8447b6f9ec6SBoris Brezillon
845de854881SBoris Brezillon kfree(queue);
846de854881SBoris Brezillon }
847de854881SBoris Brezillon
group_release_work(struct work_struct * work)848de854881SBoris Brezillon static void group_release_work(struct work_struct *work)
849de854881SBoris Brezillon {
850de854881SBoris Brezillon struct panthor_group *group = container_of(work,
851de854881SBoris Brezillon struct panthor_group,
852de854881SBoris Brezillon release_work);
853de854881SBoris Brezillon u32 i;
854de854881SBoris Brezillon
855de854881SBoris Brezillon for (i = 0; i < group->queue_count; i++)
856de854881SBoris Brezillon group_free_queue(group, group->queues[i]);
857de854881SBoris Brezillon
858ff60c8daSBoris Brezillon panthor_kernel_bo_destroy(group->suspend_buf);
859ff60c8daSBoris Brezillon panthor_kernel_bo_destroy(group->protm_suspend_buf);
860ff60c8daSBoris Brezillon panthor_kernel_bo_destroy(group->syncobjs);
861de854881SBoris Brezillon
862de854881SBoris Brezillon panthor_vm_put(group->vm);
863de854881SBoris Brezillon kfree(group);
864de854881SBoris Brezillon }
865de854881SBoris Brezillon
group_release(struct kref * kref)866de854881SBoris Brezillon static void group_release(struct kref *kref)
867de854881SBoris Brezillon {
868de854881SBoris Brezillon struct panthor_group *group = container_of(kref,
869de854881SBoris Brezillon struct panthor_group,
870de854881SBoris Brezillon refcount);
871de854881SBoris Brezillon struct panthor_device *ptdev = group->ptdev;
872de854881SBoris Brezillon
873de854881SBoris Brezillon drm_WARN_ON(&ptdev->base, group->csg_id >= 0);
874de854881SBoris Brezillon drm_WARN_ON(&ptdev->base, !list_empty(&group->run_node));
875de854881SBoris Brezillon drm_WARN_ON(&ptdev->base, !list_empty(&group->wait_node));
876de854881SBoris Brezillon
877de854881SBoris Brezillon queue_work(panthor_cleanup_wq, &group->release_work);
878de854881SBoris Brezillon }
879de854881SBoris Brezillon
group_put(struct panthor_group * group)880de854881SBoris Brezillon static void group_put(struct panthor_group *group)
881de854881SBoris Brezillon {
882de854881SBoris Brezillon if (group)
883de854881SBoris Brezillon kref_put(&group->refcount, group_release);
884de854881SBoris Brezillon }
885de854881SBoris Brezillon
886de854881SBoris Brezillon static struct panthor_group *
group_get(struct panthor_group * group)887de854881SBoris Brezillon group_get(struct panthor_group *group)
888de854881SBoris Brezillon {
889de854881SBoris Brezillon if (group)
890de854881SBoris Brezillon kref_get(&group->refcount);
891de854881SBoris Brezillon
892de854881SBoris Brezillon return group;
893de854881SBoris Brezillon }
894de854881SBoris Brezillon
895de854881SBoris Brezillon /**
896de854881SBoris Brezillon * group_bind_locked() - Bind a group to a group slot
897de854881SBoris Brezillon * @group: Group.
898de854881SBoris Brezillon * @csg_id: Slot.
899de854881SBoris Brezillon *
900de854881SBoris Brezillon * Return: 0 on success, a negative error code otherwise.
901de854881SBoris Brezillon */
902de854881SBoris Brezillon static int
group_bind_locked(struct panthor_group * group,u32 csg_id)903de854881SBoris Brezillon group_bind_locked(struct panthor_group *group, u32 csg_id)
904de854881SBoris Brezillon {
905de854881SBoris Brezillon struct panthor_device *ptdev = group->ptdev;
906de854881SBoris Brezillon struct panthor_csg_slot *csg_slot;
907de854881SBoris Brezillon int ret;
908de854881SBoris Brezillon
909de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
910de854881SBoris Brezillon
911de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS ||
912de854881SBoris Brezillon ptdev->scheduler->csg_slots[csg_id].group))
913de854881SBoris Brezillon return -EINVAL;
914de854881SBoris Brezillon
915de854881SBoris Brezillon ret = panthor_vm_active(group->vm);
916de854881SBoris Brezillon if (ret)
917de854881SBoris Brezillon return ret;
918de854881SBoris Brezillon
919de854881SBoris Brezillon csg_slot = &ptdev->scheduler->csg_slots[csg_id];
920de854881SBoris Brezillon group_get(group);
921de854881SBoris Brezillon group->csg_id = csg_id;
922de854881SBoris Brezillon
923de854881SBoris Brezillon /* Dummy doorbell allocation: doorbell is assigned to the group and
924de854881SBoris Brezillon * all queues use the same doorbell.
925de854881SBoris Brezillon *
926de854881SBoris Brezillon * TODO: Implement LRU-based doorbell assignment, so the most often
927de854881SBoris Brezillon * updated queues get their own doorbell, thus avoiding useless checks
928de854881SBoris Brezillon * on queues belonging to the same group that are rarely updated.
929de854881SBoris Brezillon */
930de854881SBoris Brezillon for (u32 i = 0; i < group->queue_count; i++)
931de854881SBoris Brezillon group->queues[i]->doorbell_id = csg_id + 1;
932de854881SBoris Brezillon
933de854881SBoris Brezillon csg_slot->group = group;
934de854881SBoris Brezillon
935de854881SBoris Brezillon return 0;
936de854881SBoris Brezillon }
937de854881SBoris Brezillon
938de854881SBoris Brezillon /**
939de854881SBoris Brezillon * group_unbind_locked() - Unbind a group from a slot.
940de854881SBoris Brezillon * @group: Group to unbind.
941de854881SBoris Brezillon *
942de854881SBoris Brezillon * Return: 0 on success, a negative error code otherwise.
943de854881SBoris Brezillon */
944de854881SBoris Brezillon static int
group_unbind_locked(struct panthor_group * group)945de854881SBoris Brezillon group_unbind_locked(struct panthor_group *group)
946de854881SBoris Brezillon {
947de854881SBoris Brezillon struct panthor_device *ptdev = group->ptdev;
948de854881SBoris Brezillon struct panthor_csg_slot *slot;
949de854881SBoris Brezillon
950de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
951de854881SBoris Brezillon
952de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS))
953de854881SBoris Brezillon return -EINVAL;
954de854881SBoris Brezillon
955de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE))
956de854881SBoris Brezillon return -EINVAL;
957de854881SBoris Brezillon
958de854881SBoris Brezillon slot = &ptdev->scheduler->csg_slots[group->csg_id];
959de854881SBoris Brezillon panthor_vm_idle(group->vm);
960de854881SBoris Brezillon group->csg_id = -1;
961de854881SBoris Brezillon
962de854881SBoris Brezillon /* Tiler OOM events will be re-issued next time the group is scheduled. */
963de854881SBoris Brezillon atomic_set(&group->tiler_oom, 0);
964de854881SBoris Brezillon cancel_work(&group->tiler_oom_work);
965de854881SBoris Brezillon
966de854881SBoris Brezillon for (u32 i = 0; i < group->queue_count; i++)
967de854881SBoris Brezillon group->queues[i]->doorbell_id = -1;
968de854881SBoris Brezillon
969de854881SBoris Brezillon slot->group = NULL;
970de854881SBoris Brezillon
971de854881SBoris Brezillon group_put(group);
972de854881SBoris Brezillon return 0;
973de854881SBoris Brezillon }
974de854881SBoris Brezillon
975de854881SBoris Brezillon /**
976de854881SBoris Brezillon * cs_slot_prog_locked() - Program a queue slot
977de854881SBoris Brezillon * @ptdev: Device.
978de854881SBoris Brezillon * @csg_id: Group slot ID.
979de854881SBoris Brezillon * @cs_id: Queue slot ID.
980de854881SBoris Brezillon *
981de854881SBoris Brezillon * Program a queue slot with the queue information so things can start being
982de854881SBoris Brezillon * executed on this queue.
983de854881SBoris Brezillon *
984de854881SBoris Brezillon * The group slot must have a group bound to it already (group_bind_locked()).
985de854881SBoris Brezillon */
986de854881SBoris Brezillon static void
cs_slot_prog_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)987de854881SBoris Brezillon cs_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
988de854881SBoris Brezillon {
989de854881SBoris Brezillon struct panthor_queue *queue = ptdev->scheduler->csg_slots[csg_id].group->queues[cs_id];
990de854881SBoris Brezillon struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
991de854881SBoris Brezillon
992de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
993de854881SBoris Brezillon
994de854881SBoris Brezillon queue->iface.input->extract = queue->iface.output->extract;
995de854881SBoris Brezillon drm_WARN_ON(&ptdev->base, queue->iface.input->insert < queue->iface.input->extract);
996de854881SBoris Brezillon
997de854881SBoris Brezillon cs_iface->input->ringbuf_base = panthor_kernel_bo_gpuva(queue->ringbuf);
998de854881SBoris Brezillon cs_iface->input->ringbuf_size = panthor_kernel_bo_size(queue->ringbuf);
999de854881SBoris Brezillon cs_iface->input->ringbuf_input = queue->iface.input_fw_va;
1000de854881SBoris Brezillon cs_iface->input->ringbuf_output = queue->iface.output_fw_va;
1001de854881SBoris Brezillon cs_iface->input->config = CS_CONFIG_PRIORITY(queue->priority) |
1002de854881SBoris Brezillon CS_CONFIG_DOORBELL(queue->doorbell_id);
1003de854881SBoris Brezillon cs_iface->input->ack_irq_mask = ~0;
1004de854881SBoris Brezillon panthor_fw_update_reqs(cs_iface, req,
1005de854881SBoris Brezillon CS_IDLE_SYNC_WAIT |
1006de854881SBoris Brezillon CS_IDLE_EMPTY |
1007de854881SBoris Brezillon CS_STATE_START |
1008de854881SBoris Brezillon CS_EXTRACT_EVENT,
1009de854881SBoris Brezillon CS_IDLE_SYNC_WAIT |
1010de854881SBoris Brezillon CS_IDLE_EMPTY |
1011de854881SBoris Brezillon CS_STATE_MASK |
1012de854881SBoris Brezillon CS_EXTRACT_EVENT);
1013de854881SBoris Brezillon if (queue->iface.input->insert != queue->iface.input->extract && queue->timeout_suspended) {
1014de854881SBoris Brezillon drm_sched_resume_timeout(&queue->scheduler, queue->remaining_time);
1015de854881SBoris Brezillon queue->timeout_suspended = false;
1016de854881SBoris Brezillon }
1017de854881SBoris Brezillon }
1018de854881SBoris Brezillon
1019de854881SBoris Brezillon /**
1020be7ffc82SLiviu Dudau * cs_slot_reset_locked() - Reset a queue slot
1021de854881SBoris Brezillon * @ptdev: Device.
1022de854881SBoris Brezillon * @csg_id: Group slot.
1023de854881SBoris Brezillon * @cs_id: Queue slot.
1024de854881SBoris Brezillon *
1025de854881SBoris Brezillon * Change the queue slot state to STOP and suspend the queue timeout if
1026de854881SBoris Brezillon * the queue is not blocked.
1027de854881SBoris Brezillon *
1028de854881SBoris Brezillon * The group slot must have a group bound to it (group_bind_locked()).
1029de854881SBoris Brezillon */
1030de854881SBoris Brezillon static int
cs_slot_reset_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1031de854881SBoris Brezillon cs_slot_reset_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
1032de854881SBoris Brezillon {
1033de854881SBoris Brezillon struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1034de854881SBoris Brezillon struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group;
1035de854881SBoris Brezillon struct panthor_queue *queue = group->queues[cs_id];
1036de854881SBoris Brezillon
1037de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
1038de854881SBoris Brezillon
1039de854881SBoris Brezillon panthor_fw_update_reqs(cs_iface, req,
1040de854881SBoris Brezillon CS_STATE_STOP,
1041de854881SBoris Brezillon CS_STATE_MASK);
1042de854881SBoris Brezillon
1043de854881SBoris Brezillon /* If the queue is blocked, we want to keep the timeout running, so
1044de854881SBoris Brezillon * we can detect unbounded waits and kill the group when that happens.
1045de854881SBoris Brezillon */
1046de854881SBoris Brezillon if (!(group->blocked_queues & BIT(cs_id)) && !queue->timeout_suspended) {
1047de854881SBoris Brezillon queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler);
1048de854881SBoris Brezillon queue->timeout_suspended = true;
1049de854881SBoris Brezillon WARN_ON(queue->remaining_time > msecs_to_jiffies(JOB_TIMEOUT_MS));
1050de854881SBoris Brezillon }
1051de854881SBoris Brezillon
1052de854881SBoris Brezillon return 0;
1053de854881SBoris Brezillon }
1054de854881SBoris Brezillon
1055de854881SBoris Brezillon /**
1056de854881SBoris Brezillon * csg_slot_sync_priority_locked() - Synchronize the group slot priority
1057de854881SBoris Brezillon * @ptdev: Device.
1058de854881SBoris Brezillon * @csg_id: Group slot ID.
1059de854881SBoris Brezillon *
1060de854881SBoris Brezillon * Group slot priority update happens asynchronously. When we receive a
1061de854881SBoris Brezillon * %CSG_ENDPOINT_CONFIG, we know the update is effective, and can
1062de854881SBoris Brezillon * reflect it to our panthor_csg_slot object.
1063de854881SBoris Brezillon */
1064de854881SBoris Brezillon static void
csg_slot_sync_priority_locked(struct panthor_device * ptdev,u32 csg_id)1065de854881SBoris Brezillon csg_slot_sync_priority_locked(struct panthor_device *ptdev, u32 csg_id)
1066de854881SBoris Brezillon {
1067de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1068de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface;
1069de854881SBoris Brezillon
1070de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
1071de854881SBoris Brezillon
1072de854881SBoris Brezillon csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1073de854881SBoris Brezillon csg_slot->priority = (csg_iface->input->endpoint_req & CSG_EP_REQ_PRIORITY_MASK) >> 28;
1074de854881SBoris Brezillon }
1075de854881SBoris Brezillon
1076de854881SBoris Brezillon /**
1077de854881SBoris Brezillon * cs_slot_sync_queue_state_locked() - Synchronize the queue slot priority
1078de854881SBoris Brezillon * @ptdev: Device.
1079de854881SBoris Brezillon * @csg_id: Group slot.
1080de854881SBoris Brezillon * @cs_id: Queue slot.
1081de854881SBoris Brezillon *
1082de854881SBoris Brezillon * Queue state is updated on group suspend or STATUS_UPDATE event.
1083de854881SBoris Brezillon */
1084de854881SBoris Brezillon static void
cs_slot_sync_queue_state_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1085de854881SBoris Brezillon cs_slot_sync_queue_state_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
1086de854881SBoris Brezillon {
1087de854881SBoris Brezillon struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group;
1088de854881SBoris Brezillon struct panthor_queue *queue = group->queues[cs_id];
1089de854881SBoris Brezillon struct panthor_fw_cs_iface *cs_iface =
1090de854881SBoris Brezillon panthor_fw_get_cs_iface(group->ptdev, csg_id, cs_id);
1091de854881SBoris Brezillon
1092de854881SBoris Brezillon u32 status_wait_cond;
1093de854881SBoris Brezillon
1094de854881SBoris Brezillon switch (cs_iface->output->status_blocked_reason) {
1095de854881SBoris Brezillon case CS_STATUS_BLOCKED_REASON_UNBLOCKED:
1096de854881SBoris Brezillon if (queue->iface.input->insert == queue->iface.output->extract &&
1097de854881SBoris Brezillon cs_iface->output->status_scoreboards == 0)
1098de854881SBoris Brezillon group->idle_queues |= BIT(cs_id);
1099de854881SBoris Brezillon break;
1100de854881SBoris Brezillon
1101de854881SBoris Brezillon case CS_STATUS_BLOCKED_REASON_SYNC_WAIT:
1102de854881SBoris Brezillon if (list_empty(&group->wait_node)) {
1103de854881SBoris Brezillon list_move_tail(&group->wait_node,
1104de854881SBoris Brezillon &group->ptdev->scheduler->groups.waiting);
1105de854881SBoris Brezillon }
1106de854881SBoris Brezillon group->blocked_queues |= BIT(cs_id);
1107de854881SBoris Brezillon queue->syncwait.gpu_va = cs_iface->output->status_wait_sync_ptr;
1108de854881SBoris Brezillon queue->syncwait.ref = cs_iface->output->status_wait_sync_value;
1109de854881SBoris Brezillon status_wait_cond = cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_COND_MASK;
1110de854881SBoris Brezillon queue->syncwait.gt = status_wait_cond == CS_STATUS_WAIT_SYNC_COND_GT;
1111de854881SBoris Brezillon if (cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_64B) {
1112de854881SBoris Brezillon u64 sync_val_hi = cs_iface->output->status_wait_sync_value_hi;
1113de854881SBoris Brezillon
1114de854881SBoris Brezillon queue->syncwait.sync64 = true;
1115de854881SBoris Brezillon queue->syncwait.ref |= sync_val_hi << 32;
1116de854881SBoris Brezillon } else {
1117de854881SBoris Brezillon queue->syncwait.sync64 = false;
1118de854881SBoris Brezillon }
1119de854881SBoris Brezillon break;
1120de854881SBoris Brezillon
1121de854881SBoris Brezillon default:
1122de854881SBoris Brezillon /* Other reasons are not blocking. Consider the queue as runnable
1123de854881SBoris Brezillon * in those cases.
1124de854881SBoris Brezillon */
1125de854881SBoris Brezillon break;
1126de854881SBoris Brezillon }
1127de854881SBoris Brezillon }
1128de854881SBoris Brezillon
1129de854881SBoris Brezillon static void
csg_slot_sync_queues_state_locked(struct panthor_device * ptdev,u32 csg_id)1130de854881SBoris Brezillon csg_slot_sync_queues_state_locked(struct panthor_device *ptdev, u32 csg_id)
1131de854881SBoris Brezillon {
1132de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1133de854881SBoris Brezillon struct panthor_group *group = csg_slot->group;
1134de854881SBoris Brezillon u32 i;
1135de854881SBoris Brezillon
1136de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
1137de854881SBoris Brezillon
1138de854881SBoris Brezillon group->idle_queues = 0;
1139de854881SBoris Brezillon group->blocked_queues = 0;
1140de854881SBoris Brezillon
1141de854881SBoris Brezillon for (i = 0; i < group->queue_count; i++) {
1142de854881SBoris Brezillon if (group->queues[i])
1143de854881SBoris Brezillon cs_slot_sync_queue_state_locked(ptdev, csg_id, i);
1144de854881SBoris Brezillon }
1145de854881SBoris Brezillon }
1146de854881SBoris Brezillon
1147de854881SBoris Brezillon static void
csg_slot_sync_state_locked(struct panthor_device * ptdev,u32 csg_id)1148de854881SBoris Brezillon csg_slot_sync_state_locked(struct panthor_device *ptdev, u32 csg_id)
1149de854881SBoris Brezillon {
1150de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1151de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface;
1152de854881SBoris Brezillon struct panthor_group *group;
1153de854881SBoris Brezillon enum panthor_group_state new_state, old_state;
11548bdbd8b5SBoris Brezillon u32 csg_state;
1155de854881SBoris Brezillon
1156de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
1157de854881SBoris Brezillon
1158de854881SBoris Brezillon csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1159de854881SBoris Brezillon group = csg_slot->group;
1160de854881SBoris Brezillon
1161de854881SBoris Brezillon if (!group)
1162de854881SBoris Brezillon return;
1163de854881SBoris Brezillon
1164de854881SBoris Brezillon old_state = group->state;
11658bdbd8b5SBoris Brezillon csg_state = csg_iface->output->ack & CSG_STATE_MASK;
11668bdbd8b5SBoris Brezillon switch (csg_state) {
1167de854881SBoris Brezillon case CSG_STATE_START:
1168de854881SBoris Brezillon case CSG_STATE_RESUME:
1169de854881SBoris Brezillon new_state = PANTHOR_CS_GROUP_ACTIVE;
1170de854881SBoris Brezillon break;
1171de854881SBoris Brezillon case CSG_STATE_TERMINATE:
1172de854881SBoris Brezillon new_state = PANTHOR_CS_GROUP_TERMINATED;
1173de854881SBoris Brezillon break;
1174de854881SBoris Brezillon case CSG_STATE_SUSPEND:
1175de854881SBoris Brezillon new_state = PANTHOR_CS_GROUP_SUSPENDED;
1176de854881SBoris Brezillon break;
11778bdbd8b5SBoris Brezillon default:
11788bdbd8b5SBoris Brezillon /* The unknown state might be caused by a FW state corruption,
11798bdbd8b5SBoris Brezillon * which means the group metadata can't be trusted anymore, and
11808bdbd8b5SBoris Brezillon * the SUSPEND operation might propagate the corruption to the
11818bdbd8b5SBoris Brezillon * suspend buffers. Flag the group state as unknown to make
11828bdbd8b5SBoris Brezillon * sure it's unusable after that point.
11838bdbd8b5SBoris Brezillon */
11848bdbd8b5SBoris Brezillon drm_err(&ptdev->base, "Invalid state on CSG %d (state=%d)",
11858bdbd8b5SBoris Brezillon csg_id, csg_state);
11868bdbd8b5SBoris Brezillon new_state = PANTHOR_CS_GROUP_UNKNOWN_STATE;
11878bdbd8b5SBoris Brezillon break;
1188de854881SBoris Brezillon }
1189de854881SBoris Brezillon
1190de854881SBoris Brezillon if (old_state == new_state)
1191de854881SBoris Brezillon return;
1192de854881SBoris Brezillon
11938bdbd8b5SBoris Brezillon /* The unknown state might be caused by a FW issue, reset the FW to
11948bdbd8b5SBoris Brezillon * take a fresh start.
11958bdbd8b5SBoris Brezillon */
11968bdbd8b5SBoris Brezillon if (new_state == PANTHOR_CS_GROUP_UNKNOWN_STATE)
11978bdbd8b5SBoris Brezillon panthor_device_schedule_reset(ptdev);
11988bdbd8b5SBoris Brezillon
1199de854881SBoris Brezillon if (new_state == PANTHOR_CS_GROUP_SUSPENDED)
1200de854881SBoris Brezillon csg_slot_sync_queues_state_locked(ptdev, csg_id);
1201de854881SBoris Brezillon
1202de854881SBoris Brezillon if (old_state == PANTHOR_CS_GROUP_ACTIVE) {
1203de854881SBoris Brezillon u32 i;
1204de854881SBoris Brezillon
1205de854881SBoris Brezillon /* Reset the queue slots so we start from a clean
1206de854881SBoris Brezillon * state when starting/resuming a new group on this
1207de854881SBoris Brezillon * CSG slot. No wait needed here, and no ringbell
1208de854881SBoris Brezillon * either, since the CS slot will only be re-used
1209de854881SBoris Brezillon * on the next CSG start operation.
1210de854881SBoris Brezillon */
1211de854881SBoris Brezillon for (i = 0; i < group->queue_count; i++) {
1212de854881SBoris Brezillon if (group->queues[i])
1213de854881SBoris Brezillon cs_slot_reset_locked(ptdev, csg_id, i);
1214de854881SBoris Brezillon }
1215de854881SBoris Brezillon }
1216de854881SBoris Brezillon
1217de854881SBoris Brezillon group->state = new_state;
1218de854881SBoris Brezillon }
1219de854881SBoris Brezillon
1220de854881SBoris Brezillon static int
csg_slot_prog_locked(struct panthor_device * ptdev,u32 csg_id,u32 priority)1221de854881SBoris Brezillon csg_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 priority)
1222de854881SBoris Brezillon {
1223de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface;
1224de854881SBoris Brezillon struct panthor_csg_slot *csg_slot;
1225de854881SBoris Brezillon struct panthor_group *group;
1226de854881SBoris Brezillon u32 queue_mask = 0, i;
1227de854881SBoris Brezillon
1228de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
1229de854881SBoris Brezillon
1230de854881SBoris Brezillon if (priority > MAX_CSG_PRIO)
1231de854881SBoris Brezillon return -EINVAL;
1232de854881SBoris Brezillon
1233de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, csg_id >= MAX_CSGS))
1234de854881SBoris Brezillon return -EINVAL;
1235de854881SBoris Brezillon
1236de854881SBoris Brezillon csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1237de854881SBoris Brezillon group = csg_slot->group;
1238de854881SBoris Brezillon if (!group || group->state == PANTHOR_CS_GROUP_ACTIVE)
1239de854881SBoris Brezillon return 0;
1240de854881SBoris Brezillon
1241de854881SBoris Brezillon csg_iface = panthor_fw_get_csg_iface(group->ptdev, csg_id);
1242de854881SBoris Brezillon
1243de854881SBoris Brezillon for (i = 0; i < group->queue_count; i++) {
1244de854881SBoris Brezillon if (group->queues[i]) {
1245de854881SBoris Brezillon cs_slot_prog_locked(ptdev, csg_id, i);
1246de854881SBoris Brezillon queue_mask |= BIT(i);
1247de854881SBoris Brezillon }
1248de854881SBoris Brezillon }
1249de854881SBoris Brezillon
1250de854881SBoris Brezillon csg_iface->input->allow_compute = group->compute_core_mask;
1251de854881SBoris Brezillon csg_iface->input->allow_fragment = group->fragment_core_mask;
1252de854881SBoris Brezillon csg_iface->input->allow_other = group->tiler_core_mask;
1253de854881SBoris Brezillon csg_iface->input->endpoint_req = CSG_EP_REQ_COMPUTE(group->max_compute_cores) |
1254de854881SBoris Brezillon CSG_EP_REQ_FRAGMENT(group->max_fragment_cores) |
1255de854881SBoris Brezillon CSG_EP_REQ_TILER(group->max_tiler_cores) |
1256de854881SBoris Brezillon CSG_EP_REQ_PRIORITY(priority);
1257de854881SBoris Brezillon csg_iface->input->config = panthor_vm_as(group->vm);
1258de854881SBoris Brezillon
1259de854881SBoris Brezillon if (group->suspend_buf)
1260de854881SBoris Brezillon csg_iface->input->suspend_buf = panthor_kernel_bo_gpuva(group->suspend_buf);
1261de854881SBoris Brezillon else
1262de854881SBoris Brezillon csg_iface->input->suspend_buf = 0;
1263de854881SBoris Brezillon
1264de854881SBoris Brezillon if (group->protm_suspend_buf) {
1265de854881SBoris Brezillon csg_iface->input->protm_suspend_buf =
1266de854881SBoris Brezillon panthor_kernel_bo_gpuva(group->protm_suspend_buf);
1267de854881SBoris Brezillon } else {
1268de854881SBoris Brezillon csg_iface->input->protm_suspend_buf = 0;
1269de854881SBoris Brezillon }
1270de854881SBoris Brezillon
1271de854881SBoris Brezillon csg_iface->input->ack_irq_mask = ~0;
1272de854881SBoris Brezillon panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, queue_mask);
1273de854881SBoris Brezillon return 0;
1274de854881SBoris Brezillon }
1275de854881SBoris Brezillon
1276de854881SBoris Brezillon static void
cs_slot_process_fatal_event_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1277de854881SBoris Brezillon cs_slot_process_fatal_event_locked(struct panthor_device *ptdev,
1278de854881SBoris Brezillon u32 csg_id, u32 cs_id)
1279de854881SBoris Brezillon {
1280de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
1281de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
1282de854881SBoris Brezillon struct panthor_group *group = csg_slot->group;
1283de854881SBoris Brezillon struct panthor_fw_cs_iface *cs_iface;
1284de854881SBoris Brezillon u32 fatal;
1285de854881SBoris Brezillon u64 info;
1286de854881SBoris Brezillon
1287de854881SBoris Brezillon lockdep_assert_held(&sched->lock);
1288de854881SBoris Brezillon
1289de854881SBoris Brezillon cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1290de854881SBoris Brezillon fatal = cs_iface->output->fatal;
1291de854881SBoris Brezillon info = cs_iface->output->fatal_info;
1292de854881SBoris Brezillon
1293de854881SBoris Brezillon if (group)
1294de854881SBoris Brezillon group->fatal_queues |= BIT(cs_id);
1295de854881SBoris Brezillon
12962b2a26b3SBoris Brezillon if (CS_EXCEPTION_TYPE(fatal) == DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE) {
12972b2a26b3SBoris Brezillon /* If this exception is unrecoverable, queue a reset, and make
12982b2a26b3SBoris Brezillon * sure we stop scheduling groups until the reset has happened.
12992b2a26b3SBoris Brezillon */
13002b2a26b3SBoris Brezillon panthor_device_schedule_reset(ptdev);
13012b2a26b3SBoris Brezillon cancel_delayed_work(&sched->tick_work);
13022b2a26b3SBoris Brezillon } else {
1303de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, 0);
13042b2a26b3SBoris Brezillon }
13052b2a26b3SBoris Brezillon
1306de854881SBoris Brezillon drm_warn(&ptdev->base,
1307de854881SBoris Brezillon "CSG slot %d CS slot: %d\n"
1308de854881SBoris Brezillon "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n"
1309de854881SBoris Brezillon "CS_FATAL.EXCEPTION_DATA: 0x%x\n"
1310de854881SBoris Brezillon "CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n",
1311de854881SBoris Brezillon csg_id, cs_id,
1312de854881SBoris Brezillon (unsigned int)CS_EXCEPTION_TYPE(fatal),
1313de854881SBoris Brezillon panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fatal)),
1314de854881SBoris Brezillon (unsigned int)CS_EXCEPTION_DATA(fatal),
1315de854881SBoris Brezillon info);
1316de854881SBoris Brezillon }
1317de854881SBoris Brezillon
1318de854881SBoris Brezillon static void
cs_slot_process_fault_event_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1319de854881SBoris Brezillon cs_slot_process_fault_event_locked(struct panthor_device *ptdev,
1320de854881SBoris Brezillon u32 csg_id, u32 cs_id)
1321de854881SBoris Brezillon {
1322de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
1323de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
1324de854881SBoris Brezillon struct panthor_group *group = csg_slot->group;
1325de854881SBoris Brezillon struct panthor_queue *queue = group && cs_id < group->queue_count ?
1326de854881SBoris Brezillon group->queues[cs_id] : NULL;
1327de854881SBoris Brezillon struct panthor_fw_cs_iface *cs_iface;
1328de854881SBoris Brezillon u32 fault;
1329de854881SBoris Brezillon u64 info;
1330de854881SBoris Brezillon
1331de854881SBoris Brezillon lockdep_assert_held(&sched->lock);
1332de854881SBoris Brezillon
1333de854881SBoris Brezillon cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1334de854881SBoris Brezillon fault = cs_iface->output->fault;
1335de854881SBoris Brezillon info = cs_iface->output->fault_info;
1336de854881SBoris Brezillon
1337de854881SBoris Brezillon if (queue && CS_EXCEPTION_TYPE(fault) == DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT) {
1338de854881SBoris Brezillon u64 cs_extract = queue->iface.output->extract;
1339de854881SBoris Brezillon struct panthor_job *job;
1340de854881SBoris Brezillon
1341de854881SBoris Brezillon spin_lock(&queue->fence_ctx.lock);
1342de854881SBoris Brezillon list_for_each_entry(job, &queue->fence_ctx.in_flight_jobs, node) {
1343de854881SBoris Brezillon if (cs_extract >= job->ringbuf.end)
1344de854881SBoris Brezillon continue;
1345de854881SBoris Brezillon
1346de854881SBoris Brezillon if (cs_extract < job->ringbuf.start)
1347de854881SBoris Brezillon break;
1348de854881SBoris Brezillon
1349de854881SBoris Brezillon dma_fence_set_error(job->done_fence, -EINVAL);
1350de854881SBoris Brezillon }
1351de854881SBoris Brezillon spin_unlock(&queue->fence_ctx.lock);
1352de854881SBoris Brezillon }
1353de854881SBoris Brezillon
1354de854881SBoris Brezillon drm_warn(&ptdev->base,
1355de854881SBoris Brezillon "CSG slot %d CS slot: %d\n"
1356de854881SBoris Brezillon "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n"
1357de854881SBoris Brezillon "CS_FAULT.EXCEPTION_DATA: 0x%x\n"
1358de854881SBoris Brezillon "CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n",
1359de854881SBoris Brezillon csg_id, cs_id,
1360de854881SBoris Brezillon (unsigned int)CS_EXCEPTION_TYPE(fault),
1361de854881SBoris Brezillon panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fault)),
1362de854881SBoris Brezillon (unsigned int)CS_EXCEPTION_DATA(fault),
1363de854881SBoris Brezillon info);
1364de854881SBoris Brezillon }
1365de854881SBoris Brezillon
group_process_tiler_oom(struct panthor_group * group,u32 cs_id)1366de854881SBoris Brezillon static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id)
1367de854881SBoris Brezillon {
1368de854881SBoris Brezillon struct panthor_device *ptdev = group->ptdev;
1369de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
1370de854881SBoris Brezillon u32 renderpasses_in_flight, pending_frag_count;
1371de854881SBoris Brezillon struct panthor_heap_pool *heaps = NULL;
1372de854881SBoris Brezillon u64 heap_address, new_chunk_va = 0;
1373de854881SBoris Brezillon u32 vt_start, vt_end, frag_end;
1374de854881SBoris Brezillon int ret, csg_id;
1375de854881SBoris Brezillon
1376de854881SBoris Brezillon mutex_lock(&sched->lock);
1377de854881SBoris Brezillon csg_id = group->csg_id;
1378de854881SBoris Brezillon if (csg_id >= 0) {
1379de854881SBoris Brezillon struct panthor_fw_cs_iface *cs_iface;
1380de854881SBoris Brezillon
1381de854881SBoris Brezillon cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1382de854881SBoris Brezillon heaps = panthor_vm_get_heap_pool(group->vm, false);
1383de854881SBoris Brezillon heap_address = cs_iface->output->heap_address;
1384de854881SBoris Brezillon vt_start = cs_iface->output->heap_vt_start;
1385de854881SBoris Brezillon vt_end = cs_iface->output->heap_vt_end;
1386de854881SBoris Brezillon frag_end = cs_iface->output->heap_frag_end;
1387de854881SBoris Brezillon renderpasses_in_flight = vt_start - frag_end;
1388de854881SBoris Brezillon pending_frag_count = vt_end - frag_end;
1389de854881SBoris Brezillon }
1390de854881SBoris Brezillon mutex_unlock(&sched->lock);
1391de854881SBoris Brezillon
1392de854881SBoris Brezillon /* The group got scheduled out, we stop here. We will get a new tiler OOM event
1393de854881SBoris Brezillon * when it's scheduled again.
1394de854881SBoris Brezillon */
1395de854881SBoris Brezillon if (unlikely(csg_id < 0))
1396de854881SBoris Brezillon return 0;
1397de854881SBoris Brezillon
139845c734fdSHarshit Mogalapalli if (IS_ERR(heaps) || frag_end > vt_end || vt_end >= vt_start) {
1399de854881SBoris Brezillon ret = -EINVAL;
1400de854881SBoris Brezillon } else {
1401de854881SBoris Brezillon /* We do the allocation without holding the scheduler lock to avoid
1402de854881SBoris Brezillon * blocking the scheduling.
1403de854881SBoris Brezillon */
1404de854881SBoris Brezillon ret = panthor_heap_grow(heaps, heap_address,
1405de854881SBoris Brezillon renderpasses_in_flight,
1406de854881SBoris Brezillon pending_frag_count, &new_chunk_va);
1407de854881SBoris Brezillon }
1408de854881SBoris Brezillon
1409d2143297SAntonino Maniscalco /* If the heap context doesn't have memory for us, we want to let the
1410d2143297SAntonino Maniscalco * FW try to reclaim memory by waiting for fragment jobs to land or by
1411d2143297SAntonino Maniscalco * executing the tiler OOM exception handler, which is supposed to
1412d2143297SAntonino Maniscalco * implement incremental rendering.
1413d2143297SAntonino Maniscalco */
1414d2143297SAntonino Maniscalco if (ret && ret != -ENOMEM) {
1415de854881SBoris Brezillon drm_warn(&ptdev->base, "Failed to extend the tiler heap\n");
1416de854881SBoris Brezillon group->fatal_queues |= BIT(cs_id);
1417de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, 0);
1418de854881SBoris Brezillon goto out_put_heap_pool;
1419de854881SBoris Brezillon }
1420de854881SBoris Brezillon
1421de854881SBoris Brezillon mutex_lock(&sched->lock);
1422de854881SBoris Brezillon csg_id = group->csg_id;
1423de854881SBoris Brezillon if (csg_id >= 0) {
1424de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface;
1425de854881SBoris Brezillon struct panthor_fw_cs_iface *cs_iface;
1426de854881SBoris Brezillon
1427de854881SBoris Brezillon csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1428de854881SBoris Brezillon cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1429de854881SBoris Brezillon
1430de854881SBoris Brezillon cs_iface->input->heap_start = new_chunk_va;
1431de854881SBoris Brezillon cs_iface->input->heap_end = new_chunk_va;
1432de854881SBoris Brezillon panthor_fw_update_reqs(cs_iface, req, cs_iface->output->ack, CS_TILER_OOM);
1433de854881SBoris Brezillon panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, BIT(cs_id));
1434de854881SBoris Brezillon panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id));
1435de854881SBoris Brezillon }
1436de854881SBoris Brezillon mutex_unlock(&sched->lock);
1437de854881SBoris Brezillon
1438de854881SBoris Brezillon /* We allocated a chunck, but couldn't link it to the heap
1439de854881SBoris Brezillon * context because the group was scheduled out while we were
1440de854881SBoris Brezillon * allocating memory. We need to return this chunk to the heap.
1441de854881SBoris Brezillon */
1442de854881SBoris Brezillon if (unlikely(csg_id < 0 && new_chunk_va))
1443de854881SBoris Brezillon panthor_heap_return_chunk(heaps, heap_address, new_chunk_va);
1444de854881SBoris Brezillon
1445de854881SBoris Brezillon ret = 0;
1446de854881SBoris Brezillon
1447de854881SBoris Brezillon out_put_heap_pool:
1448de854881SBoris Brezillon panthor_heap_pool_put(heaps);
1449de854881SBoris Brezillon return ret;
1450de854881SBoris Brezillon }
1451de854881SBoris Brezillon
group_tiler_oom_work(struct work_struct * work)1452de854881SBoris Brezillon static void group_tiler_oom_work(struct work_struct *work)
1453de854881SBoris Brezillon {
1454de854881SBoris Brezillon struct panthor_group *group =
1455de854881SBoris Brezillon container_of(work, struct panthor_group, tiler_oom_work);
1456de854881SBoris Brezillon u32 tiler_oom = atomic_xchg(&group->tiler_oom, 0);
1457de854881SBoris Brezillon
1458de854881SBoris Brezillon while (tiler_oom) {
1459de854881SBoris Brezillon u32 cs_id = ffs(tiler_oom) - 1;
1460de854881SBoris Brezillon
1461de854881SBoris Brezillon group_process_tiler_oom(group, cs_id);
1462de854881SBoris Brezillon tiler_oom &= ~BIT(cs_id);
1463de854881SBoris Brezillon }
1464de854881SBoris Brezillon
1465de854881SBoris Brezillon group_put(group);
1466de854881SBoris Brezillon }
1467de854881SBoris Brezillon
1468de854881SBoris Brezillon static void
cs_slot_process_tiler_oom_event_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1469de854881SBoris Brezillon cs_slot_process_tiler_oom_event_locked(struct panthor_device *ptdev,
1470de854881SBoris Brezillon u32 csg_id, u32 cs_id)
1471de854881SBoris Brezillon {
1472de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
1473de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
1474de854881SBoris Brezillon struct panthor_group *group = csg_slot->group;
1475de854881SBoris Brezillon
1476de854881SBoris Brezillon lockdep_assert_held(&sched->lock);
1477de854881SBoris Brezillon
1478de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, !group))
1479de854881SBoris Brezillon return;
1480de854881SBoris Brezillon
1481de854881SBoris Brezillon atomic_or(BIT(cs_id), &group->tiler_oom);
1482de854881SBoris Brezillon
1483de854881SBoris Brezillon /* We don't use group_queue_work() here because we want to queue the
1484de854881SBoris Brezillon * work item to the heap_alloc_wq.
1485de854881SBoris Brezillon */
1486de854881SBoris Brezillon group_get(group);
1487de854881SBoris Brezillon if (!queue_work(sched->heap_alloc_wq, &group->tiler_oom_work))
1488de854881SBoris Brezillon group_put(group);
1489de854881SBoris Brezillon }
1490de854881SBoris Brezillon
cs_slot_process_irq_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1491de854881SBoris Brezillon static bool cs_slot_process_irq_locked(struct panthor_device *ptdev,
1492de854881SBoris Brezillon u32 csg_id, u32 cs_id)
1493de854881SBoris Brezillon {
1494de854881SBoris Brezillon struct panthor_fw_cs_iface *cs_iface;
1495de854881SBoris Brezillon u32 req, ack, events;
1496de854881SBoris Brezillon
1497de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
1498de854881SBoris Brezillon
1499de854881SBoris Brezillon cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1500de854881SBoris Brezillon req = cs_iface->input->req;
1501de854881SBoris Brezillon ack = cs_iface->output->ack;
1502de854881SBoris Brezillon events = (req ^ ack) & CS_EVT_MASK;
1503de854881SBoris Brezillon
1504de854881SBoris Brezillon if (events & CS_FATAL)
1505de854881SBoris Brezillon cs_slot_process_fatal_event_locked(ptdev, csg_id, cs_id);
1506de854881SBoris Brezillon
1507de854881SBoris Brezillon if (events & CS_FAULT)
1508de854881SBoris Brezillon cs_slot_process_fault_event_locked(ptdev, csg_id, cs_id);
1509de854881SBoris Brezillon
1510de854881SBoris Brezillon if (events & CS_TILER_OOM)
1511de854881SBoris Brezillon cs_slot_process_tiler_oom_event_locked(ptdev, csg_id, cs_id);
1512de854881SBoris Brezillon
1513de854881SBoris Brezillon /* We don't acknowledge the TILER_OOM event since its handling is
1514de854881SBoris Brezillon * deferred to a separate work.
1515de854881SBoris Brezillon */
1516de854881SBoris Brezillon panthor_fw_update_reqs(cs_iface, req, ack, CS_FATAL | CS_FAULT);
1517de854881SBoris Brezillon
1518de854881SBoris Brezillon return (events & (CS_FAULT | CS_TILER_OOM)) != 0;
1519de854881SBoris Brezillon }
1520de854881SBoris Brezillon
csg_slot_sync_idle_state_locked(struct panthor_device * ptdev,u32 csg_id)1521de854881SBoris Brezillon static void csg_slot_sync_idle_state_locked(struct panthor_device *ptdev, u32 csg_id)
1522de854881SBoris Brezillon {
1523de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1524de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface;
1525de854881SBoris Brezillon
1526de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
1527de854881SBoris Brezillon
1528de854881SBoris Brezillon csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1529de854881SBoris Brezillon csg_slot->idle = csg_iface->output->status_state & CSG_STATUS_STATE_IS_IDLE;
1530de854881SBoris Brezillon }
1531de854881SBoris Brezillon
csg_slot_process_idle_event_locked(struct panthor_device * ptdev,u32 csg_id)1532de854881SBoris Brezillon static void csg_slot_process_idle_event_locked(struct panthor_device *ptdev, u32 csg_id)
1533de854881SBoris Brezillon {
1534de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
1535de854881SBoris Brezillon
1536de854881SBoris Brezillon lockdep_assert_held(&sched->lock);
1537de854881SBoris Brezillon
1538de854881SBoris Brezillon sched->might_have_idle_groups = true;
1539de854881SBoris Brezillon
1540de854881SBoris Brezillon /* Schedule a tick so we can evict idle groups and schedule non-idle
1541de854881SBoris Brezillon * ones. This will also update runtime PM and devfreq busy/idle states,
1542de854881SBoris Brezillon * so the device can lower its frequency or get suspended.
1543de854881SBoris Brezillon */
1544de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, 0);
1545de854881SBoris Brezillon }
1546de854881SBoris Brezillon
csg_slot_sync_update_locked(struct panthor_device * ptdev,u32 csg_id)1547de854881SBoris Brezillon static void csg_slot_sync_update_locked(struct panthor_device *ptdev,
1548de854881SBoris Brezillon u32 csg_id)
1549de854881SBoris Brezillon {
1550de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1551de854881SBoris Brezillon struct panthor_group *group = csg_slot->group;
1552de854881SBoris Brezillon
1553de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
1554de854881SBoris Brezillon
1555de854881SBoris Brezillon if (group)
1556de854881SBoris Brezillon group_queue_work(group, sync_upd);
1557de854881SBoris Brezillon
1558de854881SBoris Brezillon sched_queue_work(ptdev->scheduler, sync_upd);
1559de854881SBoris Brezillon }
1560de854881SBoris Brezillon
1561de854881SBoris Brezillon static void
csg_slot_process_progress_timer_event_locked(struct panthor_device * ptdev,u32 csg_id)1562de854881SBoris Brezillon csg_slot_process_progress_timer_event_locked(struct panthor_device *ptdev, u32 csg_id)
1563de854881SBoris Brezillon {
1564de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
1565de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
1566de854881SBoris Brezillon struct panthor_group *group = csg_slot->group;
1567de854881SBoris Brezillon
1568de854881SBoris Brezillon lockdep_assert_held(&sched->lock);
1569de854881SBoris Brezillon
1570de854881SBoris Brezillon drm_warn(&ptdev->base, "CSG slot %d progress timeout\n", csg_id);
1571de854881SBoris Brezillon
1572de854881SBoris Brezillon group = csg_slot->group;
1573de854881SBoris Brezillon if (!drm_WARN_ON(&ptdev->base, !group))
1574de854881SBoris Brezillon group->timedout = true;
1575de854881SBoris Brezillon
1576de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, 0);
1577de854881SBoris Brezillon }
1578de854881SBoris Brezillon
sched_process_csg_irq_locked(struct panthor_device * ptdev,u32 csg_id)1579de854881SBoris Brezillon static void sched_process_csg_irq_locked(struct panthor_device *ptdev, u32 csg_id)
1580de854881SBoris Brezillon {
1581de854881SBoris Brezillon u32 req, ack, cs_irq_req, cs_irq_ack, cs_irqs, csg_events;
1582de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface;
1583de854881SBoris Brezillon u32 ring_cs_db_mask = 0;
1584de854881SBoris Brezillon
1585de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
1586de854881SBoris Brezillon
1587de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count))
1588de854881SBoris Brezillon return;
1589de854881SBoris Brezillon
1590de854881SBoris Brezillon csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1591de854881SBoris Brezillon req = READ_ONCE(csg_iface->input->req);
1592de854881SBoris Brezillon ack = READ_ONCE(csg_iface->output->ack);
1593de854881SBoris Brezillon cs_irq_req = READ_ONCE(csg_iface->output->cs_irq_req);
1594de854881SBoris Brezillon cs_irq_ack = READ_ONCE(csg_iface->input->cs_irq_ack);
1595de854881SBoris Brezillon csg_events = (req ^ ack) & CSG_EVT_MASK;
1596de854881SBoris Brezillon
1597de854881SBoris Brezillon /* There may not be any pending CSG/CS interrupts to process */
1598de854881SBoris Brezillon if (req == ack && cs_irq_req == cs_irq_ack)
1599de854881SBoris Brezillon return;
1600de854881SBoris Brezillon
1601de854881SBoris Brezillon /* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before
1602de854881SBoris Brezillon * examining the CS_ACK & CS_REQ bits. This would ensure that Host
1603de854881SBoris Brezillon * doesn't miss an interrupt for the CS in the race scenario where
1604de854881SBoris Brezillon * whilst Host is servicing an interrupt for the CS, firmware sends
1605de854881SBoris Brezillon * another interrupt for that CS.
1606de854881SBoris Brezillon */
1607de854881SBoris Brezillon csg_iface->input->cs_irq_ack = cs_irq_req;
1608de854881SBoris Brezillon
1609de854881SBoris Brezillon panthor_fw_update_reqs(csg_iface, req, ack,
1610de854881SBoris Brezillon CSG_SYNC_UPDATE |
1611de854881SBoris Brezillon CSG_IDLE |
1612de854881SBoris Brezillon CSG_PROGRESS_TIMER_EVENT);
1613de854881SBoris Brezillon
1614de854881SBoris Brezillon if (csg_events & CSG_IDLE)
1615de854881SBoris Brezillon csg_slot_process_idle_event_locked(ptdev, csg_id);
1616de854881SBoris Brezillon
1617de854881SBoris Brezillon if (csg_events & CSG_PROGRESS_TIMER_EVENT)
1618de854881SBoris Brezillon csg_slot_process_progress_timer_event_locked(ptdev, csg_id);
1619de854881SBoris Brezillon
1620de854881SBoris Brezillon cs_irqs = cs_irq_req ^ cs_irq_ack;
1621de854881SBoris Brezillon while (cs_irqs) {
1622de854881SBoris Brezillon u32 cs_id = ffs(cs_irqs) - 1;
1623de854881SBoris Brezillon
1624de854881SBoris Brezillon if (cs_slot_process_irq_locked(ptdev, csg_id, cs_id))
1625de854881SBoris Brezillon ring_cs_db_mask |= BIT(cs_id);
1626de854881SBoris Brezillon
1627de854881SBoris Brezillon cs_irqs &= ~BIT(cs_id);
1628de854881SBoris Brezillon }
1629de854881SBoris Brezillon
1630de854881SBoris Brezillon if (csg_events & CSG_SYNC_UPDATE)
1631de854881SBoris Brezillon csg_slot_sync_update_locked(ptdev, csg_id);
1632de854881SBoris Brezillon
1633de854881SBoris Brezillon if (ring_cs_db_mask)
1634de854881SBoris Brezillon panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, ring_cs_db_mask);
1635de854881SBoris Brezillon
1636de854881SBoris Brezillon panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id));
1637de854881SBoris Brezillon }
1638de854881SBoris Brezillon
sched_process_idle_event_locked(struct panthor_device * ptdev)1639de854881SBoris Brezillon static void sched_process_idle_event_locked(struct panthor_device *ptdev)
1640de854881SBoris Brezillon {
1641de854881SBoris Brezillon struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
1642de854881SBoris Brezillon
1643de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
1644de854881SBoris Brezillon
1645de854881SBoris Brezillon /* Acknowledge the idle event and schedule a tick. */
1646de854881SBoris Brezillon panthor_fw_update_reqs(glb_iface, req, glb_iface->output->ack, GLB_IDLE);
1647de854881SBoris Brezillon sched_queue_delayed_work(ptdev->scheduler, tick, 0);
1648de854881SBoris Brezillon }
1649de854881SBoris Brezillon
1650de854881SBoris Brezillon /**
1651be7ffc82SLiviu Dudau * sched_process_global_irq_locked() - Process the scheduling part of a global IRQ
1652de854881SBoris Brezillon * @ptdev: Device.
1653de854881SBoris Brezillon */
sched_process_global_irq_locked(struct panthor_device * ptdev)1654de854881SBoris Brezillon static void sched_process_global_irq_locked(struct panthor_device *ptdev)
1655de854881SBoris Brezillon {
1656de854881SBoris Brezillon struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
1657de854881SBoris Brezillon u32 req, ack, evts;
1658de854881SBoris Brezillon
1659de854881SBoris Brezillon lockdep_assert_held(&ptdev->scheduler->lock);
1660de854881SBoris Brezillon
1661de854881SBoris Brezillon req = READ_ONCE(glb_iface->input->req);
1662de854881SBoris Brezillon ack = READ_ONCE(glb_iface->output->ack);
1663de854881SBoris Brezillon evts = (req ^ ack) & GLB_EVT_MASK;
1664de854881SBoris Brezillon
1665de854881SBoris Brezillon if (evts & GLB_IDLE)
1666de854881SBoris Brezillon sched_process_idle_event_locked(ptdev);
1667de854881SBoris Brezillon }
1668de854881SBoris Brezillon
process_fw_events_work(struct work_struct * work)1669de854881SBoris Brezillon static void process_fw_events_work(struct work_struct *work)
1670de854881SBoris Brezillon {
1671de854881SBoris Brezillon struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler,
1672de854881SBoris Brezillon fw_events_work);
1673de854881SBoris Brezillon u32 events = atomic_xchg(&sched->fw_events, 0);
1674de854881SBoris Brezillon struct panthor_device *ptdev = sched->ptdev;
1675de854881SBoris Brezillon
1676de854881SBoris Brezillon mutex_lock(&sched->lock);
1677de854881SBoris Brezillon
1678de854881SBoris Brezillon if (events & JOB_INT_GLOBAL_IF) {
1679de854881SBoris Brezillon sched_process_global_irq_locked(ptdev);
1680de854881SBoris Brezillon events &= ~JOB_INT_GLOBAL_IF;
1681de854881SBoris Brezillon }
1682de854881SBoris Brezillon
1683de854881SBoris Brezillon while (events) {
1684de854881SBoris Brezillon u32 csg_id = ffs(events) - 1;
1685de854881SBoris Brezillon
1686de854881SBoris Brezillon sched_process_csg_irq_locked(ptdev, csg_id);
1687de854881SBoris Brezillon events &= ~BIT(csg_id);
1688de854881SBoris Brezillon }
1689de854881SBoris Brezillon
1690de854881SBoris Brezillon mutex_unlock(&sched->lock);
1691de854881SBoris Brezillon }
1692de854881SBoris Brezillon
1693de854881SBoris Brezillon /**
1694de854881SBoris Brezillon * panthor_sched_report_fw_events() - Report FW events to the scheduler.
1695de854881SBoris Brezillon */
panthor_sched_report_fw_events(struct panthor_device * ptdev,u32 events)1696de854881SBoris Brezillon void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events)
1697de854881SBoris Brezillon {
1698de854881SBoris Brezillon if (!ptdev->scheduler)
1699de854881SBoris Brezillon return;
1700de854881SBoris Brezillon
1701de854881SBoris Brezillon atomic_or(events, &ptdev->scheduler->fw_events);
1702de854881SBoris Brezillon sched_queue_work(ptdev->scheduler, fw_events);
1703de854881SBoris Brezillon }
1704de854881SBoris Brezillon
fence_get_driver_name(struct dma_fence * fence)1705de854881SBoris Brezillon static const char *fence_get_driver_name(struct dma_fence *fence)
1706de854881SBoris Brezillon {
1707de854881SBoris Brezillon return "panthor";
1708de854881SBoris Brezillon }
1709de854881SBoris Brezillon
queue_fence_get_timeline_name(struct dma_fence * fence)1710de854881SBoris Brezillon static const char *queue_fence_get_timeline_name(struct dma_fence *fence)
1711de854881SBoris Brezillon {
1712de854881SBoris Brezillon return "queue-fence";
1713de854881SBoris Brezillon }
1714de854881SBoris Brezillon
1715de854881SBoris Brezillon static const struct dma_fence_ops panthor_queue_fence_ops = {
1716de854881SBoris Brezillon .get_driver_name = fence_get_driver_name,
1717de854881SBoris Brezillon .get_timeline_name = queue_fence_get_timeline_name,
1718de854881SBoris Brezillon };
1719de854881SBoris Brezillon
1720de854881SBoris Brezillon struct panthor_csg_slots_upd_ctx {
1721de854881SBoris Brezillon u32 update_mask;
1722de854881SBoris Brezillon u32 timedout_mask;
1723de854881SBoris Brezillon struct {
1724de854881SBoris Brezillon u32 value;
1725de854881SBoris Brezillon u32 mask;
1726de854881SBoris Brezillon } requests[MAX_CSGS];
1727de854881SBoris Brezillon };
1728de854881SBoris Brezillon
csgs_upd_ctx_init(struct panthor_csg_slots_upd_ctx * ctx)1729de854881SBoris Brezillon static void csgs_upd_ctx_init(struct panthor_csg_slots_upd_ctx *ctx)
1730de854881SBoris Brezillon {
1731de854881SBoris Brezillon memset(ctx, 0, sizeof(*ctx));
1732de854881SBoris Brezillon }
1733de854881SBoris Brezillon
csgs_upd_ctx_queue_reqs(struct panthor_device * ptdev,struct panthor_csg_slots_upd_ctx * ctx,u32 csg_id,u32 value,u32 mask)1734de854881SBoris Brezillon static void csgs_upd_ctx_queue_reqs(struct panthor_device *ptdev,
1735de854881SBoris Brezillon struct panthor_csg_slots_upd_ctx *ctx,
1736de854881SBoris Brezillon u32 csg_id, u32 value, u32 mask)
1737de854881SBoris Brezillon {
1738de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, !mask) ||
1739de854881SBoris Brezillon drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count))
1740de854881SBoris Brezillon return;
1741de854881SBoris Brezillon
1742de854881SBoris Brezillon ctx->requests[csg_id].value = (ctx->requests[csg_id].value & ~mask) | (value & mask);
1743de854881SBoris Brezillon ctx->requests[csg_id].mask |= mask;
1744de854881SBoris Brezillon ctx->update_mask |= BIT(csg_id);
1745de854881SBoris Brezillon }
1746de854881SBoris Brezillon
csgs_upd_ctx_apply_locked(struct panthor_device * ptdev,struct panthor_csg_slots_upd_ctx * ctx)1747de854881SBoris Brezillon static int csgs_upd_ctx_apply_locked(struct panthor_device *ptdev,
1748de854881SBoris Brezillon struct panthor_csg_slots_upd_ctx *ctx)
1749de854881SBoris Brezillon {
1750de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
1751de854881SBoris Brezillon u32 update_slots = ctx->update_mask;
1752de854881SBoris Brezillon
1753de854881SBoris Brezillon lockdep_assert_held(&sched->lock);
1754de854881SBoris Brezillon
1755de854881SBoris Brezillon if (!ctx->update_mask)
1756de854881SBoris Brezillon return 0;
1757de854881SBoris Brezillon
1758de854881SBoris Brezillon while (update_slots) {
1759de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface;
1760de854881SBoris Brezillon u32 csg_id = ffs(update_slots) - 1;
1761de854881SBoris Brezillon
1762de854881SBoris Brezillon update_slots &= ~BIT(csg_id);
1763de854881SBoris Brezillon csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1764de854881SBoris Brezillon panthor_fw_update_reqs(csg_iface, req,
1765de854881SBoris Brezillon ctx->requests[csg_id].value,
1766de854881SBoris Brezillon ctx->requests[csg_id].mask);
1767de854881SBoris Brezillon }
1768de854881SBoris Brezillon
1769de854881SBoris Brezillon panthor_fw_ring_csg_doorbells(ptdev, ctx->update_mask);
1770de854881SBoris Brezillon
1771de854881SBoris Brezillon update_slots = ctx->update_mask;
1772de854881SBoris Brezillon while (update_slots) {
1773de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface;
1774de854881SBoris Brezillon u32 csg_id = ffs(update_slots) - 1;
1775de854881SBoris Brezillon u32 req_mask = ctx->requests[csg_id].mask, acked;
1776de854881SBoris Brezillon int ret;
1777de854881SBoris Brezillon
1778de854881SBoris Brezillon update_slots &= ~BIT(csg_id);
1779de854881SBoris Brezillon csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1780de854881SBoris Brezillon
1781de854881SBoris Brezillon ret = panthor_fw_csg_wait_acks(ptdev, csg_id, req_mask, &acked, 100);
1782de854881SBoris Brezillon
1783de854881SBoris Brezillon if (acked & CSG_ENDPOINT_CONFIG)
1784de854881SBoris Brezillon csg_slot_sync_priority_locked(ptdev, csg_id);
1785de854881SBoris Brezillon
1786de854881SBoris Brezillon if (acked & CSG_STATE_MASK)
1787de854881SBoris Brezillon csg_slot_sync_state_locked(ptdev, csg_id);
1788de854881SBoris Brezillon
1789de854881SBoris Brezillon if (acked & CSG_STATUS_UPDATE) {
1790de854881SBoris Brezillon csg_slot_sync_queues_state_locked(ptdev, csg_id);
1791de854881SBoris Brezillon csg_slot_sync_idle_state_locked(ptdev, csg_id);
1792de854881SBoris Brezillon }
1793de854881SBoris Brezillon
1794de854881SBoris Brezillon if (ret && acked != req_mask &&
1795de854881SBoris Brezillon ((csg_iface->input->req ^ csg_iface->output->ack) & req_mask) != 0) {
1796de854881SBoris Brezillon drm_err(&ptdev->base, "CSG %d update request timedout", csg_id);
1797de854881SBoris Brezillon ctx->timedout_mask |= BIT(csg_id);
1798de854881SBoris Brezillon }
1799de854881SBoris Brezillon }
1800de854881SBoris Brezillon
1801de854881SBoris Brezillon if (ctx->timedout_mask)
1802de854881SBoris Brezillon return -ETIMEDOUT;
1803de854881SBoris Brezillon
1804de854881SBoris Brezillon return 0;
1805de854881SBoris Brezillon }
1806de854881SBoris Brezillon
1807de854881SBoris Brezillon struct panthor_sched_tick_ctx {
1808de854881SBoris Brezillon struct list_head old_groups[PANTHOR_CSG_PRIORITY_COUNT];
1809de854881SBoris Brezillon struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT];
1810de854881SBoris Brezillon u32 idle_group_count;
1811de854881SBoris Brezillon u32 group_count;
1812de854881SBoris Brezillon enum panthor_csg_priority min_priority;
1813de854881SBoris Brezillon struct panthor_vm *vms[MAX_CS_PER_CSG];
1814de854881SBoris Brezillon u32 as_count;
1815de854881SBoris Brezillon bool immediate_tick;
1816de854881SBoris Brezillon u32 csg_upd_failed_mask;
1817de854881SBoris Brezillon };
1818de854881SBoris Brezillon
1819de854881SBoris Brezillon static bool
tick_ctx_is_full(const struct panthor_scheduler * sched,const struct panthor_sched_tick_ctx * ctx)1820de854881SBoris Brezillon tick_ctx_is_full(const struct panthor_scheduler *sched,
1821de854881SBoris Brezillon const struct panthor_sched_tick_ctx *ctx)
1822de854881SBoris Brezillon {
1823de854881SBoris Brezillon return ctx->group_count == sched->csg_slot_count;
1824de854881SBoris Brezillon }
1825de854881SBoris Brezillon
1826de854881SBoris Brezillon static bool
group_is_idle(struct panthor_group * group)1827de854881SBoris Brezillon group_is_idle(struct panthor_group *group)
1828de854881SBoris Brezillon {
1829de854881SBoris Brezillon struct panthor_device *ptdev = group->ptdev;
1830de854881SBoris Brezillon u32 inactive_queues;
1831de854881SBoris Brezillon
1832de854881SBoris Brezillon if (group->csg_id >= 0)
1833de854881SBoris Brezillon return ptdev->scheduler->csg_slots[group->csg_id].idle;
1834de854881SBoris Brezillon
1835de854881SBoris Brezillon inactive_queues = group->idle_queues | group->blocked_queues;
1836de854881SBoris Brezillon return hweight32(inactive_queues) == group->queue_count;
1837de854881SBoris Brezillon }
1838de854881SBoris Brezillon
1839de854881SBoris Brezillon static bool
group_can_run(struct panthor_group * group)1840de854881SBoris Brezillon group_can_run(struct panthor_group *group)
1841de854881SBoris Brezillon {
1842de854881SBoris Brezillon return group->state != PANTHOR_CS_GROUP_TERMINATED &&
18438bdbd8b5SBoris Brezillon group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE &&
1844de854881SBoris Brezillon !group->destroyed && group->fatal_queues == 0 &&
1845de854881SBoris Brezillon !group->timedout;
1846de854881SBoris Brezillon }
1847de854881SBoris Brezillon
1848de854881SBoris Brezillon static void
tick_ctx_pick_groups_from_list(const struct panthor_scheduler * sched,struct panthor_sched_tick_ctx * ctx,struct list_head * queue,bool skip_idle_groups,bool owned_by_tick_ctx)1849de854881SBoris Brezillon tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched,
1850de854881SBoris Brezillon struct panthor_sched_tick_ctx *ctx,
1851de854881SBoris Brezillon struct list_head *queue,
1852de854881SBoris Brezillon bool skip_idle_groups,
1853de854881SBoris Brezillon bool owned_by_tick_ctx)
1854de854881SBoris Brezillon {
1855de854881SBoris Brezillon struct panthor_group *group, *tmp;
1856de854881SBoris Brezillon
1857de854881SBoris Brezillon if (tick_ctx_is_full(sched, ctx))
1858de854881SBoris Brezillon return;
1859de854881SBoris Brezillon
1860de854881SBoris Brezillon list_for_each_entry_safe(group, tmp, queue, run_node) {
1861de854881SBoris Brezillon u32 i;
1862de854881SBoris Brezillon
1863de854881SBoris Brezillon if (!group_can_run(group))
1864de854881SBoris Brezillon continue;
1865de854881SBoris Brezillon
1866de854881SBoris Brezillon if (skip_idle_groups && group_is_idle(group))
1867de854881SBoris Brezillon continue;
1868de854881SBoris Brezillon
1869de854881SBoris Brezillon for (i = 0; i < ctx->as_count; i++) {
1870de854881SBoris Brezillon if (ctx->vms[i] == group->vm)
1871de854881SBoris Brezillon break;
1872de854881SBoris Brezillon }
1873de854881SBoris Brezillon
1874de854881SBoris Brezillon if (i == ctx->as_count && ctx->as_count == sched->as_slot_count)
1875de854881SBoris Brezillon continue;
1876de854881SBoris Brezillon
1877de854881SBoris Brezillon if (!owned_by_tick_ctx)
1878de854881SBoris Brezillon group_get(group);
1879de854881SBoris Brezillon
1880de854881SBoris Brezillon list_move_tail(&group->run_node, &ctx->groups[group->priority]);
1881de854881SBoris Brezillon ctx->group_count++;
1882de854881SBoris Brezillon if (group_is_idle(group))
1883de854881SBoris Brezillon ctx->idle_group_count++;
1884de854881SBoris Brezillon
1885de854881SBoris Brezillon if (i == ctx->as_count)
1886de854881SBoris Brezillon ctx->vms[ctx->as_count++] = group->vm;
1887de854881SBoris Brezillon
1888de854881SBoris Brezillon if (ctx->min_priority > group->priority)
1889de854881SBoris Brezillon ctx->min_priority = group->priority;
1890de854881SBoris Brezillon
1891de854881SBoris Brezillon if (tick_ctx_is_full(sched, ctx))
1892de854881SBoris Brezillon return;
1893de854881SBoris Brezillon }
1894de854881SBoris Brezillon }
1895de854881SBoris Brezillon
1896de854881SBoris Brezillon static void
tick_ctx_insert_old_group(struct panthor_scheduler * sched,struct panthor_sched_tick_ctx * ctx,struct panthor_group * group,bool full_tick)1897de854881SBoris Brezillon tick_ctx_insert_old_group(struct panthor_scheduler *sched,
1898de854881SBoris Brezillon struct panthor_sched_tick_ctx *ctx,
1899de854881SBoris Brezillon struct panthor_group *group,
1900de854881SBoris Brezillon bool full_tick)
1901de854881SBoris Brezillon {
1902de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id];
1903de854881SBoris Brezillon struct panthor_group *other_group;
1904de854881SBoris Brezillon
1905de854881SBoris Brezillon if (!full_tick) {
1906de854881SBoris Brezillon list_add_tail(&group->run_node, &ctx->old_groups[group->priority]);
1907de854881SBoris Brezillon return;
1908de854881SBoris Brezillon }
1909de854881SBoris Brezillon
1910de854881SBoris Brezillon /* Rotate to make sure groups with lower CSG slot
1911de854881SBoris Brezillon * priorities have a chance to get a higher CSG slot
1912de854881SBoris Brezillon * priority next time they get picked. This priority
1913de854881SBoris Brezillon * has an impact on resource request ordering, so it's
1914de854881SBoris Brezillon * important to make sure we don't let one group starve
1915de854881SBoris Brezillon * all other groups with the same group priority.
1916de854881SBoris Brezillon */
1917de854881SBoris Brezillon list_for_each_entry(other_group,
1918de854881SBoris Brezillon &ctx->old_groups[csg_slot->group->priority],
1919de854881SBoris Brezillon run_node) {
1920de854881SBoris Brezillon struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id];
1921de854881SBoris Brezillon
1922de854881SBoris Brezillon if (other_csg_slot->priority > csg_slot->priority) {
1923de854881SBoris Brezillon list_add_tail(&csg_slot->group->run_node, &other_group->run_node);
1924de854881SBoris Brezillon return;
1925de854881SBoris Brezillon }
1926de854881SBoris Brezillon }
1927de854881SBoris Brezillon
1928de854881SBoris Brezillon list_add_tail(&group->run_node, &ctx->old_groups[group->priority]);
1929de854881SBoris Brezillon }
1930de854881SBoris Brezillon
1931de854881SBoris Brezillon static void
tick_ctx_init(struct panthor_scheduler * sched,struct panthor_sched_tick_ctx * ctx,bool full_tick)1932de854881SBoris Brezillon tick_ctx_init(struct panthor_scheduler *sched,
1933de854881SBoris Brezillon struct panthor_sched_tick_ctx *ctx,
1934de854881SBoris Brezillon bool full_tick)
1935de854881SBoris Brezillon {
1936de854881SBoris Brezillon struct panthor_device *ptdev = sched->ptdev;
1937de854881SBoris Brezillon struct panthor_csg_slots_upd_ctx upd_ctx;
1938de854881SBoris Brezillon int ret;
1939de854881SBoris Brezillon u32 i;
1940de854881SBoris Brezillon
1941de854881SBoris Brezillon memset(ctx, 0, sizeof(*ctx));
1942de854881SBoris Brezillon csgs_upd_ctx_init(&upd_ctx);
1943de854881SBoris Brezillon
1944de854881SBoris Brezillon ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT;
1945de854881SBoris Brezillon for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) {
1946de854881SBoris Brezillon INIT_LIST_HEAD(&ctx->groups[i]);
1947de854881SBoris Brezillon INIT_LIST_HEAD(&ctx->old_groups[i]);
1948de854881SBoris Brezillon }
1949de854881SBoris Brezillon
1950de854881SBoris Brezillon for (i = 0; i < sched->csg_slot_count; i++) {
1951de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
1952de854881SBoris Brezillon struct panthor_group *group = csg_slot->group;
1953de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface;
1954de854881SBoris Brezillon
1955de854881SBoris Brezillon if (!group)
1956de854881SBoris Brezillon continue;
1957de854881SBoris Brezillon
1958de854881SBoris Brezillon csg_iface = panthor_fw_get_csg_iface(ptdev, i);
1959de854881SBoris Brezillon group_get(group);
1960de854881SBoris Brezillon
1961de854881SBoris Brezillon /* If there was unhandled faults on the VM, force processing of
1962de854881SBoris Brezillon * CSG IRQs, so we can flag the faulty queue.
1963de854881SBoris Brezillon */
1964de854881SBoris Brezillon if (panthor_vm_has_unhandled_faults(group->vm)) {
1965de854881SBoris Brezillon sched_process_csg_irq_locked(ptdev, i);
1966de854881SBoris Brezillon
1967de854881SBoris Brezillon /* No fatal fault reported, flag all queues as faulty. */
1968de854881SBoris Brezillon if (!group->fatal_queues)
1969de854881SBoris Brezillon group->fatal_queues |= GENMASK(group->queue_count - 1, 0);
1970de854881SBoris Brezillon }
1971de854881SBoris Brezillon
1972de854881SBoris Brezillon tick_ctx_insert_old_group(sched, ctx, group, full_tick);
1973de854881SBoris Brezillon csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i,
1974de854881SBoris Brezillon csg_iface->output->ack ^ CSG_STATUS_UPDATE,
1975de854881SBoris Brezillon CSG_STATUS_UPDATE);
1976de854881SBoris Brezillon }
1977de854881SBoris Brezillon
1978de854881SBoris Brezillon ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
1979de854881SBoris Brezillon if (ret) {
1980de854881SBoris Brezillon panthor_device_schedule_reset(ptdev);
1981de854881SBoris Brezillon ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
1982de854881SBoris Brezillon }
1983de854881SBoris Brezillon }
1984de854881SBoris Brezillon
1985de854881SBoris Brezillon #define NUM_INSTRS_PER_SLOT 16
1986de854881SBoris Brezillon
1987de854881SBoris Brezillon static void
group_term_post_processing(struct panthor_group * group)1988de854881SBoris Brezillon group_term_post_processing(struct panthor_group *group)
1989de854881SBoris Brezillon {
1990de854881SBoris Brezillon struct panthor_job *job, *tmp;
1991de854881SBoris Brezillon LIST_HEAD(faulty_jobs);
1992de854881SBoris Brezillon bool cookie;
1993de854881SBoris Brezillon u32 i = 0;
1994de854881SBoris Brezillon
1995de854881SBoris Brezillon if (drm_WARN_ON(&group->ptdev->base, group_can_run(group)))
1996de854881SBoris Brezillon return;
1997de854881SBoris Brezillon
1998de854881SBoris Brezillon cookie = dma_fence_begin_signalling();
1999de854881SBoris Brezillon for (i = 0; i < group->queue_count; i++) {
2000de854881SBoris Brezillon struct panthor_queue *queue = group->queues[i];
2001de854881SBoris Brezillon struct panthor_syncobj_64b *syncobj;
2002de854881SBoris Brezillon int err;
2003de854881SBoris Brezillon
2004de854881SBoris Brezillon if (group->fatal_queues & BIT(i))
2005de854881SBoris Brezillon err = -EINVAL;
2006de854881SBoris Brezillon else if (group->timedout)
2007de854881SBoris Brezillon err = -ETIMEDOUT;
2008de854881SBoris Brezillon else
2009de854881SBoris Brezillon err = -ECANCELED;
2010de854881SBoris Brezillon
2011de854881SBoris Brezillon if (!queue)
2012de854881SBoris Brezillon continue;
2013de854881SBoris Brezillon
2014de854881SBoris Brezillon spin_lock(&queue->fence_ctx.lock);
2015de854881SBoris Brezillon list_for_each_entry_safe(job, tmp, &queue->fence_ctx.in_flight_jobs, node) {
2016de854881SBoris Brezillon list_move_tail(&job->node, &faulty_jobs);
2017de854881SBoris Brezillon dma_fence_set_error(job->done_fence, err);
2018de854881SBoris Brezillon dma_fence_signal_locked(job->done_fence);
2019de854881SBoris Brezillon }
2020de854881SBoris Brezillon spin_unlock(&queue->fence_ctx.lock);
2021de854881SBoris Brezillon
2022de854881SBoris Brezillon /* Manually update the syncobj seqno to unblock waiters. */
2023de854881SBoris Brezillon syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj));
2024de854881SBoris Brezillon syncobj->status = ~0;
2025de854881SBoris Brezillon syncobj->seqno = atomic64_read(&queue->fence_ctx.seqno);
2026de854881SBoris Brezillon sched_queue_work(group->ptdev->scheduler, sync_upd);
2027de854881SBoris Brezillon }
2028de854881SBoris Brezillon dma_fence_end_signalling(cookie);
2029de854881SBoris Brezillon
2030de854881SBoris Brezillon list_for_each_entry_safe(job, tmp, &faulty_jobs, node) {
2031de854881SBoris Brezillon list_del_init(&job->node);
2032de854881SBoris Brezillon panthor_job_put(&job->base);
2033de854881SBoris Brezillon }
2034de854881SBoris Brezillon }
2035de854881SBoris Brezillon
group_term_work(struct work_struct * work)2036de854881SBoris Brezillon static void group_term_work(struct work_struct *work)
2037de854881SBoris Brezillon {
2038de854881SBoris Brezillon struct panthor_group *group =
2039de854881SBoris Brezillon container_of(work, struct panthor_group, term_work);
2040de854881SBoris Brezillon
2041de854881SBoris Brezillon group_term_post_processing(group);
2042de854881SBoris Brezillon group_put(group);
2043de854881SBoris Brezillon }
2044de854881SBoris Brezillon
2045de854881SBoris Brezillon static void
tick_ctx_cleanup(struct panthor_scheduler * sched,struct panthor_sched_tick_ctx * ctx)2046de854881SBoris Brezillon tick_ctx_cleanup(struct panthor_scheduler *sched,
2047de854881SBoris Brezillon struct panthor_sched_tick_ctx *ctx)
2048de854881SBoris Brezillon {
2049de854881SBoris Brezillon struct panthor_group *group, *tmp;
2050de854881SBoris Brezillon u32 i;
2051de854881SBoris Brezillon
2052de854881SBoris Brezillon for (i = 0; i < ARRAY_SIZE(ctx->old_groups); i++) {
2053de854881SBoris Brezillon list_for_each_entry_safe(group, tmp, &ctx->old_groups[i], run_node) {
2054de854881SBoris Brezillon /* If everything went fine, we should only have groups
2055de854881SBoris Brezillon * to be terminated in the old_groups lists.
2056de854881SBoris Brezillon */
2057de854881SBoris Brezillon drm_WARN_ON(&group->ptdev->base, !ctx->csg_upd_failed_mask &&
2058de854881SBoris Brezillon group_can_run(group));
2059de854881SBoris Brezillon
2060de854881SBoris Brezillon if (!group_can_run(group)) {
2061de854881SBoris Brezillon list_del_init(&group->run_node);
2062de854881SBoris Brezillon list_del_init(&group->wait_node);
2063de854881SBoris Brezillon group_queue_work(group, term);
2064de854881SBoris Brezillon } else if (group->csg_id >= 0) {
2065de854881SBoris Brezillon list_del_init(&group->run_node);
2066de854881SBoris Brezillon } else {
2067de854881SBoris Brezillon list_move(&group->run_node,
2068de854881SBoris Brezillon group_is_idle(group) ?
2069de854881SBoris Brezillon &sched->groups.idle[group->priority] :
2070de854881SBoris Brezillon &sched->groups.runnable[group->priority]);
2071de854881SBoris Brezillon }
2072de854881SBoris Brezillon group_put(group);
2073de854881SBoris Brezillon }
2074de854881SBoris Brezillon }
2075de854881SBoris Brezillon
2076de854881SBoris Brezillon for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) {
2077de854881SBoris Brezillon /* If everything went fine, the groups to schedule lists should
2078de854881SBoris Brezillon * be empty.
2079de854881SBoris Brezillon */
2080de854881SBoris Brezillon drm_WARN_ON(&group->ptdev->base,
2081de854881SBoris Brezillon !ctx->csg_upd_failed_mask && !list_empty(&ctx->groups[i]));
2082de854881SBoris Brezillon
2083de854881SBoris Brezillon list_for_each_entry_safe(group, tmp, &ctx->groups[i], run_node) {
2084de854881SBoris Brezillon if (group->csg_id >= 0) {
2085de854881SBoris Brezillon list_del_init(&group->run_node);
2086de854881SBoris Brezillon } else {
2087de854881SBoris Brezillon list_move(&group->run_node,
2088de854881SBoris Brezillon group_is_idle(group) ?
2089de854881SBoris Brezillon &sched->groups.idle[group->priority] :
2090de854881SBoris Brezillon &sched->groups.runnable[group->priority]);
2091de854881SBoris Brezillon }
2092de854881SBoris Brezillon group_put(group);
2093de854881SBoris Brezillon }
2094de854881SBoris Brezillon }
2095de854881SBoris Brezillon }
2096de854881SBoris Brezillon
2097de854881SBoris Brezillon static void
tick_ctx_apply(struct panthor_scheduler * sched,struct panthor_sched_tick_ctx * ctx)2098de854881SBoris Brezillon tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx)
2099de854881SBoris Brezillon {
2100de854881SBoris Brezillon struct panthor_group *group, *tmp;
2101de854881SBoris Brezillon struct panthor_device *ptdev = sched->ptdev;
2102de854881SBoris Brezillon struct panthor_csg_slot *csg_slot;
2103de854881SBoris Brezillon int prio, new_csg_prio = MAX_CSG_PRIO, i;
2104d76653c3SNathan Chancellor u32 free_csg_slots = 0;
2105de854881SBoris Brezillon struct panthor_csg_slots_upd_ctx upd_ctx;
2106de854881SBoris Brezillon int ret;
2107de854881SBoris Brezillon
2108de854881SBoris Brezillon csgs_upd_ctx_init(&upd_ctx);
2109de854881SBoris Brezillon
2110de854881SBoris Brezillon for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
2111de854881SBoris Brezillon /* Suspend or terminate evicted groups. */
2112de854881SBoris Brezillon list_for_each_entry(group, &ctx->old_groups[prio], run_node) {
2113de854881SBoris Brezillon bool term = !group_can_run(group);
2114de854881SBoris Brezillon int csg_id = group->csg_id;
2115de854881SBoris Brezillon
2116de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, csg_id < 0))
2117de854881SBoris Brezillon continue;
2118de854881SBoris Brezillon
2119de854881SBoris Brezillon csg_slot = &sched->csg_slots[csg_id];
2120de854881SBoris Brezillon csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
2121de854881SBoris Brezillon term ? CSG_STATE_TERMINATE : CSG_STATE_SUSPEND,
2122de854881SBoris Brezillon CSG_STATE_MASK);
2123de854881SBoris Brezillon }
2124de854881SBoris Brezillon
2125de854881SBoris Brezillon /* Update priorities on already running groups. */
2126de854881SBoris Brezillon list_for_each_entry(group, &ctx->groups[prio], run_node) {
2127de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface;
2128de854881SBoris Brezillon int csg_id = group->csg_id;
2129de854881SBoris Brezillon
2130de854881SBoris Brezillon if (csg_id < 0) {
2131de854881SBoris Brezillon new_csg_prio--;
2132de854881SBoris Brezillon continue;
2133de854881SBoris Brezillon }
2134de854881SBoris Brezillon
2135de854881SBoris Brezillon csg_slot = &sched->csg_slots[csg_id];
2136de854881SBoris Brezillon csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
2137de854881SBoris Brezillon if (csg_slot->priority == new_csg_prio) {
2138de854881SBoris Brezillon new_csg_prio--;
2139de854881SBoris Brezillon continue;
2140de854881SBoris Brezillon }
2141de854881SBoris Brezillon
2142de854881SBoris Brezillon panthor_fw_update_reqs(csg_iface, endpoint_req,
2143de854881SBoris Brezillon CSG_EP_REQ_PRIORITY(new_csg_prio),
2144de854881SBoris Brezillon CSG_EP_REQ_PRIORITY_MASK);
2145de854881SBoris Brezillon csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
2146de854881SBoris Brezillon csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG,
2147de854881SBoris Brezillon CSG_ENDPOINT_CONFIG);
2148de854881SBoris Brezillon new_csg_prio--;
2149de854881SBoris Brezillon }
2150de854881SBoris Brezillon }
2151de854881SBoris Brezillon
2152de854881SBoris Brezillon ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
2153de854881SBoris Brezillon if (ret) {
2154de854881SBoris Brezillon panthor_device_schedule_reset(ptdev);
2155de854881SBoris Brezillon ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
2156de854881SBoris Brezillon return;
2157de854881SBoris Brezillon }
2158de854881SBoris Brezillon
2159de854881SBoris Brezillon /* Unbind evicted groups. */
2160de854881SBoris Brezillon for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
2161de854881SBoris Brezillon list_for_each_entry(group, &ctx->old_groups[prio], run_node) {
2162de854881SBoris Brezillon /* This group is gone. Process interrupts to clear
2163de854881SBoris Brezillon * any pending interrupts before we start the new
2164de854881SBoris Brezillon * group.
2165de854881SBoris Brezillon */
2166de854881SBoris Brezillon if (group->csg_id >= 0)
2167de854881SBoris Brezillon sched_process_csg_irq_locked(ptdev, group->csg_id);
2168de854881SBoris Brezillon
2169de854881SBoris Brezillon group_unbind_locked(group);
2170de854881SBoris Brezillon }
2171de854881SBoris Brezillon }
2172de854881SBoris Brezillon
2173de854881SBoris Brezillon for (i = 0; i < sched->csg_slot_count; i++) {
2174de854881SBoris Brezillon if (!sched->csg_slots[i].group)
2175de854881SBoris Brezillon free_csg_slots |= BIT(i);
2176de854881SBoris Brezillon }
2177de854881SBoris Brezillon
2178de854881SBoris Brezillon csgs_upd_ctx_init(&upd_ctx);
2179de854881SBoris Brezillon new_csg_prio = MAX_CSG_PRIO;
2180de854881SBoris Brezillon
2181de854881SBoris Brezillon /* Start new groups. */
2182de854881SBoris Brezillon for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
2183de854881SBoris Brezillon list_for_each_entry(group, &ctx->groups[prio], run_node) {
2184de854881SBoris Brezillon int csg_id = group->csg_id;
2185de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface;
2186de854881SBoris Brezillon
2187de854881SBoris Brezillon if (csg_id >= 0) {
2188de854881SBoris Brezillon new_csg_prio--;
2189de854881SBoris Brezillon continue;
2190de854881SBoris Brezillon }
2191de854881SBoris Brezillon
2192de854881SBoris Brezillon csg_id = ffs(free_csg_slots) - 1;
2193de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, csg_id < 0))
2194de854881SBoris Brezillon break;
2195de854881SBoris Brezillon
2196de854881SBoris Brezillon csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
2197de854881SBoris Brezillon csg_slot = &sched->csg_slots[csg_id];
2198de854881SBoris Brezillon group_bind_locked(group, csg_id);
2199de854881SBoris Brezillon csg_slot_prog_locked(ptdev, csg_id, new_csg_prio--);
2200de854881SBoris Brezillon csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
2201de854881SBoris Brezillon group->state == PANTHOR_CS_GROUP_SUSPENDED ?
2202de854881SBoris Brezillon CSG_STATE_RESUME : CSG_STATE_START,
2203de854881SBoris Brezillon CSG_STATE_MASK);
2204de854881SBoris Brezillon csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
2205de854881SBoris Brezillon csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG,
2206de854881SBoris Brezillon CSG_ENDPOINT_CONFIG);
2207de854881SBoris Brezillon free_csg_slots &= ~BIT(csg_id);
2208de854881SBoris Brezillon }
2209de854881SBoris Brezillon }
2210de854881SBoris Brezillon
2211de854881SBoris Brezillon ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
2212de854881SBoris Brezillon if (ret) {
2213de854881SBoris Brezillon panthor_device_schedule_reset(ptdev);
2214de854881SBoris Brezillon ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
2215de854881SBoris Brezillon return;
2216de854881SBoris Brezillon }
2217de854881SBoris Brezillon
2218de854881SBoris Brezillon for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
2219de854881SBoris Brezillon list_for_each_entry_safe(group, tmp, &ctx->groups[prio], run_node) {
2220de854881SBoris Brezillon list_del_init(&group->run_node);
2221de854881SBoris Brezillon
2222de854881SBoris Brezillon /* If the group has been destroyed while we were
2223de854881SBoris Brezillon * scheduling, ask for an immediate tick to
2224de854881SBoris Brezillon * re-evaluate as soon as possible and get rid of
2225de854881SBoris Brezillon * this dangling group.
2226de854881SBoris Brezillon */
2227de854881SBoris Brezillon if (group->destroyed)
2228de854881SBoris Brezillon ctx->immediate_tick = true;
2229de854881SBoris Brezillon group_put(group);
2230de854881SBoris Brezillon }
2231de854881SBoris Brezillon
2232de854881SBoris Brezillon /* Return evicted groups to the idle or run queues. Groups
2233de854881SBoris Brezillon * that can no longer be run (because they've been destroyed
2234de854881SBoris Brezillon * or experienced an unrecoverable error) will be scheduled
2235de854881SBoris Brezillon * for destruction in tick_ctx_cleanup().
2236de854881SBoris Brezillon */
2237de854881SBoris Brezillon list_for_each_entry_safe(group, tmp, &ctx->old_groups[prio], run_node) {
2238de854881SBoris Brezillon if (!group_can_run(group))
2239de854881SBoris Brezillon continue;
2240de854881SBoris Brezillon
2241de854881SBoris Brezillon if (group_is_idle(group))
2242de854881SBoris Brezillon list_move_tail(&group->run_node, &sched->groups.idle[prio]);
2243de854881SBoris Brezillon else
2244de854881SBoris Brezillon list_move_tail(&group->run_node, &sched->groups.runnable[prio]);
2245de854881SBoris Brezillon group_put(group);
2246de854881SBoris Brezillon }
2247de854881SBoris Brezillon }
2248de854881SBoris Brezillon
2249de854881SBoris Brezillon sched->used_csg_slot_count = ctx->group_count;
2250de854881SBoris Brezillon sched->might_have_idle_groups = ctx->idle_group_count > 0;
2251de854881SBoris Brezillon }
2252de854881SBoris Brezillon
2253de854881SBoris Brezillon static u64
tick_ctx_update_resched_target(struct panthor_scheduler * sched,const struct panthor_sched_tick_ctx * ctx)2254de854881SBoris Brezillon tick_ctx_update_resched_target(struct panthor_scheduler *sched,
2255de854881SBoris Brezillon const struct panthor_sched_tick_ctx *ctx)
2256de854881SBoris Brezillon {
2257de854881SBoris Brezillon /* We had space left, no need to reschedule until some external event happens. */
2258de854881SBoris Brezillon if (!tick_ctx_is_full(sched, ctx))
2259de854881SBoris Brezillon goto no_tick;
2260de854881SBoris Brezillon
2261de854881SBoris Brezillon /* If idle groups were scheduled, no need to wake up until some external
2262de854881SBoris Brezillon * event happens (group unblocked, new job submitted, ...).
2263de854881SBoris Brezillon */
2264de854881SBoris Brezillon if (ctx->idle_group_count)
2265de854881SBoris Brezillon goto no_tick;
2266de854881SBoris Brezillon
2267de854881SBoris Brezillon if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT))
2268de854881SBoris Brezillon goto no_tick;
2269de854881SBoris Brezillon
2270de854881SBoris Brezillon /* If there are groups of the same priority waiting, we need to
2271de854881SBoris Brezillon * keep the scheduler ticking, otherwise, we'll just wait for
2272de854881SBoris Brezillon * new groups with higher priority to be queued.
2273de854881SBoris Brezillon */
2274de854881SBoris Brezillon if (!list_empty(&sched->groups.runnable[ctx->min_priority])) {
2275de854881SBoris Brezillon u64 resched_target = sched->last_tick + sched->tick_period;
2276de854881SBoris Brezillon
2277de854881SBoris Brezillon if (time_before64(sched->resched_target, sched->last_tick) ||
2278de854881SBoris Brezillon time_before64(resched_target, sched->resched_target))
2279de854881SBoris Brezillon sched->resched_target = resched_target;
2280de854881SBoris Brezillon
2281de854881SBoris Brezillon return sched->resched_target - sched->last_tick;
2282de854881SBoris Brezillon }
2283de854881SBoris Brezillon
2284de854881SBoris Brezillon no_tick:
2285de854881SBoris Brezillon sched->resched_target = U64_MAX;
2286de854881SBoris Brezillon return U64_MAX;
2287de854881SBoris Brezillon }
2288de854881SBoris Brezillon
tick_work(struct work_struct * work)2289de854881SBoris Brezillon static void tick_work(struct work_struct *work)
2290de854881SBoris Brezillon {
2291de854881SBoris Brezillon struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler,
2292de854881SBoris Brezillon tick_work.work);
2293de854881SBoris Brezillon struct panthor_device *ptdev = sched->ptdev;
2294de854881SBoris Brezillon struct panthor_sched_tick_ctx ctx;
2295de854881SBoris Brezillon u64 remaining_jiffies = 0, resched_delay;
2296de854881SBoris Brezillon u64 now = get_jiffies_64();
2297de854881SBoris Brezillon int prio, ret, cookie;
2298de854881SBoris Brezillon
2299de854881SBoris Brezillon if (!drm_dev_enter(&ptdev->base, &cookie))
2300de854881SBoris Brezillon return;
2301de854881SBoris Brezillon
2302de854881SBoris Brezillon ret = pm_runtime_resume_and_get(ptdev->base.dev);
2303de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, ret))
2304de854881SBoris Brezillon goto out_dev_exit;
2305de854881SBoris Brezillon
2306de854881SBoris Brezillon if (time_before64(now, sched->resched_target))
2307de854881SBoris Brezillon remaining_jiffies = sched->resched_target - now;
2308de854881SBoris Brezillon
2309de854881SBoris Brezillon mutex_lock(&sched->lock);
2310de854881SBoris Brezillon if (panthor_device_reset_is_pending(sched->ptdev))
2311de854881SBoris Brezillon goto out_unlock;
2312de854881SBoris Brezillon
2313de854881SBoris Brezillon tick_ctx_init(sched, &ctx, remaining_jiffies != 0);
2314de854881SBoris Brezillon if (ctx.csg_upd_failed_mask)
2315de854881SBoris Brezillon goto out_cleanup_ctx;
2316de854881SBoris Brezillon
2317de854881SBoris Brezillon if (remaining_jiffies) {
2318de854881SBoris Brezillon /* Scheduling forced in the middle of a tick. Only RT groups
2319de854881SBoris Brezillon * can preempt non-RT ones. Currently running RT groups can't be
2320de854881SBoris Brezillon * preempted.
2321de854881SBoris Brezillon */
2322de854881SBoris Brezillon for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
2323de854881SBoris Brezillon prio >= 0 && !tick_ctx_is_full(sched, &ctx);
2324de854881SBoris Brezillon prio--) {
2325de854881SBoris Brezillon tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio],
2326de854881SBoris Brezillon true, true);
2327de854881SBoris Brezillon if (prio == PANTHOR_CSG_PRIORITY_RT) {
2328de854881SBoris Brezillon tick_ctx_pick_groups_from_list(sched, &ctx,
2329de854881SBoris Brezillon &sched->groups.runnable[prio],
2330de854881SBoris Brezillon true, false);
2331de854881SBoris Brezillon }
2332de854881SBoris Brezillon }
2333de854881SBoris Brezillon }
2334de854881SBoris Brezillon
2335de854881SBoris Brezillon /* First pick non-idle groups */
2336de854881SBoris Brezillon for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
2337de854881SBoris Brezillon prio >= 0 && !tick_ctx_is_full(sched, &ctx);
2338de854881SBoris Brezillon prio--) {
2339de854881SBoris Brezillon tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio],
2340de854881SBoris Brezillon true, false);
2341de854881SBoris Brezillon tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true);
2342de854881SBoris Brezillon }
2343de854881SBoris Brezillon
2344de854881SBoris Brezillon /* If we have free CSG slots left, pick idle groups */
2345de854881SBoris Brezillon for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
2346de854881SBoris Brezillon prio >= 0 && !tick_ctx_is_full(sched, &ctx);
2347de854881SBoris Brezillon prio--) {
2348de854881SBoris Brezillon /* Check the old_group queue first to avoid reprogramming the slots */
2349de854881SBoris Brezillon tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], false, true);
2350de854881SBoris Brezillon tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.idle[prio],
2351de854881SBoris Brezillon false, false);
2352de854881SBoris Brezillon }
2353de854881SBoris Brezillon
2354de854881SBoris Brezillon tick_ctx_apply(sched, &ctx);
2355de854881SBoris Brezillon if (ctx.csg_upd_failed_mask)
2356de854881SBoris Brezillon goto out_cleanup_ctx;
2357de854881SBoris Brezillon
2358de854881SBoris Brezillon if (ctx.idle_group_count == ctx.group_count) {
2359de854881SBoris Brezillon panthor_devfreq_record_idle(sched->ptdev);
2360de854881SBoris Brezillon if (sched->pm.has_ref) {
2361de854881SBoris Brezillon pm_runtime_put_autosuspend(ptdev->base.dev);
2362de854881SBoris Brezillon sched->pm.has_ref = false;
2363de854881SBoris Brezillon }
2364de854881SBoris Brezillon } else {
2365de854881SBoris Brezillon panthor_devfreq_record_busy(sched->ptdev);
2366de854881SBoris Brezillon if (!sched->pm.has_ref) {
2367de854881SBoris Brezillon pm_runtime_get(ptdev->base.dev);
2368de854881SBoris Brezillon sched->pm.has_ref = true;
2369de854881SBoris Brezillon }
2370de854881SBoris Brezillon }
2371de854881SBoris Brezillon
2372de854881SBoris Brezillon sched->last_tick = now;
2373de854881SBoris Brezillon resched_delay = tick_ctx_update_resched_target(sched, &ctx);
2374de854881SBoris Brezillon if (ctx.immediate_tick)
2375de854881SBoris Brezillon resched_delay = 0;
2376de854881SBoris Brezillon
2377de854881SBoris Brezillon if (resched_delay != U64_MAX)
2378de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, resched_delay);
2379de854881SBoris Brezillon
2380de854881SBoris Brezillon out_cleanup_ctx:
2381de854881SBoris Brezillon tick_ctx_cleanup(sched, &ctx);
2382de854881SBoris Brezillon
2383de854881SBoris Brezillon out_unlock:
2384de854881SBoris Brezillon mutex_unlock(&sched->lock);
2385de854881SBoris Brezillon pm_runtime_mark_last_busy(ptdev->base.dev);
2386de854881SBoris Brezillon pm_runtime_put_autosuspend(ptdev->base.dev);
2387de854881SBoris Brezillon
2388de854881SBoris Brezillon out_dev_exit:
2389de854881SBoris Brezillon drm_dev_exit(cookie);
2390de854881SBoris Brezillon }
2391de854881SBoris Brezillon
panthor_queue_eval_syncwait(struct panthor_group * group,u8 queue_idx)2392de854881SBoris Brezillon static int panthor_queue_eval_syncwait(struct panthor_group *group, u8 queue_idx)
2393de854881SBoris Brezillon {
2394de854881SBoris Brezillon struct panthor_queue *queue = group->queues[queue_idx];
2395de854881SBoris Brezillon union {
2396de854881SBoris Brezillon struct panthor_syncobj_64b sync64;
2397de854881SBoris Brezillon struct panthor_syncobj_32b sync32;
2398de854881SBoris Brezillon } *syncobj;
2399de854881SBoris Brezillon bool result;
2400de854881SBoris Brezillon u64 value;
2401de854881SBoris Brezillon
2402de854881SBoris Brezillon syncobj = panthor_queue_get_syncwait_obj(group, queue);
2403de854881SBoris Brezillon if (!syncobj)
2404de854881SBoris Brezillon return -EINVAL;
2405de854881SBoris Brezillon
2406de854881SBoris Brezillon value = queue->syncwait.sync64 ?
2407de854881SBoris Brezillon syncobj->sync64.seqno :
2408de854881SBoris Brezillon syncobj->sync32.seqno;
2409de854881SBoris Brezillon
2410de854881SBoris Brezillon if (queue->syncwait.gt)
2411de854881SBoris Brezillon result = value > queue->syncwait.ref;
2412de854881SBoris Brezillon else
2413de854881SBoris Brezillon result = value <= queue->syncwait.ref;
2414de854881SBoris Brezillon
2415de854881SBoris Brezillon if (result)
2416de854881SBoris Brezillon panthor_queue_put_syncwait_obj(queue);
2417de854881SBoris Brezillon
2418de854881SBoris Brezillon return result;
2419de854881SBoris Brezillon }
2420de854881SBoris Brezillon
sync_upd_work(struct work_struct * work)2421de854881SBoris Brezillon static void sync_upd_work(struct work_struct *work)
2422de854881SBoris Brezillon {
2423de854881SBoris Brezillon struct panthor_scheduler *sched = container_of(work,
2424de854881SBoris Brezillon struct panthor_scheduler,
2425de854881SBoris Brezillon sync_upd_work);
2426de854881SBoris Brezillon struct panthor_group *group, *tmp;
2427de854881SBoris Brezillon bool immediate_tick = false;
2428de854881SBoris Brezillon
2429de854881SBoris Brezillon mutex_lock(&sched->lock);
2430de854881SBoris Brezillon list_for_each_entry_safe(group, tmp, &sched->groups.waiting, wait_node) {
2431de854881SBoris Brezillon u32 tested_queues = group->blocked_queues;
2432de854881SBoris Brezillon u32 unblocked_queues = 0;
2433de854881SBoris Brezillon
2434de854881SBoris Brezillon while (tested_queues) {
2435de854881SBoris Brezillon u32 cs_id = ffs(tested_queues) - 1;
2436de854881SBoris Brezillon int ret;
2437de854881SBoris Brezillon
2438de854881SBoris Brezillon ret = panthor_queue_eval_syncwait(group, cs_id);
2439de854881SBoris Brezillon drm_WARN_ON(&group->ptdev->base, ret < 0);
2440de854881SBoris Brezillon if (ret)
2441de854881SBoris Brezillon unblocked_queues |= BIT(cs_id);
2442de854881SBoris Brezillon
2443de854881SBoris Brezillon tested_queues &= ~BIT(cs_id);
2444de854881SBoris Brezillon }
2445de854881SBoris Brezillon
2446de854881SBoris Brezillon if (unblocked_queues) {
2447de854881SBoris Brezillon group->blocked_queues &= ~unblocked_queues;
2448de854881SBoris Brezillon
2449de854881SBoris Brezillon if (group->csg_id < 0) {
2450de854881SBoris Brezillon list_move(&group->run_node,
2451de854881SBoris Brezillon &sched->groups.runnable[group->priority]);
2452de854881SBoris Brezillon if (group->priority == PANTHOR_CSG_PRIORITY_RT)
2453de854881SBoris Brezillon immediate_tick = true;
2454de854881SBoris Brezillon }
2455de854881SBoris Brezillon }
2456de854881SBoris Brezillon
2457de854881SBoris Brezillon if (!group->blocked_queues)
2458de854881SBoris Brezillon list_del_init(&group->wait_node);
2459de854881SBoris Brezillon }
2460de854881SBoris Brezillon mutex_unlock(&sched->lock);
2461de854881SBoris Brezillon
2462de854881SBoris Brezillon if (immediate_tick)
2463de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, 0);
2464de854881SBoris Brezillon }
2465de854881SBoris Brezillon
group_schedule_locked(struct panthor_group * group,u32 queue_mask)2466de854881SBoris Brezillon static void group_schedule_locked(struct panthor_group *group, u32 queue_mask)
2467de854881SBoris Brezillon {
2468de854881SBoris Brezillon struct panthor_device *ptdev = group->ptdev;
2469de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
2470de854881SBoris Brezillon struct list_head *queue = &sched->groups.runnable[group->priority];
2471de854881SBoris Brezillon u64 delay_jiffies = 0;
2472de854881SBoris Brezillon bool was_idle;
2473de854881SBoris Brezillon u64 now;
2474de854881SBoris Brezillon
2475de854881SBoris Brezillon if (!group_can_run(group))
2476de854881SBoris Brezillon return;
2477de854881SBoris Brezillon
2478de854881SBoris Brezillon /* All updated queues are blocked, no need to wake up the scheduler. */
2479de854881SBoris Brezillon if ((queue_mask & group->blocked_queues) == queue_mask)
2480de854881SBoris Brezillon return;
2481de854881SBoris Brezillon
2482de854881SBoris Brezillon was_idle = group_is_idle(group);
2483de854881SBoris Brezillon group->idle_queues &= ~queue_mask;
2484de854881SBoris Brezillon
2485de854881SBoris Brezillon /* Don't mess up with the lists if we're in a middle of a reset. */
2486de854881SBoris Brezillon if (atomic_read(&sched->reset.in_progress))
2487de854881SBoris Brezillon return;
2488de854881SBoris Brezillon
2489de854881SBoris Brezillon if (was_idle && !group_is_idle(group))
2490de854881SBoris Brezillon list_move_tail(&group->run_node, queue);
2491de854881SBoris Brezillon
2492de854881SBoris Brezillon /* RT groups are preemptive. */
2493de854881SBoris Brezillon if (group->priority == PANTHOR_CSG_PRIORITY_RT) {
2494de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, 0);
2495de854881SBoris Brezillon return;
2496de854881SBoris Brezillon }
2497de854881SBoris Brezillon
2498de854881SBoris Brezillon /* Some groups might be idle, force an immediate tick to
2499de854881SBoris Brezillon * re-evaluate.
2500de854881SBoris Brezillon */
2501de854881SBoris Brezillon if (sched->might_have_idle_groups) {
2502de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, 0);
2503de854881SBoris Brezillon return;
2504de854881SBoris Brezillon }
2505de854881SBoris Brezillon
2506de854881SBoris Brezillon /* Scheduler is ticking, nothing to do. */
2507de854881SBoris Brezillon if (sched->resched_target != U64_MAX) {
2508de854881SBoris Brezillon /* If there are free slots, force immediating ticking. */
2509de854881SBoris Brezillon if (sched->used_csg_slot_count < sched->csg_slot_count)
2510de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, 0);
2511de854881SBoris Brezillon
2512de854881SBoris Brezillon return;
2513de854881SBoris Brezillon }
2514de854881SBoris Brezillon
2515de854881SBoris Brezillon /* Scheduler tick was off, recalculate the resched_target based on the
2516de854881SBoris Brezillon * last tick event, and queue the scheduler work.
2517de854881SBoris Brezillon */
2518de854881SBoris Brezillon now = get_jiffies_64();
2519de854881SBoris Brezillon sched->resched_target = sched->last_tick + sched->tick_period;
2520de854881SBoris Brezillon if (sched->used_csg_slot_count == sched->csg_slot_count &&
2521de854881SBoris Brezillon time_before64(now, sched->resched_target))
2522de854881SBoris Brezillon delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX);
2523de854881SBoris Brezillon
2524de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, delay_jiffies);
2525de854881SBoris Brezillon }
2526de854881SBoris Brezillon
queue_stop(struct panthor_queue * queue,struct panthor_job * bad_job)2527de854881SBoris Brezillon static void queue_stop(struct panthor_queue *queue,
2528de854881SBoris Brezillon struct panthor_job *bad_job)
2529de854881SBoris Brezillon {
2530de854881SBoris Brezillon drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL);
2531de854881SBoris Brezillon }
2532de854881SBoris Brezillon
queue_start(struct panthor_queue * queue)2533de854881SBoris Brezillon static void queue_start(struct panthor_queue *queue)
2534de854881SBoris Brezillon {
2535de854881SBoris Brezillon struct panthor_job *job;
2536de854881SBoris Brezillon
2537de854881SBoris Brezillon /* Re-assign the parent fences. */
2538de854881SBoris Brezillon list_for_each_entry(job, &queue->scheduler.pending_list, base.list)
2539de854881SBoris Brezillon job->base.s_fence->parent = dma_fence_get(job->done_fence);
2540de854881SBoris Brezillon
2541de854881SBoris Brezillon drm_sched_start(&queue->scheduler);
2542de854881SBoris Brezillon }
2543de854881SBoris Brezillon
panthor_group_stop(struct panthor_group * group)2544de854881SBoris Brezillon static void panthor_group_stop(struct panthor_group *group)
2545de854881SBoris Brezillon {
2546de854881SBoris Brezillon struct panthor_scheduler *sched = group->ptdev->scheduler;
2547de854881SBoris Brezillon
2548de854881SBoris Brezillon lockdep_assert_held(&sched->reset.lock);
2549de854881SBoris Brezillon
2550de854881SBoris Brezillon for (u32 i = 0; i < group->queue_count; i++)
2551de854881SBoris Brezillon queue_stop(group->queues[i], NULL);
2552de854881SBoris Brezillon
2553de854881SBoris Brezillon group_get(group);
2554de854881SBoris Brezillon list_move_tail(&group->run_node, &sched->reset.stopped_groups);
2555de854881SBoris Brezillon }
2556de854881SBoris Brezillon
panthor_group_start(struct panthor_group * group)2557de854881SBoris Brezillon static void panthor_group_start(struct panthor_group *group)
2558de854881SBoris Brezillon {
2559de854881SBoris Brezillon struct panthor_scheduler *sched = group->ptdev->scheduler;
2560de854881SBoris Brezillon
2561de854881SBoris Brezillon lockdep_assert_held(&group->ptdev->scheduler->reset.lock);
2562de854881SBoris Brezillon
2563de854881SBoris Brezillon for (u32 i = 0; i < group->queue_count; i++)
2564de854881SBoris Brezillon queue_start(group->queues[i]);
2565de854881SBoris Brezillon
2566de854881SBoris Brezillon if (group_can_run(group)) {
2567de854881SBoris Brezillon list_move_tail(&group->run_node,
2568de854881SBoris Brezillon group_is_idle(group) ?
2569de854881SBoris Brezillon &sched->groups.idle[group->priority] :
2570de854881SBoris Brezillon &sched->groups.runnable[group->priority]);
2571de854881SBoris Brezillon } else {
2572de854881SBoris Brezillon list_del_init(&group->run_node);
2573de854881SBoris Brezillon list_del_init(&group->wait_node);
2574de854881SBoris Brezillon group_queue_work(group, term);
2575de854881SBoris Brezillon }
2576de854881SBoris Brezillon
2577de854881SBoris Brezillon group_put(group);
2578de854881SBoris Brezillon }
2579de854881SBoris Brezillon
panthor_sched_immediate_tick(struct panthor_device * ptdev)2580de854881SBoris Brezillon static void panthor_sched_immediate_tick(struct panthor_device *ptdev)
2581de854881SBoris Brezillon {
2582de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
2583de854881SBoris Brezillon
2584de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, 0);
2585de854881SBoris Brezillon }
2586de854881SBoris Brezillon
2587de854881SBoris Brezillon /**
2588de854881SBoris Brezillon * panthor_sched_report_mmu_fault() - Report MMU faults to the scheduler.
2589de854881SBoris Brezillon */
panthor_sched_report_mmu_fault(struct panthor_device * ptdev)2590de854881SBoris Brezillon void panthor_sched_report_mmu_fault(struct panthor_device *ptdev)
2591de854881SBoris Brezillon {
2592de854881SBoris Brezillon /* Force a tick to immediately kill faulty groups. */
2593de854881SBoris Brezillon if (ptdev->scheduler)
2594de854881SBoris Brezillon panthor_sched_immediate_tick(ptdev);
2595de854881SBoris Brezillon }
2596de854881SBoris Brezillon
panthor_sched_resume(struct panthor_device * ptdev)2597de854881SBoris Brezillon void panthor_sched_resume(struct panthor_device *ptdev)
2598de854881SBoris Brezillon {
2599de854881SBoris Brezillon /* Force a tick to re-evaluate after a resume. */
2600de854881SBoris Brezillon panthor_sched_immediate_tick(ptdev);
2601de854881SBoris Brezillon }
2602de854881SBoris Brezillon
panthor_sched_suspend(struct panthor_device * ptdev)2603de854881SBoris Brezillon void panthor_sched_suspend(struct panthor_device *ptdev)
2604de854881SBoris Brezillon {
2605de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
2606de854881SBoris Brezillon struct panthor_csg_slots_upd_ctx upd_ctx;
2607de854881SBoris Brezillon struct panthor_group *group;
2608be2d3e9dSBoris Brezillon u32 suspended_slots;
2609de854881SBoris Brezillon u32 i;
2610de854881SBoris Brezillon
2611de854881SBoris Brezillon mutex_lock(&sched->lock);
2612de854881SBoris Brezillon csgs_upd_ctx_init(&upd_ctx);
2613de854881SBoris Brezillon for (i = 0; i < sched->csg_slot_count; i++) {
2614de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
2615de854881SBoris Brezillon
2616de854881SBoris Brezillon if (csg_slot->group) {
2617de854881SBoris Brezillon csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i,
26188bdbd8b5SBoris Brezillon group_can_run(csg_slot->group) ?
26198bdbd8b5SBoris Brezillon CSG_STATE_SUSPEND : CSG_STATE_TERMINATE,
2620de854881SBoris Brezillon CSG_STATE_MASK);
2621de854881SBoris Brezillon }
2622de854881SBoris Brezillon }
2623de854881SBoris Brezillon
2624de854881SBoris Brezillon suspended_slots = upd_ctx.update_mask;
2625de854881SBoris Brezillon
2626de854881SBoris Brezillon csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
2627de854881SBoris Brezillon suspended_slots &= ~upd_ctx.timedout_mask;
2628de854881SBoris Brezillon
2629be2d3e9dSBoris Brezillon if (upd_ctx.timedout_mask) {
2630be2d3e9dSBoris Brezillon u32 slot_mask = upd_ctx.timedout_mask;
2631de854881SBoris Brezillon
2632de854881SBoris Brezillon drm_err(&ptdev->base, "CSG suspend failed, escalating to termination");
2633de854881SBoris Brezillon csgs_upd_ctx_init(&upd_ctx);
2634de854881SBoris Brezillon while (slot_mask) {
2635de854881SBoris Brezillon u32 csg_id = ffs(slot_mask) - 1;
2636de854881SBoris Brezillon
2637de854881SBoris Brezillon csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
2638de854881SBoris Brezillon CSG_STATE_TERMINATE,
2639de854881SBoris Brezillon CSG_STATE_MASK);
2640de854881SBoris Brezillon slot_mask &= ~BIT(csg_id);
2641de854881SBoris Brezillon }
2642de854881SBoris Brezillon
2643de854881SBoris Brezillon csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
2644de854881SBoris Brezillon
2645de854881SBoris Brezillon slot_mask = upd_ctx.timedout_mask;
2646de854881SBoris Brezillon while (slot_mask) {
2647de854881SBoris Brezillon u32 csg_id = ffs(slot_mask) - 1;
2648de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
2649de854881SBoris Brezillon
2650de854881SBoris Brezillon /* Terminate command timedout, but the soft-reset will
2651de854881SBoris Brezillon * automatically terminate all active groups, so let's
2652de854881SBoris Brezillon * force the state to halted here.
2653de854881SBoris Brezillon */
2654de854881SBoris Brezillon if (csg_slot->group->state != PANTHOR_CS_GROUP_TERMINATED)
2655de854881SBoris Brezillon csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED;
2656de854881SBoris Brezillon slot_mask &= ~BIT(csg_id);
2657de854881SBoris Brezillon }
2658de854881SBoris Brezillon }
2659de854881SBoris Brezillon
2660de854881SBoris Brezillon /* Flush L2 and LSC caches to make sure suspend state is up-to-date.
2661de854881SBoris Brezillon * If the flush fails, flag all queues for termination.
2662de854881SBoris Brezillon */
2663de854881SBoris Brezillon if (suspended_slots) {
2664de854881SBoris Brezillon bool flush_caches_failed = false;
2665de854881SBoris Brezillon u32 slot_mask = suspended_slots;
2666de854881SBoris Brezillon
2667de854881SBoris Brezillon if (panthor_gpu_flush_caches(ptdev, CACHE_CLEAN, CACHE_CLEAN, 0))
2668de854881SBoris Brezillon flush_caches_failed = true;
2669de854881SBoris Brezillon
2670de854881SBoris Brezillon while (slot_mask) {
2671de854881SBoris Brezillon u32 csg_id = ffs(slot_mask) - 1;
2672de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
2673de854881SBoris Brezillon
2674de854881SBoris Brezillon if (flush_caches_failed)
2675de854881SBoris Brezillon csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED;
2676de854881SBoris Brezillon else
2677de854881SBoris Brezillon csg_slot_sync_update_locked(ptdev, csg_id);
2678de854881SBoris Brezillon
2679de854881SBoris Brezillon slot_mask &= ~BIT(csg_id);
2680de854881SBoris Brezillon }
2681de854881SBoris Brezillon }
2682de854881SBoris Brezillon
2683de854881SBoris Brezillon for (i = 0; i < sched->csg_slot_count; i++) {
2684de854881SBoris Brezillon struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
2685de854881SBoris Brezillon
2686de854881SBoris Brezillon group = csg_slot->group;
2687de854881SBoris Brezillon if (!group)
2688de854881SBoris Brezillon continue;
2689de854881SBoris Brezillon
2690de854881SBoris Brezillon group_get(group);
2691de854881SBoris Brezillon
2692de854881SBoris Brezillon if (group->csg_id >= 0)
2693de854881SBoris Brezillon sched_process_csg_irq_locked(ptdev, group->csg_id);
2694de854881SBoris Brezillon
2695de854881SBoris Brezillon group_unbind_locked(group);
2696de854881SBoris Brezillon
2697de854881SBoris Brezillon drm_WARN_ON(&group->ptdev->base, !list_empty(&group->run_node));
2698de854881SBoris Brezillon
2699de854881SBoris Brezillon if (group_can_run(group)) {
2700de854881SBoris Brezillon list_add(&group->run_node,
2701de854881SBoris Brezillon &sched->groups.idle[group->priority]);
2702de854881SBoris Brezillon } else {
2703de854881SBoris Brezillon /* We don't bother stopping the scheduler if the group is
2704de854881SBoris Brezillon * faulty, the group termination work will finish the job.
2705de854881SBoris Brezillon */
2706de854881SBoris Brezillon list_del_init(&group->wait_node);
2707de854881SBoris Brezillon group_queue_work(group, term);
2708de854881SBoris Brezillon }
2709de854881SBoris Brezillon group_put(group);
2710de854881SBoris Brezillon }
2711de854881SBoris Brezillon mutex_unlock(&sched->lock);
2712de854881SBoris Brezillon }
2713de854881SBoris Brezillon
panthor_sched_pre_reset(struct panthor_device * ptdev)2714de854881SBoris Brezillon void panthor_sched_pre_reset(struct panthor_device *ptdev)
2715de854881SBoris Brezillon {
2716de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
2717de854881SBoris Brezillon struct panthor_group *group, *group_tmp;
2718de854881SBoris Brezillon u32 i;
2719de854881SBoris Brezillon
2720de854881SBoris Brezillon mutex_lock(&sched->reset.lock);
2721de854881SBoris Brezillon atomic_set(&sched->reset.in_progress, true);
2722de854881SBoris Brezillon
2723de854881SBoris Brezillon /* Cancel all scheduler works. Once this is done, these works can't be
2724de854881SBoris Brezillon * scheduled again until the reset operation is complete.
2725de854881SBoris Brezillon */
2726de854881SBoris Brezillon cancel_work_sync(&sched->sync_upd_work);
2727de854881SBoris Brezillon cancel_delayed_work_sync(&sched->tick_work);
2728de854881SBoris Brezillon
2729de854881SBoris Brezillon panthor_sched_suspend(ptdev);
2730de854881SBoris Brezillon
2731de854881SBoris Brezillon /* Stop all groups that might still accept jobs, so we don't get passed
2732de854881SBoris Brezillon * new jobs while we're resetting.
2733de854881SBoris Brezillon */
2734de854881SBoris Brezillon for (i = 0; i < ARRAY_SIZE(sched->groups.runnable); i++) {
2735de854881SBoris Brezillon /* All groups should be in the idle lists. */
2736de854881SBoris Brezillon drm_WARN_ON(&ptdev->base, !list_empty(&sched->groups.runnable[i]));
2737de854881SBoris Brezillon list_for_each_entry_safe(group, group_tmp, &sched->groups.runnable[i], run_node)
2738de854881SBoris Brezillon panthor_group_stop(group);
2739de854881SBoris Brezillon }
2740de854881SBoris Brezillon
2741de854881SBoris Brezillon for (i = 0; i < ARRAY_SIZE(sched->groups.idle); i++) {
2742de854881SBoris Brezillon list_for_each_entry_safe(group, group_tmp, &sched->groups.idle[i], run_node)
2743de854881SBoris Brezillon panthor_group_stop(group);
2744de854881SBoris Brezillon }
2745de854881SBoris Brezillon
2746de854881SBoris Brezillon mutex_unlock(&sched->reset.lock);
2747de854881SBoris Brezillon }
2748de854881SBoris Brezillon
panthor_sched_post_reset(struct panthor_device * ptdev,bool reset_failed)27493ce4322bSBoris Brezillon void panthor_sched_post_reset(struct panthor_device *ptdev, bool reset_failed)
2750de854881SBoris Brezillon {
2751de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
2752de854881SBoris Brezillon struct panthor_group *group, *group_tmp;
2753de854881SBoris Brezillon
2754de854881SBoris Brezillon mutex_lock(&sched->reset.lock);
2755de854881SBoris Brezillon
27563ce4322bSBoris Brezillon list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) {
27573ce4322bSBoris Brezillon /* Consider all previously running group as terminated if the
27583ce4322bSBoris Brezillon * reset failed.
27593ce4322bSBoris Brezillon */
27603ce4322bSBoris Brezillon if (reset_failed)
27613ce4322bSBoris Brezillon group->state = PANTHOR_CS_GROUP_TERMINATED;
27623ce4322bSBoris Brezillon
2763de854881SBoris Brezillon panthor_group_start(group);
27643ce4322bSBoris Brezillon }
2765de854881SBoris Brezillon
2766de854881SBoris Brezillon /* We're done resetting the GPU, clear the reset.in_progress bit so we can
2767de854881SBoris Brezillon * kick the scheduler.
2768de854881SBoris Brezillon */
2769de854881SBoris Brezillon atomic_set(&sched->reset.in_progress, false);
2770de854881SBoris Brezillon mutex_unlock(&sched->reset.lock);
2771de854881SBoris Brezillon
27723ce4322bSBoris Brezillon /* No need to queue a tick and update syncs if the reset failed. */
27733ce4322bSBoris Brezillon if (!reset_failed) {
2774de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, 0);
2775de854881SBoris Brezillon sched_queue_work(sched, sync_upd);
2776de854881SBoris Brezillon }
27773ce4322bSBoris Brezillon }
2778de854881SBoris Brezillon
group_sync_upd_work(struct work_struct * work)2779de854881SBoris Brezillon static void group_sync_upd_work(struct work_struct *work)
2780de854881SBoris Brezillon {
2781de854881SBoris Brezillon struct panthor_group *group =
2782de854881SBoris Brezillon container_of(work, struct panthor_group, sync_upd_work);
2783de854881SBoris Brezillon struct panthor_job *job, *job_tmp;
2784de854881SBoris Brezillon LIST_HEAD(done_jobs);
2785de854881SBoris Brezillon u32 queue_idx;
2786de854881SBoris Brezillon bool cookie;
2787de854881SBoris Brezillon
2788de854881SBoris Brezillon cookie = dma_fence_begin_signalling();
2789de854881SBoris Brezillon for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) {
2790de854881SBoris Brezillon struct panthor_queue *queue = group->queues[queue_idx];
2791de854881SBoris Brezillon struct panthor_syncobj_64b *syncobj;
2792de854881SBoris Brezillon
2793de854881SBoris Brezillon if (!queue)
2794de854881SBoris Brezillon continue;
2795de854881SBoris Brezillon
2796de854881SBoris Brezillon syncobj = group->syncobjs->kmap + (queue_idx * sizeof(*syncobj));
2797de854881SBoris Brezillon
2798de854881SBoris Brezillon spin_lock(&queue->fence_ctx.lock);
2799de854881SBoris Brezillon list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) {
2800de854881SBoris Brezillon if (syncobj->seqno < job->done_fence->seqno)
2801de854881SBoris Brezillon break;
2802de854881SBoris Brezillon
2803de854881SBoris Brezillon list_move_tail(&job->node, &done_jobs);
2804de854881SBoris Brezillon dma_fence_signal_locked(job->done_fence);
2805de854881SBoris Brezillon }
2806de854881SBoris Brezillon spin_unlock(&queue->fence_ctx.lock);
2807de854881SBoris Brezillon }
2808de854881SBoris Brezillon dma_fence_end_signalling(cookie);
2809de854881SBoris Brezillon
2810de854881SBoris Brezillon list_for_each_entry_safe(job, job_tmp, &done_jobs, node) {
2811de854881SBoris Brezillon list_del_init(&job->node);
2812de854881SBoris Brezillon panthor_job_put(&job->base);
2813de854881SBoris Brezillon }
2814de854881SBoris Brezillon
2815de854881SBoris Brezillon group_put(group);
2816de854881SBoris Brezillon }
2817de854881SBoris Brezillon
2818de854881SBoris Brezillon static struct dma_fence *
queue_run_job(struct drm_sched_job * sched_job)2819de854881SBoris Brezillon queue_run_job(struct drm_sched_job *sched_job)
2820de854881SBoris Brezillon {
2821de854881SBoris Brezillon struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
2822de854881SBoris Brezillon struct panthor_group *group = job->group;
2823de854881SBoris Brezillon struct panthor_queue *queue = group->queues[job->queue_idx];
2824de854881SBoris Brezillon struct panthor_device *ptdev = group->ptdev;
2825de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
2826de854881SBoris Brezillon u32 ringbuf_size = panthor_kernel_bo_size(queue->ringbuf);
2827de854881SBoris Brezillon u32 ringbuf_insert = queue->iface.input->insert & (ringbuf_size - 1);
2828de854881SBoris Brezillon u64 addr_reg = ptdev->csif_info.cs_reg_count -
2829de854881SBoris Brezillon ptdev->csif_info.unpreserved_cs_reg_count;
2830de854881SBoris Brezillon u64 val_reg = addr_reg + 2;
2831de854881SBoris Brezillon u64 sync_addr = panthor_kernel_bo_gpuva(group->syncobjs) +
2832de854881SBoris Brezillon job->queue_idx * sizeof(struct panthor_syncobj_64b);
2833de854881SBoris Brezillon u32 waitall_mask = GENMASK(sched->sb_slot_count - 1, 0);
2834de854881SBoris Brezillon struct dma_fence *done_fence;
2835de854881SBoris Brezillon int ret;
2836de854881SBoris Brezillon
2837de854881SBoris Brezillon u64 call_instrs[NUM_INSTRS_PER_SLOT] = {
2838de854881SBoris Brezillon /* MOV32 rX+2, cs.latest_flush */
2839de854881SBoris Brezillon (2ull << 56) | (val_reg << 48) | job->call_info.latest_flush,
2840de854881SBoris Brezillon
2841de854881SBoris Brezillon /* FLUSH_CACHE2.clean_inv_all.no_wait.signal(0) rX+2 */
2842de854881SBoris Brezillon (36ull << 56) | (0ull << 48) | (val_reg << 40) | (0 << 16) | 0x233,
2843de854881SBoris Brezillon
2844de854881SBoris Brezillon /* MOV48 rX:rX+1, cs.start */
2845de854881SBoris Brezillon (1ull << 56) | (addr_reg << 48) | job->call_info.start,
2846de854881SBoris Brezillon
2847de854881SBoris Brezillon /* MOV32 rX+2, cs.size */
2848de854881SBoris Brezillon (2ull << 56) | (val_reg << 48) | job->call_info.size,
2849de854881SBoris Brezillon
2850de854881SBoris Brezillon /* WAIT(0) => waits for FLUSH_CACHE2 instruction */
2851de854881SBoris Brezillon (3ull << 56) | (1 << 16),
2852de854881SBoris Brezillon
2853de854881SBoris Brezillon /* CALL rX:rX+1, rX+2 */
2854de854881SBoris Brezillon (32ull << 56) | (addr_reg << 40) | (val_reg << 32),
2855de854881SBoris Brezillon
2856de854881SBoris Brezillon /* MOV48 rX:rX+1, sync_addr */
2857de854881SBoris Brezillon (1ull << 56) | (addr_reg << 48) | sync_addr,
2858de854881SBoris Brezillon
2859de854881SBoris Brezillon /* MOV48 rX+2, #1 */
2860de854881SBoris Brezillon (1ull << 56) | (val_reg << 48) | 1,
2861de854881SBoris Brezillon
2862de854881SBoris Brezillon /* WAIT(all) */
2863de854881SBoris Brezillon (3ull << 56) | (waitall_mask << 16),
2864de854881SBoris Brezillon
2865de854881SBoris Brezillon /* SYNC_ADD64.system_scope.propage_err.nowait rX:rX+1, rX+2*/
2866de854881SBoris Brezillon (51ull << 56) | (0ull << 48) | (addr_reg << 40) | (val_reg << 32) | (0 << 16) | 1,
2867de854881SBoris Brezillon
2868de854881SBoris Brezillon /* ERROR_BARRIER, so we can recover from faults at job
2869de854881SBoris Brezillon * boundaries.
2870de854881SBoris Brezillon */
2871de854881SBoris Brezillon (47ull << 56),
2872de854881SBoris Brezillon };
2873de854881SBoris Brezillon
2874de854881SBoris Brezillon /* Need to be cacheline aligned to please the prefetcher. */
2875de854881SBoris Brezillon static_assert(sizeof(call_instrs) % 64 == 0,
2876de854881SBoris Brezillon "call_instrs is not aligned on a cacheline");
2877de854881SBoris Brezillon
28787b6f9ec6SBoris Brezillon /* Stream size is zero, nothing to do except making sure all previously
28797b6f9ec6SBoris Brezillon * submitted jobs are done before we signal the
28807b6f9ec6SBoris Brezillon * drm_sched_job::s_fence::finished fence.
2881de854881SBoris Brezillon */
28827b6f9ec6SBoris Brezillon if (!job->call_info.size) {
28837b6f9ec6SBoris Brezillon job->done_fence = dma_fence_get(queue->fence_ctx.last_fence);
28847b6f9ec6SBoris Brezillon return dma_fence_get(job->done_fence);
28857b6f9ec6SBoris Brezillon }
2886de854881SBoris Brezillon
2887de854881SBoris Brezillon ret = pm_runtime_resume_and_get(ptdev->base.dev);
2888de854881SBoris Brezillon if (drm_WARN_ON(&ptdev->base, ret))
2889de854881SBoris Brezillon return ERR_PTR(ret);
2890de854881SBoris Brezillon
2891de854881SBoris Brezillon mutex_lock(&sched->lock);
2892de854881SBoris Brezillon if (!group_can_run(group)) {
2893de854881SBoris Brezillon done_fence = ERR_PTR(-ECANCELED);
2894de854881SBoris Brezillon goto out_unlock;
2895de854881SBoris Brezillon }
2896de854881SBoris Brezillon
2897de854881SBoris Brezillon dma_fence_init(job->done_fence,
2898de854881SBoris Brezillon &panthor_queue_fence_ops,
2899de854881SBoris Brezillon &queue->fence_ctx.lock,
2900de854881SBoris Brezillon queue->fence_ctx.id,
2901de854881SBoris Brezillon atomic64_inc_return(&queue->fence_ctx.seqno));
2902de854881SBoris Brezillon
2903de854881SBoris Brezillon memcpy(queue->ringbuf->kmap + ringbuf_insert,
2904de854881SBoris Brezillon call_instrs, sizeof(call_instrs));
2905de854881SBoris Brezillon
2906de854881SBoris Brezillon panthor_job_get(&job->base);
2907de854881SBoris Brezillon spin_lock(&queue->fence_ctx.lock);
2908de854881SBoris Brezillon list_add_tail(&job->node, &queue->fence_ctx.in_flight_jobs);
2909de854881SBoris Brezillon spin_unlock(&queue->fence_ctx.lock);
2910de854881SBoris Brezillon
2911de854881SBoris Brezillon job->ringbuf.start = queue->iface.input->insert;
2912de854881SBoris Brezillon job->ringbuf.end = job->ringbuf.start + sizeof(call_instrs);
2913de854881SBoris Brezillon
2914de854881SBoris Brezillon /* Make sure the ring buffer is updated before the INSERT
2915de854881SBoris Brezillon * register.
2916de854881SBoris Brezillon */
2917de854881SBoris Brezillon wmb();
2918de854881SBoris Brezillon
2919de854881SBoris Brezillon queue->iface.input->extract = queue->iface.output->extract;
2920de854881SBoris Brezillon queue->iface.input->insert = job->ringbuf.end;
2921de854881SBoris Brezillon
2922de854881SBoris Brezillon if (group->csg_id < 0) {
2923de854881SBoris Brezillon /* If the queue is blocked, we want to keep the timeout running, so we
2924de854881SBoris Brezillon * can detect unbounded waits and kill the group when that happens.
2925de854881SBoris Brezillon * Otherwise, we suspend the timeout so the time we spend waiting for
2926de854881SBoris Brezillon * a CSG slot is not counted.
2927de854881SBoris Brezillon */
2928de854881SBoris Brezillon if (!(group->blocked_queues & BIT(job->queue_idx)) &&
2929de854881SBoris Brezillon !queue->timeout_suspended) {
2930de854881SBoris Brezillon queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler);
2931de854881SBoris Brezillon queue->timeout_suspended = true;
2932de854881SBoris Brezillon }
2933de854881SBoris Brezillon
2934de854881SBoris Brezillon group_schedule_locked(group, BIT(job->queue_idx));
2935de854881SBoris Brezillon } else {
2936de854881SBoris Brezillon gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1);
2937de854881SBoris Brezillon if (!sched->pm.has_ref &&
2938de854881SBoris Brezillon !(group->blocked_queues & BIT(job->queue_idx))) {
2939de854881SBoris Brezillon pm_runtime_get(ptdev->base.dev);
2940de854881SBoris Brezillon sched->pm.has_ref = true;
2941de854881SBoris Brezillon }
2942896868edSSteven Price panthor_devfreq_record_busy(sched->ptdev);
2943de854881SBoris Brezillon }
2944de854881SBoris Brezillon
29457b6f9ec6SBoris Brezillon /* Update the last fence. */
29467b6f9ec6SBoris Brezillon dma_fence_put(queue->fence_ctx.last_fence);
29477b6f9ec6SBoris Brezillon queue->fence_ctx.last_fence = dma_fence_get(job->done_fence);
29487b6f9ec6SBoris Brezillon
2949de854881SBoris Brezillon done_fence = dma_fence_get(job->done_fence);
2950de854881SBoris Brezillon
2951de854881SBoris Brezillon out_unlock:
2952de854881SBoris Brezillon mutex_unlock(&sched->lock);
2953de854881SBoris Brezillon pm_runtime_mark_last_busy(ptdev->base.dev);
2954de854881SBoris Brezillon pm_runtime_put_autosuspend(ptdev->base.dev);
2955de854881SBoris Brezillon
2956de854881SBoris Brezillon return done_fence;
2957de854881SBoris Brezillon }
2958de854881SBoris Brezillon
2959de854881SBoris Brezillon static enum drm_gpu_sched_stat
queue_timedout_job(struct drm_sched_job * sched_job)2960de854881SBoris Brezillon queue_timedout_job(struct drm_sched_job *sched_job)
2961de854881SBoris Brezillon {
2962de854881SBoris Brezillon struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
2963de854881SBoris Brezillon struct panthor_group *group = job->group;
2964de854881SBoris Brezillon struct panthor_device *ptdev = group->ptdev;
2965de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
2966de854881SBoris Brezillon struct panthor_queue *queue = group->queues[job->queue_idx];
2967de854881SBoris Brezillon
2968de854881SBoris Brezillon drm_warn(&ptdev->base, "job timeout\n");
2969de854881SBoris Brezillon
2970de854881SBoris Brezillon drm_WARN_ON(&ptdev->base, atomic_read(&sched->reset.in_progress));
2971de854881SBoris Brezillon
2972de854881SBoris Brezillon queue_stop(queue, job);
2973de854881SBoris Brezillon
2974de854881SBoris Brezillon mutex_lock(&sched->lock);
2975de854881SBoris Brezillon group->timedout = true;
2976de854881SBoris Brezillon if (group->csg_id >= 0) {
2977de854881SBoris Brezillon sched_queue_delayed_work(ptdev->scheduler, tick, 0);
2978de854881SBoris Brezillon } else {
2979de854881SBoris Brezillon /* Remove from the run queues, so the scheduler can't
2980de854881SBoris Brezillon * pick the group on the next tick.
2981de854881SBoris Brezillon */
2982de854881SBoris Brezillon list_del_init(&group->run_node);
2983de854881SBoris Brezillon list_del_init(&group->wait_node);
2984de854881SBoris Brezillon
2985de854881SBoris Brezillon group_queue_work(group, term);
2986de854881SBoris Brezillon }
2987de854881SBoris Brezillon mutex_unlock(&sched->lock);
2988de854881SBoris Brezillon
2989de854881SBoris Brezillon queue_start(queue);
2990de854881SBoris Brezillon
2991de854881SBoris Brezillon return DRM_GPU_SCHED_STAT_NOMINAL;
2992de854881SBoris Brezillon }
2993de854881SBoris Brezillon
queue_free_job(struct drm_sched_job * sched_job)2994de854881SBoris Brezillon static void queue_free_job(struct drm_sched_job *sched_job)
2995de854881SBoris Brezillon {
2996de854881SBoris Brezillon drm_sched_job_cleanup(sched_job);
2997de854881SBoris Brezillon panthor_job_put(sched_job);
2998de854881SBoris Brezillon }
2999de854881SBoris Brezillon
3000de854881SBoris Brezillon static const struct drm_sched_backend_ops panthor_queue_sched_ops = {
3001de854881SBoris Brezillon .run_job = queue_run_job,
3002de854881SBoris Brezillon .timedout_job = queue_timedout_job,
3003de854881SBoris Brezillon .free_job = queue_free_job,
3004de854881SBoris Brezillon };
3005de854881SBoris Brezillon
3006de854881SBoris Brezillon static struct panthor_queue *
group_create_queue(struct panthor_group * group,const struct drm_panthor_queue_create * args)3007de854881SBoris Brezillon group_create_queue(struct panthor_group *group,
3008de854881SBoris Brezillon const struct drm_panthor_queue_create *args)
3009de854881SBoris Brezillon {
3010de854881SBoris Brezillon struct drm_gpu_scheduler *drm_sched;
3011de854881SBoris Brezillon struct panthor_queue *queue;
3012de854881SBoris Brezillon int ret;
3013de854881SBoris Brezillon
3014de854881SBoris Brezillon if (args->pad[0] || args->pad[1] || args->pad[2])
3015de854881SBoris Brezillon return ERR_PTR(-EINVAL);
3016de854881SBoris Brezillon
3017de854881SBoris Brezillon if (args->ringbuf_size < SZ_4K || args->ringbuf_size > SZ_64K ||
3018de854881SBoris Brezillon !is_power_of_2(args->ringbuf_size))
3019de854881SBoris Brezillon return ERR_PTR(-EINVAL);
3020de854881SBoris Brezillon
3021de854881SBoris Brezillon if (args->priority > CSF_MAX_QUEUE_PRIO)
3022de854881SBoris Brezillon return ERR_PTR(-EINVAL);
3023de854881SBoris Brezillon
3024de854881SBoris Brezillon queue = kzalloc(sizeof(*queue), GFP_KERNEL);
3025de854881SBoris Brezillon if (!queue)
3026de854881SBoris Brezillon return ERR_PTR(-ENOMEM);
3027de854881SBoris Brezillon
3028de854881SBoris Brezillon queue->fence_ctx.id = dma_fence_context_alloc(1);
3029de854881SBoris Brezillon spin_lock_init(&queue->fence_ctx.lock);
3030de854881SBoris Brezillon INIT_LIST_HEAD(&queue->fence_ctx.in_flight_jobs);
3031de854881SBoris Brezillon
3032de854881SBoris Brezillon queue->priority = args->priority;
3033de854881SBoris Brezillon
3034de854881SBoris Brezillon queue->ringbuf = panthor_kernel_bo_create(group->ptdev, group->vm,
3035de854881SBoris Brezillon args->ringbuf_size,
3036de854881SBoris Brezillon DRM_PANTHOR_BO_NO_MMAP,
3037de854881SBoris Brezillon DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |
3038de854881SBoris Brezillon DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,
3039de854881SBoris Brezillon PANTHOR_VM_KERNEL_AUTO_VA);
3040de854881SBoris Brezillon if (IS_ERR(queue->ringbuf)) {
3041de854881SBoris Brezillon ret = PTR_ERR(queue->ringbuf);
3042de854881SBoris Brezillon goto err_free_queue;
3043de854881SBoris Brezillon }
3044de854881SBoris Brezillon
3045de854881SBoris Brezillon ret = panthor_kernel_bo_vmap(queue->ringbuf);
3046de854881SBoris Brezillon if (ret)
3047de854881SBoris Brezillon goto err_free_queue;
3048de854881SBoris Brezillon
3049de854881SBoris Brezillon queue->iface.mem = panthor_fw_alloc_queue_iface_mem(group->ptdev,
3050de854881SBoris Brezillon &queue->iface.input,
3051de854881SBoris Brezillon &queue->iface.output,
3052de854881SBoris Brezillon &queue->iface.input_fw_va,
3053de854881SBoris Brezillon &queue->iface.output_fw_va);
3054de854881SBoris Brezillon if (IS_ERR(queue->iface.mem)) {
3055de854881SBoris Brezillon ret = PTR_ERR(queue->iface.mem);
3056de854881SBoris Brezillon goto err_free_queue;
3057de854881SBoris Brezillon }
3058de854881SBoris Brezillon
3059de854881SBoris Brezillon ret = drm_sched_init(&queue->scheduler, &panthor_queue_sched_ops,
3060de854881SBoris Brezillon group->ptdev->scheduler->wq, 1,
3061de854881SBoris Brezillon args->ringbuf_size / (NUM_INSTRS_PER_SLOT * sizeof(u64)),
3062de854881SBoris Brezillon 0, msecs_to_jiffies(JOB_TIMEOUT_MS),
3063de854881SBoris Brezillon group->ptdev->reset.wq,
3064de854881SBoris Brezillon NULL, "panthor-queue", group->ptdev->base.dev);
3065de854881SBoris Brezillon if (ret)
3066de854881SBoris Brezillon goto err_free_queue;
3067de854881SBoris Brezillon
3068de854881SBoris Brezillon drm_sched = &queue->scheduler;
3069de854881SBoris Brezillon ret = drm_sched_entity_init(&queue->entity, 0, &drm_sched, 1, NULL);
3070de854881SBoris Brezillon
3071de854881SBoris Brezillon return queue;
3072de854881SBoris Brezillon
3073de854881SBoris Brezillon err_free_queue:
3074de854881SBoris Brezillon group_free_queue(group, queue);
3075de854881SBoris Brezillon return ERR_PTR(ret);
3076de854881SBoris Brezillon }
3077de854881SBoris Brezillon
3078de854881SBoris Brezillon #define MAX_GROUPS_PER_POOL 128
3079de854881SBoris Brezillon
panthor_group_create(struct panthor_file * pfile,const struct drm_panthor_group_create * group_args,const struct drm_panthor_queue_create * queue_args)3080de854881SBoris Brezillon int panthor_group_create(struct panthor_file *pfile,
3081de854881SBoris Brezillon const struct drm_panthor_group_create *group_args,
3082de854881SBoris Brezillon const struct drm_panthor_queue_create *queue_args)
3083de854881SBoris Brezillon {
3084de854881SBoris Brezillon struct panthor_device *ptdev = pfile->ptdev;
3085de854881SBoris Brezillon struct panthor_group_pool *gpool = pfile->groups;
3086de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
3087de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0);
3088de854881SBoris Brezillon struct panthor_group *group = NULL;
3089de854881SBoris Brezillon u32 gid, i, suspend_size;
3090de854881SBoris Brezillon int ret;
3091de854881SBoris Brezillon
3092de854881SBoris Brezillon if (group_args->pad)
3093de854881SBoris Brezillon return -EINVAL;
3094de854881SBoris Brezillon
3095*5f776204SMary Guillemard if (group_args->priority >= PANTHOR_CSG_PRIORITY_COUNT)
3096de854881SBoris Brezillon return -EINVAL;
3097de854881SBoris Brezillon
3098de854881SBoris Brezillon if ((group_args->compute_core_mask & ~ptdev->gpu_info.shader_present) ||
3099de854881SBoris Brezillon (group_args->fragment_core_mask & ~ptdev->gpu_info.shader_present) ||
3100de854881SBoris Brezillon (group_args->tiler_core_mask & ~ptdev->gpu_info.tiler_present))
3101de854881SBoris Brezillon return -EINVAL;
3102de854881SBoris Brezillon
3103de854881SBoris Brezillon if (hweight64(group_args->compute_core_mask) < group_args->max_compute_cores ||
3104de854881SBoris Brezillon hweight64(group_args->fragment_core_mask) < group_args->max_fragment_cores ||
3105de854881SBoris Brezillon hweight64(group_args->tiler_core_mask) < group_args->max_tiler_cores)
3106de854881SBoris Brezillon return -EINVAL;
3107de854881SBoris Brezillon
3108de854881SBoris Brezillon group = kzalloc(sizeof(*group), GFP_KERNEL);
3109de854881SBoris Brezillon if (!group)
3110de854881SBoris Brezillon return -ENOMEM;
3111de854881SBoris Brezillon
3112de854881SBoris Brezillon spin_lock_init(&group->fatal_lock);
3113de854881SBoris Brezillon kref_init(&group->refcount);
3114de854881SBoris Brezillon group->state = PANTHOR_CS_GROUP_CREATED;
3115de854881SBoris Brezillon group->csg_id = -1;
3116de854881SBoris Brezillon
3117de854881SBoris Brezillon group->ptdev = ptdev;
3118de854881SBoris Brezillon group->max_compute_cores = group_args->max_compute_cores;
3119de854881SBoris Brezillon group->compute_core_mask = group_args->compute_core_mask;
3120de854881SBoris Brezillon group->max_fragment_cores = group_args->max_fragment_cores;
3121de854881SBoris Brezillon group->fragment_core_mask = group_args->fragment_core_mask;
3122de854881SBoris Brezillon group->max_tiler_cores = group_args->max_tiler_cores;
3123de854881SBoris Brezillon group->tiler_core_mask = group_args->tiler_core_mask;
3124de854881SBoris Brezillon group->priority = group_args->priority;
3125de854881SBoris Brezillon
3126de854881SBoris Brezillon INIT_LIST_HEAD(&group->wait_node);
3127de854881SBoris Brezillon INIT_LIST_HEAD(&group->run_node);
3128de854881SBoris Brezillon INIT_WORK(&group->term_work, group_term_work);
3129de854881SBoris Brezillon INIT_WORK(&group->sync_upd_work, group_sync_upd_work);
3130de854881SBoris Brezillon INIT_WORK(&group->tiler_oom_work, group_tiler_oom_work);
3131de854881SBoris Brezillon INIT_WORK(&group->release_work, group_release_work);
3132de854881SBoris Brezillon
3133de854881SBoris Brezillon group->vm = panthor_vm_pool_get_vm(pfile->vms, group_args->vm_id);
3134de854881SBoris Brezillon if (!group->vm) {
3135de854881SBoris Brezillon ret = -EINVAL;
3136de854881SBoris Brezillon goto err_put_group;
3137de854881SBoris Brezillon }
3138de854881SBoris Brezillon
3139de854881SBoris Brezillon suspend_size = csg_iface->control->suspend_size;
3140de854881SBoris Brezillon group->suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size);
3141de854881SBoris Brezillon if (IS_ERR(group->suspend_buf)) {
3142de854881SBoris Brezillon ret = PTR_ERR(group->suspend_buf);
3143de854881SBoris Brezillon group->suspend_buf = NULL;
3144de854881SBoris Brezillon goto err_put_group;
3145de854881SBoris Brezillon }
3146de854881SBoris Brezillon
3147de854881SBoris Brezillon suspend_size = csg_iface->control->protm_suspend_size;
3148de854881SBoris Brezillon group->protm_suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size);
3149de854881SBoris Brezillon if (IS_ERR(group->protm_suspend_buf)) {
3150de854881SBoris Brezillon ret = PTR_ERR(group->protm_suspend_buf);
3151de854881SBoris Brezillon group->protm_suspend_buf = NULL;
3152de854881SBoris Brezillon goto err_put_group;
3153de854881SBoris Brezillon }
3154de854881SBoris Brezillon
3155de854881SBoris Brezillon group->syncobjs = panthor_kernel_bo_create(ptdev, group->vm,
3156de854881SBoris Brezillon group_args->queues.count *
3157de854881SBoris Brezillon sizeof(struct panthor_syncobj_64b),
3158de854881SBoris Brezillon DRM_PANTHOR_BO_NO_MMAP,
3159de854881SBoris Brezillon DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |
3160de854881SBoris Brezillon DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,
3161de854881SBoris Brezillon PANTHOR_VM_KERNEL_AUTO_VA);
3162de854881SBoris Brezillon if (IS_ERR(group->syncobjs)) {
3163de854881SBoris Brezillon ret = PTR_ERR(group->syncobjs);
3164de854881SBoris Brezillon goto err_put_group;
3165de854881SBoris Brezillon }
3166de854881SBoris Brezillon
3167de854881SBoris Brezillon ret = panthor_kernel_bo_vmap(group->syncobjs);
3168de854881SBoris Brezillon if (ret)
3169de854881SBoris Brezillon goto err_put_group;
3170de854881SBoris Brezillon
3171de854881SBoris Brezillon memset(group->syncobjs->kmap, 0,
3172de854881SBoris Brezillon group_args->queues.count * sizeof(struct panthor_syncobj_64b));
3173de854881SBoris Brezillon
3174de854881SBoris Brezillon for (i = 0; i < group_args->queues.count; i++) {
3175de854881SBoris Brezillon group->queues[i] = group_create_queue(group, &queue_args[i]);
3176de854881SBoris Brezillon if (IS_ERR(group->queues[i])) {
3177de854881SBoris Brezillon ret = PTR_ERR(group->queues[i]);
3178de854881SBoris Brezillon group->queues[i] = NULL;
3179de854881SBoris Brezillon goto err_put_group;
3180de854881SBoris Brezillon }
3181de854881SBoris Brezillon
3182de854881SBoris Brezillon group->queue_count++;
3183de854881SBoris Brezillon }
3184de854881SBoris Brezillon
3185de854881SBoris Brezillon group->idle_queues = GENMASK(group->queue_count - 1, 0);
3186de854881SBoris Brezillon
3187de854881SBoris Brezillon ret = xa_alloc(&gpool->xa, &gid, group, XA_LIMIT(1, MAX_GROUPS_PER_POOL), GFP_KERNEL);
3188de854881SBoris Brezillon if (ret)
3189de854881SBoris Brezillon goto err_put_group;
3190de854881SBoris Brezillon
3191de854881SBoris Brezillon mutex_lock(&sched->reset.lock);
3192de854881SBoris Brezillon if (atomic_read(&sched->reset.in_progress)) {
3193de854881SBoris Brezillon panthor_group_stop(group);
3194de854881SBoris Brezillon } else {
3195de854881SBoris Brezillon mutex_lock(&sched->lock);
3196de854881SBoris Brezillon list_add_tail(&group->run_node,
3197de854881SBoris Brezillon &sched->groups.idle[group->priority]);
3198de854881SBoris Brezillon mutex_unlock(&sched->lock);
3199de854881SBoris Brezillon }
3200de854881SBoris Brezillon mutex_unlock(&sched->reset.lock);
3201de854881SBoris Brezillon
3202de854881SBoris Brezillon return gid;
3203de854881SBoris Brezillon
3204de854881SBoris Brezillon err_put_group:
3205de854881SBoris Brezillon group_put(group);
3206de854881SBoris Brezillon return ret;
3207de854881SBoris Brezillon }
3208de854881SBoris Brezillon
panthor_group_destroy(struct panthor_file * pfile,u32 group_handle)3209de854881SBoris Brezillon int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle)
3210de854881SBoris Brezillon {
3211de854881SBoris Brezillon struct panthor_group_pool *gpool = pfile->groups;
3212de854881SBoris Brezillon struct panthor_device *ptdev = pfile->ptdev;
3213de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
3214de854881SBoris Brezillon struct panthor_group *group;
3215de854881SBoris Brezillon
3216de854881SBoris Brezillon group = xa_erase(&gpool->xa, group_handle);
3217de854881SBoris Brezillon if (!group)
3218de854881SBoris Brezillon return -EINVAL;
3219de854881SBoris Brezillon
3220de854881SBoris Brezillon for (u32 i = 0; i < group->queue_count; i++) {
3221de854881SBoris Brezillon if (group->queues[i])
3222de854881SBoris Brezillon drm_sched_entity_destroy(&group->queues[i]->entity);
3223de854881SBoris Brezillon }
3224de854881SBoris Brezillon
3225de854881SBoris Brezillon mutex_lock(&sched->reset.lock);
3226de854881SBoris Brezillon mutex_lock(&sched->lock);
3227de854881SBoris Brezillon group->destroyed = true;
3228de854881SBoris Brezillon if (group->csg_id >= 0) {
3229de854881SBoris Brezillon sched_queue_delayed_work(sched, tick, 0);
3230de854881SBoris Brezillon } else if (!atomic_read(&sched->reset.in_progress)) {
3231de854881SBoris Brezillon /* Remove from the run queues, so the scheduler can't
3232de854881SBoris Brezillon * pick the group on the next tick.
3233de854881SBoris Brezillon */
3234de854881SBoris Brezillon list_del_init(&group->run_node);
3235de854881SBoris Brezillon list_del_init(&group->wait_node);
3236de854881SBoris Brezillon group_queue_work(group, term);
3237de854881SBoris Brezillon }
3238de854881SBoris Brezillon mutex_unlock(&sched->lock);
3239de854881SBoris Brezillon mutex_unlock(&sched->reset.lock);
3240de854881SBoris Brezillon
3241de854881SBoris Brezillon group_put(group);
3242de854881SBoris Brezillon return 0;
3243de854881SBoris Brezillon }
3244de854881SBoris Brezillon
panthor_group_get_state(struct panthor_file * pfile,struct drm_panthor_group_get_state * get_state)3245de854881SBoris Brezillon int panthor_group_get_state(struct panthor_file *pfile,
3246de854881SBoris Brezillon struct drm_panthor_group_get_state *get_state)
3247de854881SBoris Brezillon {
3248de854881SBoris Brezillon struct panthor_group_pool *gpool = pfile->groups;
3249de854881SBoris Brezillon struct panthor_device *ptdev = pfile->ptdev;
3250de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
3251de854881SBoris Brezillon struct panthor_group *group;
3252de854881SBoris Brezillon
3253de854881SBoris Brezillon if (get_state->pad)
3254de854881SBoris Brezillon return -EINVAL;
3255de854881SBoris Brezillon
3256de854881SBoris Brezillon group = group_get(xa_load(&gpool->xa, get_state->group_handle));
3257de854881SBoris Brezillon if (!group)
3258de854881SBoris Brezillon return -EINVAL;
3259de854881SBoris Brezillon
3260de854881SBoris Brezillon memset(get_state, 0, sizeof(*get_state));
3261de854881SBoris Brezillon
3262de854881SBoris Brezillon mutex_lock(&sched->lock);
3263de854881SBoris Brezillon if (group->timedout)
3264de854881SBoris Brezillon get_state->state |= DRM_PANTHOR_GROUP_STATE_TIMEDOUT;
3265de854881SBoris Brezillon if (group->fatal_queues) {
3266de854881SBoris Brezillon get_state->state |= DRM_PANTHOR_GROUP_STATE_FATAL_FAULT;
3267de854881SBoris Brezillon get_state->fatal_queues = group->fatal_queues;
3268de854881SBoris Brezillon }
3269de854881SBoris Brezillon mutex_unlock(&sched->lock);
3270de854881SBoris Brezillon
3271de854881SBoris Brezillon group_put(group);
3272de854881SBoris Brezillon return 0;
3273de854881SBoris Brezillon }
3274de854881SBoris Brezillon
panthor_group_pool_create(struct panthor_file * pfile)3275de854881SBoris Brezillon int panthor_group_pool_create(struct panthor_file *pfile)
3276de854881SBoris Brezillon {
3277de854881SBoris Brezillon struct panthor_group_pool *gpool;
3278de854881SBoris Brezillon
3279de854881SBoris Brezillon gpool = kzalloc(sizeof(*gpool), GFP_KERNEL);
3280de854881SBoris Brezillon if (!gpool)
3281de854881SBoris Brezillon return -ENOMEM;
3282de854881SBoris Brezillon
3283de854881SBoris Brezillon xa_init_flags(&gpool->xa, XA_FLAGS_ALLOC1);
3284de854881SBoris Brezillon pfile->groups = gpool;
3285de854881SBoris Brezillon return 0;
3286de854881SBoris Brezillon }
3287de854881SBoris Brezillon
panthor_group_pool_destroy(struct panthor_file * pfile)3288de854881SBoris Brezillon void panthor_group_pool_destroy(struct panthor_file *pfile)
3289de854881SBoris Brezillon {
3290de854881SBoris Brezillon struct panthor_group_pool *gpool = pfile->groups;
3291de854881SBoris Brezillon struct panthor_group *group;
3292de854881SBoris Brezillon unsigned long i;
3293de854881SBoris Brezillon
3294de854881SBoris Brezillon if (IS_ERR_OR_NULL(gpool))
3295de854881SBoris Brezillon return;
3296de854881SBoris Brezillon
3297de854881SBoris Brezillon xa_for_each(&gpool->xa, i, group)
3298de854881SBoris Brezillon panthor_group_destroy(pfile, i);
3299de854881SBoris Brezillon
3300de854881SBoris Brezillon xa_destroy(&gpool->xa);
3301de854881SBoris Brezillon kfree(gpool);
3302de854881SBoris Brezillon pfile->groups = NULL;
3303de854881SBoris Brezillon }
3304de854881SBoris Brezillon
job_release(struct kref * ref)3305de854881SBoris Brezillon static void job_release(struct kref *ref)
3306de854881SBoris Brezillon {
3307de854881SBoris Brezillon struct panthor_job *job = container_of(ref, struct panthor_job, refcount);
3308de854881SBoris Brezillon
3309de854881SBoris Brezillon drm_WARN_ON(&job->group->ptdev->base, !list_empty(&job->node));
3310de854881SBoris Brezillon
3311de854881SBoris Brezillon if (job->base.s_fence)
3312de854881SBoris Brezillon drm_sched_job_cleanup(&job->base);
3313de854881SBoris Brezillon
3314de854881SBoris Brezillon if (job->done_fence && job->done_fence->ops)
3315de854881SBoris Brezillon dma_fence_put(job->done_fence);
3316de854881SBoris Brezillon else
3317de854881SBoris Brezillon dma_fence_free(job->done_fence);
3318de854881SBoris Brezillon
3319de854881SBoris Brezillon group_put(job->group);
3320de854881SBoris Brezillon
3321de854881SBoris Brezillon kfree(job);
3322de854881SBoris Brezillon }
3323de854881SBoris Brezillon
panthor_job_get(struct drm_sched_job * sched_job)3324de854881SBoris Brezillon struct drm_sched_job *panthor_job_get(struct drm_sched_job *sched_job)
3325de854881SBoris Brezillon {
3326de854881SBoris Brezillon if (sched_job) {
3327de854881SBoris Brezillon struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
3328de854881SBoris Brezillon
3329de854881SBoris Brezillon kref_get(&job->refcount);
3330de854881SBoris Brezillon }
3331de854881SBoris Brezillon
3332de854881SBoris Brezillon return sched_job;
3333de854881SBoris Brezillon }
3334de854881SBoris Brezillon
panthor_job_put(struct drm_sched_job * sched_job)3335de854881SBoris Brezillon void panthor_job_put(struct drm_sched_job *sched_job)
3336de854881SBoris Brezillon {
3337de854881SBoris Brezillon struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
3338de854881SBoris Brezillon
3339de854881SBoris Brezillon if (sched_job)
3340de854881SBoris Brezillon kref_put(&job->refcount, job_release);
3341de854881SBoris Brezillon }
3342de854881SBoris Brezillon
panthor_job_vm(struct drm_sched_job * sched_job)3343de854881SBoris Brezillon struct panthor_vm *panthor_job_vm(struct drm_sched_job *sched_job)
3344de854881SBoris Brezillon {
3345de854881SBoris Brezillon struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
3346de854881SBoris Brezillon
3347de854881SBoris Brezillon return job->group->vm;
3348de854881SBoris Brezillon }
3349de854881SBoris Brezillon
3350de854881SBoris Brezillon struct drm_sched_job *
panthor_job_create(struct panthor_file * pfile,u16 group_handle,const struct drm_panthor_queue_submit * qsubmit)3351de854881SBoris Brezillon panthor_job_create(struct panthor_file *pfile,
3352de854881SBoris Brezillon u16 group_handle,
3353de854881SBoris Brezillon const struct drm_panthor_queue_submit *qsubmit)
3354de854881SBoris Brezillon {
3355de854881SBoris Brezillon struct panthor_group_pool *gpool = pfile->groups;
3356de854881SBoris Brezillon struct panthor_job *job;
3357de854881SBoris Brezillon int ret;
3358de854881SBoris Brezillon
3359de854881SBoris Brezillon if (qsubmit->pad)
3360de854881SBoris Brezillon return ERR_PTR(-EINVAL);
3361de854881SBoris Brezillon
3362de854881SBoris Brezillon /* If stream_addr is zero, so stream_size should be. */
3363de854881SBoris Brezillon if ((qsubmit->stream_size == 0) != (qsubmit->stream_addr == 0))
3364de854881SBoris Brezillon return ERR_PTR(-EINVAL);
3365de854881SBoris Brezillon
3366de854881SBoris Brezillon /* Make sure the address is aligned on 64-byte (cacheline) and the size is
3367de854881SBoris Brezillon * aligned on 8-byte (instruction size).
3368de854881SBoris Brezillon */
3369de854881SBoris Brezillon if ((qsubmit->stream_addr & 63) || (qsubmit->stream_size & 7))
3370de854881SBoris Brezillon return ERR_PTR(-EINVAL);
3371de854881SBoris Brezillon
3372de854881SBoris Brezillon /* bits 24:30 must be zero. */
3373de854881SBoris Brezillon if (qsubmit->latest_flush & GENMASK(30, 24))
3374de854881SBoris Brezillon return ERR_PTR(-EINVAL);
3375de854881SBoris Brezillon
3376de854881SBoris Brezillon job = kzalloc(sizeof(*job), GFP_KERNEL);
3377de854881SBoris Brezillon if (!job)
3378de854881SBoris Brezillon return ERR_PTR(-ENOMEM);
3379de854881SBoris Brezillon
3380de854881SBoris Brezillon kref_init(&job->refcount);
3381de854881SBoris Brezillon job->queue_idx = qsubmit->queue_index;
3382de854881SBoris Brezillon job->call_info.size = qsubmit->stream_size;
3383de854881SBoris Brezillon job->call_info.start = qsubmit->stream_addr;
3384de854881SBoris Brezillon job->call_info.latest_flush = qsubmit->latest_flush;
3385de854881SBoris Brezillon INIT_LIST_HEAD(&job->node);
3386de854881SBoris Brezillon
3387de854881SBoris Brezillon job->group = group_get(xa_load(&gpool->xa, group_handle));
3388de854881SBoris Brezillon if (!job->group) {
3389de854881SBoris Brezillon ret = -EINVAL;
3390de854881SBoris Brezillon goto err_put_job;
3391de854881SBoris Brezillon }
3392de854881SBoris Brezillon
3393de854881SBoris Brezillon if (job->queue_idx >= job->group->queue_count ||
3394de854881SBoris Brezillon !job->group->queues[job->queue_idx]) {
3395de854881SBoris Brezillon ret = -EINVAL;
3396de854881SBoris Brezillon goto err_put_job;
3397de854881SBoris Brezillon }
3398de854881SBoris Brezillon
33997b6f9ec6SBoris Brezillon /* Empty command streams don't need a fence, they'll pick the one from
34007b6f9ec6SBoris Brezillon * the previously submitted job.
34017b6f9ec6SBoris Brezillon */
34027b6f9ec6SBoris Brezillon if (job->call_info.size) {
3403de854881SBoris Brezillon job->done_fence = kzalloc(sizeof(*job->done_fence), GFP_KERNEL);
3404de854881SBoris Brezillon if (!job->done_fence) {
3405de854881SBoris Brezillon ret = -ENOMEM;
3406de854881SBoris Brezillon goto err_put_job;
3407de854881SBoris Brezillon }
34087b6f9ec6SBoris Brezillon }
3409de854881SBoris Brezillon
3410de854881SBoris Brezillon ret = drm_sched_job_init(&job->base,
3411de854881SBoris Brezillon &job->group->queues[job->queue_idx]->entity,
3412de854881SBoris Brezillon 1, job->group);
3413de854881SBoris Brezillon if (ret)
3414de854881SBoris Brezillon goto err_put_job;
3415de854881SBoris Brezillon
3416de854881SBoris Brezillon return &job->base;
3417de854881SBoris Brezillon
3418de854881SBoris Brezillon err_put_job:
3419de854881SBoris Brezillon panthor_job_put(&job->base);
3420de854881SBoris Brezillon return ERR_PTR(ret);
3421de854881SBoris Brezillon }
3422de854881SBoris Brezillon
panthor_job_update_resvs(struct drm_exec * exec,struct drm_sched_job * sched_job)3423de854881SBoris Brezillon void panthor_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *sched_job)
3424de854881SBoris Brezillon {
3425de854881SBoris Brezillon struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
3426de854881SBoris Brezillon
3427de854881SBoris Brezillon /* Still not sure why we want USAGE_WRITE for external objects, since I
3428de854881SBoris Brezillon * was assuming this would be handled through explicit syncs being imported
3429de854881SBoris Brezillon * to external BOs with DMA_BUF_IOCTL_IMPORT_SYNC_FILE, but other drivers
3430de854881SBoris Brezillon * seem to pass DMA_RESV_USAGE_WRITE, so there must be a good reason.
3431de854881SBoris Brezillon */
3432de854881SBoris Brezillon panthor_vm_update_resvs(job->group->vm, exec, &sched_job->s_fence->finished,
3433de854881SBoris Brezillon DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_WRITE);
3434de854881SBoris Brezillon }
3435de854881SBoris Brezillon
panthor_sched_unplug(struct panthor_device * ptdev)3436de854881SBoris Brezillon void panthor_sched_unplug(struct panthor_device *ptdev)
3437de854881SBoris Brezillon {
3438de854881SBoris Brezillon struct panthor_scheduler *sched = ptdev->scheduler;
3439de854881SBoris Brezillon
3440de854881SBoris Brezillon cancel_delayed_work_sync(&sched->tick_work);
3441de854881SBoris Brezillon
3442de854881SBoris Brezillon mutex_lock(&sched->lock);
3443de854881SBoris Brezillon if (sched->pm.has_ref) {
3444de854881SBoris Brezillon pm_runtime_put(ptdev->base.dev);
3445de854881SBoris Brezillon sched->pm.has_ref = false;
3446de854881SBoris Brezillon }
3447de854881SBoris Brezillon mutex_unlock(&sched->lock);
3448de854881SBoris Brezillon }
3449de854881SBoris Brezillon
panthor_sched_fini(struct drm_device * ddev,void * res)3450de854881SBoris Brezillon static void panthor_sched_fini(struct drm_device *ddev, void *res)
3451de854881SBoris Brezillon {
3452de854881SBoris Brezillon struct panthor_scheduler *sched = res;
3453de854881SBoris Brezillon int prio;
3454de854881SBoris Brezillon
3455de854881SBoris Brezillon if (!sched || !sched->csg_slot_count)
3456de854881SBoris Brezillon return;
3457de854881SBoris Brezillon
3458de854881SBoris Brezillon cancel_delayed_work_sync(&sched->tick_work);
3459de854881SBoris Brezillon
3460de854881SBoris Brezillon if (sched->wq)
3461de854881SBoris Brezillon destroy_workqueue(sched->wq);
3462de854881SBoris Brezillon
3463de854881SBoris Brezillon if (sched->heap_alloc_wq)
3464de854881SBoris Brezillon destroy_workqueue(sched->heap_alloc_wq);
3465de854881SBoris Brezillon
3466de854881SBoris Brezillon for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
3467de854881SBoris Brezillon drm_WARN_ON(ddev, !list_empty(&sched->groups.runnable[prio]));
3468de854881SBoris Brezillon drm_WARN_ON(ddev, !list_empty(&sched->groups.idle[prio]));
3469de854881SBoris Brezillon }
3470de854881SBoris Brezillon
3471de854881SBoris Brezillon drm_WARN_ON(ddev, !list_empty(&sched->groups.waiting));
3472de854881SBoris Brezillon }
3473de854881SBoris Brezillon
panthor_sched_init(struct panthor_device * ptdev)3474de854881SBoris Brezillon int panthor_sched_init(struct panthor_device *ptdev)
3475de854881SBoris Brezillon {
3476de854881SBoris Brezillon struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
3477de854881SBoris Brezillon struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0);
3478de854881SBoris Brezillon struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, 0, 0);
3479de854881SBoris Brezillon struct panthor_scheduler *sched;
3480de854881SBoris Brezillon u32 gpu_as_count, num_groups;
3481de854881SBoris Brezillon int prio, ret;
3482de854881SBoris Brezillon
3483de854881SBoris Brezillon sched = drmm_kzalloc(&ptdev->base, sizeof(*sched), GFP_KERNEL);
3484de854881SBoris Brezillon if (!sched)
3485de854881SBoris Brezillon return -ENOMEM;
3486de854881SBoris Brezillon
3487de854881SBoris Brezillon /* The highest bit in JOB_INT_* is reserved for globabl IRQs. That
3488de854881SBoris Brezillon * leaves 31 bits for CSG IRQs, hence the MAX_CSGS clamp here.
3489de854881SBoris Brezillon */
3490de854881SBoris Brezillon num_groups = min_t(u32, MAX_CSGS, glb_iface->control->group_num);
3491de854881SBoris Brezillon
3492de854881SBoris Brezillon /* The FW-side scheduler might deadlock if two groups with the same
3493de854881SBoris Brezillon * priority try to access a set of resources that overlaps, with part
3494de854881SBoris Brezillon * of the resources being allocated to one group and the other part to
3495de854881SBoris Brezillon * the other group, both groups waiting for the remaining resources to
3496de854881SBoris Brezillon * be allocated. To avoid that, it is recommended to assign each CSG a
3497de854881SBoris Brezillon * different priority. In theory we could allow several groups to have
3498de854881SBoris Brezillon * the same CSG priority if they don't request the same resources, but
3499de854881SBoris Brezillon * that makes the scheduling logic more complicated, so let's clamp
3500de854881SBoris Brezillon * the number of CSG slots to MAX_CSG_PRIO + 1 for now.
3501de854881SBoris Brezillon */
3502de854881SBoris Brezillon num_groups = min_t(u32, MAX_CSG_PRIO + 1, num_groups);
3503de854881SBoris Brezillon
3504de854881SBoris Brezillon /* We need at least one AS for the MCU and one for the GPU contexts. */
3505de854881SBoris Brezillon gpu_as_count = hweight32(ptdev->gpu_info.as_present & GENMASK(31, 1));
3506de854881SBoris Brezillon if (!gpu_as_count) {
3507de854881SBoris Brezillon drm_err(&ptdev->base, "Not enough AS (%d, expected at least 2)",
3508de854881SBoris Brezillon gpu_as_count + 1);
3509de854881SBoris Brezillon return -EINVAL;
3510de854881SBoris Brezillon }
3511de854881SBoris Brezillon
3512de854881SBoris Brezillon sched->ptdev = ptdev;
3513de854881SBoris Brezillon sched->sb_slot_count = CS_FEATURES_SCOREBOARDS(cs_iface->control->features);
3514de854881SBoris Brezillon sched->csg_slot_count = num_groups;
3515de854881SBoris Brezillon sched->cs_slot_count = csg_iface->control->stream_num;
3516de854881SBoris Brezillon sched->as_slot_count = gpu_as_count;
3517de854881SBoris Brezillon ptdev->csif_info.csg_slot_count = sched->csg_slot_count;
3518de854881SBoris Brezillon ptdev->csif_info.cs_slot_count = sched->cs_slot_count;
3519de854881SBoris Brezillon ptdev->csif_info.scoreboard_slot_count = sched->sb_slot_count;
3520de854881SBoris Brezillon
3521de854881SBoris Brezillon sched->last_tick = 0;
3522de854881SBoris Brezillon sched->resched_target = U64_MAX;
3523de854881SBoris Brezillon sched->tick_period = msecs_to_jiffies(10);
3524de854881SBoris Brezillon INIT_DELAYED_WORK(&sched->tick_work, tick_work);
3525de854881SBoris Brezillon INIT_WORK(&sched->sync_upd_work, sync_upd_work);
3526de854881SBoris Brezillon INIT_WORK(&sched->fw_events_work, process_fw_events_work);
3527de854881SBoris Brezillon
3528de854881SBoris Brezillon ret = drmm_mutex_init(&ptdev->base, &sched->lock);
3529de854881SBoris Brezillon if (ret)
3530de854881SBoris Brezillon return ret;
3531de854881SBoris Brezillon
3532de854881SBoris Brezillon for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
3533de854881SBoris Brezillon INIT_LIST_HEAD(&sched->groups.runnable[prio]);
3534de854881SBoris Brezillon INIT_LIST_HEAD(&sched->groups.idle[prio]);
3535de854881SBoris Brezillon }
3536de854881SBoris Brezillon INIT_LIST_HEAD(&sched->groups.waiting);
3537de854881SBoris Brezillon
3538de854881SBoris Brezillon ret = drmm_mutex_init(&ptdev->base, &sched->reset.lock);
3539de854881SBoris Brezillon if (ret)
3540de854881SBoris Brezillon return ret;
3541de854881SBoris Brezillon
3542de854881SBoris Brezillon INIT_LIST_HEAD(&sched->reset.stopped_groups);
3543de854881SBoris Brezillon
3544de854881SBoris Brezillon /* sched->heap_alloc_wq will be used for heap chunk allocation on
3545de854881SBoris Brezillon * tiler OOM events, which means we can't use the same workqueue for
3546de854881SBoris Brezillon * the scheduler because works queued by the scheduler are in
3547de854881SBoris Brezillon * the dma-signalling path. Allocate a dedicated heap_alloc_wq to
3548de854881SBoris Brezillon * work around this limitation.
3549de854881SBoris Brezillon *
3550de854881SBoris Brezillon * FIXME: Ultimately, what we need is a failable/non-blocking GEM
3551de854881SBoris Brezillon * allocation path that we can call when a heap OOM is reported. The
3552de854881SBoris Brezillon * FW is smart enough to fall back on other methods if the kernel can't
3553de854881SBoris Brezillon * allocate memory, and fail the tiling job if none of these
3554de854881SBoris Brezillon * countermeasures worked.
3555de854881SBoris Brezillon *
3556de854881SBoris Brezillon * Set WQ_MEM_RECLAIM on sched->wq to unblock the situation when the
3557de854881SBoris Brezillon * system is running out of memory.
3558de854881SBoris Brezillon */
3559de854881SBoris Brezillon sched->heap_alloc_wq = alloc_workqueue("panthor-heap-alloc", WQ_UNBOUND, 0);
3560de854881SBoris Brezillon sched->wq = alloc_workqueue("panthor-csf-sched", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
3561de854881SBoris Brezillon if (!sched->wq || !sched->heap_alloc_wq) {
3562de854881SBoris Brezillon panthor_sched_fini(&ptdev->base, sched);
3563de854881SBoris Brezillon drm_err(&ptdev->base, "Failed to allocate the workqueues");
3564de854881SBoris Brezillon return -ENOMEM;
3565de854881SBoris Brezillon }
3566de854881SBoris Brezillon
3567de854881SBoris Brezillon ret = drmm_add_action_or_reset(&ptdev->base, panthor_sched_fini, sched);
3568de854881SBoris Brezillon if (ret)
3569de854881SBoris Brezillon return ret;
3570de854881SBoris Brezillon
3571de854881SBoris Brezillon ptdev->scheduler = sched;
3572de854881SBoris Brezillon return 0;
3573de854881SBoris Brezillon }
3574