xref: /linux/drivers/gpu/drm/xe/xe_cpu_bind.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2026 Intel Corporation
4  */
5 
6 #include <drm/drm_managed.h>
7 #include <linux/mutex.h>
8 
9 #include "xe_cpu_bind.h"
10 #include "xe_device_types.h"
11 #include "xe_exec_queue.h"
12 #include "xe_pt.h"
13 #include "xe_sched_job.h"
14 #include "xe_trace_bo.h"
15 #include "xe_vm.h"
16 
17 /**
18  * struct xe_cpu_bind - cpu_bind context.
19  */
20 struct xe_cpu_bind {
21 	/** @xe: Xe device */
22 	struct xe_device *xe;
23 	/** @q: Default exec queue used for kernel binds */
24 	struct xe_exec_queue *q;
25 	/** @job_mutex: Timeline mutex for @q. */
26 	struct mutex job_mutex;
27 };
28 
29 static bool is_cpu_bind_queue(struct xe_cpu_bind *cpu_bind,
30 			      struct xe_exec_queue *q)
31 {
32 	return cpu_bind->q == q;
33 }
34 
35 static void xe_cpu_bind_fini(void *arg)
36 {
37 	struct xe_cpu_bind *cpu_bind = arg;
38 
39 	mutex_destroy(&cpu_bind->job_mutex);
40 	xe_exec_queue_put(cpu_bind->q);
41 }
42 
43 /**
44  * xe_cpu_bind_init() - Initialize a cpu_bind context
45  * @xe: &struct xe_device
46  *
47  * Return: 0 if successful, negative error code on failure
48  */
49 int xe_cpu_bind_init(struct xe_device *xe)
50 {
51 	struct xe_cpu_bind *cpu_bind =
52 		drmm_kzalloc(&xe->drm, sizeof(*cpu_bind), GFP_KERNEL);
53 	struct xe_exec_queue *q;
54 
55 	if (!cpu_bind)
56 		return -ENOMEM;
57 
58 	q = xe_exec_queue_create_bind(xe, xe_device_get_root_tile(xe), NULL,
59 				      EXEC_QUEUE_FLAG_KERNEL |
60 				      EXEC_QUEUE_FLAG_MIGRATE, 0);
61 	if (IS_ERR(q))
62 		return PTR_ERR(q);
63 
64 	cpu_bind->xe = xe;
65 	cpu_bind->q = q;
66 	xe->cpu_bind = cpu_bind;
67 
68 	mutex_init(&cpu_bind->job_mutex);
69 
70 	fs_reclaim_acquire(GFP_KERNEL);
71 	might_lock(&cpu_bind->job_mutex);
72 	fs_reclaim_release(GFP_KERNEL);
73 
74 	return devm_add_action_or_reset(cpu_bind->xe->drm.dev, xe_cpu_bind_fini,
75 					cpu_bind);
76 }
77 
78 /**
79  * xe_cpu_bind_queue() - Get the bind queue from cpu_bind context.
80  * @cpu_bind: The cpu bind context.
81  *
82  * Return: Pointer to bind queue.
83  */
84 struct xe_exec_queue *xe_cpu_bind_queue(struct xe_cpu_bind *cpu_bind)
85 {
86 	return cpu_bind->q;
87 }
88 
89 /**
90  * xe_cpu_bind_update_pgtables_execute() - Update a VM's PTEs via the CPU
91  * @vm: The VM being updated
92  * @tile: The tile being updated
93  * @ops: The CPU bind PT update ops
94  * @pt_op: The VM PT update op
95  * @num_ops: The number of The VM PT update ops
96  * @force_clear: Force clear operation
97  *
98  * Execute the VM PT update ops array which results in a VM's PTEs being updated
99  * via the CPU.
100  */
101 void
102 xe_cpu_bind_update_pgtables_execute(struct xe_vm *vm, struct xe_tile *tile,
103 				    const struct xe_cpu_bind_pt_update_ops *ops,
104 				    struct xe_vm_pgtable_update_op *pt_op,
105 				    u32 num_ops, bool force_clear)
106 {
107 	u32 j, i;
108 
109 	for (j = 0; j < num_ops; ++j, ++pt_op) {
110 		for (i = 0; i < pt_op->num_entries; i++) {
111 			const struct xe_vm_pgtable_update *update =
112 				&pt_op->entries[i];
113 
114 			xe_assert(vm->xe, !iosys_map_is_null(&update->pt_bo->vmap));
115 
116 			if (pt_op->bind && !force_clear)
117 				ops->populate(tile, &update->pt_bo->vmap,
118 					      update);
119 			else
120 				ops->clear(vm, tile, &update->pt_bo->vmap,
121 					   update);
122 		}
123 	}
124 
125 	trace_xe_vm_cpu_bind(vm);
126 	xe_device_wmb(vm->xe);
127 }
128 
129 static struct dma_fence *
130 xe_cpu_bind_update_pgtables_no_job(struct xe_cpu_bind *cpu_bind,
131 				   struct xe_cpu_bind_pt_update *pt_update)
132 {
133 	const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops;
134 	struct xe_vm *vm = pt_update->vops->vm;
135 	struct xe_tile *tile;
136 	int err, id;
137 
138 	if (ops->pre_commit) {
139 		pt_update->job = NULL;
140 		err = ops->pre_commit(pt_update);
141 		if (err)
142 			return ERR_PTR(err);
143 	}
144 
145 	for_each_tile(tile, vm->xe, id) {
146 		struct xe_vm_pgtable_update_ops *pt_update_ops =
147 			&pt_update->vops->pt_update_ops[tile->id];
148 
149 		if (!pt_update_ops->pt_job_ops)
150 			continue;
151 
152 		xe_cpu_bind_update_pgtables_execute(vm, tile, ops,
153 						    pt_update_ops->pt_job_ops->ops,
154 						    pt_update_ops->pt_job_ops->current_op,
155 						    false);
156 	}
157 
158 	return dma_fence_get_stub();
159 }
160 
161 static struct dma_fence *
162 xe_cpu_bind_update_pgtables_job(struct xe_cpu_bind *cpu_bind,
163 				struct xe_cpu_bind_pt_update *pt_update)
164 {
165 	const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops;
166 	struct xe_exec_queue *q = pt_update->vops->q;
167 	struct xe_device *xe = cpu_bind->xe;
168 	struct xe_sched_job *job;
169 	struct dma_fence *fence;
170 	struct xe_tile *tile;
171 	int err, id;
172 	bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
173 
174 	job = xe_sched_job_create(q, NULL);
175 	if (IS_ERR(job))
176 		return ERR_CAST(job);
177 
178 	xe_assert(xe, job->is_pt_job);
179 
180 	if (ops->pre_commit) {
181 		pt_update->job = job;
182 		err = ops->pre_commit(pt_update);
183 		if (err)
184 			goto err_job;
185 	}
186 
187 	if (is_cpu_bind)
188 		mutex_lock(&cpu_bind->job_mutex);
189 
190 	job->pt_update[0].vm = pt_update->vops->vm;
191 	job->pt_update[0].ops = ops;
192 	for_each_tile(tile, xe, id) {
193 		struct xe_vm_pgtable_update_ops *pt_update_ops =
194 			&pt_update->vops->pt_update_ops[tile->id];
195 
196 		job->pt_update[0].pt_job_ops[tile->id] =
197 			xe_pt_job_ops_get(pt_update_ops->pt_job_ops);
198 	}
199 
200 	xe_sched_job_arm(job);
201 	fence = dma_fence_get(&job->drm.s_fence->finished);
202 	xe_sched_job_push(job);
203 
204 	if (is_cpu_bind)
205 		mutex_unlock(&cpu_bind->job_mutex);
206 
207 	return fence;
208 
209 err_job:
210 	xe_sched_job_put(job);
211 	return ERR_PTR(err);
212 }
213 
214 /**
215  * xe_cpu_bind_update_pgtables() - Pipelined page-table update
216  * @cpu_bind: The cpu bind context.
217  * @pt_update: PT update arguments
218  *
219  * Perform a pipelined page-table update. The update descriptors are typically
220  * built under the same lock critical section as a call to this function. If
221  * using the default engine for the updates, they will be performed in the
222  * order they grab the job_mutex. If different engines are used, external
223  * synchronization is needed for overlapping updates to maintain page-table
224  * consistency. Note that the meaning of "overlapping" is that the updates
225  * touch the same page-table, which might be a higher-level page-directory.
226  * If no pipelining is needed, then updates may be performed by the cpu.
227  *
228  * Return: A dma_fence that, when signaled, indicates the update completion.
229  */
230 struct dma_fence *
231 xe_cpu_bind_update_pgtables(struct xe_cpu_bind *cpu_bind,
232 			    struct xe_cpu_bind_pt_update *pt_update)
233 {
234 	struct dma_fence *fence;
235 
236 	fence = xe_cpu_bind_update_pgtables_no_job(cpu_bind, pt_update);
237 
238 	/* -ETIME indicates a job is needed, anything else is legit error */
239 	if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME)
240 		return fence;
241 
242 	return xe_cpu_bind_update_pgtables_job(cpu_bind, pt_update);
243 }
244 
245 /**
246  * xe_cpu_bind_job_lock() - Lock cpu_bind job lock
247  * @cpu_bind: The cpu bind context.
248  * @q: Queue associated with the operation which requires a lock
249  *
250  * Lock the cpu_bind job lock if the queue is a cpu bind queue, otherwise
251  * assert the VM's dma-resv is held (user queue's have own locking).
252  */
253 void xe_cpu_bind_job_lock(struct xe_cpu_bind *cpu_bind,
254 			  struct xe_exec_queue *q)
255 {
256 	bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
257 
258 	if (is_cpu_bind)
259 		mutex_lock(&cpu_bind->job_mutex);
260 	else
261 		xe_vm_assert_held(q->user_vm);	/* User queues VM's should be locked */
262 }
263 
264 /**
265  * xe_cpu_bind_job_unlock() - Unlock cpu_bind job lock
266  * @cpu_bind: The cpu bind context.
267  * @q: Queue associated with the operation which requires a lock
268  *
269  * Unlock the cpu_bind job lock if the queue is a cpu bind queue, otherwise
270  * assert the VM's dma-resv is held (user queue's have own locking).
271  */
272 void xe_cpu_bind_job_unlock(struct xe_cpu_bind *cpu_bind,
273 			    struct xe_exec_queue *q)
274 {
275 	bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
276 
277 	if (is_cpu_bind)
278 		mutex_unlock(&cpu_bind->job_mutex);
279 	else
280 		xe_vm_assert_held(q->user_vm);	/* User queues VM's should be locked */
281 }
282 
283 #if IS_ENABLED(CONFIG_PROVE_LOCKING)
284 /**
285  * xe_cpu_bind_job_lock_assert() - Assert cpu_bind job lock held of queue
286  * @q: cpu bind queue
287  */
288 void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q)
289 {
290 	struct xe_device *xe = gt_to_xe(q->gt);
291 	struct xe_cpu_bind *cpu_bind = xe->cpu_bind;
292 
293 	xe_assert(xe, q == cpu_bind->q);
294 	lockdep_assert_held(&cpu_bind->job_mutex);
295 }
296 #endif
297