1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2026 Intel Corporation 4 */ 5 6 #include <drm/drm_managed.h> 7 #include <linux/mutex.h> 8 9 #include "xe_cpu_bind.h" 10 #include "xe_device_types.h" 11 #include "xe_exec_queue.h" 12 #include "xe_pt.h" 13 #include "xe_sched_job.h" 14 #include "xe_trace_bo.h" 15 #include "xe_vm.h" 16 17 /** 18 * struct xe_cpu_bind - cpu_bind context. 19 */ 20 struct xe_cpu_bind { 21 /** @xe: Xe device */ 22 struct xe_device *xe; 23 /** @q: Default exec queue used for kernel binds */ 24 struct xe_exec_queue *q; 25 /** @job_mutex: Timeline mutex for @q. */ 26 struct mutex job_mutex; 27 }; 28 29 static bool is_cpu_bind_queue(struct xe_cpu_bind *cpu_bind, 30 struct xe_exec_queue *q) 31 { 32 return cpu_bind->q == q; 33 } 34 35 static void xe_cpu_bind_fini(void *arg) 36 { 37 struct xe_cpu_bind *cpu_bind = arg; 38 39 mutex_destroy(&cpu_bind->job_mutex); 40 xe_exec_queue_put(cpu_bind->q); 41 } 42 43 /** 44 * xe_cpu_bind_init() - Initialize a cpu_bind context 45 * @xe: &struct xe_device 46 * 47 * Return: 0 if successful, negative error code on failure 48 */ 49 int xe_cpu_bind_init(struct xe_device *xe) 50 { 51 struct xe_cpu_bind *cpu_bind = 52 drmm_kzalloc(&xe->drm, sizeof(*cpu_bind), GFP_KERNEL); 53 struct xe_exec_queue *q; 54 55 if (!cpu_bind) 56 return -ENOMEM; 57 58 q = xe_exec_queue_create_bind(xe, xe_device_get_root_tile(xe), NULL, 59 EXEC_QUEUE_FLAG_KERNEL | 60 EXEC_QUEUE_FLAG_MIGRATE, 0); 61 if (IS_ERR(q)) 62 return PTR_ERR(q); 63 64 cpu_bind->xe = xe; 65 cpu_bind->q = q; 66 xe->cpu_bind = cpu_bind; 67 68 mutex_init(&cpu_bind->job_mutex); 69 70 fs_reclaim_acquire(GFP_KERNEL); 71 might_lock(&cpu_bind->job_mutex); 72 fs_reclaim_release(GFP_KERNEL); 73 74 return devm_add_action_or_reset(cpu_bind->xe->drm.dev, xe_cpu_bind_fini, 75 cpu_bind); 76 } 77 78 /** 79 * xe_cpu_bind_queue() - Get the bind queue from cpu_bind context. 80 * @cpu_bind: The cpu bind context. 81 * 82 * Return: Pointer to bind queue. 83 */ 84 struct xe_exec_queue *xe_cpu_bind_queue(struct xe_cpu_bind *cpu_bind) 85 { 86 return cpu_bind->q; 87 } 88 89 /** 90 * xe_cpu_bind_update_pgtables_execute() - Update a VM's PTEs via the CPU 91 * @vm: The VM being updated 92 * @tile: The tile being updated 93 * @ops: The CPU bind PT update ops 94 * @pt_op: The VM PT update op 95 * @num_ops: The number of The VM PT update ops 96 * @force_clear: Force clear operation 97 * 98 * Execute the VM PT update ops array which results in a VM's PTEs being updated 99 * via the CPU. 100 */ 101 void 102 xe_cpu_bind_update_pgtables_execute(struct xe_vm *vm, struct xe_tile *tile, 103 const struct xe_cpu_bind_pt_update_ops *ops, 104 struct xe_vm_pgtable_update_op *pt_op, 105 u32 num_ops, bool force_clear) 106 { 107 u32 j, i; 108 109 for (j = 0; j < num_ops; ++j, ++pt_op) { 110 for (i = 0; i < pt_op->num_entries; i++) { 111 const struct xe_vm_pgtable_update *update = 112 &pt_op->entries[i]; 113 114 xe_assert(vm->xe, !iosys_map_is_null(&update->pt_bo->vmap)); 115 116 if (pt_op->bind && !force_clear) 117 ops->populate(tile, &update->pt_bo->vmap, 118 update); 119 else 120 ops->clear(vm, tile, &update->pt_bo->vmap, 121 update); 122 } 123 } 124 125 trace_xe_vm_cpu_bind(vm); 126 xe_device_wmb(vm->xe); 127 } 128 129 static struct dma_fence * 130 xe_cpu_bind_update_pgtables_no_job(struct xe_cpu_bind *cpu_bind, 131 struct xe_cpu_bind_pt_update *pt_update) 132 { 133 const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops; 134 struct xe_vm *vm = pt_update->vops->vm; 135 struct xe_tile *tile; 136 int err, id; 137 138 if (ops->pre_commit) { 139 pt_update->job = NULL; 140 err = ops->pre_commit(pt_update); 141 if (err) 142 return ERR_PTR(err); 143 } 144 145 for_each_tile(tile, vm->xe, id) { 146 struct xe_vm_pgtable_update_ops *pt_update_ops = 147 &pt_update->vops->pt_update_ops[tile->id]; 148 149 if (!pt_update_ops->pt_job_ops) 150 continue; 151 152 xe_cpu_bind_update_pgtables_execute(vm, tile, ops, 153 pt_update_ops->pt_job_ops->ops, 154 pt_update_ops->pt_job_ops->current_op, 155 false); 156 } 157 158 return dma_fence_get_stub(); 159 } 160 161 static struct dma_fence * 162 xe_cpu_bind_update_pgtables_job(struct xe_cpu_bind *cpu_bind, 163 struct xe_cpu_bind_pt_update *pt_update) 164 { 165 const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops; 166 struct xe_exec_queue *q = pt_update->vops->q; 167 struct xe_device *xe = cpu_bind->xe; 168 struct xe_sched_job *job; 169 struct dma_fence *fence; 170 struct xe_tile *tile; 171 int err, id; 172 bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q); 173 174 job = xe_sched_job_create(q, NULL); 175 if (IS_ERR(job)) 176 return ERR_CAST(job); 177 178 xe_assert(xe, job->is_pt_job); 179 180 if (ops->pre_commit) { 181 pt_update->job = job; 182 err = ops->pre_commit(pt_update); 183 if (err) 184 goto err_job; 185 } 186 187 if (is_cpu_bind) 188 mutex_lock(&cpu_bind->job_mutex); 189 190 job->pt_update[0].vm = pt_update->vops->vm; 191 job->pt_update[0].ops = ops; 192 for_each_tile(tile, xe, id) { 193 struct xe_vm_pgtable_update_ops *pt_update_ops = 194 &pt_update->vops->pt_update_ops[tile->id]; 195 196 job->pt_update[0].pt_job_ops[tile->id] = 197 xe_pt_job_ops_get(pt_update_ops->pt_job_ops); 198 } 199 200 xe_sched_job_arm(job); 201 fence = dma_fence_get(&job->drm.s_fence->finished); 202 xe_sched_job_push(job); 203 204 if (is_cpu_bind) 205 mutex_unlock(&cpu_bind->job_mutex); 206 207 return fence; 208 209 err_job: 210 xe_sched_job_put(job); 211 return ERR_PTR(err); 212 } 213 214 /** 215 * xe_cpu_bind_update_pgtables() - Pipelined page-table update 216 * @cpu_bind: The cpu bind context. 217 * @pt_update: PT update arguments 218 * 219 * Perform a pipelined page-table update. The update descriptors are typically 220 * built under the same lock critical section as a call to this function. If 221 * using the default engine for the updates, they will be performed in the 222 * order they grab the job_mutex. If different engines are used, external 223 * synchronization is needed for overlapping updates to maintain page-table 224 * consistency. Note that the meaning of "overlapping" is that the updates 225 * touch the same page-table, which might be a higher-level page-directory. 226 * If no pipelining is needed, then updates may be performed by the cpu. 227 * 228 * Return: A dma_fence that, when signaled, indicates the update completion. 229 */ 230 struct dma_fence * 231 xe_cpu_bind_update_pgtables(struct xe_cpu_bind *cpu_bind, 232 struct xe_cpu_bind_pt_update *pt_update) 233 { 234 struct dma_fence *fence; 235 236 fence = xe_cpu_bind_update_pgtables_no_job(cpu_bind, pt_update); 237 238 /* -ETIME indicates a job is needed, anything else is legit error */ 239 if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME) 240 return fence; 241 242 return xe_cpu_bind_update_pgtables_job(cpu_bind, pt_update); 243 } 244 245 /** 246 * xe_cpu_bind_job_lock() - Lock cpu_bind job lock 247 * @cpu_bind: The cpu bind context. 248 * @q: Queue associated with the operation which requires a lock 249 * 250 * Lock the cpu_bind job lock if the queue is a cpu bind queue, otherwise 251 * assert the VM's dma-resv is held (user queue's have own locking). 252 */ 253 void xe_cpu_bind_job_lock(struct xe_cpu_bind *cpu_bind, 254 struct xe_exec_queue *q) 255 { 256 bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q); 257 258 if (is_cpu_bind) 259 mutex_lock(&cpu_bind->job_mutex); 260 else 261 xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ 262 } 263 264 /** 265 * xe_cpu_bind_job_unlock() - Unlock cpu_bind job lock 266 * @cpu_bind: The cpu bind context. 267 * @q: Queue associated with the operation which requires a lock 268 * 269 * Unlock the cpu_bind job lock if the queue is a cpu bind queue, otherwise 270 * assert the VM's dma-resv is held (user queue's have own locking). 271 */ 272 void xe_cpu_bind_job_unlock(struct xe_cpu_bind *cpu_bind, 273 struct xe_exec_queue *q) 274 { 275 bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q); 276 277 if (is_cpu_bind) 278 mutex_unlock(&cpu_bind->job_mutex); 279 else 280 xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ 281 } 282 283 #if IS_ENABLED(CONFIG_PROVE_LOCKING) 284 /** 285 * xe_cpu_bind_job_lock_assert() - Assert cpu_bind job lock held of queue 286 * @q: cpu bind queue 287 */ 288 void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q) 289 { 290 struct xe_device *xe = gt_to_xe(q->gt); 291 struct xe_cpu_bind *cpu_bind = xe->cpu_bind; 292 293 xe_assert(xe, q == cpu_bind->q); 294 lockdep_assert_held(&cpu_bind->job_mutex); 295 } 296 #endif 297