1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include "xe_assert.h" 7 #include "xe_dep_job_types.h" 8 #include "xe_dep_scheduler.h" 9 #include "xe_exec_queue.h" 10 #include "xe_gt_types.h" 11 #include "xe_tlb_inval.h" 12 #include "xe_tlb_inval_job.h" 13 #include "xe_migrate.h" 14 #include "xe_pm.h" 15 #include "xe_vm.h" 16 17 /** struct xe_tlb_inval_job - TLB invalidation job */ 18 struct xe_tlb_inval_job { 19 /** @dep: base generic dependency Xe job */ 20 struct xe_dep_job dep; 21 /** @tlb_inval: TLB invalidation client */ 22 struct xe_tlb_inval *tlb_inval; 23 /** @q: exec queue issuing the invalidate */ 24 struct xe_exec_queue *q; 25 /** @vm: VM which TLB invalidation is being issued for */ 26 struct xe_vm *vm; 27 /** @refcount: ref count of this job */ 28 struct kref refcount; 29 /** 30 * @fence: dma fence to indicate completion. 1 way relationship - job 31 * can safely reference fence, fence cannot safely reference job. 32 */ 33 struct dma_fence *fence; 34 /** @start: Start address to invalidate */ 35 u64 start; 36 /** @end: End address to invalidate */ 37 u64 end; 38 /** @type: GT type */ 39 int type; 40 /** @fence_armed: Fence has been armed */ 41 bool fence_armed; 42 }; 43 44 static struct dma_fence *xe_tlb_inval_job_run(struct xe_dep_job *dep_job) 45 { 46 struct xe_tlb_inval_job *job = 47 container_of(dep_job, typeof(*job), dep); 48 struct xe_tlb_inval_fence *ifence = 49 container_of(job->fence, typeof(*ifence), base); 50 51 xe_tlb_inval_range(job->tlb_inval, ifence, job->start, 52 job->end, job->vm->usm.asid); 53 54 return job->fence; 55 } 56 57 static void xe_tlb_inval_job_free(struct xe_dep_job *dep_job) 58 { 59 struct xe_tlb_inval_job *job = 60 container_of(dep_job, typeof(*job), dep); 61 62 /* Pairs with get in xe_tlb_inval_job_push */ 63 xe_tlb_inval_job_put(job); 64 } 65 66 static const struct xe_dep_job_ops dep_job_ops = { 67 .run_job = xe_tlb_inval_job_run, 68 .free_job = xe_tlb_inval_job_free, 69 }; 70 71 /** 72 * xe_tlb_inval_job_create() - TLB invalidation job create 73 * @q: exec queue issuing the invalidate 74 * @tlb_inval: TLB invalidation client 75 * @dep_scheduler: Dependency scheduler for job 76 * @vm: VM which TLB invalidation is being issued for 77 * @start: Start address to invalidate 78 * @end: End address to invalidate 79 * @type: GT type 80 * 81 * Create a TLB invalidation job and initialize internal fields. The caller is 82 * responsible for releasing the creation reference. 83 * 84 * Return: TLB invalidation job object on success, ERR_PTR failure 85 */ 86 struct xe_tlb_inval_job * 87 xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval, 88 struct xe_dep_scheduler *dep_scheduler, 89 struct xe_vm *vm, u64 start, u64 end, int type) 90 { 91 struct xe_tlb_inval_job *job; 92 struct drm_sched_entity *entity = 93 xe_dep_scheduler_entity(dep_scheduler); 94 struct xe_tlb_inval_fence *ifence; 95 int err; 96 97 xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || 98 type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); 99 100 job = kmalloc(sizeof(*job), GFP_KERNEL); 101 if (!job) 102 return ERR_PTR(-ENOMEM); 103 104 job->q = q; 105 job->vm = vm; 106 job->tlb_inval = tlb_inval; 107 job->start = start; 108 job->end = end; 109 job->fence_armed = false; 110 job->dep.ops = &dep_job_ops; 111 job->type = type; 112 kref_init(&job->refcount); 113 xe_exec_queue_get(q); /* Pairs with put in xe_tlb_inval_job_destroy */ 114 xe_vm_get(vm); /* Pairs with put in xe_tlb_inval_job_destroy */ 115 116 ifence = kmalloc(sizeof(*ifence), GFP_KERNEL); 117 if (!ifence) { 118 err = -ENOMEM; 119 goto err_job; 120 } 121 job->fence = &ifence->base; 122 123 err = drm_sched_job_init(&job->dep.drm, entity, 1, NULL, 124 q->xef ? q->xef->drm->client_id : 0); 125 if (err) 126 goto err_fence; 127 128 /* Pairs with put in xe_tlb_inval_job_destroy */ 129 xe_pm_runtime_get_noresume(gt_to_xe(q->gt)); 130 131 return job; 132 133 err_fence: 134 kfree(ifence); 135 err_job: 136 xe_vm_put(vm); 137 xe_exec_queue_put(q); 138 kfree(job); 139 140 return ERR_PTR(err); 141 } 142 143 static void xe_tlb_inval_job_destroy(struct kref *ref) 144 { 145 struct xe_tlb_inval_job *job = container_of(ref, typeof(*job), 146 refcount); 147 struct xe_tlb_inval_fence *ifence = 148 container_of(job->fence, typeof(*ifence), base); 149 struct xe_exec_queue *q = job->q; 150 struct xe_device *xe = gt_to_xe(q->gt); 151 struct xe_vm *vm = job->vm; 152 153 if (!job->fence_armed) 154 kfree(ifence); 155 else 156 /* Ref from xe_tlb_inval_fence_init */ 157 dma_fence_put(job->fence); 158 159 drm_sched_job_cleanup(&job->dep.drm); 160 kfree(job); 161 xe_vm_put(vm); /* Pairs with get from xe_tlb_inval_job_create */ 162 xe_exec_queue_put(q); /* Pairs with get from xe_tlb_inval_job_create */ 163 xe_pm_runtime_put(xe); /* Pairs with get from xe_tlb_inval_job_create */ 164 } 165 166 /** 167 * xe_tlb_inval_job_alloc_dep() - TLB invalidation job alloc dependency 168 * @job: TLB invalidation job to alloc dependency for 169 * 170 * Allocate storage for a dependency in the TLB invalidation fence. This 171 * function should be called at most once per job and must be paired with 172 * xe_tlb_inval_job_push being called with a real fence. 173 * 174 * Return: 0 on success, -errno on failure 175 */ 176 int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job) 177 { 178 xe_assert(gt_to_xe(job->q->gt), !xa_load(&job->dep.drm.dependencies, 0)); 179 might_alloc(GFP_KERNEL); 180 181 return drm_sched_job_add_dependency(&job->dep.drm, 182 dma_fence_get_stub()); 183 } 184 185 /** 186 * xe_tlb_inval_job_push() - TLB invalidation job push 187 * @job: TLB invalidation job to push 188 * @m: The migration object being used 189 * @fence: Dependency for TLB invalidation job 190 * 191 * Pushes a TLB invalidation job for execution, using @fence as a dependency. 192 * Storage for @fence must be preallocated with xe_tlb_inval_job_alloc_dep 193 * prior to this call if @fence is not signaled. Takes a reference to the job’s 194 * finished fence, which the caller is responsible for releasing, and return it 195 * to the caller. This function is safe to be called in the path of reclaim. 196 * 197 * Return: Job's finished fence on success, cannot fail 198 */ 199 struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job, 200 struct xe_migrate *m, 201 struct dma_fence *fence) 202 { 203 struct xe_tlb_inval_fence *ifence = 204 container_of(job->fence, typeof(*ifence), base); 205 206 if (!dma_fence_is_signaled(fence)) { 207 void *ptr; 208 209 /* 210 * Can be in path of reclaim, hence the preallocation of fence 211 * storage in xe_tlb_inval_job_alloc_dep. Verify caller did 212 * this correctly. 213 */ 214 xe_assert(gt_to_xe(job->q->gt), 215 xa_load(&job->dep.drm.dependencies, 0) == 216 dma_fence_get_stub()); 217 218 dma_fence_get(fence); /* ref released once dependency processed by scheduler */ 219 ptr = xa_store(&job->dep.drm.dependencies, 0, fence, 220 GFP_ATOMIC); 221 xe_assert(gt_to_xe(job->q->gt), !xa_is_err(ptr)); 222 } 223 224 xe_tlb_inval_job_get(job); /* Pairs with put in free_job */ 225 job->fence_armed = true; 226 227 /* 228 * We need the migration lock to protect the job's seqno and the spsc 229 * queue, only taken on migration queue, user queues protected dma-resv 230 * VM lock. 231 */ 232 xe_migrate_job_lock(m, job->q); 233 234 /* Creation ref pairs with put in xe_tlb_inval_job_destroy */ 235 xe_tlb_inval_fence_init(job->tlb_inval, ifence, false); 236 dma_fence_get(job->fence); /* Pairs with put in DRM scheduler */ 237 238 drm_sched_job_arm(&job->dep.drm); 239 /* 240 * caller ref, get must be done before job push as it could immediately 241 * signal and free. 242 */ 243 dma_fence_get(&job->dep.drm.s_fence->finished); 244 drm_sched_entity_push_job(&job->dep.drm); 245 246 /* Let the upper layers fish this out */ 247 xe_exec_queue_tlb_inval_last_fence_set(job->q, job->vm, 248 &job->dep.drm.s_fence->finished, 249 job->type); 250 251 xe_migrate_job_unlock(m, job->q); 252 253 /* 254 * Not using job->fence, as it has its own dma-fence context, which does 255 * not allow TLB invalidation fences on the same queue, GT tuple to 256 * be squashed in dma-resv/DRM scheduler. Instead, we use the DRM scheduler 257 * context and job's finished fence, which enables squashing. 258 */ 259 return &job->dep.drm.s_fence->finished; 260 } 261 262 /** 263 * xe_tlb_inval_job_get() - Get a reference to TLB invalidation job 264 * @job: TLB invalidation job object 265 * 266 * Increment the TLB invalidation job's reference count 267 */ 268 void xe_tlb_inval_job_get(struct xe_tlb_inval_job *job) 269 { 270 kref_get(&job->refcount); 271 } 272 273 /** 274 * xe_tlb_inval_job_put() - Put a reference to TLB invalidation job 275 * @job: TLB invalidation job object 276 * 277 * Decrement the TLB invalidation job's reference count, call 278 * xe_tlb_inval_job_destroy when reference count == 0. Skips decrement if 279 * input @job is NULL or IS_ERR. 280 */ 281 void xe_tlb_inval_job_put(struct xe_tlb_inval_job *job) 282 { 283 if (!IS_ERR_OR_NULL(job)) 284 kref_put(&job->refcount, xe_tlb_inval_job_destroy); 285 } 286