1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_sync.h" 7 8 #include <linux/dma-fence-array.h> 9 #include <linux/kthread.h> 10 #include <linux/sched/mm.h> 11 #include <linux/uaccess.h> 12 13 #include <drm/drm_print.h> 14 #include <drm/drm_syncobj.h> 15 #include <uapi/drm/xe_drm.h> 16 17 #include "xe_device.h" 18 #include "xe_exec_queue.h" 19 #include "xe_macros.h" 20 #include "xe_sched_job_types.h" 21 22 struct xe_user_fence { 23 struct xe_device *xe; 24 struct kref refcount; 25 struct dma_fence_cb cb; 26 struct work_struct worker; 27 struct mm_struct *mm; 28 u64 __user *addr; 29 u64 value; 30 int signalled; 31 }; 32 33 static void user_fence_destroy(struct kref *kref) 34 { 35 struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence, 36 refcount); 37 38 mmdrop(ufence->mm); 39 kfree(ufence); 40 } 41 42 static void user_fence_get(struct xe_user_fence *ufence) 43 { 44 kref_get(&ufence->refcount); 45 } 46 47 static void user_fence_put(struct xe_user_fence *ufence) 48 { 49 kref_put(&ufence->refcount, user_fence_destroy); 50 } 51 52 static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr, 53 u64 value) 54 { 55 struct xe_user_fence *ufence; 56 u64 __user *ptr = u64_to_user_ptr(addr); 57 u64 __maybe_unused prefetch_val; 58 59 if (get_user(prefetch_val, ptr)) 60 return ERR_PTR(-EFAULT); 61 62 ufence = kzalloc_obj(*ufence); 63 if (!ufence) 64 return ERR_PTR(-ENOMEM); 65 66 ufence->xe = xe; 67 kref_init(&ufence->refcount); 68 ufence->addr = ptr; 69 ufence->value = value; 70 ufence->mm = current->mm; 71 mmgrab(ufence->mm); 72 73 return ufence; 74 } 75 76 static void user_fence_worker(struct work_struct *w) 77 { 78 struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker); 79 80 WRITE_ONCE(ufence->signalled, 1); 81 if (mmget_not_zero(ufence->mm)) { 82 kthread_use_mm(ufence->mm); 83 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) 84 XE_WARN_ON("Copy to user failed"); 85 kthread_unuse_mm(ufence->mm); 86 mmput(ufence->mm); 87 } else { 88 drm_dbg(&ufence->xe->drm, "mmget_not_zero() failed, ufence wasn't signaled\n"); 89 } 90 91 /* 92 * Wake up waiters only after updating the ufence state, allowing the UMD 93 * to safely reuse the same ufence without encountering -EBUSY errors. 94 */ 95 wake_up_all(&ufence->xe->ufence_wq); 96 user_fence_put(ufence); 97 } 98 99 static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence) 100 { 101 INIT_WORK(&ufence->worker, user_fence_worker); 102 queue_work(ufence->xe->ordered_wq, &ufence->worker); 103 dma_fence_put(fence); 104 } 105 106 static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 107 { 108 struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb); 109 110 kick_ufence(ufence, fence); 111 } 112 113 int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, 114 struct xe_sync_entry *sync, 115 struct drm_xe_sync __user *sync_user, 116 struct drm_syncobj *ufence_syncobj, 117 u64 ufence_timeline_value, 118 unsigned int flags) 119 { 120 struct drm_xe_sync sync_in; 121 int err; 122 bool exec = flags & SYNC_PARSE_FLAG_EXEC; 123 bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE; 124 bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE; 125 bool signal; 126 127 if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) 128 return -EFAULT; 129 130 if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || 131 XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) 132 return -EINVAL; 133 134 signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; 135 switch (sync_in.type) { 136 case DRM_XE_SYNC_TYPE_SYNCOBJ: 137 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 138 return -EOPNOTSUPP; 139 140 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 141 return -EINVAL; 142 143 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 144 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 145 return -ENOENT; 146 147 if (!signal) { 148 sync->fence = drm_syncobj_fence_get(sync->syncobj); 149 if (XE_IOCTL_DBG(xe, !sync->fence)) { 150 err = -EINVAL; 151 goto free_sync; 152 } 153 } 154 break; 155 156 case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: 157 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 158 return -EOPNOTSUPP; 159 160 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 161 return -EINVAL; 162 163 if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0)) 164 return -EINVAL; 165 166 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 167 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 168 return -ENOENT; 169 170 if (signal) { 171 sync->chain_fence = dma_fence_chain_alloc(); 172 if (!sync->chain_fence) { 173 err = -ENOMEM; 174 goto free_sync; 175 } 176 } else { 177 sync->fence = drm_syncobj_fence_get(sync->syncobj); 178 if (XE_IOCTL_DBG(xe, !sync->fence)) { 179 err = -EINVAL; 180 goto free_sync; 181 } 182 183 err = dma_fence_chain_find_seqno(&sync->fence, 184 sync_in.timeline_value); 185 if (err) 186 goto free_sync; 187 } 188 break; 189 190 case DRM_XE_SYNC_TYPE_USER_FENCE: 191 if (XE_IOCTL_DBG(xe, disallow_user_fence)) 192 return -EOPNOTSUPP; 193 194 if (XE_IOCTL_DBG(xe, !signal)) 195 return -EOPNOTSUPP; 196 197 if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7)) 198 return -EINVAL; 199 200 if (exec) { 201 sync->addr = sync_in.addr; 202 } else { 203 sync->ufence_timeline_value = ufence_timeline_value; 204 sync->ufence = user_fence_create(xe, sync_in.addr, 205 sync_in.timeline_value); 206 if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) 207 return PTR_ERR(sync->ufence); 208 sync->ufence_chain_fence = dma_fence_chain_alloc(); 209 if (!sync->ufence_chain_fence) { 210 err = -ENOMEM; 211 goto free_sync; 212 } 213 sync->ufence_syncobj = ufence_syncobj; 214 } 215 216 break; 217 218 default: 219 return -EINVAL; 220 } 221 222 sync->type = sync_in.type; 223 sync->flags = sync_in.flags; 224 sync->timeline_value = sync_in.timeline_value; 225 226 return 0; 227 228 free_sync: 229 xe_sync_entry_cleanup(sync); 230 return err; 231 } 232 ALLOW_ERROR_INJECTION(xe_sync_entry_parse, ERRNO); 233 234 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) 235 { 236 if (sync->fence) 237 return drm_sched_job_add_dependency(&job->drm, 238 dma_fence_get(sync->fence)); 239 240 return 0; 241 } 242 243 /** 244 * xe_sync_entry_wait() - Wait on in-sync 245 * @sync: Sync object 246 * 247 * If the sync is in an in-sync, wait on the sync to signal. 248 * 249 * Return: 0 on success, -ERESTARTSYS on failure (interruption) 250 */ 251 int xe_sync_entry_wait(struct xe_sync_entry *sync) 252 { 253 return xe_sync_needs_wait(sync) ? 254 dma_fence_wait(sync->fence, true) : 0; 255 } 256 257 /** 258 * xe_sync_needs_wait() - Sync needs a wait (input dma-fence not signaled) 259 * @sync: Sync object 260 * 261 * Return: True if sync needs a wait, False otherwise 262 */ 263 bool xe_sync_needs_wait(struct xe_sync_entry *sync) 264 { 265 return sync->fence && 266 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags); 267 } 268 269 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) 270 { 271 if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) 272 return; 273 274 if (sync->chain_fence) { 275 drm_syncobj_add_point(sync->syncobj, sync->chain_fence, 276 fence, sync->timeline_value); 277 /* 278 * The chain's ownership is transferred to the 279 * timeline. 280 */ 281 sync->chain_fence = NULL; 282 } else if (sync->syncobj) { 283 drm_syncobj_replace_fence(sync->syncobj, fence); 284 } else if (sync->ufence) { 285 int err; 286 287 drm_syncobj_add_point(sync->ufence_syncobj, 288 sync->ufence_chain_fence, 289 fence, sync->ufence_timeline_value); 290 sync->ufence_chain_fence = NULL; 291 292 fence = drm_syncobj_fence_get(sync->ufence_syncobj); 293 user_fence_get(sync->ufence); 294 err = dma_fence_add_callback(fence, &sync->ufence->cb, 295 user_fence_cb); 296 if (err == -ENOENT) { 297 kick_ufence(sync->ufence, fence); 298 } else if (err) { 299 XE_WARN_ON("failed to add user fence"); 300 user_fence_put(sync->ufence); 301 dma_fence_put(fence); 302 } 303 } 304 } 305 306 void xe_sync_entry_cleanup(struct xe_sync_entry *sync) 307 { 308 if (sync->syncobj) 309 drm_syncobj_put(sync->syncobj); 310 dma_fence_put(sync->fence); 311 dma_fence_chain_free(sync->chain_fence); 312 dma_fence_chain_free(sync->ufence_chain_fence); 313 if (!IS_ERR_OR_NULL(sync->ufence)) 314 user_fence_put(sync->ufence); 315 } 316 317 /** 318 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM 319 * @sync: input syncs 320 * @num_sync: number of syncs 321 * @q: exec queue 322 * @vm: VM 323 * 324 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create 325 * and return a composite fence of all in-fences + last fence. If no in-fences 326 * return last fence on input exec queue. Caller must drop reference to 327 * returned fence. 328 * 329 * Return: fence on success, ERR_PTR(-ENOMEM) on failure 330 */ 331 struct dma_fence * 332 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, 333 struct xe_exec_queue *q, struct xe_vm *vm) 334 { 335 struct dma_fence **fences = NULL; 336 struct dma_fence_array *cf = NULL; 337 struct dma_fence *fence; 338 int i, num_fence = 0, current_fence = 0; 339 340 lockdep_assert_held(&vm->lock); 341 342 /* Reject in fences */ 343 for (i = 0; i < num_sync; ++i) 344 if (sync[i].fence) 345 return ERR_PTR(-EOPNOTSUPP); 346 347 if (q->flags & EXEC_QUEUE_FLAG_VM) { 348 struct xe_exec_queue *__q; 349 struct xe_tile *tile; 350 u8 id; 351 352 for_each_tile(tile, vm->xe, id) { 353 num_fence++; 354 for_each_tlb_inval(i) 355 num_fence++; 356 } 357 358 fences = kmalloc_objs(*fences, num_fence); 359 if (!fences) 360 return ERR_PTR(-ENOMEM); 361 362 fences[current_fence++] = 363 xe_exec_queue_last_fence_get(q, vm); 364 for_each_tlb_inval(i) 365 fences[current_fence++] = 366 xe_exec_queue_tlb_inval_last_fence_get(q, vm, i); 367 list_for_each_entry(__q, &q->multi_gt_list, 368 multi_gt_link) { 369 fences[current_fence++] = 370 xe_exec_queue_last_fence_get(__q, vm); 371 for_each_tlb_inval(i) 372 fences[current_fence++] = 373 xe_exec_queue_tlb_inval_last_fence_get(__q, vm, i); 374 } 375 376 xe_assert(vm->xe, current_fence == num_fence); 377 cf = dma_fence_array_create(num_fence, fences, 378 dma_fence_context_alloc(1), 379 1, false); 380 if (!cf) 381 goto err_out; 382 383 return &cf->base; 384 } 385 386 fence = xe_exec_queue_last_fence_get(q, vm); 387 return fence; 388 389 err_out: 390 while (current_fence) 391 dma_fence_put(fences[--current_fence]); 392 kfree(fences); 393 394 return ERR_PTR(-ENOMEM); 395 } 396 397 /** 398 * __xe_sync_ufence_get() - Get user fence from user fence 399 * @ufence: input user fence 400 * 401 * Get a user fence reference from user fence 402 * 403 * Return: xe_user_fence pointer with reference 404 */ 405 struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence) 406 { 407 user_fence_get(ufence); 408 409 return ufence; 410 } 411 412 /** 413 * xe_sync_ufence_get() - Get user fence from sync 414 * @sync: input sync 415 * 416 * Get a user fence reference from sync. 417 * 418 * Return: xe_user_fence pointer with reference 419 */ 420 struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) 421 { 422 user_fence_get(sync->ufence); 423 424 return sync->ufence; 425 } 426 427 /** 428 * xe_sync_ufence_put() - Put user fence reference 429 * @ufence: user fence reference 430 * 431 */ 432 void xe_sync_ufence_put(struct xe_user_fence *ufence) 433 { 434 user_fence_put(ufence); 435 } 436 437 /** 438 * xe_sync_ufence_get_status() - Get user fence status 439 * @ufence: user fence 440 * 441 * Return: 1 if signalled, 0 not signalled, <0 on error 442 */ 443 int xe_sync_ufence_get_status(struct xe_user_fence *ufence) 444 { 445 return READ_ONCE(ufence->signalled); 446 } 447