1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_sync.h" 7 8 #include <linux/dma-fence-array.h> 9 #include <linux/error-injection.h> 10 #include <linux/kthread.h> 11 #include <linux/sched/mm.h> 12 #include <linux/uaccess.h> 13 14 #include <drm/drm_print.h> 15 #include <drm/drm_syncobj.h> 16 #include <uapi/drm/xe_drm.h> 17 18 #include "xe_device.h" 19 #include "xe_exec_queue.h" 20 #include "xe_macros.h" 21 #include "xe_sched_job_types.h" 22 23 struct xe_user_fence { 24 struct xe_device *xe; 25 struct kref refcount; 26 struct dma_fence_cb cb; 27 struct work_struct worker; 28 struct mm_struct *mm; 29 u64 __user *addr; 30 u64 value; 31 int signalled; 32 }; 33 34 static void user_fence_destroy(struct kref *kref) 35 { 36 struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence, 37 refcount); 38 39 mmdrop(ufence->mm); 40 kfree(ufence); 41 } 42 43 static void user_fence_get(struct xe_user_fence *ufence) 44 { 45 kref_get(&ufence->refcount); 46 } 47 48 static void user_fence_put(struct xe_user_fence *ufence) 49 { 50 kref_put(&ufence->refcount, user_fence_destroy); 51 } 52 53 static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr, 54 u64 value) 55 { 56 struct xe_user_fence *ufence; 57 u64 __user *ptr = u64_to_user_ptr(addr); 58 u64 __maybe_unused prefetch_val; 59 60 if (get_user(prefetch_val, ptr)) 61 return ERR_PTR(-EFAULT); 62 63 ufence = kzalloc_obj(*ufence); 64 if (!ufence) 65 return ERR_PTR(-ENOMEM); 66 67 ufence->xe = xe; 68 kref_init(&ufence->refcount); 69 ufence->addr = ptr; 70 ufence->value = value; 71 ufence->mm = current->mm; 72 mmgrab(ufence->mm); 73 74 return ufence; 75 } 76 77 static void user_fence_worker(struct work_struct *w) 78 { 79 struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker); 80 81 WRITE_ONCE(ufence->signalled, 1); 82 if (mmget_not_zero(ufence->mm)) { 83 kthread_use_mm(ufence->mm); 84 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) 85 XE_WARN_ON("Copy to user failed"); 86 kthread_unuse_mm(ufence->mm); 87 mmput(ufence->mm); 88 } else { 89 drm_dbg(&ufence->xe->drm, "mmget_not_zero() failed, ufence wasn't signaled\n"); 90 } 91 92 /* 93 * Wake up waiters only after updating the ufence state, allowing the UMD 94 * to safely reuse the same ufence without encountering -EBUSY errors. 95 */ 96 wake_up_all(&ufence->xe->ufence_wq); 97 user_fence_put(ufence); 98 } 99 100 static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence) 101 { 102 INIT_WORK(&ufence->worker, user_fence_worker); 103 queue_work(ufence->xe->ordered_wq, &ufence->worker); 104 dma_fence_put(fence); 105 } 106 107 static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 108 { 109 struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb); 110 111 kick_ufence(ufence, fence); 112 } 113 114 int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, 115 struct xe_sync_entry *sync, 116 struct drm_xe_sync __user *sync_user, 117 struct drm_syncobj *ufence_syncobj, 118 u64 ufence_timeline_value, 119 unsigned int flags) 120 { 121 struct drm_xe_sync sync_in; 122 int err; 123 bool exec = flags & SYNC_PARSE_FLAG_EXEC; 124 bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE; 125 bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE; 126 bool signal; 127 128 if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) 129 return -EFAULT; 130 131 if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || 132 XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) 133 return -EINVAL; 134 135 signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; 136 switch (sync_in.type) { 137 case DRM_XE_SYNC_TYPE_SYNCOBJ: 138 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 139 return -EOPNOTSUPP; 140 141 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 142 return -EINVAL; 143 144 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 145 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 146 return -ENOENT; 147 148 if (!signal) { 149 sync->fence = drm_syncobj_fence_get(sync->syncobj); 150 if (XE_IOCTL_DBG(xe, !sync->fence)) { 151 err = -EINVAL; 152 goto free_sync; 153 } 154 } 155 break; 156 157 case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: 158 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 159 return -EOPNOTSUPP; 160 161 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 162 return -EINVAL; 163 164 if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0)) 165 return -EINVAL; 166 167 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 168 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 169 return -ENOENT; 170 171 if (signal) { 172 sync->chain_fence = dma_fence_chain_alloc(); 173 if (!sync->chain_fence) { 174 err = -ENOMEM; 175 goto free_sync; 176 } 177 } else { 178 sync->fence = drm_syncobj_fence_get(sync->syncobj); 179 if (XE_IOCTL_DBG(xe, !sync->fence)) { 180 err = -EINVAL; 181 goto free_sync; 182 } 183 184 err = dma_fence_chain_find_seqno(&sync->fence, 185 sync_in.timeline_value); 186 if (err) 187 goto free_sync; 188 } 189 break; 190 191 case DRM_XE_SYNC_TYPE_USER_FENCE: 192 if (XE_IOCTL_DBG(xe, disallow_user_fence)) 193 return -EOPNOTSUPP; 194 195 if (XE_IOCTL_DBG(xe, !signal)) 196 return -EOPNOTSUPP; 197 198 if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7)) 199 return -EINVAL; 200 201 if (exec) { 202 sync->addr = sync_in.addr; 203 } else { 204 sync->ufence_timeline_value = ufence_timeline_value; 205 sync->ufence = user_fence_create(xe, sync_in.addr, 206 sync_in.timeline_value); 207 if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) 208 return PTR_ERR(sync->ufence); 209 sync->ufence_chain_fence = dma_fence_chain_alloc(); 210 if (!sync->ufence_chain_fence) { 211 err = -ENOMEM; 212 goto free_sync; 213 } 214 sync->ufence_syncobj = ufence_syncobj; 215 } 216 217 break; 218 219 default: 220 return -EINVAL; 221 } 222 223 sync->type = sync_in.type; 224 sync->flags = sync_in.flags; 225 sync->timeline_value = sync_in.timeline_value; 226 227 return 0; 228 229 free_sync: 230 xe_sync_entry_cleanup(sync); 231 return err; 232 } 233 ALLOW_ERROR_INJECTION(xe_sync_entry_parse, ERRNO); 234 235 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) 236 { 237 if (sync->fence) 238 return drm_sched_job_add_dependency(&job->drm, 239 dma_fence_get(sync->fence)); 240 241 return 0; 242 } 243 244 /** 245 * xe_sync_entry_wait() - Wait on in-sync 246 * @sync: Sync object 247 * 248 * If the sync is in an in-sync, wait on the sync to signal. 249 * 250 * Return: 0 on success, -ERESTARTSYS on failure (interruption) 251 */ 252 int xe_sync_entry_wait(struct xe_sync_entry *sync) 253 { 254 return xe_sync_needs_wait(sync) ? 255 dma_fence_wait(sync->fence, true) : 0; 256 } 257 258 /** 259 * xe_sync_needs_wait() - Sync needs a wait (input dma-fence not signaled) 260 * @sync: Sync object 261 * 262 * Return: True if sync needs a wait, False otherwise 263 */ 264 bool xe_sync_needs_wait(struct xe_sync_entry *sync) 265 { 266 return sync->fence && 267 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags); 268 } 269 270 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) 271 { 272 if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) 273 return; 274 275 if (sync->chain_fence) { 276 drm_syncobj_add_point(sync->syncobj, sync->chain_fence, 277 fence, sync->timeline_value); 278 /* 279 * The chain's ownership is transferred to the 280 * timeline. 281 */ 282 sync->chain_fence = NULL; 283 } else if (sync->syncobj) { 284 drm_syncobj_replace_fence(sync->syncobj, fence); 285 } else if (sync->ufence) { 286 int err; 287 288 drm_syncobj_add_point(sync->ufence_syncobj, 289 sync->ufence_chain_fence, 290 fence, sync->ufence_timeline_value); 291 sync->ufence_chain_fence = NULL; 292 293 fence = drm_syncobj_fence_get(sync->ufence_syncobj); 294 user_fence_get(sync->ufence); 295 err = dma_fence_add_callback(fence, &sync->ufence->cb, 296 user_fence_cb); 297 if (err == -ENOENT) { 298 kick_ufence(sync->ufence, fence); 299 } else if (err) { 300 XE_WARN_ON("failed to add user fence"); 301 user_fence_put(sync->ufence); 302 dma_fence_put(fence); 303 } 304 } 305 } 306 307 void xe_sync_entry_cleanup(struct xe_sync_entry *sync) 308 { 309 if (sync->syncobj) 310 drm_syncobj_put(sync->syncobj); 311 dma_fence_put(sync->fence); 312 dma_fence_chain_free(sync->chain_fence); 313 dma_fence_chain_free(sync->ufence_chain_fence); 314 if (!IS_ERR_OR_NULL(sync->ufence)) 315 user_fence_put(sync->ufence); 316 } 317 318 /** 319 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM 320 * @sync: input syncs 321 * @num_sync: number of syncs 322 * @q: exec queue 323 * @vm: VM 324 * 325 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create 326 * and return a composite fence of all in-fences + last fence. If no in-fences 327 * return last fence on input exec queue. Caller must drop reference to 328 * returned fence. 329 * 330 * Return: fence on success, ERR_PTR(-ENOMEM) on failure 331 */ 332 struct dma_fence * 333 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, 334 struct xe_exec_queue *q, struct xe_vm *vm) 335 { 336 struct dma_fence **fences = NULL; 337 struct dma_fence_array *cf = NULL; 338 struct dma_fence *fence; 339 int i, num_fence = 0, current_fence = 0; 340 341 lockdep_assert_held(&vm->lock); 342 343 /* Reject in fences */ 344 for (i = 0; i < num_sync; ++i) 345 if (sync[i].fence) 346 return ERR_PTR(-EOPNOTSUPP); 347 348 if (q->flags & EXEC_QUEUE_FLAG_VM) { 349 num_fence++; 350 for_each_tlb_inval(q, i) 351 num_fence++; 352 353 fences = kmalloc_objs(*fences, num_fence); 354 if (!fences) 355 return ERR_PTR(-ENOMEM); 356 357 fences[current_fence++] = 358 xe_exec_queue_last_fence_get(q, vm); 359 for_each_tlb_inval(q, i) 360 fences[current_fence++] = 361 xe_exec_queue_tlb_inval_last_fence_get(q, vm, i); 362 363 xe_assert(vm->xe, current_fence == num_fence); 364 cf = dma_fence_array_create(num_fence, fences, 365 dma_fence_context_alloc(1), 366 1); 367 if (!cf) 368 goto err_out; 369 370 return &cf->base; 371 } 372 373 fence = xe_exec_queue_last_fence_get(q, vm); 374 return fence; 375 376 err_out: 377 while (current_fence) 378 dma_fence_put(fences[--current_fence]); 379 kfree(fences); 380 381 return ERR_PTR(-ENOMEM); 382 } 383 384 /** 385 * __xe_sync_ufence_get() - Get user fence from user fence 386 * @ufence: input user fence 387 * 388 * Get a user fence reference from user fence 389 * 390 * Return: xe_user_fence pointer with reference 391 */ 392 struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence) 393 { 394 user_fence_get(ufence); 395 396 return ufence; 397 } 398 399 /** 400 * xe_sync_ufence_get() - Get user fence from sync 401 * @sync: input sync 402 * 403 * Get a user fence reference from sync. 404 * 405 * Return: xe_user_fence pointer with reference 406 */ 407 struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) 408 { 409 user_fence_get(sync->ufence); 410 411 return sync->ufence; 412 } 413 414 /** 415 * xe_sync_ufence_put() - Put user fence reference 416 * @ufence: user fence reference 417 * 418 */ 419 void xe_sync_ufence_put(struct xe_user_fence *ufence) 420 { 421 user_fence_put(ufence); 422 } 423 424 /** 425 * xe_sync_ufence_get_status() - Get user fence status 426 * @ufence: user fence 427 * 428 * Return: 1 if signalled, 0 not signalled, <0 on error 429 */ 430 int xe_sync_ufence_get_status(struct xe_user_fence *ufence) 431 { 432 return READ_ONCE(ufence->signalled); 433 } 434