1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_sync.h" 7 8 #include <linux/dma-fence-array.h> 9 #include <linux/kthread.h> 10 #include <linux/sched/mm.h> 11 #include <linux/uaccess.h> 12 13 #include <drm/drm_print.h> 14 #include <drm/drm_syncobj.h> 15 #include <drm/xe_drm.h> 16 17 #include "xe_device_types.h" 18 #include "xe_exec_queue.h" 19 #include "xe_macros.h" 20 #include "xe_sched_job_types.h" 21 22 struct xe_user_fence { 23 struct xe_device *xe; 24 struct kref refcount; 25 struct dma_fence_cb cb; 26 struct work_struct worker; 27 struct mm_struct *mm; 28 u64 __user *addr; 29 u64 value; 30 int signalled; 31 }; 32 33 static void user_fence_destroy(struct kref *kref) 34 { 35 struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence, 36 refcount); 37 38 mmdrop(ufence->mm); 39 kfree(ufence); 40 } 41 42 static void user_fence_get(struct xe_user_fence *ufence) 43 { 44 kref_get(&ufence->refcount); 45 } 46 47 static void user_fence_put(struct xe_user_fence *ufence) 48 { 49 kref_put(&ufence->refcount, user_fence_destroy); 50 } 51 52 static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr, 53 u64 value) 54 { 55 struct xe_user_fence *ufence; 56 u64 __user *ptr = u64_to_user_ptr(addr); 57 58 if (!access_ok(ptr, sizeof(*ptr))) 59 return ERR_PTR(-EFAULT); 60 61 ufence = kmalloc(sizeof(*ufence), GFP_KERNEL); 62 if (!ufence) 63 return ERR_PTR(-ENOMEM); 64 65 ufence->xe = xe; 66 kref_init(&ufence->refcount); 67 ufence->addr = ptr; 68 ufence->value = value; 69 ufence->mm = current->mm; 70 mmgrab(ufence->mm); 71 72 return ufence; 73 } 74 75 static void user_fence_worker(struct work_struct *w) 76 { 77 struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker); 78 79 if (mmget_not_zero(ufence->mm)) { 80 kthread_use_mm(ufence->mm); 81 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) 82 XE_WARN_ON("Copy to user failed"); 83 kthread_unuse_mm(ufence->mm); 84 mmput(ufence->mm); 85 } 86 87 wake_up_all(&ufence->xe->ufence_wq); 88 WRITE_ONCE(ufence->signalled, 1); 89 user_fence_put(ufence); 90 } 91 92 static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence) 93 { 94 INIT_WORK(&ufence->worker, user_fence_worker); 95 queue_work(ufence->xe->ordered_wq, &ufence->worker); 96 dma_fence_put(fence); 97 } 98 99 static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 100 { 101 struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb); 102 103 kick_ufence(ufence, fence); 104 } 105 106 int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, 107 struct xe_sync_entry *sync, 108 struct drm_xe_sync __user *sync_user, 109 unsigned int flags) 110 { 111 struct drm_xe_sync sync_in; 112 int err; 113 bool exec = flags & SYNC_PARSE_FLAG_EXEC; 114 bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE; 115 bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE; 116 bool signal; 117 118 if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) 119 return -EFAULT; 120 121 if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || 122 XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) 123 return -EINVAL; 124 125 signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; 126 switch (sync_in.type) { 127 case DRM_XE_SYNC_TYPE_SYNCOBJ: 128 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 129 return -EOPNOTSUPP; 130 131 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 132 return -EINVAL; 133 134 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 135 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 136 return -ENOENT; 137 138 if (!signal) { 139 sync->fence = drm_syncobj_fence_get(sync->syncobj); 140 if (XE_IOCTL_DBG(xe, !sync->fence)) 141 return -EINVAL; 142 } 143 break; 144 145 case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: 146 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 147 return -EOPNOTSUPP; 148 149 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 150 return -EINVAL; 151 152 if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0)) 153 return -EINVAL; 154 155 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 156 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 157 return -ENOENT; 158 159 if (signal) { 160 sync->chain_fence = dma_fence_chain_alloc(); 161 if (!sync->chain_fence) 162 return -ENOMEM; 163 } else { 164 sync->fence = drm_syncobj_fence_get(sync->syncobj); 165 if (XE_IOCTL_DBG(xe, !sync->fence)) 166 return -EINVAL; 167 168 err = dma_fence_chain_find_seqno(&sync->fence, 169 sync_in.timeline_value); 170 if (err) 171 return err; 172 } 173 break; 174 175 case DRM_XE_SYNC_TYPE_USER_FENCE: 176 if (XE_IOCTL_DBG(xe, disallow_user_fence)) 177 return -EOPNOTSUPP; 178 179 if (XE_IOCTL_DBG(xe, !signal)) 180 return -EOPNOTSUPP; 181 182 if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7)) 183 return -EINVAL; 184 185 if (exec) { 186 sync->addr = sync_in.addr; 187 } else { 188 sync->ufence = user_fence_create(xe, sync_in.addr, 189 sync_in.timeline_value); 190 if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) 191 return PTR_ERR(sync->ufence); 192 } 193 194 break; 195 196 default: 197 return -EINVAL; 198 } 199 200 sync->type = sync_in.type; 201 sync->flags = sync_in.flags; 202 sync->timeline_value = sync_in.timeline_value; 203 204 return 0; 205 } 206 207 int xe_sync_entry_wait(struct xe_sync_entry *sync) 208 { 209 if (sync->fence) 210 dma_fence_wait(sync->fence, true); 211 212 return 0; 213 } 214 215 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) 216 { 217 int err; 218 219 if (sync->fence) { 220 err = drm_sched_job_add_dependency(&job->drm, 221 dma_fence_get(sync->fence)); 222 if (err) { 223 dma_fence_put(sync->fence); 224 return err; 225 } 226 } 227 228 return 0; 229 } 230 231 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) 232 { 233 if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) 234 return; 235 236 if (sync->chain_fence) { 237 drm_syncobj_add_point(sync->syncobj, sync->chain_fence, 238 fence, sync->timeline_value); 239 /* 240 * The chain's ownership is transferred to the 241 * timeline. 242 */ 243 sync->chain_fence = NULL; 244 } else if (sync->syncobj) { 245 drm_syncobj_replace_fence(sync->syncobj, fence); 246 } else if (sync->ufence) { 247 int err; 248 249 dma_fence_get(fence); 250 user_fence_get(sync->ufence); 251 err = dma_fence_add_callback(fence, &sync->ufence->cb, 252 user_fence_cb); 253 if (err == -ENOENT) { 254 kick_ufence(sync->ufence, fence); 255 } else if (err) { 256 XE_WARN_ON("failed to add user fence"); 257 user_fence_put(sync->ufence); 258 dma_fence_put(fence); 259 } 260 } 261 } 262 263 void xe_sync_entry_cleanup(struct xe_sync_entry *sync) 264 { 265 if (sync->syncobj) 266 drm_syncobj_put(sync->syncobj); 267 if (sync->fence) 268 dma_fence_put(sync->fence); 269 if (sync->chain_fence) 270 dma_fence_chain_free(sync->chain_fence); 271 if (sync->ufence) 272 user_fence_put(sync->ufence); 273 } 274 275 /** 276 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM 277 * @sync: input syncs 278 * @num_sync: number of syncs 279 * @q: exec queue 280 * @vm: VM 281 * 282 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create 283 * and return a composite fence of all in-fences + last fence. If no in-fences 284 * return last fence on input exec queue. Caller must drop reference to 285 * returned fence. 286 * 287 * Return: fence on success, ERR_PTR(-ENOMEM) on failure 288 */ 289 struct dma_fence * 290 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, 291 struct xe_exec_queue *q, struct xe_vm *vm) 292 { 293 struct dma_fence **fences = NULL; 294 struct dma_fence_array *cf = NULL; 295 struct dma_fence *fence; 296 int i, num_in_fence = 0, current_fence = 0; 297 298 lockdep_assert_held(&vm->lock); 299 300 /* Count in-fences */ 301 for (i = 0; i < num_sync; ++i) { 302 if (sync[i].fence) { 303 ++num_in_fence; 304 fence = sync[i].fence; 305 } 306 } 307 308 /* Easy case... */ 309 if (!num_in_fence) { 310 fence = xe_exec_queue_last_fence_get(q, vm); 311 return fence; 312 } 313 314 /* Create composite fence */ 315 fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL); 316 if (!fences) 317 return ERR_PTR(-ENOMEM); 318 for (i = 0; i < num_sync; ++i) { 319 if (sync[i].fence) { 320 dma_fence_get(sync[i].fence); 321 fences[current_fence++] = sync[i].fence; 322 } 323 } 324 fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm); 325 cf = dma_fence_array_create(num_in_fence, fences, 326 vm->composite_fence_ctx, 327 vm->composite_fence_seqno++, 328 false); 329 if (!cf) { 330 --vm->composite_fence_seqno; 331 goto err_out; 332 } 333 334 return &cf->base; 335 336 err_out: 337 while (current_fence) 338 dma_fence_put(fences[--current_fence]); 339 kfree(fences); 340 kfree(cf); 341 342 return ERR_PTR(-ENOMEM); 343 } 344 345 /** 346 * __xe_sync_ufence_get() - Get user fence from user fence 347 * @ufence: input user fence 348 * 349 * Get a user fence reference from user fence 350 * 351 * Return: xe_user_fence pointer with reference 352 */ 353 struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence) 354 { 355 user_fence_get(ufence); 356 357 return ufence; 358 } 359 360 /** 361 * xe_sync_ufence_get() - Get user fence from sync 362 * @sync: input sync 363 * 364 * Get a user fence reference from sync. 365 * 366 * Return: xe_user_fence pointer with reference 367 */ 368 struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) 369 { 370 user_fence_get(sync->ufence); 371 372 return sync->ufence; 373 } 374 375 /** 376 * xe_sync_ufence_put() - Put user fence reference 377 * @ufence: user fence reference 378 * 379 */ 380 void xe_sync_ufence_put(struct xe_user_fence *ufence) 381 { 382 user_fence_put(ufence); 383 } 384 385 /** 386 * xe_sync_ufence_get_status() - Get user fence status 387 * @ufence: user fence 388 * 389 * Return: 1 if signalled, 0 not signalled, <0 on error 390 */ 391 int xe_sync_ufence_get_status(struct xe_user_fence *ufence) 392 { 393 return READ_ONCE(ufence->signalled); 394 } 395