1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_sync.h" 7 8 #include <linux/dma-fence-array.h> 9 #include <linux/kthread.h> 10 #include <linux/sched/mm.h> 11 #include <linux/uaccess.h> 12 13 #include <drm/drm_print.h> 14 #include <drm/drm_syncobj.h> 15 #include <uapi/drm/xe_drm.h> 16 17 #include "xe_device_types.h" 18 #include "xe_exec_queue.h" 19 #include "xe_macros.h" 20 #include "xe_sched_job_types.h" 21 22 struct xe_user_fence { 23 struct xe_device *xe; 24 struct kref refcount; 25 struct dma_fence_cb cb; 26 struct work_struct worker; 27 struct mm_struct *mm; 28 u64 __user *addr; 29 u64 value; 30 int signalled; 31 }; 32 33 static void user_fence_destroy(struct kref *kref) 34 { 35 struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence, 36 refcount); 37 38 mmdrop(ufence->mm); 39 kfree(ufence); 40 } 41 42 static void user_fence_get(struct xe_user_fence *ufence) 43 { 44 kref_get(&ufence->refcount); 45 } 46 47 static void user_fence_put(struct xe_user_fence *ufence) 48 { 49 kref_put(&ufence->refcount, user_fence_destroy); 50 } 51 52 static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr, 53 u64 value) 54 { 55 struct xe_user_fence *ufence; 56 u64 __user *ptr = u64_to_user_ptr(addr); 57 u64 __maybe_unused prefetch_val; 58 59 if (get_user(prefetch_val, ptr)) 60 return ERR_PTR(-EFAULT); 61 62 ufence = kzalloc(sizeof(*ufence), GFP_KERNEL); 63 if (!ufence) 64 return ERR_PTR(-ENOMEM); 65 66 ufence->xe = xe; 67 kref_init(&ufence->refcount); 68 ufence->addr = ptr; 69 ufence->value = value; 70 ufence->mm = current->mm; 71 mmgrab(ufence->mm); 72 73 return ufence; 74 } 75 76 static void user_fence_worker(struct work_struct *w) 77 { 78 struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker); 79 80 if (mmget_not_zero(ufence->mm)) { 81 kthread_use_mm(ufence->mm); 82 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) 83 XE_WARN_ON("Copy to user failed"); 84 kthread_unuse_mm(ufence->mm); 85 mmput(ufence->mm); 86 } 87 88 wake_up_all(&ufence->xe->ufence_wq); 89 WRITE_ONCE(ufence->signalled, 1); 90 user_fence_put(ufence); 91 } 92 93 static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence) 94 { 95 INIT_WORK(&ufence->worker, user_fence_worker); 96 queue_work(ufence->xe->ordered_wq, &ufence->worker); 97 dma_fence_put(fence); 98 } 99 100 static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 101 { 102 struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb); 103 104 kick_ufence(ufence, fence); 105 } 106 107 int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, 108 struct xe_sync_entry *sync, 109 struct drm_xe_sync __user *sync_user, 110 unsigned int flags) 111 { 112 struct drm_xe_sync sync_in; 113 int err; 114 bool exec = flags & SYNC_PARSE_FLAG_EXEC; 115 bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE; 116 bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE; 117 bool signal; 118 119 if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) 120 return -EFAULT; 121 122 if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || 123 XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) 124 return -EINVAL; 125 126 signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; 127 switch (sync_in.type) { 128 case DRM_XE_SYNC_TYPE_SYNCOBJ: 129 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 130 return -EOPNOTSUPP; 131 132 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 133 return -EINVAL; 134 135 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 136 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 137 return -ENOENT; 138 139 if (!signal) { 140 sync->fence = drm_syncobj_fence_get(sync->syncobj); 141 if (XE_IOCTL_DBG(xe, !sync->fence)) 142 return -EINVAL; 143 } 144 break; 145 146 case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: 147 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 148 return -EOPNOTSUPP; 149 150 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 151 return -EINVAL; 152 153 if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0)) 154 return -EINVAL; 155 156 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 157 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 158 return -ENOENT; 159 160 if (signal) { 161 sync->chain_fence = dma_fence_chain_alloc(); 162 if (!sync->chain_fence) 163 return -ENOMEM; 164 } else { 165 sync->fence = drm_syncobj_fence_get(sync->syncobj); 166 if (XE_IOCTL_DBG(xe, !sync->fence)) 167 return -EINVAL; 168 169 err = dma_fence_chain_find_seqno(&sync->fence, 170 sync_in.timeline_value); 171 if (err) 172 return err; 173 } 174 break; 175 176 case DRM_XE_SYNC_TYPE_USER_FENCE: 177 if (XE_IOCTL_DBG(xe, disallow_user_fence)) 178 return -EOPNOTSUPP; 179 180 if (XE_IOCTL_DBG(xe, !signal)) 181 return -EOPNOTSUPP; 182 183 if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7)) 184 return -EINVAL; 185 186 if (exec) { 187 sync->addr = sync_in.addr; 188 } else { 189 sync->ufence = user_fence_create(xe, sync_in.addr, 190 sync_in.timeline_value); 191 if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence))) 192 return PTR_ERR(sync->ufence); 193 } 194 195 break; 196 197 default: 198 return -EINVAL; 199 } 200 201 sync->type = sync_in.type; 202 sync->flags = sync_in.flags; 203 sync->timeline_value = sync_in.timeline_value; 204 205 return 0; 206 } 207 208 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) 209 { 210 if (sync->fence) 211 return drm_sched_job_add_dependency(&job->drm, 212 dma_fence_get(sync->fence)); 213 214 return 0; 215 } 216 217 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) 218 { 219 if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) 220 return; 221 222 if (sync->chain_fence) { 223 drm_syncobj_add_point(sync->syncobj, sync->chain_fence, 224 fence, sync->timeline_value); 225 /* 226 * The chain's ownership is transferred to the 227 * timeline. 228 */ 229 sync->chain_fence = NULL; 230 } else if (sync->syncobj) { 231 drm_syncobj_replace_fence(sync->syncobj, fence); 232 } else if (sync->ufence) { 233 int err; 234 235 dma_fence_get(fence); 236 user_fence_get(sync->ufence); 237 err = dma_fence_add_callback(fence, &sync->ufence->cb, 238 user_fence_cb); 239 if (err == -ENOENT) { 240 kick_ufence(sync->ufence, fence); 241 } else if (err) { 242 XE_WARN_ON("failed to add user fence"); 243 user_fence_put(sync->ufence); 244 dma_fence_put(fence); 245 } 246 } 247 } 248 249 void xe_sync_entry_cleanup(struct xe_sync_entry *sync) 250 { 251 if (sync->syncobj) 252 drm_syncobj_put(sync->syncobj); 253 dma_fence_put(sync->fence); 254 dma_fence_chain_free(sync->chain_fence); 255 if (sync->ufence) 256 user_fence_put(sync->ufence); 257 } 258 259 /** 260 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM 261 * @sync: input syncs 262 * @num_sync: number of syncs 263 * @q: exec queue 264 * @vm: VM 265 * 266 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create 267 * and return a composite fence of all in-fences + last fence. If no in-fences 268 * return last fence on input exec queue. Caller must drop reference to 269 * returned fence. 270 * 271 * Return: fence on success, ERR_PTR(-ENOMEM) on failure 272 */ 273 struct dma_fence * 274 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, 275 struct xe_exec_queue *q, struct xe_vm *vm) 276 { 277 struct dma_fence **fences = NULL; 278 struct dma_fence_array *cf = NULL; 279 struct dma_fence *fence; 280 int i, num_in_fence = 0, current_fence = 0; 281 282 lockdep_assert_held(&vm->lock); 283 284 /* Count in-fences */ 285 for (i = 0; i < num_sync; ++i) { 286 if (sync[i].fence) { 287 ++num_in_fence; 288 fence = sync[i].fence; 289 } 290 } 291 292 /* Easy case... */ 293 if (!num_in_fence) { 294 fence = xe_exec_queue_last_fence_get(q, vm); 295 return fence; 296 } 297 298 /* Create composite fence */ 299 fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL); 300 if (!fences) 301 return ERR_PTR(-ENOMEM); 302 for (i = 0; i < num_sync; ++i) { 303 if (sync[i].fence) { 304 dma_fence_get(sync[i].fence); 305 fences[current_fence++] = sync[i].fence; 306 } 307 } 308 fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm); 309 cf = dma_fence_array_create(num_in_fence, fences, 310 vm->composite_fence_ctx, 311 vm->composite_fence_seqno++, 312 false); 313 if (!cf) { 314 --vm->composite_fence_seqno; 315 goto err_out; 316 } 317 318 return &cf->base; 319 320 err_out: 321 while (current_fence) 322 dma_fence_put(fences[--current_fence]); 323 kfree(fences); 324 kfree(cf); 325 326 return ERR_PTR(-ENOMEM); 327 } 328 329 /** 330 * __xe_sync_ufence_get() - Get user fence from user fence 331 * @ufence: input user fence 332 * 333 * Get a user fence reference from user fence 334 * 335 * Return: xe_user_fence pointer with reference 336 */ 337 struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence) 338 { 339 user_fence_get(ufence); 340 341 return ufence; 342 } 343 344 /** 345 * xe_sync_ufence_get() - Get user fence from sync 346 * @sync: input sync 347 * 348 * Get a user fence reference from sync. 349 * 350 * Return: xe_user_fence pointer with reference 351 */ 352 struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) 353 { 354 user_fence_get(sync->ufence); 355 356 return sync->ufence; 357 } 358 359 /** 360 * xe_sync_ufence_put() - Put user fence reference 361 * @ufence: user fence reference 362 * 363 */ 364 void xe_sync_ufence_put(struct xe_user_fence *ufence) 365 { 366 user_fence_put(ufence); 367 } 368 369 /** 370 * xe_sync_ufence_get_status() - Get user fence status 371 * @ufence: user fence 372 * 373 * Return: 1 if signalled, 0 not signalled, <0 on error 374 */ 375 int xe_sync_ufence_get_status(struct xe_user_fence *ufence) 376 { 377 return READ_ONCE(ufence->signalled); 378 } 379