1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_sync.h" 7 8 #include <linux/dma-fence-array.h> 9 #include <linux/kthread.h> 10 #include <linux/sched/mm.h> 11 #include <linux/uaccess.h> 12 13 #include <drm/drm_print.h> 14 #include <drm/drm_syncobj.h> 15 #include <drm/xe_drm.h> 16 17 #include "xe_device_types.h" 18 #include "xe_exec_queue.h" 19 #include "xe_macros.h" 20 #include "xe_sched_job_types.h" 21 22 struct xe_user_fence { 23 struct xe_device *xe; 24 struct kref refcount; 25 struct dma_fence_cb cb; 26 struct work_struct worker; 27 struct mm_struct *mm; 28 u64 __user *addr; 29 u64 value; 30 int signalled; 31 }; 32 33 static void user_fence_destroy(struct kref *kref) 34 { 35 struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence, 36 refcount); 37 38 mmdrop(ufence->mm); 39 kfree(ufence); 40 } 41 42 static void user_fence_get(struct xe_user_fence *ufence) 43 { 44 kref_get(&ufence->refcount); 45 } 46 47 static void user_fence_put(struct xe_user_fence *ufence) 48 { 49 kref_put(&ufence->refcount, user_fence_destroy); 50 } 51 52 static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr, 53 u64 value) 54 { 55 struct xe_user_fence *ufence; 56 57 ufence = kmalloc(sizeof(*ufence), GFP_KERNEL); 58 if (!ufence) 59 return NULL; 60 61 ufence->xe = xe; 62 kref_init(&ufence->refcount); 63 ufence->addr = u64_to_user_ptr(addr); 64 ufence->value = value; 65 ufence->mm = current->mm; 66 mmgrab(ufence->mm); 67 68 return ufence; 69 } 70 71 static void user_fence_worker(struct work_struct *w) 72 { 73 struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker); 74 75 if (mmget_not_zero(ufence->mm)) { 76 kthread_use_mm(ufence->mm); 77 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) 78 XE_WARN_ON("Copy to user failed"); 79 kthread_unuse_mm(ufence->mm); 80 mmput(ufence->mm); 81 } 82 83 wake_up_all(&ufence->xe->ufence_wq); 84 WRITE_ONCE(ufence->signalled, 1); 85 user_fence_put(ufence); 86 } 87 88 static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence) 89 { 90 INIT_WORK(&ufence->worker, user_fence_worker); 91 queue_work(ufence->xe->ordered_wq, &ufence->worker); 92 dma_fence_put(fence); 93 } 94 95 static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 96 { 97 struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb); 98 99 kick_ufence(ufence, fence); 100 } 101 102 int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, 103 struct xe_sync_entry *sync, 104 struct drm_xe_sync __user *sync_user, 105 unsigned int flags) 106 { 107 struct drm_xe_sync sync_in; 108 int err; 109 bool exec = flags & SYNC_PARSE_FLAG_EXEC; 110 bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE; 111 bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE; 112 bool signal; 113 114 if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) 115 return -EFAULT; 116 117 if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || 118 XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) 119 return -EINVAL; 120 121 signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; 122 switch (sync_in.type) { 123 case DRM_XE_SYNC_TYPE_SYNCOBJ: 124 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 125 return -EOPNOTSUPP; 126 127 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 128 return -EINVAL; 129 130 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 131 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 132 return -ENOENT; 133 134 if (!signal) { 135 sync->fence = drm_syncobj_fence_get(sync->syncobj); 136 if (XE_IOCTL_DBG(xe, !sync->fence)) 137 return -EINVAL; 138 } 139 break; 140 141 case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: 142 if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) 143 return -EOPNOTSUPP; 144 145 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr))) 146 return -EINVAL; 147 148 if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0)) 149 return -EINVAL; 150 151 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 152 if (XE_IOCTL_DBG(xe, !sync->syncobj)) 153 return -ENOENT; 154 155 if (signal) { 156 sync->chain_fence = dma_fence_chain_alloc(); 157 if (!sync->chain_fence) 158 return -ENOMEM; 159 } else { 160 sync->fence = drm_syncobj_fence_get(sync->syncobj); 161 if (XE_IOCTL_DBG(xe, !sync->fence)) 162 return -EINVAL; 163 164 err = dma_fence_chain_find_seqno(&sync->fence, 165 sync_in.timeline_value); 166 if (err) 167 return err; 168 } 169 break; 170 171 case DRM_XE_SYNC_TYPE_USER_FENCE: 172 if (XE_IOCTL_DBG(xe, disallow_user_fence)) 173 return -EOPNOTSUPP; 174 175 if (XE_IOCTL_DBG(xe, !signal)) 176 return -EOPNOTSUPP; 177 178 if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7)) 179 return -EINVAL; 180 181 if (exec) { 182 sync->addr = sync_in.addr; 183 } else { 184 sync->ufence = user_fence_create(xe, sync_in.addr, 185 sync_in.timeline_value); 186 if (XE_IOCTL_DBG(xe, !sync->ufence)) 187 return -ENOMEM; 188 } 189 190 break; 191 192 default: 193 return -EINVAL; 194 } 195 196 sync->type = sync_in.type; 197 sync->flags = sync_in.flags; 198 sync->timeline_value = sync_in.timeline_value; 199 200 return 0; 201 } 202 203 int xe_sync_entry_wait(struct xe_sync_entry *sync) 204 { 205 if (sync->fence) 206 dma_fence_wait(sync->fence, true); 207 208 return 0; 209 } 210 211 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) 212 { 213 int err; 214 215 if (sync->fence) { 216 err = drm_sched_job_add_dependency(&job->drm, 217 dma_fence_get(sync->fence)); 218 if (err) { 219 dma_fence_put(sync->fence); 220 return err; 221 } 222 } 223 224 return 0; 225 } 226 227 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence) 228 { 229 if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) 230 return; 231 232 if (sync->chain_fence) { 233 drm_syncobj_add_point(sync->syncobj, sync->chain_fence, 234 fence, sync->timeline_value); 235 /* 236 * The chain's ownership is transferred to the 237 * timeline. 238 */ 239 sync->chain_fence = NULL; 240 } else if (sync->syncobj) { 241 drm_syncobj_replace_fence(sync->syncobj, fence); 242 } else if (sync->ufence) { 243 int err; 244 245 dma_fence_get(fence); 246 user_fence_get(sync->ufence); 247 err = dma_fence_add_callback(fence, &sync->ufence->cb, 248 user_fence_cb); 249 if (err == -ENOENT) { 250 kick_ufence(sync->ufence, fence); 251 } else if (err) { 252 XE_WARN_ON("failed to add user fence"); 253 user_fence_put(sync->ufence); 254 dma_fence_put(fence); 255 } 256 } 257 } 258 259 void xe_sync_entry_cleanup(struct xe_sync_entry *sync) 260 { 261 if (sync->syncobj) 262 drm_syncobj_put(sync->syncobj); 263 if (sync->fence) 264 dma_fence_put(sync->fence); 265 if (sync->chain_fence) 266 dma_fence_put(&sync->chain_fence->base); 267 if (sync->ufence) 268 user_fence_put(sync->ufence); 269 } 270 271 /** 272 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM 273 * @sync: input syncs 274 * @num_sync: number of syncs 275 * @q: exec queue 276 * @vm: VM 277 * 278 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create 279 * and return a composite fence of all in-fences + last fence. If no in-fences 280 * return last fence on input exec queue. Caller must drop reference to 281 * returned fence. 282 * 283 * Return: fence on success, ERR_PTR(-ENOMEM) on failure 284 */ 285 struct dma_fence * 286 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, 287 struct xe_exec_queue *q, struct xe_vm *vm) 288 { 289 struct dma_fence **fences = NULL; 290 struct dma_fence_array *cf = NULL; 291 struct dma_fence *fence; 292 int i, num_in_fence = 0, current_fence = 0; 293 294 lockdep_assert_held(&vm->lock); 295 296 /* Count in-fences */ 297 for (i = 0; i < num_sync; ++i) { 298 if (sync[i].fence) { 299 ++num_in_fence; 300 fence = sync[i].fence; 301 } 302 } 303 304 /* Easy case... */ 305 if (!num_in_fence) { 306 fence = xe_exec_queue_last_fence_get(q, vm); 307 return fence; 308 } 309 310 /* Create composite fence */ 311 fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL); 312 if (!fences) 313 return ERR_PTR(-ENOMEM); 314 for (i = 0; i < num_sync; ++i) { 315 if (sync[i].fence) { 316 dma_fence_get(sync[i].fence); 317 fences[current_fence++] = sync[i].fence; 318 } 319 } 320 fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm); 321 cf = dma_fence_array_create(num_in_fence, fences, 322 vm->composite_fence_ctx, 323 vm->composite_fence_seqno++, 324 false); 325 if (!cf) { 326 --vm->composite_fence_seqno; 327 goto err_out; 328 } 329 330 return &cf->base; 331 332 err_out: 333 while (current_fence) 334 dma_fence_put(fences[--current_fence]); 335 kfree(fences); 336 kfree(cf); 337 338 return ERR_PTR(-ENOMEM); 339 } 340 341 /** 342 * xe_sync_ufence_get() - Get user fence from sync 343 * @sync: input sync 344 * 345 * Get a user fence reference from sync. 346 * 347 * Return: xe_user_fence pointer with reference 348 */ 349 struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync) 350 { 351 user_fence_get(sync->ufence); 352 353 return sync->ufence; 354 } 355 356 /** 357 * xe_sync_ufence_put() - Put user fence reference 358 * @ufence: user fence reference 359 * 360 */ 361 void xe_sync_ufence_put(struct xe_user_fence *ufence) 362 { 363 user_fence_put(ufence); 364 } 365 366 /** 367 * xe_sync_ufence_get_status() - Get user fence status 368 * @ufence: user fence 369 * 370 * Return: 1 if signalled, 0 not signalled, <0 on error 371 */ 372 int xe_sync_ufence_get_status(struct xe_user_fence *ufence) 373 { 374 return READ_ONCE(ufence->signalled); 375 } 376