1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_sync.h" 7 8 #include <linux/kthread.h> 9 #include <linux/sched/mm.h> 10 #include <linux/uaccess.h> 11 #include <drm/xe_drm.h> 12 #include <drm/drm_print.h> 13 #include <drm/drm_syncobj.h> 14 15 #include "xe_device_types.h" 16 #include "xe_sched_job_types.h" 17 #include "xe_macros.h" 18 19 #define SYNC_FLAGS_TYPE_MASK 0x3 20 #define SYNC_FLAGS_FENCE_INSTALLED 0x10000 21 22 struct user_fence { 23 struct xe_device *xe; 24 struct kref refcount; 25 struct dma_fence_cb cb; 26 struct work_struct worker; 27 struct mm_struct *mm; 28 u64 __user *addr; 29 u64 value; 30 }; 31 32 static void user_fence_destroy(struct kref *kref) 33 { 34 struct user_fence *ufence = container_of(kref, struct user_fence, 35 refcount); 36 37 mmdrop(ufence->mm); 38 kfree(ufence); 39 } 40 41 static void user_fence_get(struct user_fence *ufence) 42 { 43 kref_get(&ufence->refcount); 44 } 45 46 static void user_fence_put(struct user_fence *ufence) 47 { 48 kref_put(&ufence->refcount, user_fence_destroy); 49 } 50 51 static struct user_fence *user_fence_create(struct xe_device *xe, u64 addr, 52 u64 value) 53 { 54 struct user_fence *ufence; 55 56 ufence = kmalloc(sizeof(*ufence), GFP_KERNEL); 57 if (!ufence) 58 return NULL; 59 60 ufence->xe = xe; 61 kref_init(&ufence->refcount); 62 ufence->addr = u64_to_user_ptr(addr); 63 ufence->value = value; 64 ufence->mm = current->mm; 65 mmgrab(ufence->mm); 66 67 return ufence; 68 } 69 70 static void user_fence_worker(struct work_struct *w) 71 { 72 struct user_fence *ufence = container_of(w, struct user_fence, worker); 73 74 if (mmget_not_zero(ufence->mm)) { 75 kthread_use_mm(ufence->mm); 76 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) 77 XE_WARN_ON("Copy to user failed"); 78 kthread_unuse_mm(ufence->mm); 79 mmput(ufence->mm); 80 } 81 82 wake_up_all(&ufence->xe->ufence_wq); 83 user_fence_put(ufence); 84 } 85 86 static void kick_ufence(struct user_fence *ufence, struct dma_fence *fence) 87 { 88 INIT_WORK(&ufence->worker, user_fence_worker); 89 queue_work(ufence->xe->ordered_wq, &ufence->worker); 90 dma_fence_put(fence); 91 } 92 93 static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 94 { 95 struct user_fence *ufence = container_of(cb, struct user_fence, cb); 96 97 kick_ufence(ufence, fence); 98 } 99 100 int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, 101 struct xe_sync_entry *sync, 102 struct drm_xe_sync __user *sync_user, 103 bool exec, bool no_dma_fences) 104 { 105 struct drm_xe_sync sync_in; 106 int err; 107 108 if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) 109 return -EFAULT; 110 111 if (XE_IOCTL_ERR(xe, sync_in.flags & 112 ~(SYNC_FLAGS_TYPE_MASK | DRM_XE_SYNC_SIGNAL))) 113 return -EINVAL; 114 115 switch (sync_in.flags & SYNC_FLAGS_TYPE_MASK) { 116 case DRM_XE_SYNC_SYNCOBJ: 117 if (XE_IOCTL_ERR(xe, no_dma_fences)) 118 return -ENOTSUPP; 119 120 if (XE_IOCTL_ERR(xe, upper_32_bits(sync_in.addr))) 121 return -EINVAL; 122 123 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 124 if (XE_IOCTL_ERR(xe, !sync->syncobj)) 125 return -ENOENT; 126 127 if (!(sync_in.flags & DRM_XE_SYNC_SIGNAL)) { 128 sync->fence = drm_syncobj_fence_get(sync->syncobj); 129 if (XE_IOCTL_ERR(xe, !sync->fence)) 130 return -EINVAL; 131 } 132 break; 133 134 case DRM_XE_SYNC_TIMELINE_SYNCOBJ: 135 if (XE_IOCTL_ERR(xe, no_dma_fences)) 136 return -ENOTSUPP; 137 138 if (XE_IOCTL_ERR(xe, upper_32_bits(sync_in.addr))) 139 return -EINVAL; 140 141 if (XE_IOCTL_ERR(xe, sync_in.timeline_value == 0)) 142 return -EINVAL; 143 144 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 145 if (XE_IOCTL_ERR(xe, !sync->syncobj)) 146 return -ENOENT; 147 148 if (sync_in.flags & DRM_XE_SYNC_SIGNAL) { 149 sync->chain_fence = dma_fence_chain_alloc(); 150 if (!sync->chain_fence) 151 return -ENOMEM; 152 } else { 153 sync->fence = drm_syncobj_fence_get(sync->syncobj); 154 if (XE_IOCTL_ERR(xe, !sync->fence)) 155 return -EINVAL; 156 157 err = dma_fence_chain_find_seqno(&sync->fence, 158 sync_in.timeline_value); 159 if (err) 160 return err; 161 } 162 break; 163 164 case DRM_XE_SYNC_DMA_BUF: 165 if (XE_IOCTL_ERR(xe, "TODO")) 166 return -EINVAL; 167 break; 168 169 case DRM_XE_SYNC_USER_FENCE: 170 if (XE_IOCTL_ERR(xe, !(sync_in.flags & DRM_XE_SYNC_SIGNAL))) 171 return -ENOTSUPP; 172 173 if (XE_IOCTL_ERR(xe, sync_in.addr & 0x7)) 174 return -EINVAL; 175 176 if (exec) { 177 sync->addr = sync_in.addr; 178 } else { 179 sync->ufence = user_fence_create(xe, sync_in.addr, 180 sync_in.timeline_value); 181 if (XE_IOCTL_ERR(xe, !sync->ufence)) 182 return -ENOMEM; 183 } 184 185 break; 186 187 default: 188 return -EINVAL; 189 } 190 191 sync->flags = sync_in.flags; 192 sync->timeline_value = sync_in.timeline_value; 193 194 return 0; 195 } 196 197 int xe_sync_entry_wait(struct xe_sync_entry *sync) 198 { 199 if (sync->fence) 200 dma_fence_wait(sync->fence, true); 201 202 return 0; 203 } 204 205 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) 206 { 207 int err; 208 209 if (sync->fence) { 210 err = drm_sched_job_add_dependency(&job->drm, 211 dma_fence_get(sync->fence)); 212 if (err) { 213 dma_fence_put(sync->fence); 214 return err; 215 } 216 } 217 218 return 0; 219 } 220 221 bool xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job, 222 struct dma_fence *fence) 223 { 224 if (!(sync->flags & DRM_XE_SYNC_SIGNAL) || 225 sync->flags & SYNC_FLAGS_FENCE_INSTALLED) 226 return false; 227 228 if (sync->chain_fence) { 229 drm_syncobj_add_point(sync->syncobj, sync->chain_fence, 230 fence, sync->timeline_value); 231 /* 232 * The chain's ownership is transferred to the 233 * timeline. 234 */ 235 sync->chain_fence = NULL; 236 } else if (sync->syncobj) { 237 drm_syncobj_replace_fence(sync->syncobj, fence); 238 } else if (sync->ufence) { 239 int err; 240 241 dma_fence_get(fence); 242 user_fence_get(sync->ufence); 243 err = dma_fence_add_callback(fence, &sync->ufence->cb, 244 user_fence_cb); 245 if (err == -ENOENT) { 246 kick_ufence(sync->ufence, fence); 247 } else if (err) { 248 XE_WARN_ON("failed to add user fence"); 249 user_fence_put(sync->ufence); 250 dma_fence_put(fence); 251 } 252 } else if ((sync->flags & SYNC_FLAGS_TYPE_MASK) == 253 DRM_XE_SYNC_USER_FENCE) { 254 job->user_fence.used = true; 255 job->user_fence.addr = sync->addr; 256 job->user_fence.value = sync->timeline_value; 257 } 258 259 /* TODO: external BO? */ 260 261 sync->flags |= SYNC_FLAGS_FENCE_INSTALLED; 262 263 return true; 264 } 265 266 void xe_sync_entry_cleanup(struct xe_sync_entry *sync) 267 { 268 if (sync->syncobj) 269 drm_syncobj_put(sync->syncobj); 270 if (sync->fence) 271 dma_fence_put(sync->fence); 272 if (sync->chain_fence) 273 dma_fence_put(&sync->chain_fence->base); 274 if (sync->ufence) 275 user_fence_put(sync->ufence); 276 } 277