1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_sync.h" 7 8 #include <linux/kthread.h> 9 #include <linux/sched/mm.h> 10 #include <linux/uaccess.h> 11 12 #include <drm/drm_print.h> 13 #include <drm/drm_syncobj.h> 14 #include <drm/xe_drm.h> 15 16 #include "xe_device_types.h" 17 #include "xe_macros.h" 18 #include "xe_sched_job_types.h" 19 20 #define SYNC_FLAGS_TYPE_MASK 0x3 21 #define SYNC_FLAGS_FENCE_INSTALLED 0x10000 22 23 struct user_fence { 24 struct xe_device *xe; 25 struct kref refcount; 26 struct dma_fence_cb cb; 27 struct work_struct worker; 28 struct mm_struct *mm; 29 u64 __user *addr; 30 u64 value; 31 }; 32 33 static void user_fence_destroy(struct kref *kref) 34 { 35 struct user_fence *ufence = container_of(kref, struct user_fence, 36 refcount); 37 38 mmdrop(ufence->mm); 39 kfree(ufence); 40 } 41 42 static void user_fence_get(struct user_fence *ufence) 43 { 44 kref_get(&ufence->refcount); 45 } 46 47 static void user_fence_put(struct user_fence *ufence) 48 { 49 kref_put(&ufence->refcount, user_fence_destroy); 50 } 51 52 static struct user_fence *user_fence_create(struct xe_device *xe, u64 addr, 53 u64 value) 54 { 55 struct user_fence *ufence; 56 57 ufence = kmalloc(sizeof(*ufence), GFP_KERNEL); 58 if (!ufence) 59 return NULL; 60 61 ufence->xe = xe; 62 kref_init(&ufence->refcount); 63 ufence->addr = u64_to_user_ptr(addr); 64 ufence->value = value; 65 ufence->mm = current->mm; 66 mmgrab(ufence->mm); 67 68 return ufence; 69 } 70 71 static void user_fence_worker(struct work_struct *w) 72 { 73 struct user_fence *ufence = container_of(w, struct user_fence, worker); 74 75 if (mmget_not_zero(ufence->mm)) { 76 kthread_use_mm(ufence->mm); 77 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) 78 XE_WARN_ON("Copy to user failed"); 79 kthread_unuse_mm(ufence->mm); 80 mmput(ufence->mm); 81 } 82 83 wake_up_all(&ufence->xe->ufence_wq); 84 user_fence_put(ufence); 85 } 86 87 static void kick_ufence(struct user_fence *ufence, struct dma_fence *fence) 88 { 89 INIT_WORK(&ufence->worker, user_fence_worker); 90 queue_work(ufence->xe->ordered_wq, &ufence->worker); 91 dma_fence_put(fence); 92 } 93 94 static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 95 { 96 struct user_fence *ufence = container_of(cb, struct user_fence, cb); 97 98 kick_ufence(ufence, fence); 99 } 100 101 int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, 102 struct xe_sync_entry *sync, 103 struct drm_xe_sync __user *sync_user, 104 bool exec, bool no_dma_fences) 105 { 106 struct drm_xe_sync sync_in; 107 int err; 108 bool signal; 109 110 if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) 111 return -EFAULT; 112 113 if (XE_IOCTL_ERR(xe, sync_in.flags & 114 ~(SYNC_FLAGS_TYPE_MASK | DRM_XE_SYNC_SIGNAL)) || 115 XE_IOCTL_ERR(xe, sync_in.pad) || 116 XE_IOCTL_ERR(xe, sync_in.reserved[0] || sync_in.reserved[1])) 117 return -EINVAL; 118 119 signal = sync_in.flags & DRM_XE_SYNC_SIGNAL; 120 switch (sync_in.flags & SYNC_FLAGS_TYPE_MASK) { 121 case DRM_XE_SYNC_SYNCOBJ: 122 if (XE_IOCTL_ERR(xe, no_dma_fences && signal)) 123 return -ENOTSUPP; 124 125 if (XE_IOCTL_ERR(xe, upper_32_bits(sync_in.addr))) 126 return -EINVAL; 127 128 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 129 if (XE_IOCTL_ERR(xe, !sync->syncobj)) 130 return -ENOENT; 131 132 if (!signal) { 133 sync->fence = drm_syncobj_fence_get(sync->syncobj); 134 if (XE_IOCTL_ERR(xe, !sync->fence)) 135 return -EINVAL; 136 } 137 break; 138 139 case DRM_XE_SYNC_TIMELINE_SYNCOBJ: 140 if (XE_IOCTL_ERR(xe, no_dma_fences && signal)) 141 return -ENOTSUPP; 142 143 if (XE_IOCTL_ERR(xe, upper_32_bits(sync_in.addr))) 144 return -EINVAL; 145 146 if (XE_IOCTL_ERR(xe, sync_in.timeline_value == 0)) 147 return -EINVAL; 148 149 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); 150 if (XE_IOCTL_ERR(xe, !sync->syncobj)) 151 return -ENOENT; 152 153 if (signal) { 154 sync->chain_fence = dma_fence_chain_alloc(); 155 if (!sync->chain_fence) 156 return -ENOMEM; 157 } else { 158 sync->fence = drm_syncobj_fence_get(sync->syncobj); 159 if (XE_IOCTL_ERR(xe, !sync->fence)) 160 return -EINVAL; 161 162 err = dma_fence_chain_find_seqno(&sync->fence, 163 sync_in.timeline_value); 164 if (err) 165 return err; 166 } 167 break; 168 169 case DRM_XE_SYNC_DMA_BUF: 170 if (XE_IOCTL_ERR(xe, "TODO")) 171 return -EINVAL; 172 break; 173 174 case DRM_XE_SYNC_USER_FENCE: 175 if (XE_IOCTL_ERR(xe, !signal)) 176 return -ENOTSUPP; 177 178 if (XE_IOCTL_ERR(xe, sync_in.addr & 0x7)) 179 return -EINVAL; 180 181 if (exec) { 182 sync->addr = sync_in.addr; 183 } else { 184 sync->ufence = user_fence_create(xe, sync_in.addr, 185 sync_in.timeline_value); 186 if (XE_IOCTL_ERR(xe, !sync->ufence)) 187 return -ENOMEM; 188 } 189 190 break; 191 192 default: 193 return -EINVAL; 194 } 195 196 sync->flags = sync_in.flags; 197 sync->timeline_value = sync_in.timeline_value; 198 199 return 0; 200 } 201 202 int xe_sync_entry_wait(struct xe_sync_entry *sync) 203 { 204 if (sync->fence) 205 dma_fence_wait(sync->fence, true); 206 207 return 0; 208 } 209 210 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) 211 { 212 int err; 213 214 if (sync->fence) { 215 err = drm_sched_job_add_dependency(&job->drm, 216 dma_fence_get(sync->fence)); 217 if (err) { 218 dma_fence_put(sync->fence); 219 return err; 220 } 221 } 222 223 return 0; 224 } 225 226 bool xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job, 227 struct dma_fence *fence) 228 { 229 if (!(sync->flags & DRM_XE_SYNC_SIGNAL) || 230 sync->flags & SYNC_FLAGS_FENCE_INSTALLED) 231 return false; 232 233 if (sync->chain_fence) { 234 drm_syncobj_add_point(sync->syncobj, sync->chain_fence, 235 fence, sync->timeline_value); 236 /* 237 * The chain's ownership is transferred to the 238 * timeline. 239 */ 240 sync->chain_fence = NULL; 241 } else if (sync->syncobj) { 242 drm_syncobj_replace_fence(sync->syncobj, fence); 243 } else if (sync->ufence) { 244 int err; 245 246 dma_fence_get(fence); 247 user_fence_get(sync->ufence); 248 err = dma_fence_add_callback(fence, &sync->ufence->cb, 249 user_fence_cb); 250 if (err == -ENOENT) { 251 kick_ufence(sync->ufence, fence); 252 } else if (err) { 253 XE_WARN_ON("failed to add user fence"); 254 user_fence_put(sync->ufence); 255 dma_fence_put(fence); 256 } 257 } else if ((sync->flags & SYNC_FLAGS_TYPE_MASK) == 258 DRM_XE_SYNC_USER_FENCE) { 259 job->user_fence.used = true; 260 job->user_fence.addr = sync->addr; 261 job->user_fence.value = sync->timeline_value; 262 } 263 264 /* TODO: external BO? */ 265 266 sync->flags |= SYNC_FLAGS_FENCE_INSTALLED; 267 268 return true; 269 } 270 271 void xe_sync_entry_cleanup(struct xe_sync_entry *sync) 272 { 273 if (sync->syncobj) 274 drm_syncobj_put(sync->syncobj); 275 if (sync->fence) 276 dma_fence_put(sync->fence); 277 if (sync->chain_fence) 278 dma_fence_put(&sync->chain_fence->base); 279 if (sync->ufence) 280 user_fence_put(sync->ufence); 281 } 282