1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Sync File validation framework 4 * 5 * Copyright (C) 2012 Google, Inc. 6 */ 7 8 #include <linux/file.h> 9 #include <linux/fs.h> 10 #include <linux/uaccess.h> 11 #include <linux/slab.h> 12 #include <linux/sync_file.h> 13 14 #include "sync_debug.h" 15 16 #define CREATE_TRACE_POINTS 17 #include "sync_trace.h" 18 19 /* 20 * SW SYNC validation framework 21 * 22 * A sync object driver that uses a 32bit counter to coordinate 23 * synchronization. Useful when there is no hardware primitive backing 24 * the synchronization. 25 * 26 * To start the framework just open: 27 * 28 * <debugfs>/sync/sw_sync 29 * 30 * That will create a sync timeline, all fences created under this timeline 31 * file descriptor will belong to the this timeline. 32 * 33 * The 'sw_sync' file can be opened many times as to create different 34 * timelines. 35 * 36 * Fences can be created with SW_SYNC_IOC_CREATE_FENCE ioctl with struct 37 * sw_sync_create_fence_data as parameter. 38 * 39 * To increment the timeline counter, SW_SYNC_IOC_INC ioctl should be used 40 * with the increment as u32. This will update the last signaled value 41 * from the timeline and signal any fence that has a seqno smaller or equal 42 * to it. 43 * 44 * struct sw_sync_create_fence_data 45 * @value: the seqno to initialise the fence with 46 * @name: the name of the new sync point 47 * @fence: return the fd of the new sync_file with the created fence 48 */ 49 struct sw_sync_create_fence_data { 50 __u32 value; 51 char name[32]; 52 __s32 fence; /* fd of new fence */ 53 }; 54 55 /** 56 * struct sw_sync_get_deadline - get the deadline hint of a sw_sync fence 57 * @deadline_ns: absolute time of the deadline 58 * @pad: must be zero 59 * @fence_fd: the sw_sync fence fd (in) 60 * 61 * Return the earliest deadline set on the fence. The timebase for the 62 * deadline is CLOCK_MONOTONIC (same as vblank). If there is no deadline 63 * set on the fence, this ioctl will return -ENOENT. 64 */ 65 struct sw_sync_get_deadline { 66 __u64 deadline_ns; 67 __u32 pad; 68 __s32 fence_fd; 69 }; 70 71 #define SW_SYNC_IOC_MAGIC 'W' 72 73 #define SW_SYNC_IOC_CREATE_FENCE _IOWR(SW_SYNC_IOC_MAGIC, 0,\ 74 struct sw_sync_create_fence_data) 75 76 #define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32) 77 #define SW_SYNC_GET_DEADLINE _IOWR(SW_SYNC_IOC_MAGIC, 2, \ 78 struct sw_sync_get_deadline) 79 80 81 #define SW_SYNC_HAS_DEADLINE_BIT DMA_FENCE_FLAG_USER_BITS 82 83 static const struct dma_fence_ops timeline_fence_ops; 84 85 static inline struct sync_pt *dma_fence_to_sync_pt(struct dma_fence *fence) 86 { 87 if (fence->ops != &timeline_fence_ops) 88 return NULL; 89 return container_of(fence, struct sync_pt, base); 90 } 91 92 /** 93 * sync_timeline_create() - creates a sync object 94 * @name: sync_timeline name 95 * 96 * Creates a new sync_timeline. Returns the sync_timeline object or NULL in 97 * case of error. 98 */ 99 static struct sync_timeline *sync_timeline_create(const char *name) 100 { 101 struct sync_timeline *obj; 102 103 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 104 if (!obj) 105 return NULL; 106 107 kref_init(&obj->kref); 108 obj->context = dma_fence_context_alloc(1); 109 strscpy(obj->name, name, sizeof(obj->name)); 110 111 obj->pt_tree = RB_ROOT; 112 INIT_LIST_HEAD(&obj->pt_list); 113 spin_lock_init(&obj->lock); 114 115 sync_timeline_debug_add(obj); 116 117 return obj; 118 } 119 120 static void sync_timeline_free(struct kref *kref) 121 { 122 struct sync_timeline *obj = 123 container_of(kref, struct sync_timeline, kref); 124 125 sync_timeline_debug_remove(obj); 126 127 kfree(obj); 128 } 129 130 static void sync_timeline_get(struct sync_timeline *obj) 131 { 132 kref_get(&obj->kref); 133 } 134 135 static void sync_timeline_put(struct sync_timeline *obj) 136 { 137 kref_put(&obj->kref, sync_timeline_free); 138 } 139 140 static const char *timeline_fence_get_driver_name(struct dma_fence *fence) 141 { 142 return "sw_sync"; 143 } 144 145 static const char *timeline_fence_get_timeline_name(struct dma_fence *fence) 146 { 147 struct sync_timeline *parent = dma_fence_parent(fence); 148 149 return parent->name; 150 } 151 152 static void timeline_fence_release(struct dma_fence *fence) 153 { 154 struct sync_pt *pt = dma_fence_to_sync_pt(fence); 155 struct sync_timeline *parent = dma_fence_parent(fence); 156 unsigned long flags; 157 158 spin_lock_irqsave(fence->lock, flags); 159 if (!list_empty(&pt->link)) { 160 list_del(&pt->link); 161 rb_erase(&pt->node, &parent->pt_tree); 162 } 163 spin_unlock_irqrestore(fence->lock, flags); 164 165 sync_timeline_put(parent); 166 dma_fence_free(fence); 167 } 168 169 static bool timeline_fence_signaled(struct dma_fence *fence) 170 { 171 struct sync_timeline *parent = dma_fence_parent(fence); 172 173 return !__dma_fence_is_later(fence->seqno, parent->value, fence->ops); 174 } 175 176 static void timeline_fence_value_str(struct dma_fence *fence, 177 char *str, int size) 178 { 179 snprintf(str, size, "%lld", fence->seqno); 180 } 181 182 static void timeline_fence_timeline_value_str(struct dma_fence *fence, 183 char *str, int size) 184 { 185 struct sync_timeline *parent = dma_fence_parent(fence); 186 187 snprintf(str, size, "%d", parent->value); 188 } 189 190 static void timeline_fence_set_deadline(struct dma_fence *fence, ktime_t deadline) 191 { 192 struct sync_pt *pt = dma_fence_to_sync_pt(fence); 193 unsigned long flags; 194 195 spin_lock_irqsave(fence->lock, flags); 196 if (test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) { 197 if (ktime_before(deadline, pt->deadline)) 198 pt->deadline = deadline; 199 } else { 200 pt->deadline = deadline; 201 __set_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags); 202 } 203 spin_unlock_irqrestore(fence->lock, flags); 204 } 205 206 static const struct dma_fence_ops timeline_fence_ops = { 207 .get_driver_name = timeline_fence_get_driver_name, 208 .get_timeline_name = timeline_fence_get_timeline_name, 209 .signaled = timeline_fence_signaled, 210 .release = timeline_fence_release, 211 .fence_value_str = timeline_fence_value_str, 212 .timeline_value_str = timeline_fence_timeline_value_str, 213 .set_deadline = timeline_fence_set_deadline, 214 }; 215 216 /** 217 * sync_timeline_signal() - signal a status change on a sync_timeline 218 * @obj: sync_timeline to signal 219 * @inc: num to increment on timeline->value 220 * 221 * A sync implementation should call this any time one of it's fences 222 * has signaled or has an error condition. 223 */ 224 static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc) 225 { 226 LIST_HEAD(signalled); 227 struct sync_pt *pt, *next; 228 229 trace_sync_timeline(obj); 230 231 spin_lock_irq(&obj->lock); 232 233 obj->value += inc; 234 235 list_for_each_entry_safe(pt, next, &obj->pt_list, link) { 236 if (!timeline_fence_signaled(&pt->base)) 237 break; 238 239 dma_fence_get(&pt->base); 240 241 list_move_tail(&pt->link, &signalled); 242 rb_erase(&pt->node, &obj->pt_tree); 243 244 dma_fence_signal_locked(&pt->base); 245 } 246 247 spin_unlock_irq(&obj->lock); 248 249 list_for_each_entry_safe(pt, next, &signalled, link) { 250 list_del_init(&pt->link); 251 dma_fence_put(&pt->base); 252 } 253 } 254 255 /** 256 * sync_pt_create() - creates a sync pt 257 * @obj: parent sync_timeline 258 * @value: value of the fence 259 * 260 * Creates a new sync_pt (fence) as a child of @parent. @size bytes will be 261 * allocated allowing for implementation specific data to be kept after 262 * the generic sync_timeline struct. Returns the sync_pt object or 263 * NULL in case of error. 264 */ 265 static struct sync_pt *sync_pt_create(struct sync_timeline *obj, 266 unsigned int value) 267 { 268 struct sync_pt *pt; 269 270 pt = kzalloc(sizeof(*pt), GFP_KERNEL); 271 if (!pt) 272 return NULL; 273 274 sync_timeline_get(obj); 275 dma_fence_init(&pt->base, &timeline_fence_ops, &obj->lock, 276 obj->context, value); 277 INIT_LIST_HEAD(&pt->link); 278 279 spin_lock_irq(&obj->lock); 280 if (!dma_fence_is_signaled_locked(&pt->base)) { 281 struct rb_node **p = &obj->pt_tree.rb_node; 282 struct rb_node *parent = NULL; 283 284 while (*p) { 285 struct sync_pt *other; 286 int cmp; 287 288 parent = *p; 289 other = rb_entry(parent, typeof(*pt), node); 290 cmp = value - other->base.seqno; 291 if (cmp > 0) { 292 p = &parent->rb_right; 293 } else if (cmp < 0) { 294 p = &parent->rb_left; 295 } else { 296 if (dma_fence_get_rcu(&other->base)) { 297 sync_timeline_put(obj); 298 kfree(pt); 299 pt = other; 300 goto unlock; 301 } 302 p = &parent->rb_left; 303 } 304 } 305 rb_link_node(&pt->node, parent, p); 306 rb_insert_color(&pt->node, &obj->pt_tree); 307 308 parent = rb_next(&pt->node); 309 list_add_tail(&pt->link, 310 parent ? &rb_entry(parent, typeof(*pt), node)->link : &obj->pt_list); 311 } 312 unlock: 313 spin_unlock_irq(&obj->lock); 314 315 return pt; 316 } 317 318 /* 319 * *WARNING* 320 * 321 * improper use of this can result in deadlocking kernel drivers from userspace. 322 */ 323 324 /* opening sw_sync create a new sync obj */ 325 static int sw_sync_debugfs_open(struct inode *inode, struct file *file) 326 { 327 struct sync_timeline *obj; 328 char task_comm[TASK_COMM_LEN]; 329 330 get_task_comm(task_comm, current); 331 332 obj = sync_timeline_create(task_comm); 333 if (!obj) 334 return -ENOMEM; 335 336 file->private_data = obj; 337 338 return 0; 339 } 340 341 static int sw_sync_debugfs_release(struct inode *inode, struct file *file) 342 { 343 struct sync_timeline *obj = file->private_data; 344 struct sync_pt *pt, *next; 345 346 spin_lock_irq(&obj->lock); 347 348 list_for_each_entry_safe(pt, next, &obj->pt_list, link) { 349 dma_fence_set_error(&pt->base, -ENOENT); 350 dma_fence_signal_locked(&pt->base); 351 } 352 353 spin_unlock_irq(&obj->lock); 354 355 sync_timeline_put(obj); 356 return 0; 357 } 358 359 static long sw_sync_ioctl_create_fence(struct sync_timeline *obj, 360 unsigned long arg) 361 { 362 int fd = get_unused_fd_flags(O_CLOEXEC); 363 int err; 364 struct sync_pt *pt; 365 struct sync_file *sync_file; 366 struct sw_sync_create_fence_data data; 367 368 if (fd < 0) 369 return fd; 370 371 if (copy_from_user(&data, (void __user *)arg, sizeof(data))) { 372 err = -EFAULT; 373 goto err; 374 } 375 376 pt = sync_pt_create(obj, data.value); 377 if (!pt) { 378 err = -ENOMEM; 379 goto err; 380 } 381 382 sync_file = sync_file_create(&pt->base); 383 dma_fence_put(&pt->base); 384 if (!sync_file) { 385 err = -ENOMEM; 386 goto err; 387 } 388 389 data.fence = fd; 390 if (copy_to_user((void __user *)arg, &data, sizeof(data))) { 391 fput(sync_file->file); 392 err = -EFAULT; 393 goto err; 394 } 395 396 fd_install(fd, sync_file->file); 397 398 return 0; 399 400 err: 401 put_unused_fd(fd); 402 return err; 403 } 404 405 static long sw_sync_ioctl_inc(struct sync_timeline *obj, unsigned long arg) 406 { 407 u32 value; 408 409 if (copy_from_user(&value, (void __user *)arg, sizeof(value))) 410 return -EFAULT; 411 412 while (value > INT_MAX) { 413 sync_timeline_signal(obj, INT_MAX); 414 value -= INT_MAX; 415 } 416 417 sync_timeline_signal(obj, value); 418 419 return 0; 420 } 421 422 static int sw_sync_ioctl_get_deadline(struct sync_timeline *obj, unsigned long arg) 423 { 424 struct sw_sync_get_deadline data; 425 struct dma_fence *fence; 426 unsigned long flags; 427 struct sync_pt *pt; 428 int ret = 0; 429 430 if (copy_from_user(&data, (void __user *)arg, sizeof(data))) 431 return -EFAULT; 432 433 if (data.deadline_ns || data.pad) 434 return -EINVAL; 435 436 fence = sync_file_get_fence(data.fence_fd); 437 if (!fence) 438 return -EINVAL; 439 440 pt = dma_fence_to_sync_pt(fence); 441 if (!pt) 442 return -EINVAL; 443 444 spin_lock_irqsave(fence->lock, flags); 445 if (test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) { 446 data.deadline_ns = ktime_to_ns(pt->deadline); 447 } else { 448 ret = -ENOENT; 449 } 450 spin_unlock_irqrestore(fence->lock, flags); 451 452 dma_fence_put(fence); 453 454 if (ret) 455 return ret; 456 457 if (copy_to_user((void __user *)arg, &data, sizeof(data))) 458 return -EFAULT; 459 460 return 0; 461 } 462 463 static long sw_sync_ioctl(struct file *file, unsigned int cmd, 464 unsigned long arg) 465 { 466 struct sync_timeline *obj = file->private_data; 467 468 switch (cmd) { 469 case SW_SYNC_IOC_CREATE_FENCE: 470 return sw_sync_ioctl_create_fence(obj, arg); 471 472 case SW_SYNC_IOC_INC: 473 return sw_sync_ioctl_inc(obj, arg); 474 475 case SW_SYNC_GET_DEADLINE: 476 return sw_sync_ioctl_get_deadline(obj, arg); 477 478 default: 479 return -ENOTTY; 480 } 481 } 482 483 const struct file_operations sw_sync_debugfs_fops = { 484 .open = sw_sync_debugfs_open, 485 .release = sw_sync_debugfs_release, 486 .unlocked_ioctl = sw_sync_ioctl, 487 .compat_ioctl = compat_ptr_ioctl, 488 }; 489