1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_file.h> 12 #include <uapi/drm/xe_drm.h> 13 14 #include "xe_device.h" 15 #include "xe_gt.h" 16 #include "xe_hw_engine_class_sysfs.h" 17 #include "xe_hw_engine_group.h" 18 #include "xe_hw_fence.h" 19 #include "xe_lrc.h" 20 #include "xe_macros.h" 21 #include "xe_migrate.h" 22 #include "xe_pm.h" 23 #include "xe_ring_ops_types.h" 24 #include "xe_trace.h" 25 #include "xe_vm.h" 26 27 enum xe_exec_queue_sched_prop { 28 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 29 XE_EXEC_QUEUE_TIMESLICE = 1, 30 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 31 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 32 }; 33 34 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 35 u64 extensions, int ext_number); 36 37 static void __xe_exec_queue_free(struct xe_exec_queue *q) 38 { 39 if (q->vm) 40 xe_vm_put(q->vm); 41 42 if (q->xef) 43 xe_file_put(q->xef); 44 45 kfree(q); 46 } 47 48 static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, 49 struct xe_vm *vm, 50 u32 logical_mask, 51 u16 width, struct xe_hw_engine *hwe, 52 u32 flags, u64 extensions) 53 { 54 struct xe_exec_queue *q; 55 struct xe_gt *gt = hwe->gt; 56 int err; 57 58 /* only kernel queues can be permanent */ 59 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 60 61 q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); 62 if (!q) 63 return ERR_PTR(-ENOMEM); 64 65 kref_init(&q->refcount); 66 q->flags = flags; 67 q->hwe = hwe; 68 q->gt = gt; 69 q->class = hwe->class; 70 q->width = width; 71 q->logical_mask = logical_mask; 72 q->fence_irq = >->fence_irq[hwe->class]; 73 q->ring_ops = gt->ring_ops[hwe->class]; 74 q->ops = gt->exec_queue_ops; 75 INIT_LIST_HEAD(&q->lr.link); 76 INIT_LIST_HEAD(&q->multi_gt_link); 77 INIT_LIST_HEAD(&q->hw_engine_group_link); 78 79 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 80 q->sched_props.preempt_timeout_us = 81 hwe->eclass->sched_props.preempt_timeout_us; 82 q->sched_props.job_timeout_ms = 83 hwe->eclass->sched_props.job_timeout_ms; 84 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && 85 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) 86 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; 87 else 88 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 89 90 if (vm) 91 q->vm = xe_vm_get(vm); 92 93 if (extensions) { 94 /* 95 * may set q->usm, must come before xe_lrc_create(), 96 * may overwrite q->sched_props, must come before q->ops->init() 97 */ 98 err = exec_queue_user_extensions(xe, q, extensions, 0); 99 if (err) { 100 __xe_exec_queue_free(q); 101 return ERR_PTR(err); 102 } 103 } 104 105 return q; 106 } 107 108 static int __xe_exec_queue_init(struct xe_exec_queue *q) 109 { 110 struct xe_vm *vm = q->vm; 111 int i, err; 112 113 if (vm) { 114 err = xe_vm_lock(vm, true); 115 if (err) 116 return err; 117 } 118 119 for (i = 0; i < q->width; ++i) { 120 q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K); 121 if (IS_ERR(q->lrc[i])) { 122 err = PTR_ERR(q->lrc[i]); 123 goto err_unlock; 124 } 125 } 126 127 if (vm) 128 xe_vm_unlock(vm); 129 130 err = q->ops->init(q); 131 if (err) 132 goto err_lrc; 133 134 return 0; 135 136 err_unlock: 137 if (vm) 138 xe_vm_unlock(vm); 139 err_lrc: 140 for (i = i - 1; i >= 0; --i) 141 xe_lrc_put(q->lrc[i]); 142 return err; 143 } 144 145 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 146 u32 logical_mask, u16 width, 147 struct xe_hw_engine *hwe, u32 flags, 148 u64 extensions) 149 { 150 struct xe_exec_queue *q; 151 int err; 152 153 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, 154 extensions); 155 if (IS_ERR(q)) 156 return q; 157 158 err = __xe_exec_queue_init(q); 159 if (err) 160 goto err_post_alloc; 161 162 return q; 163 164 err_post_alloc: 165 __xe_exec_queue_free(q); 166 return ERR_PTR(err); 167 } 168 169 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 170 struct xe_vm *vm, 171 enum xe_engine_class class, 172 u32 flags, u64 extensions) 173 { 174 struct xe_hw_engine *hwe, *hwe0 = NULL; 175 enum xe_hw_engine_id id; 176 u32 logical_mask = 0; 177 178 for_each_hw_engine(hwe, gt, id) { 179 if (xe_hw_engine_is_reserved(hwe)) 180 continue; 181 182 if (hwe->class == class) { 183 logical_mask |= BIT(hwe->logical_instance); 184 if (!hwe0) 185 hwe0 = hwe; 186 } 187 } 188 189 if (!logical_mask) 190 return ERR_PTR(-ENODEV); 191 192 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); 193 } 194 195 /** 196 * xe_exec_queue_create_bind() - Create bind exec queue. 197 * @xe: Xe device. 198 * @tile: tile which bind exec queue belongs to. 199 * @flags: exec queue creation flags 200 * @extensions: exec queue creation extensions 201 * 202 * Normalize bind exec queue creation. Bind exec queue is tied to migration VM 203 * for access to physical memory required for page table programming. On a 204 * faulting devices the reserved copy engine instance must be used to avoid 205 * deadlocking (user binds cannot get stuck behind faults as kernel binds which 206 * resolve faults depend on user binds). On non-faulting devices any copy engine 207 * can be used. 208 * 209 * Returns exec queue on success, ERR_PTR on failure 210 */ 211 struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, 212 struct xe_tile *tile, 213 u32 flags, u64 extensions) 214 { 215 struct xe_gt *gt = tile->primary_gt; 216 struct xe_exec_queue *q; 217 struct xe_vm *migrate_vm; 218 219 migrate_vm = xe_migrate_get_vm(tile->migrate); 220 if (xe->info.has_usm) { 221 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, 222 XE_ENGINE_CLASS_COPY, 223 gt->usm.reserved_bcs_instance, 224 false); 225 226 if (!hwe) { 227 xe_vm_put(migrate_vm); 228 return ERR_PTR(-EINVAL); 229 } 230 231 q = xe_exec_queue_create(xe, migrate_vm, 232 BIT(hwe->logical_instance), 1, hwe, 233 flags, extensions); 234 } else { 235 q = xe_exec_queue_create_class(xe, gt, migrate_vm, 236 XE_ENGINE_CLASS_COPY, flags, 237 extensions); 238 } 239 xe_vm_put(migrate_vm); 240 241 return q; 242 } 243 ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO); 244 245 void xe_exec_queue_destroy(struct kref *ref) 246 { 247 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 248 struct xe_exec_queue *eq, *next; 249 250 xe_exec_queue_last_fence_put_unlocked(q); 251 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 252 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 253 multi_gt_link) 254 xe_exec_queue_put(eq); 255 } 256 257 q->ops->fini(q); 258 } 259 260 void xe_exec_queue_fini(struct xe_exec_queue *q) 261 { 262 int i; 263 264 /* 265 * Before releasing our ref to lrc and xef, accumulate our run ticks 266 * and wakeup any waiters. 267 */ 268 xe_exec_queue_update_run_ticks(q); 269 if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal)) 270 wake_up_var(&q->xef->exec_queue.pending_removal); 271 272 for (i = 0; i < q->width; ++i) 273 xe_lrc_put(q->lrc[i]); 274 275 __xe_exec_queue_free(q); 276 } 277 278 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 279 { 280 switch (q->class) { 281 case XE_ENGINE_CLASS_RENDER: 282 snprintf(q->name, sizeof(q->name), "rcs%d", instance); 283 break; 284 case XE_ENGINE_CLASS_VIDEO_DECODE: 285 snprintf(q->name, sizeof(q->name), "vcs%d", instance); 286 break; 287 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 288 snprintf(q->name, sizeof(q->name), "vecs%d", instance); 289 break; 290 case XE_ENGINE_CLASS_COPY: 291 snprintf(q->name, sizeof(q->name), "bcs%d", instance); 292 break; 293 case XE_ENGINE_CLASS_COMPUTE: 294 snprintf(q->name, sizeof(q->name), "ccs%d", instance); 295 break; 296 case XE_ENGINE_CLASS_OTHER: 297 snprintf(q->name, sizeof(q->name), "gsccs%d", instance); 298 break; 299 default: 300 XE_WARN_ON(q->class); 301 } 302 } 303 304 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 305 { 306 struct xe_exec_queue *q; 307 308 mutex_lock(&xef->exec_queue.lock); 309 q = xa_load(&xef->exec_queue.xa, id); 310 if (q) 311 xe_exec_queue_get(q); 312 mutex_unlock(&xef->exec_queue.lock); 313 314 return q; 315 } 316 317 enum xe_exec_queue_priority 318 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 319 { 320 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 321 XE_EXEC_QUEUE_PRIORITY_NORMAL; 322 } 323 324 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 325 u64 value) 326 { 327 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 328 return -EINVAL; 329 330 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 331 return -EPERM; 332 333 q->sched_props.priority = value; 334 return 0; 335 } 336 337 static bool xe_exec_queue_enforce_schedule_limit(void) 338 { 339 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 340 return true; 341 #else 342 return !capable(CAP_SYS_NICE); 343 #endif 344 } 345 346 static void 347 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 348 enum xe_exec_queue_sched_prop prop, 349 u32 *min, u32 *max) 350 { 351 switch (prop) { 352 case XE_EXEC_QUEUE_JOB_TIMEOUT: 353 *min = eclass->sched_props.job_timeout_min; 354 *max = eclass->sched_props.job_timeout_max; 355 break; 356 case XE_EXEC_QUEUE_TIMESLICE: 357 *min = eclass->sched_props.timeslice_min; 358 *max = eclass->sched_props.timeslice_max; 359 break; 360 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 361 *min = eclass->sched_props.preempt_timeout_min; 362 *max = eclass->sched_props.preempt_timeout_max; 363 break; 364 default: 365 break; 366 } 367 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 368 if (capable(CAP_SYS_NICE)) { 369 switch (prop) { 370 case XE_EXEC_QUEUE_JOB_TIMEOUT: 371 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 372 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 373 break; 374 case XE_EXEC_QUEUE_TIMESLICE: 375 *min = XE_HW_ENGINE_TIMESLICE_MIN; 376 *max = XE_HW_ENGINE_TIMESLICE_MAX; 377 break; 378 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 379 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 380 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 381 break; 382 default: 383 break; 384 } 385 } 386 #endif 387 } 388 389 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 390 u64 value) 391 { 392 u32 min = 0, max = 0; 393 394 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 395 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 396 397 if (xe_exec_queue_enforce_schedule_limit() && 398 !xe_hw_engine_timeout_in_range(value, min, max)) 399 return -EINVAL; 400 401 q->sched_props.timeslice_us = value; 402 return 0; 403 } 404 405 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 406 struct xe_exec_queue *q, 407 u64 value); 408 409 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 410 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 411 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 412 }; 413 414 static int exec_queue_user_ext_set_property(struct xe_device *xe, 415 struct xe_exec_queue *q, 416 u64 extension) 417 { 418 u64 __user *address = u64_to_user_ptr(extension); 419 struct drm_xe_ext_set_property ext; 420 int err; 421 u32 idx; 422 423 err = __copy_from_user(&ext, address, sizeof(ext)); 424 if (XE_IOCTL_DBG(xe, err)) 425 return -EFAULT; 426 427 if (XE_IOCTL_DBG(xe, ext.property >= 428 ARRAY_SIZE(exec_queue_set_property_funcs)) || 429 XE_IOCTL_DBG(xe, ext.pad) || 430 XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && 431 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE)) 432 return -EINVAL; 433 434 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 435 if (!exec_queue_set_property_funcs[idx]) 436 return -EINVAL; 437 438 return exec_queue_set_property_funcs[idx](xe, q, ext.value); 439 } 440 441 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 442 struct xe_exec_queue *q, 443 u64 extension); 444 445 static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { 446 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 447 }; 448 449 #define MAX_USER_EXTENSIONS 16 450 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 451 u64 extensions, int ext_number) 452 { 453 u64 __user *address = u64_to_user_ptr(extensions); 454 struct drm_xe_user_extension ext; 455 int err; 456 u32 idx; 457 458 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 459 return -E2BIG; 460 461 err = __copy_from_user(&ext, address, sizeof(ext)); 462 if (XE_IOCTL_DBG(xe, err)) 463 return -EFAULT; 464 465 if (XE_IOCTL_DBG(xe, ext.pad) || 466 XE_IOCTL_DBG(xe, ext.name >= 467 ARRAY_SIZE(exec_queue_user_extension_funcs))) 468 return -EINVAL; 469 470 idx = array_index_nospec(ext.name, 471 ARRAY_SIZE(exec_queue_user_extension_funcs)); 472 err = exec_queue_user_extension_funcs[idx](xe, q, extensions); 473 if (XE_IOCTL_DBG(xe, err)) 474 return err; 475 476 if (ext.next_extension) 477 return exec_queue_user_extensions(xe, q, ext.next_extension, 478 ++ext_number); 479 480 return 0; 481 } 482 483 static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, 484 struct drm_xe_engine_class_instance *eci, 485 u16 width, u16 num_placements) 486 { 487 int len = width * num_placements; 488 int i, j, n; 489 u16 class; 490 u16 gt_id; 491 u32 return_mask = 0, prev_mask; 492 493 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 494 len > 1)) 495 return 0; 496 497 for (i = 0; i < width; ++i) { 498 u32 current_mask = 0; 499 500 for (j = 0; j < num_placements; ++j) { 501 struct xe_hw_engine *hwe; 502 503 n = j * width + i; 504 505 hwe = xe_hw_engine_lookup(xe, eci[n]); 506 if (XE_IOCTL_DBG(xe, !hwe)) 507 return 0; 508 509 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 510 return 0; 511 512 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 513 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 514 return 0; 515 516 class = eci[n].engine_class; 517 gt_id = eci[n].gt_id; 518 519 if (width == 1 || !i) 520 return_mask |= BIT(eci[n].engine_instance); 521 current_mask |= BIT(eci[n].engine_instance); 522 } 523 524 /* Parallel submissions must be logically contiguous */ 525 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 526 return 0; 527 528 prev_mask = current_mask; 529 } 530 531 return return_mask; 532 } 533 534 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 535 struct drm_file *file) 536 { 537 struct xe_device *xe = to_xe_device(dev); 538 struct xe_file *xef = to_xe_file(file); 539 struct drm_xe_exec_queue_create *args = data; 540 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 541 struct drm_xe_engine_class_instance __user *user_eci = 542 u64_to_user_ptr(args->instances); 543 struct xe_hw_engine *hwe; 544 struct xe_vm *vm; 545 struct xe_gt *gt; 546 struct xe_tile *tile; 547 struct xe_exec_queue *q = NULL; 548 u32 logical_mask; 549 u32 id; 550 u32 len; 551 int err; 552 553 if (XE_IOCTL_DBG(xe, args->flags) || 554 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 555 return -EINVAL; 556 557 len = args->width * args->num_placements; 558 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 559 return -EINVAL; 560 561 err = __copy_from_user(eci, user_eci, 562 sizeof(struct drm_xe_engine_class_instance) * 563 len); 564 if (XE_IOCTL_DBG(xe, err)) 565 return -EFAULT; 566 567 if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) 568 return -EINVAL; 569 570 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { 571 if (XE_IOCTL_DBG(xe, args->width != 1) || 572 XE_IOCTL_DBG(xe, args->num_placements != 1) || 573 XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 574 return -EINVAL; 575 576 for_each_tile(tile, xe, id) { 577 struct xe_exec_queue *new; 578 u32 flags = EXEC_QUEUE_FLAG_VM; 579 580 if (id) 581 flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; 582 583 new = xe_exec_queue_create_bind(xe, tile, flags, 584 args->extensions); 585 if (IS_ERR(new)) { 586 err = PTR_ERR(new); 587 if (q) 588 goto put_exec_queue; 589 return err; 590 } 591 if (id == 0) 592 q = new; 593 else 594 list_add_tail(&new->multi_gt_list, 595 &q->multi_gt_link); 596 } 597 } else { 598 gt = xe_device_get_gt(xe, eci[0].gt_id); 599 logical_mask = calc_validate_logical_mask(xe, gt, eci, 600 args->width, 601 args->num_placements); 602 if (XE_IOCTL_DBG(xe, !logical_mask)) 603 return -EINVAL; 604 605 hwe = xe_hw_engine_lookup(xe, eci[0]); 606 if (XE_IOCTL_DBG(xe, !hwe)) 607 return -EINVAL; 608 609 vm = xe_vm_lookup(xef, args->vm_id); 610 if (XE_IOCTL_DBG(xe, !vm)) 611 return -ENOENT; 612 613 err = down_read_interruptible(&vm->lock); 614 if (err) { 615 xe_vm_put(vm); 616 return err; 617 } 618 619 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 620 up_read(&vm->lock); 621 xe_vm_put(vm); 622 return -ENOENT; 623 } 624 625 q = xe_exec_queue_create(xe, vm, logical_mask, 626 args->width, hwe, 0, 627 args->extensions); 628 up_read(&vm->lock); 629 xe_vm_put(vm); 630 if (IS_ERR(q)) 631 return PTR_ERR(q); 632 633 if (xe_vm_in_preempt_fence_mode(vm)) { 634 q->lr.context = dma_fence_context_alloc(1); 635 636 err = xe_vm_add_compute_exec_queue(vm, q); 637 if (XE_IOCTL_DBG(xe, err)) 638 goto put_exec_queue; 639 } 640 641 if (q->vm && q->hwe->hw_engine_group) { 642 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); 643 if (err) 644 goto put_exec_queue; 645 } 646 } 647 648 q->xef = xe_file_get(xef); 649 650 /* user id alloc must always be last in ioctl to prevent UAF */ 651 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 652 if (err) 653 goto kill_exec_queue; 654 655 args->exec_queue_id = id; 656 657 return 0; 658 659 kill_exec_queue: 660 xe_exec_queue_kill(q); 661 put_exec_queue: 662 xe_exec_queue_put(q); 663 return err; 664 } 665 666 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 667 struct drm_file *file) 668 { 669 struct xe_device *xe = to_xe_device(dev); 670 struct xe_file *xef = to_xe_file(file); 671 struct drm_xe_exec_queue_get_property *args = data; 672 struct xe_exec_queue *q; 673 int ret; 674 675 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 676 return -EINVAL; 677 678 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 679 if (XE_IOCTL_DBG(xe, !q)) 680 return -ENOENT; 681 682 switch (args->property) { 683 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: 684 args->value = q->ops->reset_status(q); 685 ret = 0; 686 break; 687 default: 688 ret = -EINVAL; 689 } 690 691 xe_exec_queue_put(q); 692 693 return ret; 694 } 695 696 /** 697 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 698 * @q: The exec_queue 699 * 700 * Return: True if the exec_queue is long-running, false otherwise. 701 */ 702 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 703 { 704 return q->vm && xe_vm_in_lr_mode(q->vm) && 705 !(q->flags & EXEC_QUEUE_FLAG_VM); 706 } 707 708 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 709 { 710 return q->lrc[0]->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc[0]) - 1; 711 } 712 713 /** 714 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 715 * @q: The exec_queue 716 * 717 * Return: True if the exec_queue's ring is full, false otherwise. 718 */ 719 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 720 { 721 struct xe_lrc *lrc = q->lrc[0]; 722 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 723 724 return xe_exec_queue_num_job_inflight(q) >= max_job; 725 } 726 727 /** 728 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 729 * @q: The exec_queue 730 * 731 * FIXME: Need to determine what to use as the short-lived 732 * timeline lock for the exec_queues, so that the return value 733 * of this function becomes more than just an advisory 734 * snapshot in time. The timeline lock must protect the 735 * seqno from racing submissions on the same exec_queue. 736 * Typically vm->resv, but user-created timeline locks use the migrate vm 737 * and never grabs the migrate vm->resv so we have a race there. 738 * 739 * Return: True if the exec_queue is idle, false otherwise. 740 */ 741 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 742 { 743 if (xe_exec_queue_is_parallel(q)) { 744 int i; 745 746 for (i = 0; i < q->width; ++i) { 747 if (xe_lrc_seqno(q->lrc[i]) != 748 q->lrc[i]->fence_ctx.next_seqno - 1) 749 return false; 750 } 751 752 return true; 753 } 754 755 return xe_lrc_seqno(q->lrc[0]) == 756 q->lrc[0]->fence_ctx.next_seqno - 1; 757 } 758 759 /** 760 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue 761 * from hw 762 * @q: The exec queue 763 * 764 * Update the timestamp saved by HW for this exec queue and save run ticks 765 * calculated by using the delta from last update. 766 */ 767 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) 768 { 769 struct xe_file *xef; 770 struct xe_lrc *lrc; 771 u32 old_ts, new_ts; 772 773 /* 774 * Jobs that are run during driver load may use an exec_queue, but are 775 * not associated with a user xe file, so avoid accumulating busyness 776 * for kernel specific work. 777 */ 778 if (!q->vm || !q->vm->xef) 779 return; 780 781 xef = q->vm->xef; 782 783 /* 784 * Only sample the first LRC. For parallel submission, all of them are 785 * scheduled together and we compensate that below by multiplying by 786 * width - this may introduce errors if that premise is not true and 787 * they don't exit 100% aligned. On the other hand, looping through 788 * the LRCs and reading them in different time could also introduce 789 * errors. 790 */ 791 lrc = q->lrc[0]; 792 new_ts = xe_lrc_update_timestamp(lrc, &old_ts); 793 xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; 794 } 795 796 /** 797 * xe_exec_queue_kill - permanently stop all execution from an exec queue 798 * @q: The exec queue 799 * 800 * This function permanently stops all activity on an exec queue. If the queue 801 * is actively executing on the HW, it will be kicked off the engine; any 802 * pending jobs are discarded and all future submissions are rejected. 803 * This function is safe to call multiple times. 804 */ 805 void xe_exec_queue_kill(struct xe_exec_queue *q) 806 { 807 struct xe_exec_queue *eq = q, *next; 808 809 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 810 multi_gt_link) { 811 q->ops->kill(eq); 812 xe_vm_remove_compute_exec_queue(q->vm, eq); 813 } 814 815 q->ops->kill(q); 816 xe_vm_remove_compute_exec_queue(q->vm, q); 817 } 818 819 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 820 struct drm_file *file) 821 { 822 struct xe_device *xe = to_xe_device(dev); 823 struct xe_file *xef = to_xe_file(file); 824 struct drm_xe_exec_queue_destroy *args = data; 825 struct xe_exec_queue *q; 826 827 if (XE_IOCTL_DBG(xe, args->pad) || 828 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 829 return -EINVAL; 830 831 mutex_lock(&xef->exec_queue.lock); 832 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 833 if (q) 834 atomic_inc(&xef->exec_queue.pending_removal); 835 mutex_unlock(&xef->exec_queue.lock); 836 837 if (XE_IOCTL_DBG(xe, !q)) 838 return -ENOENT; 839 840 if (q->vm && q->hwe->hw_engine_group) 841 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 842 843 xe_exec_queue_kill(q); 844 845 trace_xe_exec_queue_close(q); 846 xe_exec_queue_put(q); 847 848 return 0; 849 } 850 851 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 852 struct xe_vm *vm) 853 { 854 if (q->flags & EXEC_QUEUE_FLAG_VM) { 855 lockdep_assert_held(&vm->lock); 856 } else { 857 xe_vm_assert_held(vm); 858 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); 859 } 860 } 861 862 /** 863 * xe_exec_queue_last_fence_put() - Drop ref to last fence 864 * @q: The exec queue 865 * @vm: The VM the engine does a bind or exec for 866 */ 867 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 868 { 869 xe_exec_queue_last_fence_lockdep_assert(q, vm); 870 871 xe_exec_queue_last_fence_put_unlocked(q); 872 } 873 874 /** 875 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 876 * @q: The exec queue 877 * 878 * Only safe to be called from xe_exec_queue_destroy(). 879 */ 880 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 881 { 882 if (q->last_fence) { 883 dma_fence_put(q->last_fence); 884 q->last_fence = NULL; 885 } 886 } 887 888 /** 889 * xe_exec_queue_last_fence_get() - Get last fence 890 * @q: The exec queue 891 * @vm: The VM the engine does a bind or exec for 892 * 893 * Get last fence, takes a ref 894 * 895 * Returns: last fence if not signaled, dma fence stub if signaled 896 */ 897 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 898 struct xe_vm *vm) 899 { 900 struct dma_fence *fence; 901 902 xe_exec_queue_last_fence_lockdep_assert(q, vm); 903 904 if (q->last_fence && 905 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 906 xe_exec_queue_last_fence_put(q, vm); 907 908 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 909 dma_fence_get(fence); 910 return fence; 911 } 912 913 /** 914 * xe_exec_queue_last_fence_get_for_resume() - Get last fence 915 * @q: The exec queue 916 * @vm: The VM the engine does a bind or exec for 917 * 918 * Get last fence, takes a ref. Only safe to be called in the context of 919 * resuming the hw engine group's long-running exec queue, when the group 920 * semaphore is held. 921 * 922 * Returns: last fence if not signaled, dma fence stub if signaled 923 */ 924 struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q, 925 struct xe_vm *vm) 926 { 927 struct dma_fence *fence; 928 929 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); 930 931 if (q->last_fence && 932 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 933 xe_exec_queue_last_fence_put_unlocked(q); 934 935 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 936 dma_fence_get(fence); 937 return fence; 938 } 939 940 /** 941 * xe_exec_queue_last_fence_set() - Set last fence 942 * @q: The exec queue 943 * @vm: The VM the engine does a bind or exec for 944 * @fence: The fence 945 * 946 * Set the last fence for the engine. Increases reference count for fence, when 947 * closing engine xe_exec_queue_last_fence_put should be called. 948 */ 949 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 950 struct dma_fence *fence) 951 { 952 xe_exec_queue_last_fence_lockdep_assert(q, vm); 953 954 xe_exec_queue_last_fence_put(q, vm); 955 q->last_fence = dma_fence_get(fence); 956 } 957 958 /** 959 * xe_exec_queue_last_fence_test_dep - Test last fence dependency of queue 960 * @q: The exec queue 961 * @vm: The VM the engine does a bind or exec for 962 * 963 * Returns: 964 * -ETIME if there exists an unsignalled last fence dependency, zero otherwise. 965 */ 966 int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) 967 { 968 struct dma_fence *fence; 969 int err = 0; 970 971 fence = xe_exec_queue_last_fence_get(q, vm); 972 if (fence) { 973 err = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 974 0 : -ETIME; 975 dma_fence_put(fence); 976 } 977 978 return err; 979 } 980