1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_file.h> 12 #include <uapi/drm/xe_drm.h> 13 14 #include "xe_device.h" 15 #include "xe_gt.h" 16 #include "xe_hw_engine_class_sysfs.h" 17 #include "xe_hw_engine_group.h" 18 #include "xe_hw_fence.h" 19 #include "xe_irq.h" 20 #include "xe_lrc.h" 21 #include "xe_macros.h" 22 #include "xe_migrate.h" 23 #include "xe_pm.h" 24 #include "xe_ring_ops_types.h" 25 #include "xe_trace.h" 26 #include "xe_vm.h" 27 28 enum xe_exec_queue_sched_prop { 29 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 30 XE_EXEC_QUEUE_TIMESLICE = 1, 31 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 32 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 33 }; 34 35 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 36 u64 extensions, int ext_number); 37 38 static void __xe_exec_queue_free(struct xe_exec_queue *q) 39 { 40 if (q->vm) 41 xe_vm_put(q->vm); 42 43 if (q->xef) 44 xe_file_put(q->xef); 45 46 kfree(q); 47 } 48 49 static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, 50 struct xe_vm *vm, 51 u32 logical_mask, 52 u16 width, struct xe_hw_engine *hwe, 53 u32 flags, u64 extensions) 54 { 55 struct xe_exec_queue *q; 56 struct xe_gt *gt = hwe->gt; 57 int err; 58 59 /* only kernel queues can be permanent */ 60 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 61 62 q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); 63 if (!q) 64 return ERR_PTR(-ENOMEM); 65 66 kref_init(&q->refcount); 67 q->flags = flags; 68 q->hwe = hwe; 69 q->gt = gt; 70 q->class = hwe->class; 71 q->width = width; 72 q->msix_vec = XE_IRQ_DEFAULT_MSIX; 73 q->logical_mask = logical_mask; 74 q->fence_irq = >->fence_irq[hwe->class]; 75 q->ring_ops = gt->ring_ops[hwe->class]; 76 q->ops = gt->exec_queue_ops; 77 INIT_LIST_HEAD(&q->lr.link); 78 INIT_LIST_HEAD(&q->multi_gt_link); 79 INIT_LIST_HEAD(&q->hw_engine_group_link); 80 81 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 82 q->sched_props.preempt_timeout_us = 83 hwe->eclass->sched_props.preempt_timeout_us; 84 q->sched_props.job_timeout_ms = 85 hwe->eclass->sched_props.job_timeout_ms; 86 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && 87 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) 88 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; 89 else 90 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 91 92 if (vm) 93 q->vm = xe_vm_get(vm); 94 95 if (extensions) { 96 /* 97 * may set q->usm, must come before xe_lrc_create(), 98 * may overwrite q->sched_props, must come before q->ops->init() 99 */ 100 err = exec_queue_user_extensions(xe, q, extensions, 0); 101 if (err) { 102 __xe_exec_queue_free(q); 103 return ERR_PTR(err); 104 } 105 } 106 107 return q; 108 } 109 110 static int __xe_exec_queue_init(struct xe_exec_queue *q) 111 { 112 struct xe_vm *vm = q->vm; 113 int i, err; 114 115 if (vm) { 116 err = xe_vm_lock(vm, true); 117 if (err) 118 return err; 119 } 120 121 for (i = 0; i < q->width; ++i) { 122 q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K, q->msix_vec); 123 if (IS_ERR(q->lrc[i])) { 124 err = PTR_ERR(q->lrc[i]); 125 goto err_unlock; 126 } 127 } 128 129 if (vm) 130 xe_vm_unlock(vm); 131 132 err = q->ops->init(q); 133 if (err) 134 goto err_lrc; 135 136 return 0; 137 138 err_unlock: 139 if (vm) 140 xe_vm_unlock(vm); 141 err_lrc: 142 for (i = i - 1; i >= 0; --i) 143 xe_lrc_put(q->lrc[i]); 144 return err; 145 } 146 147 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 148 u32 logical_mask, u16 width, 149 struct xe_hw_engine *hwe, u32 flags, 150 u64 extensions) 151 { 152 struct xe_exec_queue *q; 153 int err; 154 155 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, 156 extensions); 157 if (IS_ERR(q)) 158 return q; 159 160 err = __xe_exec_queue_init(q); 161 if (err) 162 goto err_post_alloc; 163 164 return q; 165 166 err_post_alloc: 167 __xe_exec_queue_free(q); 168 return ERR_PTR(err); 169 } 170 171 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 172 struct xe_vm *vm, 173 enum xe_engine_class class, 174 u32 flags, u64 extensions) 175 { 176 struct xe_hw_engine *hwe, *hwe0 = NULL; 177 enum xe_hw_engine_id id; 178 u32 logical_mask = 0; 179 180 for_each_hw_engine(hwe, gt, id) { 181 if (xe_hw_engine_is_reserved(hwe)) 182 continue; 183 184 if (hwe->class == class) { 185 logical_mask |= BIT(hwe->logical_instance); 186 if (!hwe0) 187 hwe0 = hwe; 188 } 189 } 190 191 if (!logical_mask) 192 return ERR_PTR(-ENODEV); 193 194 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); 195 } 196 197 /** 198 * xe_exec_queue_create_bind() - Create bind exec queue. 199 * @xe: Xe device. 200 * @tile: tile which bind exec queue belongs to. 201 * @flags: exec queue creation flags 202 * @extensions: exec queue creation extensions 203 * 204 * Normalize bind exec queue creation. Bind exec queue is tied to migration VM 205 * for access to physical memory required for page table programming. On a 206 * faulting devices the reserved copy engine instance must be used to avoid 207 * deadlocking (user binds cannot get stuck behind faults as kernel binds which 208 * resolve faults depend on user binds). On non-faulting devices any copy engine 209 * can be used. 210 * 211 * Returns exec queue on success, ERR_PTR on failure 212 */ 213 struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, 214 struct xe_tile *tile, 215 u32 flags, u64 extensions) 216 { 217 struct xe_gt *gt = tile->primary_gt; 218 struct xe_exec_queue *q; 219 struct xe_vm *migrate_vm; 220 221 migrate_vm = xe_migrate_get_vm(tile->migrate); 222 if (xe->info.has_usm) { 223 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, 224 XE_ENGINE_CLASS_COPY, 225 gt->usm.reserved_bcs_instance, 226 false); 227 228 if (!hwe) { 229 xe_vm_put(migrate_vm); 230 return ERR_PTR(-EINVAL); 231 } 232 233 q = xe_exec_queue_create(xe, migrate_vm, 234 BIT(hwe->logical_instance), 1, hwe, 235 flags, extensions); 236 } else { 237 q = xe_exec_queue_create_class(xe, gt, migrate_vm, 238 XE_ENGINE_CLASS_COPY, flags, 239 extensions); 240 } 241 xe_vm_put(migrate_vm); 242 243 return q; 244 } 245 ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO); 246 247 void xe_exec_queue_destroy(struct kref *ref) 248 { 249 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 250 struct xe_exec_queue *eq, *next; 251 252 xe_exec_queue_last_fence_put_unlocked(q); 253 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 254 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 255 multi_gt_link) 256 xe_exec_queue_put(eq); 257 } 258 259 q->ops->fini(q); 260 } 261 262 void xe_exec_queue_fini(struct xe_exec_queue *q) 263 { 264 int i; 265 266 /* 267 * Before releasing our ref to lrc and xef, accumulate our run ticks 268 * and wakeup any waiters. 269 */ 270 xe_exec_queue_update_run_ticks(q); 271 if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal)) 272 wake_up_var(&q->xef->exec_queue.pending_removal); 273 274 for (i = 0; i < q->width; ++i) 275 xe_lrc_put(q->lrc[i]); 276 277 __xe_exec_queue_free(q); 278 } 279 280 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 281 { 282 switch (q->class) { 283 case XE_ENGINE_CLASS_RENDER: 284 snprintf(q->name, sizeof(q->name), "rcs%d", instance); 285 break; 286 case XE_ENGINE_CLASS_VIDEO_DECODE: 287 snprintf(q->name, sizeof(q->name), "vcs%d", instance); 288 break; 289 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 290 snprintf(q->name, sizeof(q->name), "vecs%d", instance); 291 break; 292 case XE_ENGINE_CLASS_COPY: 293 snprintf(q->name, sizeof(q->name), "bcs%d", instance); 294 break; 295 case XE_ENGINE_CLASS_COMPUTE: 296 snprintf(q->name, sizeof(q->name), "ccs%d", instance); 297 break; 298 case XE_ENGINE_CLASS_OTHER: 299 snprintf(q->name, sizeof(q->name), "gsccs%d", instance); 300 break; 301 default: 302 XE_WARN_ON(q->class); 303 } 304 } 305 306 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 307 { 308 struct xe_exec_queue *q; 309 310 mutex_lock(&xef->exec_queue.lock); 311 q = xa_load(&xef->exec_queue.xa, id); 312 if (q) 313 xe_exec_queue_get(q); 314 mutex_unlock(&xef->exec_queue.lock); 315 316 return q; 317 } 318 319 enum xe_exec_queue_priority 320 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 321 { 322 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 323 XE_EXEC_QUEUE_PRIORITY_NORMAL; 324 } 325 326 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 327 u64 value) 328 { 329 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 330 return -EINVAL; 331 332 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 333 return -EPERM; 334 335 q->sched_props.priority = value; 336 return 0; 337 } 338 339 static bool xe_exec_queue_enforce_schedule_limit(void) 340 { 341 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 342 return true; 343 #else 344 return !capable(CAP_SYS_NICE); 345 #endif 346 } 347 348 static void 349 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 350 enum xe_exec_queue_sched_prop prop, 351 u32 *min, u32 *max) 352 { 353 switch (prop) { 354 case XE_EXEC_QUEUE_JOB_TIMEOUT: 355 *min = eclass->sched_props.job_timeout_min; 356 *max = eclass->sched_props.job_timeout_max; 357 break; 358 case XE_EXEC_QUEUE_TIMESLICE: 359 *min = eclass->sched_props.timeslice_min; 360 *max = eclass->sched_props.timeslice_max; 361 break; 362 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 363 *min = eclass->sched_props.preempt_timeout_min; 364 *max = eclass->sched_props.preempt_timeout_max; 365 break; 366 default: 367 break; 368 } 369 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 370 if (capable(CAP_SYS_NICE)) { 371 switch (prop) { 372 case XE_EXEC_QUEUE_JOB_TIMEOUT: 373 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 374 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 375 break; 376 case XE_EXEC_QUEUE_TIMESLICE: 377 *min = XE_HW_ENGINE_TIMESLICE_MIN; 378 *max = XE_HW_ENGINE_TIMESLICE_MAX; 379 break; 380 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 381 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 382 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 383 break; 384 default: 385 break; 386 } 387 } 388 #endif 389 } 390 391 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 392 u64 value) 393 { 394 u32 min = 0, max = 0; 395 396 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 397 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 398 399 if (xe_exec_queue_enforce_schedule_limit() && 400 !xe_hw_engine_timeout_in_range(value, min, max)) 401 return -EINVAL; 402 403 q->sched_props.timeslice_us = value; 404 return 0; 405 } 406 407 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 408 struct xe_exec_queue *q, 409 u64 value); 410 411 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 412 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 413 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 414 }; 415 416 static int exec_queue_user_ext_set_property(struct xe_device *xe, 417 struct xe_exec_queue *q, 418 u64 extension) 419 { 420 u64 __user *address = u64_to_user_ptr(extension); 421 struct drm_xe_ext_set_property ext; 422 int err; 423 u32 idx; 424 425 err = __copy_from_user(&ext, address, sizeof(ext)); 426 if (XE_IOCTL_DBG(xe, err)) 427 return -EFAULT; 428 429 if (XE_IOCTL_DBG(xe, ext.property >= 430 ARRAY_SIZE(exec_queue_set_property_funcs)) || 431 XE_IOCTL_DBG(xe, ext.pad) || 432 XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && 433 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE)) 434 return -EINVAL; 435 436 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 437 if (!exec_queue_set_property_funcs[idx]) 438 return -EINVAL; 439 440 return exec_queue_set_property_funcs[idx](xe, q, ext.value); 441 } 442 443 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 444 struct xe_exec_queue *q, 445 u64 extension); 446 447 static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { 448 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 449 }; 450 451 #define MAX_USER_EXTENSIONS 16 452 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 453 u64 extensions, int ext_number) 454 { 455 u64 __user *address = u64_to_user_ptr(extensions); 456 struct drm_xe_user_extension ext; 457 int err; 458 u32 idx; 459 460 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 461 return -E2BIG; 462 463 err = __copy_from_user(&ext, address, sizeof(ext)); 464 if (XE_IOCTL_DBG(xe, err)) 465 return -EFAULT; 466 467 if (XE_IOCTL_DBG(xe, ext.pad) || 468 XE_IOCTL_DBG(xe, ext.name >= 469 ARRAY_SIZE(exec_queue_user_extension_funcs))) 470 return -EINVAL; 471 472 idx = array_index_nospec(ext.name, 473 ARRAY_SIZE(exec_queue_user_extension_funcs)); 474 err = exec_queue_user_extension_funcs[idx](xe, q, extensions); 475 if (XE_IOCTL_DBG(xe, err)) 476 return err; 477 478 if (ext.next_extension) 479 return exec_queue_user_extensions(xe, q, ext.next_extension, 480 ++ext_number); 481 482 return 0; 483 } 484 485 static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, 486 struct drm_xe_engine_class_instance *eci, 487 u16 width, u16 num_placements) 488 { 489 int len = width * num_placements; 490 int i, j, n; 491 u16 class; 492 u16 gt_id; 493 u32 return_mask = 0, prev_mask; 494 495 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 496 len > 1)) 497 return 0; 498 499 for (i = 0; i < width; ++i) { 500 u32 current_mask = 0; 501 502 for (j = 0; j < num_placements; ++j) { 503 struct xe_hw_engine *hwe; 504 505 n = j * width + i; 506 507 hwe = xe_hw_engine_lookup(xe, eci[n]); 508 if (XE_IOCTL_DBG(xe, !hwe)) 509 return 0; 510 511 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 512 return 0; 513 514 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 515 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 516 return 0; 517 518 class = eci[n].engine_class; 519 gt_id = eci[n].gt_id; 520 521 if (width == 1 || !i) 522 return_mask |= BIT(eci[n].engine_instance); 523 current_mask |= BIT(eci[n].engine_instance); 524 } 525 526 /* Parallel submissions must be logically contiguous */ 527 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 528 return 0; 529 530 prev_mask = current_mask; 531 } 532 533 return return_mask; 534 } 535 536 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 537 struct drm_file *file) 538 { 539 struct xe_device *xe = to_xe_device(dev); 540 struct xe_file *xef = to_xe_file(file); 541 struct drm_xe_exec_queue_create *args = data; 542 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 543 struct drm_xe_engine_class_instance __user *user_eci = 544 u64_to_user_ptr(args->instances); 545 struct xe_hw_engine *hwe; 546 struct xe_vm *vm; 547 struct xe_gt *gt; 548 struct xe_tile *tile; 549 struct xe_exec_queue *q = NULL; 550 u32 logical_mask; 551 u32 id; 552 u32 len; 553 int err; 554 555 if (XE_IOCTL_DBG(xe, args->flags) || 556 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 557 return -EINVAL; 558 559 len = args->width * args->num_placements; 560 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 561 return -EINVAL; 562 563 err = __copy_from_user(eci, user_eci, 564 sizeof(struct drm_xe_engine_class_instance) * 565 len); 566 if (XE_IOCTL_DBG(xe, err)) 567 return -EFAULT; 568 569 if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) 570 return -EINVAL; 571 572 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { 573 if (XE_IOCTL_DBG(xe, args->width != 1) || 574 XE_IOCTL_DBG(xe, args->num_placements != 1) || 575 XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 576 return -EINVAL; 577 578 for_each_tile(tile, xe, id) { 579 struct xe_exec_queue *new; 580 u32 flags = EXEC_QUEUE_FLAG_VM; 581 582 if (id) 583 flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; 584 585 new = xe_exec_queue_create_bind(xe, tile, flags, 586 args->extensions); 587 if (IS_ERR(new)) { 588 err = PTR_ERR(new); 589 if (q) 590 goto put_exec_queue; 591 return err; 592 } 593 if (id == 0) 594 q = new; 595 else 596 list_add_tail(&new->multi_gt_list, 597 &q->multi_gt_link); 598 } 599 } else { 600 gt = xe_device_get_gt(xe, eci[0].gt_id); 601 logical_mask = calc_validate_logical_mask(xe, gt, eci, 602 args->width, 603 args->num_placements); 604 if (XE_IOCTL_DBG(xe, !logical_mask)) 605 return -EINVAL; 606 607 hwe = xe_hw_engine_lookup(xe, eci[0]); 608 if (XE_IOCTL_DBG(xe, !hwe)) 609 return -EINVAL; 610 611 vm = xe_vm_lookup(xef, args->vm_id); 612 if (XE_IOCTL_DBG(xe, !vm)) 613 return -ENOENT; 614 615 err = down_read_interruptible(&vm->lock); 616 if (err) { 617 xe_vm_put(vm); 618 return err; 619 } 620 621 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 622 up_read(&vm->lock); 623 xe_vm_put(vm); 624 return -ENOENT; 625 } 626 627 q = xe_exec_queue_create(xe, vm, logical_mask, 628 args->width, hwe, 0, 629 args->extensions); 630 up_read(&vm->lock); 631 xe_vm_put(vm); 632 if (IS_ERR(q)) 633 return PTR_ERR(q); 634 635 if (xe_vm_in_preempt_fence_mode(vm)) { 636 q->lr.context = dma_fence_context_alloc(1); 637 638 err = xe_vm_add_compute_exec_queue(vm, q); 639 if (XE_IOCTL_DBG(xe, err)) 640 goto put_exec_queue; 641 } 642 643 if (q->vm && q->hwe->hw_engine_group) { 644 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); 645 if (err) 646 goto put_exec_queue; 647 } 648 } 649 650 q->xef = xe_file_get(xef); 651 652 /* user id alloc must always be last in ioctl to prevent UAF */ 653 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 654 if (err) 655 goto kill_exec_queue; 656 657 args->exec_queue_id = id; 658 659 return 0; 660 661 kill_exec_queue: 662 xe_exec_queue_kill(q); 663 put_exec_queue: 664 xe_exec_queue_put(q); 665 return err; 666 } 667 668 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 669 struct drm_file *file) 670 { 671 struct xe_device *xe = to_xe_device(dev); 672 struct xe_file *xef = to_xe_file(file); 673 struct drm_xe_exec_queue_get_property *args = data; 674 struct xe_exec_queue *q; 675 int ret; 676 677 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 678 return -EINVAL; 679 680 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 681 if (XE_IOCTL_DBG(xe, !q)) 682 return -ENOENT; 683 684 switch (args->property) { 685 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: 686 args->value = q->ops->reset_status(q); 687 ret = 0; 688 break; 689 default: 690 ret = -EINVAL; 691 } 692 693 xe_exec_queue_put(q); 694 695 return ret; 696 } 697 698 /** 699 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 700 * @q: The exec_queue 701 * 702 * Return: True if the exec_queue is long-running, false otherwise. 703 */ 704 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 705 { 706 return q->vm && xe_vm_in_lr_mode(q->vm) && 707 !(q->flags & EXEC_QUEUE_FLAG_VM); 708 } 709 710 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 711 { 712 return q->lrc[0]->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc[0]) - 1; 713 } 714 715 /** 716 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 717 * @q: The exec_queue 718 * 719 * Return: True if the exec_queue's ring is full, false otherwise. 720 */ 721 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 722 { 723 struct xe_lrc *lrc = q->lrc[0]; 724 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 725 726 return xe_exec_queue_num_job_inflight(q) >= max_job; 727 } 728 729 /** 730 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 731 * @q: The exec_queue 732 * 733 * FIXME: Need to determine what to use as the short-lived 734 * timeline lock for the exec_queues, so that the return value 735 * of this function becomes more than just an advisory 736 * snapshot in time. The timeline lock must protect the 737 * seqno from racing submissions on the same exec_queue. 738 * Typically vm->resv, but user-created timeline locks use the migrate vm 739 * and never grabs the migrate vm->resv so we have a race there. 740 * 741 * Return: True if the exec_queue is idle, false otherwise. 742 */ 743 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 744 { 745 if (xe_exec_queue_is_parallel(q)) { 746 int i; 747 748 for (i = 0; i < q->width; ++i) { 749 if (xe_lrc_seqno(q->lrc[i]) != 750 q->lrc[i]->fence_ctx.next_seqno - 1) 751 return false; 752 } 753 754 return true; 755 } 756 757 return xe_lrc_seqno(q->lrc[0]) == 758 q->lrc[0]->fence_ctx.next_seqno - 1; 759 } 760 761 /** 762 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue 763 * from hw 764 * @q: The exec queue 765 * 766 * Update the timestamp saved by HW for this exec queue and save run ticks 767 * calculated by using the delta from last update. 768 */ 769 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) 770 { 771 struct xe_file *xef; 772 struct xe_lrc *lrc; 773 u32 old_ts, new_ts; 774 775 /* 776 * Jobs that are run during driver load may use an exec_queue, but are 777 * not associated with a user xe file, so avoid accumulating busyness 778 * for kernel specific work. 779 */ 780 if (!q->vm || !q->vm->xef) 781 return; 782 783 xef = q->vm->xef; 784 785 /* 786 * Only sample the first LRC. For parallel submission, all of them are 787 * scheduled together and we compensate that below by multiplying by 788 * width - this may introduce errors if that premise is not true and 789 * they don't exit 100% aligned. On the other hand, looping through 790 * the LRCs and reading them in different time could also introduce 791 * errors. 792 */ 793 lrc = q->lrc[0]; 794 new_ts = xe_lrc_update_timestamp(lrc, &old_ts); 795 xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; 796 } 797 798 /** 799 * xe_exec_queue_kill - permanently stop all execution from an exec queue 800 * @q: The exec queue 801 * 802 * This function permanently stops all activity on an exec queue. If the queue 803 * is actively executing on the HW, it will be kicked off the engine; any 804 * pending jobs are discarded and all future submissions are rejected. 805 * This function is safe to call multiple times. 806 */ 807 void xe_exec_queue_kill(struct xe_exec_queue *q) 808 { 809 struct xe_exec_queue *eq = q, *next; 810 811 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 812 multi_gt_link) { 813 q->ops->kill(eq); 814 xe_vm_remove_compute_exec_queue(q->vm, eq); 815 } 816 817 q->ops->kill(q); 818 xe_vm_remove_compute_exec_queue(q->vm, q); 819 } 820 821 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 822 struct drm_file *file) 823 { 824 struct xe_device *xe = to_xe_device(dev); 825 struct xe_file *xef = to_xe_file(file); 826 struct drm_xe_exec_queue_destroy *args = data; 827 struct xe_exec_queue *q; 828 829 if (XE_IOCTL_DBG(xe, args->pad) || 830 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 831 return -EINVAL; 832 833 mutex_lock(&xef->exec_queue.lock); 834 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 835 if (q) 836 atomic_inc(&xef->exec_queue.pending_removal); 837 mutex_unlock(&xef->exec_queue.lock); 838 839 if (XE_IOCTL_DBG(xe, !q)) 840 return -ENOENT; 841 842 if (q->vm && q->hwe->hw_engine_group) 843 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 844 845 xe_exec_queue_kill(q); 846 847 trace_xe_exec_queue_close(q); 848 xe_exec_queue_put(q); 849 850 return 0; 851 } 852 853 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 854 struct xe_vm *vm) 855 { 856 if (q->flags & EXEC_QUEUE_FLAG_VM) { 857 lockdep_assert_held(&vm->lock); 858 } else { 859 xe_vm_assert_held(vm); 860 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); 861 } 862 } 863 864 /** 865 * xe_exec_queue_last_fence_put() - Drop ref to last fence 866 * @q: The exec queue 867 * @vm: The VM the engine does a bind or exec for 868 */ 869 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 870 { 871 xe_exec_queue_last_fence_lockdep_assert(q, vm); 872 873 xe_exec_queue_last_fence_put_unlocked(q); 874 } 875 876 /** 877 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 878 * @q: The exec queue 879 * 880 * Only safe to be called from xe_exec_queue_destroy(). 881 */ 882 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 883 { 884 if (q->last_fence) { 885 dma_fence_put(q->last_fence); 886 q->last_fence = NULL; 887 } 888 } 889 890 /** 891 * xe_exec_queue_last_fence_get() - Get last fence 892 * @q: The exec queue 893 * @vm: The VM the engine does a bind or exec for 894 * 895 * Get last fence, takes a ref 896 * 897 * Returns: last fence if not signaled, dma fence stub if signaled 898 */ 899 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 900 struct xe_vm *vm) 901 { 902 struct dma_fence *fence; 903 904 xe_exec_queue_last_fence_lockdep_assert(q, vm); 905 906 if (q->last_fence && 907 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 908 xe_exec_queue_last_fence_put(q, vm); 909 910 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 911 dma_fence_get(fence); 912 return fence; 913 } 914 915 /** 916 * xe_exec_queue_last_fence_get_for_resume() - Get last fence 917 * @q: The exec queue 918 * @vm: The VM the engine does a bind or exec for 919 * 920 * Get last fence, takes a ref. Only safe to be called in the context of 921 * resuming the hw engine group's long-running exec queue, when the group 922 * semaphore is held. 923 * 924 * Returns: last fence if not signaled, dma fence stub if signaled 925 */ 926 struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q, 927 struct xe_vm *vm) 928 { 929 struct dma_fence *fence; 930 931 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); 932 933 if (q->last_fence && 934 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 935 xe_exec_queue_last_fence_put_unlocked(q); 936 937 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 938 dma_fence_get(fence); 939 return fence; 940 } 941 942 /** 943 * xe_exec_queue_last_fence_set() - Set last fence 944 * @q: The exec queue 945 * @vm: The VM the engine does a bind or exec for 946 * @fence: The fence 947 * 948 * Set the last fence for the engine. Increases reference count for fence, when 949 * closing engine xe_exec_queue_last_fence_put should be called. 950 */ 951 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 952 struct dma_fence *fence) 953 { 954 xe_exec_queue_last_fence_lockdep_assert(q, vm); 955 956 xe_exec_queue_last_fence_put(q, vm); 957 q->last_fence = dma_fence_get(fence); 958 } 959 960 /** 961 * xe_exec_queue_last_fence_test_dep - Test last fence dependency of queue 962 * @q: The exec queue 963 * @vm: The VM the engine does a bind or exec for 964 * 965 * Returns: 966 * -ETIME if there exists an unsignalled last fence dependency, zero otherwise. 967 */ 968 int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) 969 { 970 struct dma_fence *fence; 971 int err = 0; 972 973 fence = xe_exec_queue_last_fence_get(q, vm); 974 if (fence) { 975 err = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 976 0 : -ETIME; 977 dma_fence_put(fence); 978 } 979 980 return err; 981 } 982