1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_drv.h> 12 #include <drm/drm_file.h> 13 #include <uapi/drm/xe_drm.h> 14 15 #include "xe_device.h" 16 #include "xe_gt.h" 17 #include "xe_hw_engine_class_sysfs.h" 18 #include "xe_hw_engine_group.h" 19 #include "xe_hw_fence.h" 20 #include "xe_lrc.h" 21 #include "xe_macros.h" 22 #include "xe_migrate.h" 23 #include "xe_pm.h" 24 #include "xe_ring_ops_types.h" 25 #include "xe_trace.h" 26 #include "xe_vm.h" 27 28 enum xe_exec_queue_sched_prop { 29 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 30 XE_EXEC_QUEUE_TIMESLICE = 1, 31 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 32 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 33 }; 34 35 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 36 u64 extensions, int ext_number); 37 38 static void __xe_exec_queue_free(struct xe_exec_queue *q) 39 { 40 if (q->vm) 41 xe_vm_put(q->vm); 42 43 if (q->xef) 44 xe_file_put(q->xef); 45 46 kfree(q); 47 } 48 49 static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, 50 struct xe_vm *vm, 51 u32 logical_mask, 52 u16 width, struct xe_hw_engine *hwe, 53 u32 flags, u64 extensions) 54 { 55 struct xe_exec_queue *q; 56 struct xe_gt *gt = hwe->gt; 57 int err; 58 59 /* only kernel queues can be permanent */ 60 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 61 62 q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); 63 if (!q) 64 return ERR_PTR(-ENOMEM); 65 66 kref_init(&q->refcount); 67 q->flags = flags; 68 q->hwe = hwe; 69 q->gt = gt; 70 q->class = hwe->class; 71 q->width = width; 72 q->logical_mask = logical_mask; 73 q->fence_irq = >->fence_irq[hwe->class]; 74 q->ring_ops = gt->ring_ops[hwe->class]; 75 q->ops = gt->exec_queue_ops; 76 INIT_LIST_HEAD(&q->lr.link); 77 INIT_LIST_HEAD(&q->multi_gt_link); 78 INIT_LIST_HEAD(&q->hw_engine_group_link); 79 80 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 81 q->sched_props.preempt_timeout_us = 82 hwe->eclass->sched_props.preempt_timeout_us; 83 q->sched_props.job_timeout_ms = 84 hwe->eclass->sched_props.job_timeout_ms; 85 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && 86 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) 87 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; 88 else 89 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 90 91 if (vm) 92 q->vm = xe_vm_get(vm); 93 94 if (extensions) { 95 /* 96 * may set q->usm, must come before xe_lrc_create(), 97 * may overwrite q->sched_props, must come before q->ops->init() 98 */ 99 err = exec_queue_user_extensions(xe, q, extensions, 0); 100 if (err) { 101 __xe_exec_queue_free(q); 102 return ERR_PTR(err); 103 } 104 } 105 106 return q; 107 } 108 109 static int __xe_exec_queue_init(struct xe_exec_queue *q) 110 { 111 struct xe_vm *vm = q->vm; 112 int i, err; 113 114 if (vm) { 115 err = xe_vm_lock(vm, true); 116 if (err) 117 return err; 118 } 119 120 for (i = 0; i < q->width; ++i) { 121 q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K); 122 if (IS_ERR(q->lrc[i])) { 123 err = PTR_ERR(q->lrc[i]); 124 goto err_unlock; 125 } 126 } 127 128 if (vm) 129 xe_vm_unlock(vm); 130 131 err = q->ops->init(q); 132 if (err) 133 goto err_lrc; 134 135 return 0; 136 137 err_unlock: 138 if (vm) 139 xe_vm_unlock(vm); 140 err_lrc: 141 for (i = i - 1; i >= 0; --i) 142 xe_lrc_put(q->lrc[i]); 143 return err; 144 } 145 146 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 147 u32 logical_mask, u16 width, 148 struct xe_hw_engine *hwe, u32 flags, 149 u64 extensions) 150 { 151 struct xe_exec_queue *q; 152 int err; 153 154 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, 155 extensions); 156 if (IS_ERR(q)) 157 return q; 158 159 err = __xe_exec_queue_init(q); 160 if (err) 161 goto err_post_alloc; 162 163 return q; 164 165 err_post_alloc: 166 __xe_exec_queue_free(q); 167 return ERR_PTR(err); 168 } 169 170 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 171 struct xe_vm *vm, 172 enum xe_engine_class class, 173 u32 flags, u64 extensions) 174 { 175 struct xe_hw_engine *hwe, *hwe0 = NULL; 176 enum xe_hw_engine_id id; 177 u32 logical_mask = 0; 178 179 for_each_hw_engine(hwe, gt, id) { 180 if (xe_hw_engine_is_reserved(hwe)) 181 continue; 182 183 if (hwe->class == class) { 184 logical_mask |= BIT(hwe->logical_instance); 185 if (!hwe0) 186 hwe0 = hwe; 187 } 188 } 189 190 if (!logical_mask) 191 return ERR_PTR(-ENODEV); 192 193 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); 194 } 195 196 /** 197 * xe_exec_queue_create_bind() - Create bind exec queue. 198 * @xe: Xe device. 199 * @tile: tile which bind exec queue belongs to. 200 * @flags: exec queue creation flags 201 * @extensions: exec queue creation extensions 202 * 203 * Normalize bind exec queue creation. Bind exec queue is tied to migration VM 204 * for access to physical memory required for page table programming. On a 205 * faulting devices the reserved copy engine instance must be used to avoid 206 * deadlocking (user binds cannot get stuck behind faults as kernel binds which 207 * resolve faults depend on user binds). On non-faulting devices any copy engine 208 * can be used. 209 * 210 * Returns exec queue on success, ERR_PTR on failure 211 */ 212 struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, 213 struct xe_tile *tile, 214 u32 flags, u64 extensions) 215 { 216 struct xe_gt *gt = tile->primary_gt; 217 struct xe_exec_queue *q; 218 struct xe_vm *migrate_vm; 219 220 migrate_vm = xe_migrate_get_vm(tile->migrate); 221 if (xe->info.has_usm) { 222 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, 223 XE_ENGINE_CLASS_COPY, 224 gt->usm.reserved_bcs_instance, 225 false); 226 227 if (!hwe) { 228 xe_vm_put(migrate_vm); 229 return ERR_PTR(-EINVAL); 230 } 231 232 q = xe_exec_queue_create(xe, migrate_vm, 233 BIT(hwe->logical_instance), 1, hwe, 234 flags, extensions); 235 } else { 236 q = xe_exec_queue_create_class(xe, gt, migrate_vm, 237 XE_ENGINE_CLASS_COPY, flags, 238 extensions); 239 } 240 xe_vm_put(migrate_vm); 241 242 return q; 243 } 244 245 void xe_exec_queue_destroy(struct kref *ref) 246 { 247 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 248 struct xe_exec_queue *eq, *next; 249 250 xe_exec_queue_last_fence_put_unlocked(q); 251 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 252 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 253 multi_gt_link) 254 xe_exec_queue_put(eq); 255 } 256 257 q->ops->fini(q); 258 } 259 260 void xe_exec_queue_fini(struct xe_exec_queue *q) 261 { 262 int i; 263 264 /* 265 * Before releasing our ref to lrc and xef, accumulate our run ticks 266 */ 267 xe_exec_queue_update_run_ticks(q); 268 269 for (i = 0; i < q->width; ++i) 270 xe_lrc_put(q->lrc[i]); 271 272 __xe_exec_queue_free(q); 273 } 274 275 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 276 { 277 switch (q->class) { 278 case XE_ENGINE_CLASS_RENDER: 279 snprintf(q->name, sizeof(q->name), "rcs%d", instance); 280 break; 281 case XE_ENGINE_CLASS_VIDEO_DECODE: 282 snprintf(q->name, sizeof(q->name), "vcs%d", instance); 283 break; 284 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 285 snprintf(q->name, sizeof(q->name), "vecs%d", instance); 286 break; 287 case XE_ENGINE_CLASS_COPY: 288 snprintf(q->name, sizeof(q->name), "bcs%d", instance); 289 break; 290 case XE_ENGINE_CLASS_COMPUTE: 291 snprintf(q->name, sizeof(q->name), "ccs%d", instance); 292 break; 293 case XE_ENGINE_CLASS_OTHER: 294 snprintf(q->name, sizeof(q->name), "gsccs%d", instance); 295 break; 296 default: 297 XE_WARN_ON(q->class); 298 } 299 } 300 301 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 302 { 303 struct xe_exec_queue *q; 304 305 mutex_lock(&xef->exec_queue.lock); 306 q = xa_load(&xef->exec_queue.xa, id); 307 if (q) 308 xe_exec_queue_get(q); 309 mutex_unlock(&xef->exec_queue.lock); 310 311 return q; 312 } 313 314 enum xe_exec_queue_priority 315 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 316 { 317 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 318 XE_EXEC_QUEUE_PRIORITY_NORMAL; 319 } 320 321 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 322 u64 value) 323 { 324 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 325 return -EINVAL; 326 327 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 328 return -EPERM; 329 330 q->sched_props.priority = value; 331 return 0; 332 } 333 334 static bool xe_exec_queue_enforce_schedule_limit(void) 335 { 336 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 337 return true; 338 #else 339 return !capable(CAP_SYS_NICE); 340 #endif 341 } 342 343 static void 344 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 345 enum xe_exec_queue_sched_prop prop, 346 u32 *min, u32 *max) 347 { 348 switch (prop) { 349 case XE_EXEC_QUEUE_JOB_TIMEOUT: 350 *min = eclass->sched_props.job_timeout_min; 351 *max = eclass->sched_props.job_timeout_max; 352 break; 353 case XE_EXEC_QUEUE_TIMESLICE: 354 *min = eclass->sched_props.timeslice_min; 355 *max = eclass->sched_props.timeslice_max; 356 break; 357 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 358 *min = eclass->sched_props.preempt_timeout_min; 359 *max = eclass->sched_props.preempt_timeout_max; 360 break; 361 default: 362 break; 363 } 364 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 365 if (capable(CAP_SYS_NICE)) { 366 switch (prop) { 367 case XE_EXEC_QUEUE_JOB_TIMEOUT: 368 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 369 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 370 break; 371 case XE_EXEC_QUEUE_TIMESLICE: 372 *min = XE_HW_ENGINE_TIMESLICE_MIN; 373 *max = XE_HW_ENGINE_TIMESLICE_MAX; 374 break; 375 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 376 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 377 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 378 break; 379 default: 380 break; 381 } 382 } 383 #endif 384 } 385 386 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 387 u64 value) 388 { 389 u32 min = 0, max = 0; 390 391 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 392 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 393 394 if (xe_exec_queue_enforce_schedule_limit() && 395 !xe_hw_engine_timeout_in_range(value, min, max)) 396 return -EINVAL; 397 398 q->sched_props.timeslice_us = value; 399 return 0; 400 } 401 402 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 403 struct xe_exec_queue *q, 404 u64 value); 405 406 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 407 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 408 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 409 }; 410 411 static int exec_queue_user_ext_set_property(struct xe_device *xe, 412 struct xe_exec_queue *q, 413 u64 extension) 414 { 415 u64 __user *address = u64_to_user_ptr(extension); 416 struct drm_xe_ext_set_property ext; 417 int err; 418 u32 idx; 419 420 err = __copy_from_user(&ext, address, sizeof(ext)); 421 if (XE_IOCTL_DBG(xe, err)) 422 return -EFAULT; 423 424 if (XE_IOCTL_DBG(xe, ext.property >= 425 ARRAY_SIZE(exec_queue_set_property_funcs)) || 426 XE_IOCTL_DBG(xe, ext.pad) || 427 XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && 428 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE)) 429 return -EINVAL; 430 431 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 432 if (!exec_queue_set_property_funcs[idx]) 433 return -EINVAL; 434 435 return exec_queue_set_property_funcs[idx](xe, q, ext.value); 436 } 437 438 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 439 struct xe_exec_queue *q, 440 u64 extension); 441 442 static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { 443 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 444 }; 445 446 #define MAX_USER_EXTENSIONS 16 447 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 448 u64 extensions, int ext_number) 449 { 450 u64 __user *address = u64_to_user_ptr(extensions); 451 struct drm_xe_user_extension ext; 452 int err; 453 u32 idx; 454 455 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 456 return -E2BIG; 457 458 err = __copy_from_user(&ext, address, sizeof(ext)); 459 if (XE_IOCTL_DBG(xe, err)) 460 return -EFAULT; 461 462 if (XE_IOCTL_DBG(xe, ext.pad) || 463 XE_IOCTL_DBG(xe, ext.name >= 464 ARRAY_SIZE(exec_queue_user_extension_funcs))) 465 return -EINVAL; 466 467 idx = array_index_nospec(ext.name, 468 ARRAY_SIZE(exec_queue_user_extension_funcs)); 469 err = exec_queue_user_extension_funcs[idx](xe, q, extensions); 470 if (XE_IOCTL_DBG(xe, err)) 471 return err; 472 473 if (ext.next_extension) 474 return exec_queue_user_extensions(xe, q, ext.next_extension, 475 ++ext_number); 476 477 return 0; 478 } 479 480 static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, 481 struct drm_xe_engine_class_instance *eci, 482 u16 width, u16 num_placements) 483 { 484 int len = width * num_placements; 485 int i, j, n; 486 u16 class; 487 u16 gt_id; 488 u32 return_mask = 0, prev_mask; 489 490 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 491 len > 1)) 492 return 0; 493 494 for (i = 0; i < width; ++i) { 495 u32 current_mask = 0; 496 497 for (j = 0; j < num_placements; ++j) { 498 struct xe_hw_engine *hwe; 499 500 n = j * width + i; 501 502 hwe = xe_hw_engine_lookup(xe, eci[n]); 503 if (XE_IOCTL_DBG(xe, !hwe)) 504 return 0; 505 506 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 507 return 0; 508 509 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 510 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 511 return 0; 512 513 class = eci[n].engine_class; 514 gt_id = eci[n].gt_id; 515 516 if (width == 1 || !i) 517 return_mask |= BIT(eci[n].engine_instance); 518 current_mask |= BIT(eci[n].engine_instance); 519 } 520 521 /* Parallel submissions must be logically contiguous */ 522 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 523 return 0; 524 525 prev_mask = current_mask; 526 } 527 528 return return_mask; 529 } 530 531 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 532 struct drm_file *file) 533 { 534 struct xe_device *xe = to_xe_device(dev); 535 struct xe_file *xef = to_xe_file(file); 536 struct drm_xe_exec_queue_create *args = data; 537 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 538 struct drm_xe_engine_class_instance __user *user_eci = 539 u64_to_user_ptr(args->instances); 540 struct xe_hw_engine *hwe; 541 struct xe_vm *vm; 542 struct xe_gt *gt; 543 struct xe_tile *tile; 544 struct xe_exec_queue *q = NULL; 545 u32 logical_mask; 546 u32 id; 547 u32 len; 548 int err; 549 550 if (XE_IOCTL_DBG(xe, args->flags) || 551 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 552 return -EINVAL; 553 554 len = args->width * args->num_placements; 555 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 556 return -EINVAL; 557 558 err = __copy_from_user(eci, user_eci, 559 sizeof(struct drm_xe_engine_class_instance) * 560 len); 561 if (XE_IOCTL_DBG(xe, err)) 562 return -EFAULT; 563 564 if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) 565 return -EINVAL; 566 567 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { 568 if (XE_IOCTL_DBG(xe, args->width != 1) || 569 XE_IOCTL_DBG(xe, args->num_placements != 1) || 570 XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 571 return -EINVAL; 572 573 for_each_tile(tile, xe, id) { 574 struct xe_exec_queue *new; 575 u32 flags = EXEC_QUEUE_FLAG_VM; 576 577 if (id) 578 flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; 579 580 new = xe_exec_queue_create_bind(xe, tile, flags, 581 args->extensions); 582 if (IS_ERR(new)) { 583 err = PTR_ERR(new); 584 if (q) 585 goto put_exec_queue; 586 return err; 587 } 588 if (id == 0) 589 q = new; 590 else 591 list_add_tail(&new->multi_gt_list, 592 &q->multi_gt_link); 593 } 594 } else { 595 gt = xe_device_get_gt(xe, eci[0].gt_id); 596 logical_mask = calc_validate_logical_mask(xe, gt, eci, 597 args->width, 598 args->num_placements); 599 if (XE_IOCTL_DBG(xe, !logical_mask)) 600 return -EINVAL; 601 602 hwe = xe_hw_engine_lookup(xe, eci[0]); 603 if (XE_IOCTL_DBG(xe, !hwe)) 604 return -EINVAL; 605 606 vm = xe_vm_lookup(xef, args->vm_id); 607 if (XE_IOCTL_DBG(xe, !vm)) 608 return -ENOENT; 609 610 err = down_read_interruptible(&vm->lock); 611 if (err) { 612 xe_vm_put(vm); 613 return err; 614 } 615 616 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 617 up_read(&vm->lock); 618 xe_vm_put(vm); 619 return -ENOENT; 620 } 621 622 q = xe_exec_queue_create(xe, vm, logical_mask, 623 args->width, hwe, 0, 624 args->extensions); 625 up_read(&vm->lock); 626 xe_vm_put(vm); 627 if (IS_ERR(q)) 628 return PTR_ERR(q); 629 630 if (xe_vm_in_preempt_fence_mode(vm)) { 631 q->lr.context = dma_fence_context_alloc(1); 632 633 err = xe_vm_add_compute_exec_queue(vm, q); 634 if (XE_IOCTL_DBG(xe, err)) 635 goto put_exec_queue; 636 } 637 638 if (q->vm && q->hwe->hw_engine_group) { 639 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); 640 if (err) 641 goto put_exec_queue; 642 } 643 } 644 645 q->xef = xe_file_get(xef); 646 647 /* user id alloc must always be last in ioctl to prevent UAF */ 648 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 649 if (err) 650 goto kill_exec_queue; 651 652 args->exec_queue_id = id; 653 654 return 0; 655 656 kill_exec_queue: 657 xe_exec_queue_kill(q); 658 put_exec_queue: 659 xe_exec_queue_put(q); 660 return err; 661 } 662 663 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 664 struct drm_file *file) 665 { 666 struct xe_device *xe = to_xe_device(dev); 667 struct xe_file *xef = to_xe_file(file); 668 struct drm_xe_exec_queue_get_property *args = data; 669 struct xe_exec_queue *q; 670 int ret; 671 672 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 673 return -EINVAL; 674 675 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 676 if (XE_IOCTL_DBG(xe, !q)) 677 return -ENOENT; 678 679 switch (args->property) { 680 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: 681 args->value = q->ops->reset_status(q); 682 ret = 0; 683 break; 684 default: 685 ret = -EINVAL; 686 } 687 688 xe_exec_queue_put(q); 689 690 return ret; 691 } 692 693 /** 694 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 695 * @q: The exec_queue 696 * 697 * Return: True if the exec_queue is long-running, false otherwise. 698 */ 699 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 700 { 701 return q->vm && xe_vm_in_lr_mode(q->vm) && 702 !(q->flags & EXEC_QUEUE_FLAG_VM); 703 } 704 705 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 706 { 707 return q->lrc[0]->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc[0]) - 1; 708 } 709 710 /** 711 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 712 * @q: The exec_queue 713 * 714 * Return: True if the exec_queue's ring is full, false otherwise. 715 */ 716 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 717 { 718 struct xe_lrc *lrc = q->lrc[0]; 719 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 720 721 return xe_exec_queue_num_job_inflight(q) >= max_job; 722 } 723 724 /** 725 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 726 * @q: The exec_queue 727 * 728 * FIXME: Need to determine what to use as the short-lived 729 * timeline lock for the exec_queues, so that the return value 730 * of this function becomes more than just an advisory 731 * snapshot in time. The timeline lock must protect the 732 * seqno from racing submissions on the same exec_queue. 733 * Typically vm->resv, but user-created timeline locks use the migrate vm 734 * and never grabs the migrate vm->resv so we have a race there. 735 * 736 * Return: True if the exec_queue is idle, false otherwise. 737 */ 738 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 739 { 740 if (xe_exec_queue_is_parallel(q)) { 741 int i; 742 743 for (i = 0; i < q->width; ++i) { 744 if (xe_lrc_seqno(q->lrc[i]) != 745 q->lrc[i]->fence_ctx.next_seqno - 1) 746 return false; 747 } 748 749 return true; 750 } 751 752 return xe_lrc_seqno(q->lrc[0]) == 753 q->lrc[0]->fence_ctx.next_seqno - 1; 754 } 755 756 /** 757 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue 758 * from hw 759 * @q: The exec queue 760 * 761 * Update the timestamp saved by HW for this exec queue and save run ticks 762 * calculated by using the delta from last update. 763 */ 764 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) 765 { 766 struct xe_device *xe = gt_to_xe(q->gt); 767 struct xe_file *xef; 768 struct xe_lrc *lrc; 769 u32 old_ts, new_ts; 770 int idx; 771 772 /* 773 * Jobs that are run during driver load may use an exec_queue, but are 774 * not associated with a user xe file, so avoid accumulating busyness 775 * for kernel specific work. 776 */ 777 if (!q->vm || !q->vm->xef) 778 return; 779 780 /* Synchronize with unbind while holding the xe file open */ 781 if (!drm_dev_enter(&xe->drm, &idx)) 782 return; 783 784 xef = q->vm->xef; 785 786 /* 787 * Only sample the first LRC. For parallel submission, all of them are 788 * scheduled together and we compensate that below by multiplying by 789 * width - this may introduce errors if that premise is not true and 790 * they don't exit 100% aligned. On the other hand, looping through 791 * the LRCs and reading them in different time could also introduce 792 * errors. 793 */ 794 lrc = q->lrc[0]; 795 new_ts = xe_lrc_update_timestamp(lrc, &old_ts); 796 xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; 797 798 drm_dev_exit(idx); 799 } 800 801 /** 802 * xe_exec_queue_kill - permanently stop all execution from an exec queue 803 * @q: The exec queue 804 * 805 * This function permanently stops all activity on an exec queue. If the queue 806 * is actively executing on the HW, it will be kicked off the engine; any 807 * pending jobs are discarded and all future submissions are rejected. 808 * This function is safe to call multiple times. 809 */ 810 void xe_exec_queue_kill(struct xe_exec_queue *q) 811 { 812 struct xe_exec_queue *eq = q, *next; 813 814 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 815 multi_gt_link) { 816 q->ops->kill(eq); 817 xe_vm_remove_compute_exec_queue(q->vm, eq); 818 } 819 820 q->ops->kill(q); 821 xe_vm_remove_compute_exec_queue(q->vm, q); 822 } 823 824 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 825 struct drm_file *file) 826 { 827 struct xe_device *xe = to_xe_device(dev); 828 struct xe_file *xef = to_xe_file(file); 829 struct drm_xe_exec_queue_destroy *args = data; 830 struct xe_exec_queue *q; 831 832 if (XE_IOCTL_DBG(xe, args->pad) || 833 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 834 return -EINVAL; 835 836 mutex_lock(&xef->exec_queue.lock); 837 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 838 mutex_unlock(&xef->exec_queue.lock); 839 if (XE_IOCTL_DBG(xe, !q)) 840 return -ENOENT; 841 842 if (q->vm && q->hwe->hw_engine_group) 843 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 844 845 xe_exec_queue_kill(q); 846 847 trace_xe_exec_queue_close(q); 848 xe_exec_queue_put(q); 849 850 return 0; 851 } 852 853 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 854 struct xe_vm *vm) 855 { 856 if (q->flags & EXEC_QUEUE_FLAG_VM) { 857 lockdep_assert_held(&vm->lock); 858 } else { 859 xe_vm_assert_held(vm); 860 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); 861 } 862 } 863 864 /** 865 * xe_exec_queue_last_fence_put() - Drop ref to last fence 866 * @q: The exec queue 867 * @vm: The VM the engine does a bind or exec for 868 */ 869 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 870 { 871 xe_exec_queue_last_fence_lockdep_assert(q, vm); 872 873 xe_exec_queue_last_fence_put_unlocked(q); 874 } 875 876 /** 877 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 878 * @q: The exec queue 879 * 880 * Only safe to be called from xe_exec_queue_destroy(). 881 */ 882 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 883 { 884 if (q->last_fence) { 885 dma_fence_put(q->last_fence); 886 q->last_fence = NULL; 887 } 888 } 889 890 /** 891 * xe_exec_queue_last_fence_get() - Get last fence 892 * @q: The exec queue 893 * @vm: The VM the engine does a bind or exec for 894 * 895 * Get last fence, takes a ref 896 * 897 * Returns: last fence if not signaled, dma fence stub if signaled 898 */ 899 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 900 struct xe_vm *vm) 901 { 902 struct dma_fence *fence; 903 904 xe_exec_queue_last_fence_lockdep_assert(q, vm); 905 906 if (q->last_fence && 907 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 908 xe_exec_queue_last_fence_put(q, vm); 909 910 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 911 dma_fence_get(fence); 912 return fence; 913 } 914 915 /** 916 * xe_exec_queue_last_fence_get_for_resume() - Get last fence 917 * @q: The exec queue 918 * @vm: The VM the engine does a bind or exec for 919 * 920 * Get last fence, takes a ref. Only safe to be called in the context of 921 * resuming the hw engine group's long-running exec queue, when the group 922 * semaphore is held. 923 * 924 * Returns: last fence if not signaled, dma fence stub if signaled 925 */ 926 struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q, 927 struct xe_vm *vm) 928 { 929 struct dma_fence *fence; 930 931 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); 932 933 if (q->last_fence && 934 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 935 xe_exec_queue_last_fence_put_unlocked(q); 936 937 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 938 dma_fence_get(fence); 939 return fence; 940 } 941 942 /** 943 * xe_exec_queue_last_fence_set() - Set last fence 944 * @q: The exec queue 945 * @vm: The VM the engine does a bind or exec for 946 * @fence: The fence 947 * 948 * Set the last fence for the engine. Increases reference count for fence, when 949 * closing engine xe_exec_queue_last_fence_put should be called. 950 */ 951 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 952 struct dma_fence *fence) 953 { 954 xe_exec_queue_last_fence_lockdep_assert(q, vm); 955 956 xe_exec_queue_last_fence_put(q, vm); 957 q->last_fence = dma_fence_get(fence); 958 } 959 960 /** 961 * xe_exec_queue_last_fence_test_dep - Test last fence dependency of queue 962 * @q: The exec queue 963 * @vm: The VM the engine does a bind or exec for 964 * 965 * Returns: 966 * -ETIME if there exists an unsignalled last fence dependency, zero otherwise. 967 */ 968 int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) 969 { 970 struct dma_fence *fence; 971 int err = 0; 972 973 fence = xe_exec_queue_last_fence_get(q, vm); 974 if (fence) { 975 err = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 976 0 : -ETIME; 977 dma_fence_put(fence); 978 } 979 980 return err; 981 } 982