1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_drv.h> 12 #include <drm/drm_file.h> 13 #include <drm/drm_syncobj.h> 14 #include <uapi/drm/xe_drm.h> 15 16 #include "xe_dep_scheduler.h" 17 #include "xe_device.h" 18 #include "xe_gt.h" 19 #include "xe_hw_engine_class_sysfs.h" 20 #include "xe_hw_engine_group.h" 21 #include "xe_hw_fence.h" 22 #include "xe_irq.h" 23 #include "xe_lrc.h" 24 #include "xe_macros.h" 25 #include "xe_migrate.h" 26 #include "xe_pm.h" 27 #include "xe_ring_ops_types.h" 28 #include "xe_trace.h" 29 #include "xe_vm.h" 30 #include "xe_pxp.h" 31 32 enum xe_exec_queue_sched_prop { 33 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 34 XE_EXEC_QUEUE_TIMESLICE = 1, 35 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 36 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 37 }; 38 39 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 40 u64 extensions, int ext_number); 41 42 static void __xe_exec_queue_free(struct xe_exec_queue *q) 43 { 44 int i; 45 46 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) 47 if (q->tlb_inval[i].dep_scheduler) 48 xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); 49 50 if (xe_exec_queue_uses_pxp(q)) 51 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); 52 if (q->vm) 53 xe_vm_put(q->vm); 54 55 if (q->xef) 56 xe_file_put(q->xef); 57 58 kfree(q); 59 } 60 61 static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q) 62 { 63 struct xe_tile *tile = gt_to_tile(q->gt); 64 int i; 65 66 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) { 67 struct xe_dep_scheduler *dep_scheduler; 68 struct xe_gt *gt; 69 struct workqueue_struct *wq; 70 71 if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT) 72 gt = tile->primary_gt; 73 else 74 gt = tile->media_gt; 75 76 if (!gt) 77 continue; 78 79 wq = gt->tlb_inval.job_wq; 80 81 #define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */ 82 dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, 83 MAX_TLB_INVAL_JOBS); 84 if (IS_ERR(dep_scheduler)) 85 return PTR_ERR(dep_scheduler); 86 87 q->tlb_inval[i].dep_scheduler = dep_scheduler; 88 } 89 #undef MAX_TLB_INVAL_JOBS 90 91 return 0; 92 } 93 94 static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, 95 struct xe_vm *vm, 96 u32 logical_mask, 97 u16 width, struct xe_hw_engine *hwe, 98 u32 flags, u64 extensions) 99 { 100 struct xe_exec_queue *q; 101 struct xe_gt *gt = hwe->gt; 102 int err; 103 104 /* only kernel queues can be permanent */ 105 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 106 107 q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); 108 if (!q) 109 return ERR_PTR(-ENOMEM); 110 111 kref_init(&q->refcount); 112 q->flags = flags; 113 q->hwe = hwe; 114 q->gt = gt; 115 q->class = hwe->class; 116 q->width = width; 117 q->msix_vec = XE_IRQ_DEFAULT_MSIX; 118 q->logical_mask = logical_mask; 119 q->fence_irq = >->fence_irq[hwe->class]; 120 q->ring_ops = gt->ring_ops[hwe->class]; 121 q->ops = gt->exec_queue_ops; 122 INIT_LIST_HEAD(&q->lr.link); 123 INIT_LIST_HEAD(&q->multi_gt_link); 124 INIT_LIST_HEAD(&q->hw_engine_group_link); 125 INIT_LIST_HEAD(&q->pxp.link); 126 127 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 128 q->sched_props.preempt_timeout_us = 129 hwe->eclass->sched_props.preempt_timeout_us; 130 q->sched_props.job_timeout_ms = 131 hwe->eclass->sched_props.job_timeout_ms; 132 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && 133 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) 134 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; 135 else 136 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 137 138 if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) { 139 err = alloc_dep_schedulers(xe, q); 140 if (err) { 141 __xe_exec_queue_free(q); 142 return ERR_PTR(err); 143 } 144 } 145 146 if (vm) 147 q->vm = xe_vm_get(vm); 148 149 if (extensions) { 150 /* 151 * may set q->usm, must come before xe_lrc_create(), 152 * may overwrite q->sched_props, must come before q->ops->init() 153 */ 154 err = exec_queue_user_extensions(xe, q, extensions, 0); 155 if (err) { 156 __xe_exec_queue_free(q); 157 return ERR_PTR(err); 158 } 159 } 160 161 return q; 162 } 163 164 static int __xe_exec_queue_init(struct xe_exec_queue *q) 165 { 166 int i, err; 167 u32 flags = 0; 168 169 /* 170 * PXP workloads executing on RCS or CCS must run in isolation (i.e. no 171 * other workload can use the EUs at the same time). On MTL this is done 172 * by setting the RUNALONE bit in the LRC, while starting on Xe2 there 173 * is a dedicated bit for it. 174 */ 175 if (xe_exec_queue_uses_pxp(q) && 176 (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) { 177 if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20) 178 flags |= XE_LRC_CREATE_PXP; 179 else 180 flags |= XE_LRC_CREATE_RUNALONE; 181 } 182 183 for (i = 0; i < q->width; ++i) { 184 q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K, q->msix_vec, flags); 185 if (IS_ERR(q->lrc[i])) { 186 err = PTR_ERR(q->lrc[i]); 187 goto err_lrc; 188 } 189 } 190 191 err = q->ops->init(q); 192 if (err) 193 goto err_lrc; 194 195 return 0; 196 197 err_lrc: 198 for (i = i - 1; i >= 0; --i) 199 xe_lrc_put(q->lrc[i]); 200 return err; 201 } 202 203 static void __xe_exec_queue_fini(struct xe_exec_queue *q) 204 { 205 int i; 206 207 q->ops->fini(q); 208 209 for (i = 0; i < q->width; ++i) 210 xe_lrc_put(q->lrc[i]); 211 } 212 213 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 214 u32 logical_mask, u16 width, 215 struct xe_hw_engine *hwe, u32 flags, 216 u64 extensions) 217 { 218 struct xe_exec_queue *q; 219 int err; 220 221 /* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */ 222 xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0))); 223 224 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, 225 extensions); 226 if (IS_ERR(q)) 227 return q; 228 229 err = __xe_exec_queue_init(q); 230 if (err) 231 goto err_post_alloc; 232 233 /* 234 * We can only add the queue to the PXP list after the init is complete, 235 * because the PXP termination can call exec_queue_kill and that will 236 * go bad if the queue is only half-initialized. This means that we 237 * can't do it when we handle the PXP extension in __xe_exec_queue_alloc 238 * and we need to do it here instead. 239 */ 240 if (xe_exec_queue_uses_pxp(q)) { 241 err = xe_pxp_exec_queue_add(xe->pxp, q); 242 if (err) 243 goto err_post_init; 244 } 245 246 return q; 247 248 err_post_init: 249 __xe_exec_queue_fini(q); 250 err_post_alloc: 251 __xe_exec_queue_free(q); 252 return ERR_PTR(err); 253 } 254 ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO); 255 256 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 257 struct xe_vm *vm, 258 enum xe_engine_class class, 259 u32 flags, u64 extensions) 260 { 261 struct xe_hw_engine *hwe, *hwe0 = NULL; 262 enum xe_hw_engine_id id; 263 u32 logical_mask = 0; 264 265 for_each_hw_engine(hwe, gt, id) { 266 if (xe_hw_engine_is_reserved(hwe)) 267 continue; 268 269 if (hwe->class == class) { 270 logical_mask |= BIT(hwe->logical_instance); 271 if (!hwe0) 272 hwe0 = hwe; 273 } 274 } 275 276 if (!logical_mask) 277 return ERR_PTR(-ENODEV); 278 279 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); 280 } 281 282 /** 283 * xe_exec_queue_create_bind() - Create bind exec queue. 284 * @xe: Xe device. 285 * @tile: tile which bind exec queue belongs to. 286 * @flags: exec queue creation flags 287 * @extensions: exec queue creation extensions 288 * 289 * Normalize bind exec queue creation. Bind exec queue is tied to migration VM 290 * for access to physical memory required for page table programming. On a 291 * faulting devices the reserved copy engine instance must be used to avoid 292 * deadlocking (user binds cannot get stuck behind faults as kernel binds which 293 * resolve faults depend on user binds). On non-faulting devices any copy engine 294 * can be used. 295 * 296 * Returns exec queue on success, ERR_PTR on failure 297 */ 298 struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, 299 struct xe_tile *tile, 300 u32 flags, u64 extensions) 301 { 302 struct xe_gt *gt = tile->primary_gt; 303 struct xe_exec_queue *q; 304 struct xe_vm *migrate_vm; 305 306 migrate_vm = xe_migrate_get_vm(tile->migrate); 307 if (xe->info.has_usm) { 308 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, 309 XE_ENGINE_CLASS_COPY, 310 gt->usm.reserved_bcs_instance, 311 false); 312 313 if (!hwe) { 314 xe_vm_put(migrate_vm); 315 return ERR_PTR(-EINVAL); 316 } 317 318 q = xe_exec_queue_create(xe, migrate_vm, 319 BIT(hwe->logical_instance), 1, hwe, 320 flags, extensions); 321 } else { 322 q = xe_exec_queue_create_class(xe, gt, migrate_vm, 323 XE_ENGINE_CLASS_COPY, flags, 324 extensions); 325 } 326 xe_vm_put(migrate_vm); 327 328 if (!IS_ERR(q)) { 329 int err = drm_syncobj_create(&q->ufence_syncobj, 330 DRM_SYNCOBJ_CREATE_SIGNALED, 331 NULL); 332 if (err) { 333 xe_exec_queue_put(q); 334 return ERR_PTR(err); 335 } 336 } 337 338 return q; 339 } 340 ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO); 341 342 void xe_exec_queue_destroy(struct kref *ref) 343 { 344 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 345 struct xe_exec_queue *eq, *next; 346 347 if (q->ufence_syncobj) 348 drm_syncobj_put(q->ufence_syncobj); 349 350 if (xe_exec_queue_uses_pxp(q)) 351 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); 352 353 xe_exec_queue_last_fence_put_unlocked(q); 354 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 355 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 356 multi_gt_link) 357 xe_exec_queue_put(eq); 358 } 359 360 q->ops->destroy(q); 361 } 362 363 void xe_exec_queue_fini(struct xe_exec_queue *q) 364 { 365 /* 366 * Before releasing our ref to lrc and xef, accumulate our run ticks 367 * and wakeup any waiters. 368 */ 369 xe_exec_queue_update_run_ticks(q); 370 if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal)) 371 wake_up_var(&q->xef->exec_queue.pending_removal); 372 373 __xe_exec_queue_fini(q); 374 __xe_exec_queue_free(q); 375 } 376 377 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 378 { 379 switch (q->class) { 380 case XE_ENGINE_CLASS_RENDER: 381 snprintf(q->name, sizeof(q->name), "rcs%d", instance); 382 break; 383 case XE_ENGINE_CLASS_VIDEO_DECODE: 384 snprintf(q->name, sizeof(q->name), "vcs%d", instance); 385 break; 386 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 387 snprintf(q->name, sizeof(q->name), "vecs%d", instance); 388 break; 389 case XE_ENGINE_CLASS_COPY: 390 snprintf(q->name, sizeof(q->name), "bcs%d", instance); 391 break; 392 case XE_ENGINE_CLASS_COMPUTE: 393 snprintf(q->name, sizeof(q->name), "ccs%d", instance); 394 break; 395 case XE_ENGINE_CLASS_OTHER: 396 snprintf(q->name, sizeof(q->name), "gsccs%d", instance); 397 break; 398 default: 399 XE_WARN_ON(q->class); 400 } 401 } 402 403 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 404 { 405 struct xe_exec_queue *q; 406 407 mutex_lock(&xef->exec_queue.lock); 408 q = xa_load(&xef->exec_queue.xa, id); 409 if (q) 410 xe_exec_queue_get(q); 411 mutex_unlock(&xef->exec_queue.lock); 412 413 return q; 414 } 415 416 enum xe_exec_queue_priority 417 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 418 { 419 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 420 XE_EXEC_QUEUE_PRIORITY_NORMAL; 421 } 422 423 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 424 u64 value) 425 { 426 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 427 return -EINVAL; 428 429 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 430 return -EPERM; 431 432 q->sched_props.priority = value; 433 return 0; 434 } 435 436 static bool xe_exec_queue_enforce_schedule_limit(void) 437 { 438 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 439 return true; 440 #else 441 return !capable(CAP_SYS_NICE); 442 #endif 443 } 444 445 static void 446 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 447 enum xe_exec_queue_sched_prop prop, 448 u32 *min, u32 *max) 449 { 450 switch (prop) { 451 case XE_EXEC_QUEUE_JOB_TIMEOUT: 452 *min = eclass->sched_props.job_timeout_min; 453 *max = eclass->sched_props.job_timeout_max; 454 break; 455 case XE_EXEC_QUEUE_TIMESLICE: 456 *min = eclass->sched_props.timeslice_min; 457 *max = eclass->sched_props.timeslice_max; 458 break; 459 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 460 *min = eclass->sched_props.preempt_timeout_min; 461 *max = eclass->sched_props.preempt_timeout_max; 462 break; 463 default: 464 break; 465 } 466 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 467 if (capable(CAP_SYS_NICE)) { 468 switch (prop) { 469 case XE_EXEC_QUEUE_JOB_TIMEOUT: 470 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 471 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 472 break; 473 case XE_EXEC_QUEUE_TIMESLICE: 474 *min = XE_HW_ENGINE_TIMESLICE_MIN; 475 *max = XE_HW_ENGINE_TIMESLICE_MAX; 476 break; 477 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 478 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 479 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 480 break; 481 default: 482 break; 483 } 484 } 485 #endif 486 } 487 488 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 489 u64 value) 490 { 491 u32 min = 0, max = 0; 492 493 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 494 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 495 496 if (xe_exec_queue_enforce_schedule_limit() && 497 !xe_hw_engine_timeout_in_range(value, min, max)) 498 return -EINVAL; 499 500 q->sched_props.timeslice_us = value; 501 return 0; 502 } 503 504 static int 505 exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value) 506 { 507 if (value == DRM_XE_PXP_TYPE_NONE) 508 return 0; 509 510 /* we only support HWDRM sessions right now */ 511 if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM)) 512 return -EINVAL; 513 514 if (!xe_pxp_is_enabled(xe->pxp)) 515 return -ENODEV; 516 517 return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM); 518 } 519 520 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 521 struct xe_exec_queue *q, 522 u64 value); 523 524 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 525 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 526 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 527 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type, 528 }; 529 530 static int exec_queue_user_ext_set_property(struct xe_device *xe, 531 struct xe_exec_queue *q, 532 u64 extension) 533 { 534 u64 __user *address = u64_to_user_ptr(extension); 535 struct drm_xe_ext_set_property ext; 536 int err; 537 u32 idx; 538 539 err = copy_from_user(&ext, address, sizeof(ext)); 540 if (XE_IOCTL_DBG(xe, err)) 541 return -EFAULT; 542 543 if (XE_IOCTL_DBG(xe, ext.property >= 544 ARRAY_SIZE(exec_queue_set_property_funcs)) || 545 XE_IOCTL_DBG(xe, ext.pad) || 546 XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && 547 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE && 548 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE)) 549 return -EINVAL; 550 551 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 552 if (!exec_queue_set_property_funcs[idx]) 553 return -EINVAL; 554 555 return exec_queue_set_property_funcs[idx](xe, q, ext.value); 556 } 557 558 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 559 struct xe_exec_queue *q, 560 u64 extension); 561 562 static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { 563 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 564 }; 565 566 #define MAX_USER_EXTENSIONS 16 567 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 568 u64 extensions, int ext_number) 569 { 570 u64 __user *address = u64_to_user_ptr(extensions); 571 struct drm_xe_user_extension ext; 572 int err; 573 u32 idx; 574 575 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 576 return -E2BIG; 577 578 err = copy_from_user(&ext, address, sizeof(ext)); 579 if (XE_IOCTL_DBG(xe, err)) 580 return -EFAULT; 581 582 if (XE_IOCTL_DBG(xe, ext.pad) || 583 XE_IOCTL_DBG(xe, ext.name >= 584 ARRAY_SIZE(exec_queue_user_extension_funcs))) 585 return -EINVAL; 586 587 idx = array_index_nospec(ext.name, 588 ARRAY_SIZE(exec_queue_user_extension_funcs)); 589 err = exec_queue_user_extension_funcs[idx](xe, q, extensions); 590 if (XE_IOCTL_DBG(xe, err)) 591 return err; 592 593 if (ext.next_extension) 594 return exec_queue_user_extensions(xe, q, ext.next_extension, 595 ++ext_number); 596 597 return 0; 598 } 599 600 static u32 calc_validate_logical_mask(struct xe_device *xe, 601 struct drm_xe_engine_class_instance *eci, 602 u16 width, u16 num_placements) 603 { 604 int len = width * num_placements; 605 int i, j, n; 606 u16 class; 607 u16 gt_id; 608 u32 return_mask = 0, prev_mask; 609 610 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 611 len > 1)) 612 return 0; 613 614 for (i = 0; i < width; ++i) { 615 u32 current_mask = 0; 616 617 for (j = 0; j < num_placements; ++j) { 618 struct xe_hw_engine *hwe; 619 620 n = j * width + i; 621 622 hwe = xe_hw_engine_lookup(xe, eci[n]); 623 if (XE_IOCTL_DBG(xe, !hwe)) 624 return 0; 625 626 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 627 return 0; 628 629 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 630 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 631 return 0; 632 633 class = eci[n].engine_class; 634 gt_id = eci[n].gt_id; 635 636 if (width == 1 || !i) 637 return_mask |= BIT(eci[n].engine_instance); 638 current_mask |= BIT(eci[n].engine_instance); 639 } 640 641 /* Parallel submissions must be logically contiguous */ 642 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 643 return 0; 644 645 prev_mask = current_mask; 646 } 647 648 return return_mask; 649 } 650 651 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 652 struct drm_file *file) 653 { 654 struct xe_device *xe = to_xe_device(dev); 655 struct xe_file *xef = to_xe_file(file); 656 struct drm_xe_exec_queue_create *args = data; 657 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 658 struct drm_xe_engine_class_instance __user *user_eci = 659 u64_to_user_ptr(args->instances); 660 struct xe_hw_engine *hwe; 661 struct xe_vm *vm; 662 struct xe_tile *tile; 663 struct xe_exec_queue *q = NULL; 664 u32 logical_mask; 665 u32 flags = 0; 666 u32 id; 667 u32 len; 668 int err; 669 670 if (XE_IOCTL_DBG(xe, args->flags & ~DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) || 671 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 672 return -EINVAL; 673 674 len = args->width * args->num_placements; 675 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 676 return -EINVAL; 677 678 err = copy_from_user(eci, user_eci, 679 sizeof(struct drm_xe_engine_class_instance) * len); 680 if (XE_IOCTL_DBG(xe, err)) 681 return -EFAULT; 682 683 if (XE_IOCTL_DBG(xe, !xe_device_get_gt(xe, eci[0].gt_id))) 684 return -EINVAL; 685 686 if (args->flags & DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) 687 flags |= EXEC_QUEUE_FLAG_LOW_LATENCY; 688 689 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { 690 if (XE_IOCTL_DBG(xe, args->width != 1) || 691 XE_IOCTL_DBG(xe, args->num_placements != 1) || 692 XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 693 return -EINVAL; 694 695 for_each_tile(tile, xe, id) { 696 struct xe_exec_queue *new; 697 698 flags |= EXEC_QUEUE_FLAG_VM; 699 if (id) 700 flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; 701 702 new = xe_exec_queue_create_bind(xe, tile, flags, 703 args->extensions); 704 if (IS_ERR(new)) { 705 err = PTR_ERR(new); 706 if (q) 707 goto put_exec_queue; 708 return err; 709 } 710 if (id == 0) 711 q = new; 712 else 713 list_add_tail(&new->multi_gt_list, 714 &q->multi_gt_link); 715 } 716 } else { 717 logical_mask = calc_validate_logical_mask(xe, eci, 718 args->width, 719 args->num_placements); 720 if (XE_IOCTL_DBG(xe, !logical_mask)) 721 return -EINVAL; 722 723 hwe = xe_hw_engine_lookup(xe, eci[0]); 724 if (XE_IOCTL_DBG(xe, !hwe)) 725 return -EINVAL; 726 727 vm = xe_vm_lookup(xef, args->vm_id); 728 if (XE_IOCTL_DBG(xe, !vm)) 729 return -ENOENT; 730 731 err = down_read_interruptible(&vm->lock); 732 if (err) { 733 xe_vm_put(vm); 734 return err; 735 } 736 737 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 738 up_read(&vm->lock); 739 xe_vm_put(vm); 740 return -ENOENT; 741 } 742 743 q = xe_exec_queue_create(xe, vm, logical_mask, 744 args->width, hwe, flags, 745 args->extensions); 746 up_read(&vm->lock); 747 xe_vm_put(vm); 748 if (IS_ERR(q)) 749 return PTR_ERR(q); 750 751 if (xe_vm_in_preempt_fence_mode(vm)) { 752 q->lr.context = dma_fence_context_alloc(1); 753 754 err = xe_vm_add_compute_exec_queue(vm, q); 755 if (XE_IOCTL_DBG(xe, err)) 756 goto put_exec_queue; 757 } 758 759 if (q->vm && q->hwe->hw_engine_group) { 760 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); 761 if (err) 762 goto put_exec_queue; 763 } 764 } 765 766 q->xef = xe_file_get(xef); 767 768 /* user id alloc must always be last in ioctl to prevent UAF */ 769 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 770 if (err) 771 goto kill_exec_queue; 772 773 args->exec_queue_id = id; 774 775 return 0; 776 777 kill_exec_queue: 778 xe_exec_queue_kill(q); 779 put_exec_queue: 780 xe_exec_queue_put(q); 781 return err; 782 } 783 784 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 785 struct drm_file *file) 786 { 787 struct xe_device *xe = to_xe_device(dev); 788 struct xe_file *xef = to_xe_file(file); 789 struct drm_xe_exec_queue_get_property *args = data; 790 struct xe_exec_queue *q; 791 int ret; 792 793 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 794 return -EINVAL; 795 796 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 797 if (XE_IOCTL_DBG(xe, !q)) 798 return -ENOENT; 799 800 switch (args->property) { 801 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: 802 args->value = q->ops->reset_status(q); 803 ret = 0; 804 break; 805 default: 806 ret = -EINVAL; 807 } 808 809 xe_exec_queue_put(q); 810 811 return ret; 812 } 813 814 /** 815 * xe_exec_queue_lrc() - Get the LRC from exec queue. 816 * @q: The exec_queue. 817 * 818 * Retrieves the primary LRC for the exec queue. Note that this function 819 * returns only the first LRC instance, even when multiple parallel LRCs 820 * are configured. 821 * 822 * Return: Pointer to LRC on success, error on failure 823 */ 824 struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q) 825 { 826 return q->lrc[0]; 827 } 828 829 /** 830 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 831 * @q: The exec_queue 832 * 833 * Return: True if the exec_queue is long-running, false otherwise. 834 */ 835 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 836 { 837 return q->vm && xe_vm_in_lr_mode(q->vm) && 838 !(q->flags & EXEC_QUEUE_FLAG_VM); 839 } 840 841 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 842 { 843 return q->lrc[0]->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc[0]) - 1; 844 } 845 846 /** 847 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 848 * @q: The exec_queue 849 * 850 * Return: True if the exec_queue's ring is full, false otherwise. 851 */ 852 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 853 { 854 struct xe_lrc *lrc = q->lrc[0]; 855 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 856 857 return xe_exec_queue_num_job_inflight(q) >= max_job; 858 } 859 860 /** 861 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 862 * @q: The exec_queue 863 * 864 * FIXME: Need to determine what to use as the short-lived 865 * timeline lock for the exec_queues, so that the return value 866 * of this function becomes more than just an advisory 867 * snapshot in time. The timeline lock must protect the 868 * seqno from racing submissions on the same exec_queue. 869 * Typically vm->resv, but user-created timeline locks use the migrate vm 870 * and never grabs the migrate vm->resv so we have a race there. 871 * 872 * Return: True if the exec_queue is idle, false otherwise. 873 */ 874 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 875 { 876 if (xe_exec_queue_is_parallel(q)) { 877 int i; 878 879 for (i = 0; i < q->width; ++i) { 880 if (xe_lrc_seqno(q->lrc[i]) != 881 q->lrc[i]->fence_ctx.next_seqno - 1) 882 return false; 883 } 884 885 return true; 886 } 887 888 return xe_lrc_seqno(q->lrc[0]) == 889 q->lrc[0]->fence_ctx.next_seqno - 1; 890 } 891 892 /** 893 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue 894 * from hw 895 * @q: The exec queue 896 * 897 * Update the timestamp saved by HW for this exec queue and save run ticks 898 * calculated by using the delta from last update. 899 */ 900 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) 901 { 902 struct xe_device *xe = gt_to_xe(q->gt); 903 struct xe_lrc *lrc; 904 u64 old_ts, new_ts; 905 int idx; 906 907 /* 908 * Jobs that are executed by kernel doesn't have a corresponding xe_file 909 * and thus are not accounted. 910 */ 911 if (!q->xef) 912 return; 913 914 /* Synchronize with unbind while holding the xe file open */ 915 if (!drm_dev_enter(&xe->drm, &idx)) 916 return; 917 /* 918 * Only sample the first LRC. For parallel submission, all of them are 919 * scheduled together and we compensate that below by multiplying by 920 * width - this may introduce errors if that premise is not true and 921 * they don't exit 100% aligned. On the other hand, looping through 922 * the LRCs and reading them in different time could also introduce 923 * errors. 924 */ 925 lrc = q->lrc[0]; 926 new_ts = xe_lrc_update_timestamp(lrc, &old_ts); 927 q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; 928 929 drm_dev_exit(idx); 930 } 931 932 /** 933 * xe_exec_queue_kill - permanently stop all execution from an exec queue 934 * @q: The exec queue 935 * 936 * This function permanently stops all activity on an exec queue. If the queue 937 * is actively executing on the HW, it will be kicked off the engine; any 938 * pending jobs are discarded and all future submissions are rejected. 939 * This function is safe to call multiple times. 940 */ 941 void xe_exec_queue_kill(struct xe_exec_queue *q) 942 { 943 struct xe_exec_queue *eq = q, *next; 944 945 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 946 multi_gt_link) { 947 q->ops->kill(eq); 948 xe_vm_remove_compute_exec_queue(q->vm, eq); 949 } 950 951 q->ops->kill(q); 952 xe_vm_remove_compute_exec_queue(q->vm, q); 953 } 954 955 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 956 struct drm_file *file) 957 { 958 struct xe_device *xe = to_xe_device(dev); 959 struct xe_file *xef = to_xe_file(file); 960 struct drm_xe_exec_queue_destroy *args = data; 961 struct xe_exec_queue *q; 962 963 if (XE_IOCTL_DBG(xe, args->pad) || 964 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 965 return -EINVAL; 966 967 mutex_lock(&xef->exec_queue.lock); 968 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 969 if (q) 970 atomic_inc(&xef->exec_queue.pending_removal); 971 mutex_unlock(&xef->exec_queue.lock); 972 973 if (XE_IOCTL_DBG(xe, !q)) 974 return -ENOENT; 975 976 if (q->vm && q->hwe->hw_engine_group) 977 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 978 979 xe_exec_queue_kill(q); 980 981 trace_xe_exec_queue_close(q); 982 xe_exec_queue_put(q); 983 984 return 0; 985 } 986 987 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 988 struct xe_vm *vm) 989 { 990 if (q->flags & EXEC_QUEUE_FLAG_VM) { 991 lockdep_assert_held(&vm->lock); 992 } else { 993 xe_vm_assert_held(vm); 994 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); 995 } 996 } 997 998 /** 999 * xe_exec_queue_last_fence_put() - Drop ref to last fence 1000 * @q: The exec queue 1001 * @vm: The VM the engine does a bind or exec for 1002 */ 1003 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 1004 { 1005 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1006 1007 xe_exec_queue_last_fence_put_unlocked(q); 1008 } 1009 1010 /** 1011 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 1012 * @q: The exec queue 1013 * 1014 * Only safe to be called from xe_exec_queue_destroy(). 1015 */ 1016 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 1017 { 1018 if (q->last_fence) { 1019 dma_fence_put(q->last_fence); 1020 q->last_fence = NULL; 1021 } 1022 } 1023 1024 /** 1025 * xe_exec_queue_last_fence_get() - Get last fence 1026 * @q: The exec queue 1027 * @vm: The VM the engine does a bind or exec for 1028 * 1029 * Get last fence, takes a ref 1030 * 1031 * Returns: last fence if not signaled, dma fence stub if signaled 1032 */ 1033 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 1034 struct xe_vm *vm) 1035 { 1036 struct dma_fence *fence; 1037 1038 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1039 1040 if (q->last_fence && 1041 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 1042 xe_exec_queue_last_fence_put(q, vm); 1043 1044 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 1045 dma_fence_get(fence); 1046 return fence; 1047 } 1048 1049 /** 1050 * xe_exec_queue_last_fence_get_for_resume() - Get last fence 1051 * @q: The exec queue 1052 * @vm: The VM the engine does a bind or exec for 1053 * 1054 * Get last fence, takes a ref. Only safe to be called in the context of 1055 * resuming the hw engine group's long-running exec queue, when the group 1056 * semaphore is held. 1057 * 1058 * Returns: last fence if not signaled, dma fence stub if signaled 1059 */ 1060 struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q, 1061 struct xe_vm *vm) 1062 { 1063 struct dma_fence *fence; 1064 1065 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); 1066 1067 if (q->last_fence && 1068 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 1069 xe_exec_queue_last_fence_put_unlocked(q); 1070 1071 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 1072 dma_fence_get(fence); 1073 return fence; 1074 } 1075 1076 /** 1077 * xe_exec_queue_last_fence_set() - Set last fence 1078 * @q: The exec queue 1079 * @vm: The VM the engine does a bind or exec for 1080 * @fence: The fence 1081 * 1082 * Set the last fence for the engine. Increases reference count for fence, when 1083 * closing engine xe_exec_queue_last_fence_put should be called. 1084 */ 1085 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 1086 struct dma_fence *fence) 1087 { 1088 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1089 1090 xe_exec_queue_last_fence_put(q, vm); 1091 q->last_fence = dma_fence_get(fence); 1092 } 1093 1094 /** 1095 * xe_exec_queue_last_fence_test_dep - Test last fence dependency of queue 1096 * @q: The exec queue 1097 * @vm: The VM the engine does a bind or exec for 1098 * 1099 * Returns: 1100 * -ETIME if there exists an unsignalled last fence dependency, zero otherwise. 1101 */ 1102 int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) 1103 { 1104 struct dma_fence *fence; 1105 int err = 0; 1106 1107 fence = xe_exec_queue_last_fence_get(q, vm); 1108 if (fence) { 1109 err = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 1110 0 : -ETIME; 1111 dma_fence_put(fence); 1112 } 1113 1114 return err; 1115 } 1116 1117 /** 1118 * xe_exec_queue_contexts_hwsp_rebase - Re-compute GGTT references 1119 * within all LRCs of a queue. 1120 * @q: the &xe_exec_queue struct instance containing target LRCs 1121 * @scratch: scratch buffer to be used as temporary storage 1122 * 1123 * Returns: zero on success, negative error code on failure 1124 */ 1125 int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch) 1126 { 1127 int i; 1128 int err = 0; 1129 1130 for (i = 0; i < q->width; ++i) { 1131 xe_lrc_update_memirq_regs_with_address(q->lrc[i], q->hwe, scratch); 1132 xe_lrc_update_hwctx_regs_with_address(q->lrc[i]); 1133 err = xe_lrc_setup_wa_bb_with_scratch(q->lrc[i], q->hwe, scratch); 1134 if (err) 1135 break; 1136 } 1137 1138 return err; 1139 } 1140 1141 /** 1142 * xe_exec_queue_jobs_ring_restore - Re-emit ring commands of requests pending on given queue. 1143 * @q: the &xe_exec_queue struct instance 1144 */ 1145 void xe_exec_queue_jobs_ring_restore(struct xe_exec_queue *q) 1146 { 1147 struct xe_gpu_scheduler *sched = &q->guc->sched; 1148 struct xe_sched_job *job; 1149 1150 /* 1151 * This routine is used within VF migration recovery. This means 1152 * using the lock here introduces a restriction: we cannot wait 1153 * for any GFX HW response while the lock is taken. 1154 */ 1155 spin_lock(&sched->base.job_list_lock); 1156 list_for_each_entry(job, &sched->base.pending_list, drm.list) { 1157 if (xe_sched_job_is_error(job)) 1158 continue; 1159 1160 q->ring_ops->emit_job(job); 1161 } 1162 spin_unlock(&sched->base.job_list_lock); 1163 } 1164