1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_drv.h> 12 #include <drm/drm_file.h> 13 #include <uapi/drm/xe_drm.h> 14 15 #include "xe_dep_scheduler.h" 16 #include "xe_device.h" 17 #include "xe_gt.h" 18 #include "xe_hw_engine_class_sysfs.h" 19 #include "xe_hw_engine_group.h" 20 #include "xe_hw_fence.h" 21 #include "xe_irq.h" 22 #include "xe_lrc.h" 23 #include "xe_macros.h" 24 #include "xe_migrate.h" 25 #include "xe_pm.h" 26 #include "xe_ring_ops_types.h" 27 #include "xe_trace.h" 28 #include "xe_vm.h" 29 #include "xe_pxp.h" 30 31 /** 32 * DOC: Execution Queue 33 * 34 * An Execution queue is an interface for the HW context of execution. 35 * The user creates an execution queue, submits the GPU jobs through those 36 * queues and in the end destroys them. 37 * 38 * Execution queues can also be created by XeKMD itself for driver internal 39 * operations like object migration etc. 40 * 41 * An execution queue is associated with a specified HW engine or a group of 42 * engines (belonging to the same tile and engine class) and any GPU job 43 * submitted on the queue will be run on one of these engines. 44 * 45 * An execution queue is tied to an address space (VM). It holds a reference 46 * of the associated VM and the underlying Logical Ring Context/s (LRC/s) 47 * until the queue is destroyed. 48 * 49 * The execution queue sits on top of the submission backend. It opaquely 50 * handles the GuC and Execlist backends whichever the platform uses, and 51 * the ring operations the different engine classes support. 52 */ 53 54 enum xe_exec_queue_sched_prop { 55 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 56 XE_EXEC_QUEUE_TIMESLICE = 1, 57 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 58 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 59 }; 60 61 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 62 u64 extensions, int ext_number); 63 64 static void __xe_exec_queue_free(struct xe_exec_queue *q) 65 { 66 int i; 67 68 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) 69 if (q->tlb_inval[i].dep_scheduler) 70 xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); 71 72 if (xe_exec_queue_uses_pxp(q)) 73 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); 74 if (q->vm) 75 xe_vm_put(q->vm); 76 77 if (q->xef) 78 xe_file_put(q->xef); 79 80 kfree(q); 81 } 82 83 static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q) 84 { 85 struct xe_tile *tile = gt_to_tile(q->gt); 86 int i; 87 88 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) { 89 struct xe_dep_scheduler *dep_scheduler; 90 struct xe_gt *gt; 91 struct workqueue_struct *wq; 92 93 if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT) 94 gt = tile->primary_gt; 95 else 96 gt = tile->media_gt; 97 98 if (!gt) 99 continue; 100 101 wq = gt->tlb_inval.job_wq; 102 103 #define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */ 104 dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, 105 MAX_TLB_INVAL_JOBS); 106 if (IS_ERR(dep_scheduler)) 107 return PTR_ERR(dep_scheduler); 108 109 q->tlb_inval[i].dep_scheduler = dep_scheduler; 110 } 111 #undef MAX_TLB_INVAL_JOBS 112 113 return 0; 114 } 115 116 static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, 117 struct xe_vm *vm, 118 u32 logical_mask, 119 u16 width, struct xe_hw_engine *hwe, 120 u32 flags, u64 extensions) 121 { 122 struct xe_exec_queue *q; 123 struct xe_gt *gt = hwe->gt; 124 int err; 125 126 /* only kernel queues can be permanent */ 127 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 128 129 q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); 130 if (!q) 131 return ERR_PTR(-ENOMEM); 132 133 kref_init(&q->refcount); 134 q->flags = flags; 135 q->hwe = hwe; 136 q->gt = gt; 137 q->class = hwe->class; 138 q->width = width; 139 q->msix_vec = XE_IRQ_DEFAULT_MSIX; 140 q->logical_mask = logical_mask; 141 q->fence_irq = >->fence_irq[hwe->class]; 142 q->ring_ops = gt->ring_ops[hwe->class]; 143 q->ops = gt->exec_queue_ops; 144 INIT_LIST_HEAD(&q->lr.link); 145 INIT_LIST_HEAD(&q->multi_gt_link); 146 INIT_LIST_HEAD(&q->hw_engine_group_link); 147 INIT_LIST_HEAD(&q->pxp.link); 148 149 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 150 q->sched_props.preempt_timeout_us = 151 hwe->eclass->sched_props.preempt_timeout_us; 152 q->sched_props.job_timeout_ms = 153 hwe->eclass->sched_props.job_timeout_ms; 154 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && 155 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) 156 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; 157 else 158 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 159 160 if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) { 161 err = alloc_dep_schedulers(xe, q); 162 if (err) { 163 __xe_exec_queue_free(q); 164 return ERR_PTR(err); 165 } 166 } 167 168 if (vm) 169 q->vm = xe_vm_get(vm); 170 171 if (extensions) { 172 /* 173 * may set q->usm, must come before xe_lrc_create(), 174 * may overwrite q->sched_props, must come before q->ops->init() 175 */ 176 err = exec_queue_user_extensions(xe, q, extensions, 0); 177 if (err) { 178 __xe_exec_queue_free(q); 179 return ERR_PTR(err); 180 } 181 } 182 183 return q; 184 } 185 186 static int __xe_exec_queue_init(struct xe_exec_queue *q) 187 { 188 int i, err; 189 u32 flags = 0; 190 191 /* 192 * PXP workloads executing on RCS or CCS must run in isolation (i.e. no 193 * other workload can use the EUs at the same time). On MTL this is done 194 * by setting the RUNALONE bit in the LRC, while starting on Xe2 there 195 * is a dedicated bit for it. 196 */ 197 if (xe_exec_queue_uses_pxp(q) && 198 (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) { 199 if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20) 200 flags |= XE_LRC_CREATE_PXP; 201 else 202 flags |= XE_LRC_CREATE_RUNALONE; 203 } 204 205 for (i = 0; i < q->width; ++i) { 206 q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K, q->msix_vec, flags); 207 if (IS_ERR(q->lrc[i])) { 208 err = PTR_ERR(q->lrc[i]); 209 goto err_lrc; 210 } 211 } 212 213 err = q->ops->init(q); 214 if (err) 215 goto err_lrc; 216 217 return 0; 218 219 err_lrc: 220 for (i = i - 1; i >= 0; --i) 221 xe_lrc_put(q->lrc[i]); 222 return err; 223 } 224 225 static void __xe_exec_queue_fini(struct xe_exec_queue *q) 226 { 227 int i; 228 229 q->ops->fini(q); 230 231 for (i = 0; i < q->width; ++i) 232 xe_lrc_put(q->lrc[i]); 233 } 234 235 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 236 u32 logical_mask, u16 width, 237 struct xe_hw_engine *hwe, u32 flags, 238 u64 extensions) 239 { 240 struct xe_exec_queue *q; 241 int err; 242 243 /* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */ 244 xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0))); 245 246 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, 247 extensions); 248 if (IS_ERR(q)) 249 return q; 250 251 err = __xe_exec_queue_init(q); 252 if (err) 253 goto err_post_alloc; 254 255 /* 256 * We can only add the queue to the PXP list after the init is complete, 257 * because the PXP termination can call exec_queue_kill and that will 258 * go bad if the queue is only half-initialized. This means that we 259 * can't do it when we handle the PXP extension in __xe_exec_queue_alloc 260 * and we need to do it here instead. 261 */ 262 if (xe_exec_queue_uses_pxp(q)) { 263 err = xe_pxp_exec_queue_add(xe->pxp, q); 264 if (err) 265 goto err_post_init; 266 } 267 268 return q; 269 270 err_post_init: 271 __xe_exec_queue_fini(q); 272 err_post_alloc: 273 __xe_exec_queue_free(q); 274 return ERR_PTR(err); 275 } 276 ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO); 277 278 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 279 struct xe_vm *vm, 280 enum xe_engine_class class, 281 u32 flags, u64 extensions) 282 { 283 struct xe_hw_engine *hwe, *hwe0 = NULL; 284 enum xe_hw_engine_id id; 285 u32 logical_mask = 0; 286 287 for_each_hw_engine(hwe, gt, id) { 288 if (xe_hw_engine_is_reserved(hwe)) 289 continue; 290 291 if (hwe->class == class) { 292 logical_mask |= BIT(hwe->logical_instance); 293 if (!hwe0) 294 hwe0 = hwe; 295 } 296 } 297 298 if (!logical_mask) 299 return ERR_PTR(-ENODEV); 300 301 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); 302 } 303 304 /** 305 * xe_exec_queue_create_bind() - Create bind exec queue. 306 * @xe: Xe device. 307 * @tile: tile which bind exec queue belongs to. 308 * @flags: exec queue creation flags 309 * @extensions: exec queue creation extensions 310 * 311 * Normalize bind exec queue creation. Bind exec queue is tied to migration VM 312 * for access to physical memory required for page table programming. On a 313 * faulting devices the reserved copy engine instance must be used to avoid 314 * deadlocking (user binds cannot get stuck behind faults as kernel binds which 315 * resolve faults depend on user binds). On non-faulting devices any copy engine 316 * can be used. 317 * 318 * Returns exec queue on success, ERR_PTR on failure 319 */ 320 struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, 321 struct xe_tile *tile, 322 u32 flags, u64 extensions) 323 { 324 struct xe_gt *gt = tile->primary_gt; 325 struct xe_exec_queue *q; 326 struct xe_vm *migrate_vm; 327 328 migrate_vm = xe_migrate_get_vm(tile->migrate); 329 if (xe->info.has_usm) { 330 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, 331 XE_ENGINE_CLASS_COPY, 332 gt->usm.reserved_bcs_instance, 333 false); 334 335 if (!hwe) { 336 xe_vm_put(migrate_vm); 337 return ERR_PTR(-EINVAL); 338 } 339 340 q = xe_exec_queue_create(xe, migrate_vm, 341 BIT(hwe->logical_instance), 1, hwe, 342 flags, extensions); 343 } else { 344 q = xe_exec_queue_create_class(xe, gt, migrate_vm, 345 XE_ENGINE_CLASS_COPY, flags, 346 extensions); 347 } 348 xe_vm_put(migrate_vm); 349 350 return q; 351 } 352 ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO); 353 354 void xe_exec_queue_destroy(struct kref *ref) 355 { 356 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 357 struct xe_exec_queue *eq, *next; 358 359 if (xe_exec_queue_uses_pxp(q)) 360 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); 361 362 xe_exec_queue_last_fence_put_unlocked(q); 363 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 364 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 365 multi_gt_link) 366 xe_exec_queue_put(eq); 367 } 368 369 q->ops->destroy(q); 370 } 371 372 void xe_exec_queue_fini(struct xe_exec_queue *q) 373 { 374 /* 375 * Before releasing our ref to lrc and xef, accumulate our run ticks 376 * and wakeup any waiters. 377 */ 378 xe_exec_queue_update_run_ticks(q); 379 if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal)) 380 wake_up_var(&q->xef->exec_queue.pending_removal); 381 382 __xe_exec_queue_fini(q); 383 __xe_exec_queue_free(q); 384 } 385 386 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 387 { 388 switch (q->class) { 389 case XE_ENGINE_CLASS_RENDER: 390 snprintf(q->name, sizeof(q->name), "rcs%d", instance); 391 break; 392 case XE_ENGINE_CLASS_VIDEO_DECODE: 393 snprintf(q->name, sizeof(q->name), "vcs%d", instance); 394 break; 395 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 396 snprintf(q->name, sizeof(q->name), "vecs%d", instance); 397 break; 398 case XE_ENGINE_CLASS_COPY: 399 snprintf(q->name, sizeof(q->name), "bcs%d", instance); 400 break; 401 case XE_ENGINE_CLASS_COMPUTE: 402 snprintf(q->name, sizeof(q->name), "ccs%d", instance); 403 break; 404 case XE_ENGINE_CLASS_OTHER: 405 snprintf(q->name, sizeof(q->name), "gsccs%d", instance); 406 break; 407 default: 408 XE_WARN_ON(q->class); 409 } 410 } 411 412 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 413 { 414 struct xe_exec_queue *q; 415 416 mutex_lock(&xef->exec_queue.lock); 417 q = xa_load(&xef->exec_queue.xa, id); 418 if (q) 419 xe_exec_queue_get(q); 420 mutex_unlock(&xef->exec_queue.lock); 421 422 return q; 423 } 424 425 enum xe_exec_queue_priority 426 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 427 { 428 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 429 XE_EXEC_QUEUE_PRIORITY_NORMAL; 430 } 431 432 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 433 u64 value) 434 { 435 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 436 return -EINVAL; 437 438 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 439 return -EPERM; 440 441 q->sched_props.priority = value; 442 return 0; 443 } 444 445 static bool xe_exec_queue_enforce_schedule_limit(void) 446 { 447 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 448 return true; 449 #else 450 return !capable(CAP_SYS_NICE); 451 #endif 452 } 453 454 static void 455 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 456 enum xe_exec_queue_sched_prop prop, 457 u32 *min, u32 *max) 458 { 459 switch (prop) { 460 case XE_EXEC_QUEUE_JOB_TIMEOUT: 461 *min = eclass->sched_props.job_timeout_min; 462 *max = eclass->sched_props.job_timeout_max; 463 break; 464 case XE_EXEC_QUEUE_TIMESLICE: 465 *min = eclass->sched_props.timeslice_min; 466 *max = eclass->sched_props.timeslice_max; 467 break; 468 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 469 *min = eclass->sched_props.preempt_timeout_min; 470 *max = eclass->sched_props.preempt_timeout_max; 471 break; 472 default: 473 break; 474 } 475 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 476 if (capable(CAP_SYS_NICE)) { 477 switch (prop) { 478 case XE_EXEC_QUEUE_JOB_TIMEOUT: 479 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 480 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 481 break; 482 case XE_EXEC_QUEUE_TIMESLICE: 483 *min = XE_HW_ENGINE_TIMESLICE_MIN; 484 *max = XE_HW_ENGINE_TIMESLICE_MAX; 485 break; 486 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 487 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 488 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 489 break; 490 default: 491 break; 492 } 493 } 494 #endif 495 } 496 497 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 498 u64 value) 499 { 500 u32 min = 0, max = 0; 501 502 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 503 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 504 505 if (xe_exec_queue_enforce_schedule_limit() && 506 !xe_hw_engine_timeout_in_range(value, min, max)) 507 return -EINVAL; 508 509 q->sched_props.timeslice_us = value; 510 return 0; 511 } 512 513 static int 514 exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value) 515 { 516 if (value == DRM_XE_PXP_TYPE_NONE) 517 return 0; 518 519 /* we only support HWDRM sessions right now */ 520 if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM)) 521 return -EINVAL; 522 523 if (!xe_pxp_is_enabled(xe->pxp)) 524 return -ENODEV; 525 526 return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM); 527 } 528 529 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 530 struct xe_exec_queue *q, 531 u64 value); 532 533 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 534 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 535 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 536 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type, 537 }; 538 539 static int exec_queue_user_ext_set_property(struct xe_device *xe, 540 struct xe_exec_queue *q, 541 u64 extension) 542 { 543 u64 __user *address = u64_to_user_ptr(extension); 544 struct drm_xe_ext_set_property ext; 545 int err; 546 u32 idx; 547 548 err = copy_from_user(&ext, address, sizeof(ext)); 549 if (XE_IOCTL_DBG(xe, err)) 550 return -EFAULT; 551 552 if (XE_IOCTL_DBG(xe, ext.property >= 553 ARRAY_SIZE(exec_queue_set_property_funcs)) || 554 XE_IOCTL_DBG(xe, ext.pad) || 555 XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && 556 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE && 557 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE)) 558 return -EINVAL; 559 560 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 561 if (!exec_queue_set_property_funcs[idx]) 562 return -EINVAL; 563 564 return exec_queue_set_property_funcs[idx](xe, q, ext.value); 565 } 566 567 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 568 struct xe_exec_queue *q, 569 u64 extension); 570 571 static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { 572 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 573 }; 574 575 #define MAX_USER_EXTENSIONS 16 576 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 577 u64 extensions, int ext_number) 578 { 579 u64 __user *address = u64_to_user_ptr(extensions); 580 struct drm_xe_user_extension ext; 581 int err; 582 u32 idx; 583 584 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 585 return -E2BIG; 586 587 err = copy_from_user(&ext, address, sizeof(ext)); 588 if (XE_IOCTL_DBG(xe, err)) 589 return -EFAULT; 590 591 if (XE_IOCTL_DBG(xe, ext.pad) || 592 XE_IOCTL_DBG(xe, ext.name >= 593 ARRAY_SIZE(exec_queue_user_extension_funcs))) 594 return -EINVAL; 595 596 idx = array_index_nospec(ext.name, 597 ARRAY_SIZE(exec_queue_user_extension_funcs)); 598 err = exec_queue_user_extension_funcs[idx](xe, q, extensions); 599 if (XE_IOCTL_DBG(xe, err)) 600 return err; 601 602 if (ext.next_extension) 603 return exec_queue_user_extensions(xe, q, ext.next_extension, 604 ++ext_number); 605 606 return 0; 607 } 608 609 static u32 calc_validate_logical_mask(struct xe_device *xe, 610 struct drm_xe_engine_class_instance *eci, 611 u16 width, u16 num_placements) 612 { 613 int len = width * num_placements; 614 int i, j, n; 615 u16 class; 616 u16 gt_id; 617 u32 return_mask = 0, prev_mask; 618 619 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 620 len > 1)) 621 return 0; 622 623 for (i = 0; i < width; ++i) { 624 u32 current_mask = 0; 625 626 for (j = 0; j < num_placements; ++j) { 627 struct xe_hw_engine *hwe; 628 629 n = j * width + i; 630 631 hwe = xe_hw_engine_lookup(xe, eci[n]); 632 if (XE_IOCTL_DBG(xe, !hwe)) 633 return 0; 634 635 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 636 return 0; 637 638 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 639 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 640 return 0; 641 642 class = eci[n].engine_class; 643 gt_id = eci[n].gt_id; 644 645 if (width == 1 || !i) 646 return_mask |= BIT(eci[n].engine_instance); 647 current_mask |= BIT(eci[n].engine_instance); 648 } 649 650 /* Parallel submissions must be logically contiguous */ 651 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 652 return 0; 653 654 prev_mask = current_mask; 655 } 656 657 return return_mask; 658 } 659 660 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 661 struct drm_file *file) 662 { 663 struct xe_device *xe = to_xe_device(dev); 664 struct xe_file *xef = to_xe_file(file); 665 struct drm_xe_exec_queue_create *args = data; 666 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 667 struct drm_xe_engine_class_instance __user *user_eci = 668 u64_to_user_ptr(args->instances); 669 struct xe_hw_engine *hwe; 670 struct xe_vm *vm; 671 struct xe_tile *tile; 672 struct xe_exec_queue *q = NULL; 673 u32 logical_mask; 674 u32 flags = 0; 675 u32 id; 676 u32 len; 677 int err; 678 679 if (XE_IOCTL_DBG(xe, args->flags & ~DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) || 680 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 681 return -EINVAL; 682 683 len = args->width * args->num_placements; 684 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 685 return -EINVAL; 686 687 err = copy_from_user(eci, user_eci, 688 sizeof(struct drm_xe_engine_class_instance) * len); 689 if (XE_IOCTL_DBG(xe, err)) 690 return -EFAULT; 691 692 if (XE_IOCTL_DBG(xe, !xe_device_get_gt(xe, eci[0].gt_id))) 693 return -EINVAL; 694 695 if (args->flags & DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) 696 flags |= EXEC_QUEUE_FLAG_LOW_LATENCY; 697 698 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { 699 if (XE_IOCTL_DBG(xe, args->width != 1) || 700 XE_IOCTL_DBG(xe, args->num_placements != 1) || 701 XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 702 return -EINVAL; 703 704 for_each_tile(tile, xe, id) { 705 struct xe_exec_queue *new; 706 707 flags |= EXEC_QUEUE_FLAG_VM; 708 if (id) 709 flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; 710 711 new = xe_exec_queue_create_bind(xe, tile, flags, 712 args->extensions); 713 if (IS_ERR(new)) { 714 err = PTR_ERR(new); 715 if (q) 716 goto put_exec_queue; 717 return err; 718 } 719 if (id == 0) 720 q = new; 721 else 722 list_add_tail(&new->multi_gt_list, 723 &q->multi_gt_link); 724 } 725 } else { 726 logical_mask = calc_validate_logical_mask(xe, eci, 727 args->width, 728 args->num_placements); 729 if (XE_IOCTL_DBG(xe, !logical_mask)) 730 return -EINVAL; 731 732 hwe = xe_hw_engine_lookup(xe, eci[0]); 733 if (XE_IOCTL_DBG(xe, !hwe)) 734 return -EINVAL; 735 736 vm = xe_vm_lookup(xef, args->vm_id); 737 if (XE_IOCTL_DBG(xe, !vm)) 738 return -ENOENT; 739 740 err = down_read_interruptible(&vm->lock); 741 if (err) { 742 xe_vm_put(vm); 743 return err; 744 } 745 746 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 747 up_read(&vm->lock); 748 xe_vm_put(vm); 749 return -ENOENT; 750 } 751 752 q = xe_exec_queue_create(xe, vm, logical_mask, 753 args->width, hwe, flags, 754 args->extensions); 755 up_read(&vm->lock); 756 xe_vm_put(vm); 757 if (IS_ERR(q)) 758 return PTR_ERR(q); 759 760 if (xe_vm_in_preempt_fence_mode(vm)) { 761 q->lr.context = dma_fence_context_alloc(1); 762 763 err = xe_vm_add_compute_exec_queue(vm, q); 764 if (XE_IOCTL_DBG(xe, err)) 765 goto put_exec_queue; 766 } 767 768 if (q->vm && q->hwe->hw_engine_group) { 769 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); 770 if (err) 771 goto put_exec_queue; 772 } 773 } 774 775 q->xef = xe_file_get(xef); 776 777 /* user id alloc must always be last in ioctl to prevent UAF */ 778 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 779 if (err) 780 goto kill_exec_queue; 781 782 args->exec_queue_id = id; 783 784 return 0; 785 786 kill_exec_queue: 787 xe_exec_queue_kill(q); 788 put_exec_queue: 789 xe_exec_queue_put(q); 790 return err; 791 } 792 793 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 794 struct drm_file *file) 795 { 796 struct xe_device *xe = to_xe_device(dev); 797 struct xe_file *xef = to_xe_file(file); 798 struct drm_xe_exec_queue_get_property *args = data; 799 struct xe_exec_queue *q; 800 int ret; 801 802 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 803 return -EINVAL; 804 805 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 806 if (XE_IOCTL_DBG(xe, !q)) 807 return -ENOENT; 808 809 switch (args->property) { 810 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: 811 args->value = q->ops->reset_status(q); 812 ret = 0; 813 break; 814 default: 815 ret = -EINVAL; 816 } 817 818 xe_exec_queue_put(q); 819 820 return ret; 821 } 822 823 /** 824 * xe_exec_queue_lrc() - Get the LRC from exec queue. 825 * @q: The exec_queue. 826 * 827 * Retrieves the primary LRC for the exec queue. Note that this function 828 * returns only the first LRC instance, even when multiple parallel LRCs 829 * are configured. 830 * 831 * Return: Pointer to LRC on success, error on failure 832 */ 833 struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q) 834 { 835 return q->lrc[0]; 836 } 837 838 /** 839 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 840 * @q: The exec_queue 841 * 842 * Return: True if the exec_queue is long-running, false otherwise. 843 */ 844 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 845 { 846 return q->vm && xe_vm_in_lr_mode(q->vm) && 847 !(q->flags & EXEC_QUEUE_FLAG_VM); 848 } 849 850 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 851 { 852 return q->lrc[0]->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc[0]) - 1; 853 } 854 855 /** 856 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 857 * @q: The exec_queue 858 * 859 * Return: True if the exec_queue's ring is full, false otherwise. 860 */ 861 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 862 { 863 struct xe_lrc *lrc = q->lrc[0]; 864 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 865 866 return xe_exec_queue_num_job_inflight(q) >= max_job; 867 } 868 869 /** 870 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 871 * @q: The exec_queue 872 * 873 * FIXME: Need to determine what to use as the short-lived 874 * timeline lock for the exec_queues, so that the return value 875 * of this function becomes more than just an advisory 876 * snapshot in time. The timeline lock must protect the 877 * seqno from racing submissions on the same exec_queue. 878 * Typically vm->resv, but user-created timeline locks use the migrate vm 879 * and never grabs the migrate vm->resv so we have a race there. 880 * 881 * Return: True if the exec_queue is idle, false otherwise. 882 */ 883 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 884 { 885 if (xe_exec_queue_is_parallel(q)) { 886 int i; 887 888 for (i = 0; i < q->width; ++i) { 889 if (xe_lrc_seqno(q->lrc[i]) != 890 q->lrc[i]->fence_ctx.next_seqno - 1) 891 return false; 892 } 893 894 return true; 895 } 896 897 return xe_lrc_seqno(q->lrc[0]) == 898 q->lrc[0]->fence_ctx.next_seqno - 1; 899 } 900 901 /** 902 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue 903 * from hw 904 * @q: The exec queue 905 * 906 * Update the timestamp saved by HW for this exec queue and save run ticks 907 * calculated by using the delta from last update. 908 */ 909 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) 910 { 911 struct xe_device *xe = gt_to_xe(q->gt); 912 struct xe_lrc *lrc; 913 u64 old_ts, new_ts; 914 int idx; 915 916 /* 917 * Jobs that are executed by kernel doesn't have a corresponding xe_file 918 * and thus are not accounted. 919 */ 920 if (!q->xef) 921 return; 922 923 /* Synchronize with unbind while holding the xe file open */ 924 if (!drm_dev_enter(&xe->drm, &idx)) 925 return; 926 /* 927 * Only sample the first LRC. For parallel submission, all of them are 928 * scheduled together and we compensate that below by multiplying by 929 * width - this may introduce errors if that premise is not true and 930 * they don't exit 100% aligned. On the other hand, looping through 931 * the LRCs and reading them in different time could also introduce 932 * errors. 933 */ 934 lrc = q->lrc[0]; 935 new_ts = xe_lrc_update_timestamp(lrc, &old_ts); 936 q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; 937 938 drm_dev_exit(idx); 939 } 940 941 /** 942 * xe_exec_queue_kill - permanently stop all execution from an exec queue 943 * @q: The exec queue 944 * 945 * This function permanently stops all activity on an exec queue. If the queue 946 * is actively executing on the HW, it will be kicked off the engine; any 947 * pending jobs are discarded and all future submissions are rejected. 948 * This function is safe to call multiple times. 949 */ 950 void xe_exec_queue_kill(struct xe_exec_queue *q) 951 { 952 struct xe_exec_queue *eq = q, *next; 953 954 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 955 multi_gt_link) { 956 q->ops->kill(eq); 957 xe_vm_remove_compute_exec_queue(q->vm, eq); 958 } 959 960 q->ops->kill(q); 961 xe_vm_remove_compute_exec_queue(q->vm, q); 962 } 963 964 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 965 struct drm_file *file) 966 { 967 struct xe_device *xe = to_xe_device(dev); 968 struct xe_file *xef = to_xe_file(file); 969 struct drm_xe_exec_queue_destroy *args = data; 970 struct xe_exec_queue *q; 971 972 if (XE_IOCTL_DBG(xe, args->pad) || 973 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 974 return -EINVAL; 975 976 mutex_lock(&xef->exec_queue.lock); 977 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 978 if (q) 979 atomic_inc(&xef->exec_queue.pending_removal); 980 mutex_unlock(&xef->exec_queue.lock); 981 982 if (XE_IOCTL_DBG(xe, !q)) 983 return -ENOENT; 984 985 if (q->vm && q->hwe->hw_engine_group) 986 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 987 988 xe_exec_queue_kill(q); 989 990 trace_xe_exec_queue_close(q); 991 xe_exec_queue_put(q); 992 993 return 0; 994 } 995 996 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 997 struct xe_vm *vm) 998 { 999 if (q->flags & EXEC_QUEUE_FLAG_VM) { 1000 lockdep_assert_held(&vm->lock); 1001 } else { 1002 xe_vm_assert_held(vm); 1003 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); 1004 } 1005 } 1006 1007 /** 1008 * xe_exec_queue_last_fence_put() - Drop ref to last fence 1009 * @q: The exec queue 1010 * @vm: The VM the engine does a bind or exec for 1011 */ 1012 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 1013 { 1014 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1015 1016 xe_exec_queue_last_fence_put_unlocked(q); 1017 } 1018 1019 /** 1020 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 1021 * @q: The exec queue 1022 * 1023 * Only safe to be called from xe_exec_queue_destroy(). 1024 */ 1025 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 1026 { 1027 if (q->last_fence) { 1028 dma_fence_put(q->last_fence); 1029 q->last_fence = NULL; 1030 } 1031 } 1032 1033 /** 1034 * xe_exec_queue_last_fence_get() - Get last fence 1035 * @q: The exec queue 1036 * @vm: The VM the engine does a bind or exec for 1037 * 1038 * Get last fence, takes a ref 1039 * 1040 * Returns: last fence if not signaled, dma fence stub if signaled 1041 */ 1042 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 1043 struct xe_vm *vm) 1044 { 1045 struct dma_fence *fence; 1046 1047 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1048 1049 if (q->last_fence && 1050 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 1051 xe_exec_queue_last_fence_put(q, vm); 1052 1053 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 1054 dma_fence_get(fence); 1055 return fence; 1056 } 1057 1058 /** 1059 * xe_exec_queue_last_fence_get_for_resume() - Get last fence 1060 * @q: The exec queue 1061 * @vm: The VM the engine does a bind or exec for 1062 * 1063 * Get last fence, takes a ref. Only safe to be called in the context of 1064 * resuming the hw engine group's long-running exec queue, when the group 1065 * semaphore is held. 1066 * 1067 * Returns: last fence if not signaled, dma fence stub if signaled 1068 */ 1069 struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q, 1070 struct xe_vm *vm) 1071 { 1072 struct dma_fence *fence; 1073 1074 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); 1075 1076 if (q->last_fence && 1077 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 1078 xe_exec_queue_last_fence_put_unlocked(q); 1079 1080 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 1081 dma_fence_get(fence); 1082 return fence; 1083 } 1084 1085 /** 1086 * xe_exec_queue_last_fence_set() - Set last fence 1087 * @q: The exec queue 1088 * @vm: The VM the engine does a bind or exec for 1089 * @fence: The fence 1090 * 1091 * Set the last fence for the engine. Increases reference count for fence, when 1092 * closing engine xe_exec_queue_last_fence_put should be called. 1093 */ 1094 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 1095 struct dma_fence *fence) 1096 { 1097 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1098 1099 xe_exec_queue_last_fence_put(q, vm); 1100 q->last_fence = dma_fence_get(fence); 1101 } 1102 1103 /** 1104 * xe_exec_queue_last_fence_test_dep - Test last fence dependency of queue 1105 * @q: The exec queue 1106 * @vm: The VM the engine does a bind or exec for 1107 * 1108 * Returns: 1109 * -ETIME if there exists an unsignalled last fence dependency, zero otherwise. 1110 */ 1111 int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) 1112 { 1113 struct dma_fence *fence; 1114 int err = 0; 1115 1116 fence = xe_exec_queue_last_fence_get(q, vm); 1117 if (fence) { 1118 err = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 1119 0 : -ETIME; 1120 dma_fence_put(fence); 1121 } 1122 1123 return err; 1124 } 1125 1126 /** 1127 * xe_exec_queue_contexts_hwsp_rebase - Re-compute GGTT references 1128 * within all LRCs of a queue. 1129 * @q: the &xe_exec_queue struct instance containing target LRCs 1130 * @scratch: scratch buffer to be used as temporary storage 1131 * 1132 * Returns: zero on success, negative error code on failure 1133 */ 1134 int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch) 1135 { 1136 int i; 1137 int err = 0; 1138 1139 for (i = 0; i < q->width; ++i) { 1140 xe_lrc_update_memirq_regs_with_address(q->lrc[i], q->hwe, scratch); 1141 xe_lrc_update_hwctx_regs_with_address(q->lrc[i]); 1142 err = xe_lrc_setup_wa_bb_with_scratch(q->lrc[i], q->hwe, scratch); 1143 if (err) 1144 break; 1145 } 1146 1147 return err; 1148 } 1149 1150 /** 1151 * xe_exec_queue_jobs_ring_restore - Re-emit ring commands of requests pending on given queue. 1152 * @q: the &xe_exec_queue struct instance 1153 */ 1154 void xe_exec_queue_jobs_ring_restore(struct xe_exec_queue *q) 1155 { 1156 struct xe_gpu_scheduler *sched = &q->guc->sched; 1157 struct xe_sched_job *job; 1158 1159 /* 1160 * This routine is used within VF migration recovery. This means 1161 * using the lock here introduces a restriction: we cannot wait 1162 * for any GFX HW response while the lock is taken. 1163 */ 1164 spin_lock(&sched->base.job_list_lock); 1165 list_for_each_entry(job, &sched->base.pending_list, drm.list) { 1166 if (xe_sched_job_is_error(job)) 1167 continue; 1168 1169 q->ring_ops->emit_job(job); 1170 } 1171 spin_unlock(&sched->base.job_list_lock); 1172 } 1173