1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_drv.h> 12 #include <drm/drm_file.h> 13 #include <uapi/drm/xe_drm.h> 14 15 #include "xe_dep_scheduler.h" 16 #include "xe_device.h" 17 #include "xe_gt.h" 18 #include "xe_gt_sriov_vf.h" 19 #include "xe_hw_engine_class_sysfs.h" 20 #include "xe_hw_engine_group.h" 21 #include "xe_hw_fence.h" 22 #include "xe_irq.h" 23 #include "xe_lrc.h" 24 #include "xe_macros.h" 25 #include "xe_migrate.h" 26 #include "xe_pm.h" 27 #include "xe_ring_ops_types.h" 28 #include "xe_trace.h" 29 #include "xe_vm.h" 30 #include "xe_pxp.h" 31 32 /** 33 * DOC: Execution Queue 34 * 35 * An Execution queue is an interface for the HW context of execution. 36 * The user creates an execution queue, submits the GPU jobs through those 37 * queues and in the end destroys them. 38 * 39 * Execution queues can also be created by XeKMD itself for driver internal 40 * operations like object migration etc. 41 * 42 * An execution queue is associated with a specified HW engine or a group of 43 * engines (belonging to the same tile and engine class) and any GPU job 44 * submitted on the queue will be run on one of these engines. 45 * 46 * An execution queue is tied to an address space (VM). It holds a reference 47 * of the associated VM and the underlying Logical Ring Context/s (LRC/s) 48 * until the queue is destroyed. 49 * 50 * The execution queue sits on top of the submission backend. It opaquely 51 * handles the GuC and Execlist backends whichever the platform uses, and 52 * the ring operations the different engine classes support. 53 */ 54 55 enum xe_exec_queue_sched_prop { 56 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 57 XE_EXEC_QUEUE_TIMESLICE = 1, 58 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 59 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 60 }; 61 62 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 63 u64 extensions, int ext_number); 64 65 static void __xe_exec_queue_free(struct xe_exec_queue *q) 66 { 67 int i; 68 69 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) 70 if (q->tlb_inval[i].dep_scheduler) 71 xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); 72 73 if (xe_exec_queue_uses_pxp(q)) 74 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); 75 if (q->vm) 76 xe_vm_put(q->vm); 77 78 if (q->xef) 79 xe_file_put(q->xef); 80 81 kfree(q); 82 } 83 84 static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q) 85 { 86 struct xe_tile *tile = gt_to_tile(q->gt); 87 int i; 88 89 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) { 90 struct xe_dep_scheduler *dep_scheduler; 91 struct xe_gt *gt; 92 struct workqueue_struct *wq; 93 94 if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT) 95 gt = tile->primary_gt; 96 else 97 gt = tile->media_gt; 98 99 if (!gt) 100 continue; 101 102 wq = gt->tlb_inval.job_wq; 103 104 #define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */ 105 dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, 106 MAX_TLB_INVAL_JOBS); 107 if (IS_ERR(dep_scheduler)) 108 return PTR_ERR(dep_scheduler); 109 110 q->tlb_inval[i].dep_scheduler = dep_scheduler; 111 } 112 #undef MAX_TLB_INVAL_JOBS 113 114 return 0; 115 } 116 117 static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, 118 struct xe_vm *vm, 119 u32 logical_mask, 120 u16 width, struct xe_hw_engine *hwe, 121 u32 flags, u64 extensions) 122 { 123 struct xe_exec_queue *q; 124 struct xe_gt *gt = hwe->gt; 125 int err; 126 127 /* only kernel queues can be permanent */ 128 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 129 130 q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); 131 if (!q) 132 return ERR_PTR(-ENOMEM); 133 134 kref_init(&q->refcount); 135 q->flags = flags; 136 q->hwe = hwe; 137 q->gt = gt; 138 q->class = hwe->class; 139 q->width = width; 140 q->msix_vec = XE_IRQ_DEFAULT_MSIX; 141 q->logical_mask = logical_mask; 142 q->fence_irq = >->fence_irq[hwe->class]; 143 q->ring_ops = gt->ring_ops[hwe->class]; 144 q->ops = gt->exec_queue_ops; 145 INIT_LIST_HEAD(&q->lr.link); 146 INIT_LIST_HEAD(&q->multi_gt_link); 147 INIT_LIST_HEAD(&q->hw_engine_group_link); 148 INIT_LIST_HEAD(&q->pxp.link); 149 150 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 151 q->sched_props.preempt_timeout_us = 152 hwe->eclass->sched_props.preempt_timeout_us; 153 q->sched_props.job_timeout_ms = 154 hwe->eclass->sched_props.job_timeout_ms; 155 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && 156 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) 157 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; 158 else 159 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 160 161 if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) { 162 err = alloc_dep_schedulers(xe, q); 163 if (err) { 164 __xe_exec_queue_free(q); 165 return ERR_PTR(err); 166 } 167 } 168 169 if (vm) 170 q->vm = xe_vm_get(vm); 171 172 if (extensions) { 173 /* 174 * may set q->usm, must come before xe_lrc_create(), 175 * may overwrite q->sched_props, must come before q->ops->init() 176 */ 177 err = exec_queue_user_extensions(xe, q, extensions, 0); 178 if (err) { 179 __xe_exec_queue_free(q); 180 return ERR_PTR(err); 181 } 182 } 183 184 return q; 185 } 186 187 static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags) 188 { 189 int i, err; 190 u32 flags = 0; 191 192 /* 193 * PXP workloads executing on RCS or CCS must run in isolation (i.e. no 194 * other workload can use the EUs at the same time). On MTL this is done 195 * by setting the RUNALONE bit in the LRC, while starting on Xe2 there 196 * is a dedicated bit for it. 197 */ 198 if (xe_exec_queue_uses_pxp(q) && 199 (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) { 200 if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20) 201 flags |= XE_LRC_CREATE_PXP; 202 else 203 flags |= XE_LRC_CREATE_RUNALONE; 204 } 205 206 if (!(exec_queue_flags & EXEC_QUEUE_FLAG_KERNEL)) 207 flags |= XE_LRC_CREATE_USER_CTX; 208 209 err = q->ops->init(q); 210 if (err) 211 return err; 212 213 /* 214 * This must occur after q->ops->init to avoid race conditions during VF 215 * post-migration recovery, as the fixups for the LRC GGTT addresses 216 * depend on the queue being present in the backend tracking structure. 217 * 218 * In addition to above, we must wait on inflight GGTT changes to avoid 219 * writing out stale values here. Such wait provides a solid solution 220 * (without a race) only if the function can detect migration instantly 221 * from the moment vCPU resumes execution. 222 */ 223 for (i = 0; i < q->width; ++i) { 224 struct xe_lrc *lrc; 225 226 xe_gt_sriov_vf_wait_valid_ggtt(q->gt); 227 lrc = xe_lrc_create(q->hwe, q->vm, xe_lrc_ring_size(), 228 q->msix_vec, flags); 229 if (IS_ERR(lrc)) { 230 err = PTR_ERR(lrc); 231 goto err_lrc; 232 } 233 234 /* Pairs with READ_ONCE to xe_exec_queue_contexts_hwsp_rebase */ 235 WRITE_ONCE(q->lrc[i], lrc); 236 } 237 238 return 0; 239 240 err_lrc: 241 for (i = i - 1; i >= 0; --i) 242 xe_lrc_put(q->lrc[i]); 243 return err; 244 } 245 246 static void __xe_exec_queue_fini(struct xe_exec_queue *q) 247 { 248 int i; 249 250 q->ops->fini(q); 251 252 for (i = 0; i < q->width; ++i) 253 xe_lrc_put(q->lrc[i]); 254 } 255 256 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 257 u32 logical_mask, u16 width, 258 struct xe_hw_engine *hwe, u32 flags, 259 u64 extensions) 260 { 261 struct xe_exec_queue *q; 262 int err; 263 264 /* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */ 265 xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0))); 266 267 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, 268 extensions); 269 if (IS_ERR(q)) 270 return q; 271 272 err = __xe_exec_queue_init(q, flags); 273 if (err) 274 goto err_post_alloc; 275 276 /* 277 * We can only add the queue to the PXP list after the init is complete, 278 * because the PXP termination can call exec_queue_kill and that will 279 * go bad if the queue is only half-initialized. This means that we 280 * can't do it when we handle the PXP extension in __xe_exec_queue_alloc 281 * and we need to do it here instead. 282 */ 283 if (xe_exec_queue_uses_pxp(q)) { 284 err = xe_pxp_exec_queue_add(xe->pxp, q); 285 if (err) 286 goto err_post_init; 287 } 288 289 return q; 290 291 err_post_init: 292 __xe_exec_queue_fini(q); 293 err_post_alloc: 294 __xe_exec_queue_free(q); 295 return ERR_PTR(err); 296 } 297 ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO); 298 299 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 300 struct xe_vm *vm, 301 enum xe_engine_class class, 302 u32 flags, u64 extensions) 303 { 304 struct xe_hw_engine *hwe, *hwe0 = NULL; 305 enum xe_hw_engine_id id; 306 u32 logical_mask = 0; 307 308 for_each_hw_engine(hwe, gt, id) { 309 if (xe_hw_engine_is_reserved(hwe)) 310 continue; 311 312 if (hwe->class == class) { 313 logical_mask |= BIT(hwe->logical_instance); 314 if (!hwe0) 315 hwe0 = hwe; 316 } 317 } 318 319 if (!logical_mask) 320 return ERR_PTR(-ENODEV); 321 322 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); 323 } 324 325 /** 326 * xe_exec_queue_create_bind() - Create bind exec queue. 327 * @xe: Xe device. 328 * @tile: tile which bind exec queue belongs to. 329 * @flags: exec queue creation flags 330 * @extensions: exec queue creation extensions 331 * 332 * Normalize bind exec queue creation. Bind exec queue is tied to migration VM 333 * for access to physical memory required for page table programming. On a 334 * faulting devices the reserved copy engine instance must be used to avoid 335 * deadlocking (user binds cannot get stuck behind faults as kernel binds which 336 * resolve faults depend on user binds). On non-faulting devices any copy engine 337 * can be used. 338 * 339 * Returns exec queue on success, ERR_PTR on failure 340 */ 341 struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, 342 struct xe_tile *tile, 343 u32 flags, u64 extensions) 344 { 345 struct xe_gt *gt = tile->primary_gt; 346 struct xe_exec_queue *q; 347 struct xe_vm *migrate_vm; 348 349 migrate_vm = xe_migrate_get_vm(tile->migrate); 350 if (xe->info.has_usm) { 351 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, 352 XE_ENGINE_CLASS_COPY, 353 gt->usm.reserved_bcs_instance, 354 false); 355 356 if (!hwe) { 357 xe_vm_put(migrate_vm); 358 return ERR_PTR(-EINVAL); 359 } 360 361 q = xe_exec_queue_create(xe, migrate_vm, 362 BIT(hwe->logical_instance), 1, hwe, 363 flags, extensions); 364 } else { 365 q = xe_exec_queue_create_class(xe, gt, migrate_vm, 366 XE_ENGINE_CLASS_COPY, flags, 367 extensions); 368 } 369 xe_vm_put(migrate_vm); 370 371 return q; 372 } 373 ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO); 374 375 void xe_exec_queue_destroy(struct kref *ref) 376 { 377 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 378 struct xe_exec_queue *eq, *next; 379 380 xe_assert(gt_to_xe(q->gt), atomic_read(&q->job_cnt) == 0); 381 382 if (xe_exec_queue_uses_pxp(q)) 383 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); 384 385 xe_exec_queue_last_fence_put_unlocked(q); 386 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 387 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 388 multi_gt_link) 389 xe_exec_queue_put(eq); 390 } 391 392 q->ops->destroy(q); 393 } 394 395 void xe_exec_queue_fini(struct xe_exec_queue *q) 396 { 397 /* 398 * Before releasing our ref to lrc and xef, accumulate our run ticks 399 * and wakeup any waiters. 400 */ 401 xe_exec_queue_update_run_ticks(q); 402 if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal)) 403 wake_up_var(&q->xef->exec_queue.pending_removal); 404 405 __xe_exec_queue_fini(q); 406 __xe_exec_queue_free(q); 407 } 408 409 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 410 { 411 switch (q->class) { 412 case XE_ENGINE_CLASS_RENDER: 413 snprintf(q->name, sizeof(q->name), "rcs%d", instance); 414 break; 415 case XE_ENGINE_CLASS_VIDEO_DECODE: 416 snprintf(q->name, sizeof(q->name), "vcs%d", instance); 417 break; 418 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 419 snprintf(q->name, sizeof(q->name), "vecs%d", instance); 420 break; 421 case XE_ENGINE_CLASS_COPY: 422 snprintf(q->name, sizeof(q->name), "bcs%d", instance); 423 break; 424 case XE_ENGINE_CLASS_COMPUTE: 425 snprintf(q->name, sizeof(q->name), "ccs%d", instance); 426 break; 427 case XE_ENGINE_CLASS_OTHER: 428 snprintf(q->name, sizeof(q->name), "gsccs%d", instance); 429 break; 430 default: 431 XE_WARN_ON(q->class); 432 } 433 } 434 435 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 436 { 437 struct xe_exec_queue *q; 438 439 mutex_lock(&xef->exec_queue.lock); 440 q = xa_load(&xef->exec_queue.xa, id); 441 if (q) 442 xe_exec_queue_get(q); 443 mutex_unlock(&xef->exec_queue.lock); 444 445 return q; 446 } 447 448 enum xe_exec_queue_priority 449 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 450 { 451 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 452 XE_EXEC_QUEUE_PRIORITY_NORMAL; 453 } 454 455 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 456 u64 value) 457 { 458 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 459 return -EINVAL; 460 461 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 462 return -EPERM; 463 464 q->sched_props.priority = value; 465 return 0; 466 } 467 468 static bool xe_exec_queue_enforce_schedule_limit(void) 469 { 470 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 471 return true; 472 #else 473 return !capable(CAP_SYS_NICE); 474 #endif 475 } 476 477 static void 478 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 479 enum xe_exec_queue_sched_prop prop, 480 u32 *min, u32 *max) 481 { 482 switch (prop) { 483 case XE_EXEC_QUEUE_JOB_TIMEOUT: 484 *min = eclass->sched_props.job_timeout_min; 485 *max = eclass->sched_props.job_timeout_max; 486 break; 487 case XE_EXEC_QUEUE_TIMESLICE: 488 *min = eclass->sched_props.timeslice_min; 489 *max = eclass->sched_props.timeslice_max; 490 break; 491 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 492 *min = eclass->sched_props.preempt_timeout_min; 493 *max = eclass->sched_props.preempt_timeout_max; 494 break; 495 default: 496 break; 497 } 498 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 499 if (capable(CAP_SYS_NICE)) { 500 switch (prop) { 501 case XE_EXEC_QUEUE_JOB_TIMEOUT: 502 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 503 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 504 break; 505 case XE_EXEC_QUEUE_TIMESLICE: 506 *min = XE_HW_ENGINE_TIMESLICE_MIN; 507 *max = XE_HW_ENGINE_TIMESLICE_MAX; 508 break; 509 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 510 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 511 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 512 break; 513 default: 514 break; 515 } 516 } 517 #endif 518 } 519 520 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 521 u64 value) 522 { 523 u32 min = 0, max = 0; 524 525 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 526 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 527 528 if (xe_exec_queue_enforce_schedule_limit() && 529 !xe_hw_engine_timeout_in_range(value, min, max)) 530 return -EINVAL; 531 532 q->sched_props.timeslice_us = value; 533 return 0; 534 } 535 536 static int 537 exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value) 538 { 539 if (value == DRM_XE_PXP_TYPE_NONE) 540 return 0; 541 542 /* we only support HWDRM sessions right now */ 543 if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM)) 544 return -EINVAL; 545 546 if (!xe_pxp_is_enabled(xe->pxp)) 547 return -ENODEV; 548 549 return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM); 550 } 551 552 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 553 struct xe_exec_queue *q, 554 u64 value); 555 556 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 557 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 558 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 559 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type, 560 }; 561 562 static int exec_queue_user_ext_set_property(struct xe_device *xe, 563 struct xe_exec_queue *q, 564 u64 extension) 565 { 566 u64 __user *address = u64_to_user_ptr(extension); 567 struct drm_xe_ext_set_property ext; 568 int err; 569 u32 idx; 570 571 err = copy_from_user(&ext, address, sizeof(ext)); 572 if (XE_IOCTL_DBG(xe, err)) 573 return -EFAULT; 574 575 if (XE_IOCTL_DBG(xe, ext.property >= 576 ARRAY_SIZE(exec_queue_set_property_funcs)) || 577 XE_IOCTL_DBG(xe, ext.pad) || 578 XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && 579 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE && 580 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE)) 581 return -EINVAL; 582 583 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 584 if (!exec_queue_set_property_funcs[idx]) 585 return -EINVAL; 586 587 return exec_queue_set_property_funcs[idx](xe, q, ext.value); 588 } 589 590 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 591 struct xe_exec_queue *q, 592 u64 extension); 593 594 static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { 595 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 596 }; 597 598 #define MAX_USER_EXTENSIONS 16 599 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 600 u64 extensions, int ext_number) 601 { 602 u64 __user *address = u64_to_user_ptr(extensions); 603 struct drm_xe_user_extension ext; 604 int err; 605 u32 idx; 606 607 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 608 return -E2BIG; 609 610 err = copy_from_user(&ext, address, sizeof(ext)); 611 if (XE_IOCTL_DBG(xe, err)) 612 return -EFAULT; 613 614 if (XE_IOCTL_DBG(xe, ext.pad) || 615 XE_IOCTL_DBG(xe, ext.name >= 616 ARRAY_SIZE(exec_queue_user_extension_funcs))) 617 return -EINVAL; 618 619 idx = array_index_nospec(ext.name, 620 ARRAY_SIZE(exec_queue_user_extension_funcs)); 621 err = exec_queue_user_extension_funcs[idx](xe, q, extensions); 622 if (XE_IOCTL_DBG(xe, err)) 623 return err; 624 625 if (ext.next_extension) 626 return exec_queue_user_extensions(xe, q, ext.next_extension, 627 ++ext_number); 628 629 return 0; 630 } 631 632 static u32 calc_validate_logical_mask(struct xe_device *xe, 633 struct drm_xe_engine_class_instance *eci, 634 u16 width, u16 num_placements) 635 { 636 int len = width * num_placements; 637 int i, j, n; 638 u16 class; 639 u16 gt_id; 640 u32 return_mask = 0, prev_mask; 641 642 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 643 len > 1)) 644 return 0; 645 646 for (i = 0; i < width; ++i) { 647 u32 current_mask = 0; 648 649 for (j = 0; j < num_placements; ++j) { 650 struct xe_hw_engine *hwe; 651 652 n = j * width + i; 653 654 hwe = xe_hw_engine_lookup(xe, eci[n]); 655 if (XE_IOCTL_DBG(xe, !hwe)) 656 return 0; 657 658 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 659 return 0; 660 661 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 662 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 663 return 0; 664 665 class = eci[n].engine_class; 666 gt_id = eci[n].gt_id; 667 668 if (width == 1 || !i) 669 return_mask |= BIT(eci[n].engine_instance); 670 current_mask |= BIT(eci[n].engine_instance); 671 } 672 673 /* Parallel submissions must be logically contiguous */ 674 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 675 return 0; 676 677 prev_mask = current_mask; 678 } 679 680 return return_mask; 681 } 682 683 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 684 struct drm_file *file) 685 { 686 struct xe_device *xe = to_xe_device(dev); 687 struct xe_file *xef = to_xe_file(file); 688 struct drm_xe_exec_queue_create *args = data; 689 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 690 struct drm_xe_engine_class_instance __user *user_eci = 691 u64_to_user_ptr(args->instances); 692 struct xe_hw_engine *hwe; 693 struct xe_vm *vm; 694 struct xe_tile *tile; 695 struct xe_exec_queue *q = NULL; 696 u32 logical_mask; 697 u32 flags = 0; 698 u32 id; 699 u32 len; 700 int err; 701 702 if (XE_IOCTL_DBG(xe, args->flags & ~DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) || 703 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 704 return -EINVAL; 705 706 len = args->width * args->num_placements; 707 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 708 return -EINVAL; 709 710 err = copy_from_user(eci, user_eci, 711 sizeof(struct drm_xe_engine_class_instance) * len); 712 if (XE_IOCTL_DBG(xe, err)) 713 return -EFAULT; 714 715 if (XE_IOCTL_DBG(xe, !xe_device_get_gt(xe, eci[0].gt_id))) 716 return -EINVAL; 717 718 if (args->flags & DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) 719 flags |= EXEC_QUEUE_FLAG_LOW_LATENCY; 720 721 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { 722 if (XE_IOCTL_DBG(xe, args->width != 1) || 723 XE_IOCTL_DBG(xe, args->num_placements != 1) || 724 XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 725 return -EINVAL; 726 727 for_each_tile(tile, xe, id) { 728 struct xe_exec_queue *new; 729 730 flags |= EXEC_QUEUE_FLAG_VM; 731 if (id) 732 flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; 733 734 new = xe_exec_queue_create_bind(xe, tile, flags, 735 args->extensions); 736 if (IS_ERR(new)) { 737 err = PTR_ERR(new); 738 if (q) 739 goto put_exec_queue; 740 return err; 741 } 742 if (id == 0) 743 q = new; 744 else 745 list_add_tail(&new->multi_gt_list, 746 &q->multi_gt_link); 747 } 748 } else { 749 logical_mask = calc_validate_logical_mask(xe, eci, 750 args->width, 751 args->num_placements); 752 if (XE_IOCTL_DBG(xe, !logical_mask)) 753 return -EINVAL; 754 755 hwe = xe_hw_engine_lookup(xe, eci[0]); 756 if (XE_IOCTL_DBG(xe, !hwe)) 757 return -EINVAL; 758 759 vm = xe_vm_lookup(xef, args->vm_id); 760 if (XE_IOCTL_DBG(xe, !vm)) 761 return -ENOENT; 762 763 err = down_read_interruptible(&vm->lock); 764 if (err) { 765 xe_vm_put(vm); 766 return err; 767 } 768 769 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 770 up_read(&vm->lock); 771 xe_vm_put(vm); 772 return -ENOENT; 773 } 774 775 q = xe_exec_queue_create(xe, vm, logical_mask, 776 args->width, hwe, flags, 777 args->extensions); 778 up_read(&vm->lock); 779 xe_vm_put(vm); 780 if (IS_ERR(q)) 781 return PTR_ERR(q); 782 783 if (xe_vm_in_preempt_fence_mode(vm)) { 784 q->lr.context = dma_fence_context_alloc(1); 785 786 err = xe_vm_add_compute_exec_queue(vm, q); 787 if (XE_IOCTL_DBG(xe, err)) 788 goto put_exec_queue; 789 } 790 791 if (q->vm && q->hwe->hw_engine_group) { 792 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); 793 if (err) 794 goto put_exec_queue; 795 } 796 } 797 798 q->xef = xe_file_get(xef); 799 800 /* user id alloc must always be last in ioctl to prevent UAF */ 801 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 802 if (err) 803 goto kill_exec_queue; 804 805 args->exec_queue_id = id; 806 807 return 0; 808 809 kill_exec_queue: 810 xe_exec_queue_kill(q); 811 put_exec_queue: 812 xe_exec_queue_put(q); 813 return err; 814 } 815 816 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 817 struct drm_file *file) 818 { 819 struct xe_device *xe = to_xe_device(dev); 820 struct xe_file *xef = to_xe_file(file); 821 struct drm_xe_exec_queue_get_property *args = data; 822 struct xe_exec_queue *q; 823 int ret; 824 825 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 826 return -EINVAL; 827 828 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 829 if (XE_IOCTL_DBG(xe, !q)) 830 return -ENOENT; 831 832 switch (args->property) { 833 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: 834 args->value = q->ops->reset_status(q); 835 ret = 0; 836 break; 837 default: 838 ret = -EINVAL; 839 } 840 841 xe_exec_queue_put(q); 842 843 return ret; 844 } 845 846 /** 847 * xe_exec_queue_lrc() - Get the LRC from exec queue. 848 * @q: The exec_queue. 849 * 850 * Retrieves the primary LRC for the exec queue. Note that this function 851 * returns only the first LRC instance, even when multiple parallel LRCs 852 * are configured. 853 * 854 * Return: Pointer to LRC on success, error on failure 855 */ 856 struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q) 857 { 858 return q->lrc[0]; 859 } 860 861 /** 862 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 863 * @q: The exec_queue 864 * 865 * Return: True if the exec_queue is long-running, false otherwise. 866 */ 867 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 868 { 869 return q->vm && xe_vm_in_lr_mode(q->vm) && 870 !(q->flags & EXEC_QUEUE_FLAG_VM); 871 } 872 873 /** 874 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 875 * @q: The exec_queue 876 * 877 * FIXME: Need to determine what to use as the short-lived 878 * timeline lock for the exec_queues, so that the return value 879 * of this function becomes more than just an advisory 880 * snapshot in time. The timeline lock must protect the 881 * seqno from racing submissions on the same exec_queue. 882 * Typically vm->resv, but user-created timeline locks use the migrate vm 883 * and never grabs the migrate vm->resv so we have a race there. 884 * 885 * Return: True if the exec_queue is idle, false otherwise. 886 */ 887 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 888 { 889 if (xe_exec_queue_is_parallel(q)) { 890 int i; 891 892 for (i = 0; i < q->width; ++i) { 893 if (xe_lrc_seqno(q->lrc[i]) != 894 q->lrc[i]->fence_ctx.next_seqno - 1) 895 return false; 896 } 897 898 return true; 899 } 900 901 return xe_lrc_seqno(q->lrc[0]) == 902 q->lrc[0]->fence_ctx.next_seqno - 1; 903 } 904 905 /** 906 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue 907 * from hw 908 * @q: The exec queue 909 * 910 * Update the timestamp saved by HW for this exec queue and save run ticks 911 * calculated by using the delta from last update. 912 */ 913 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) 914 { 915 struct xe_device *xe = gt_to_xe(q->gt); 916 struct xe_lrc *lrc; 917 u64 old_ts, new_ts; 918 int idx; 919 920 /* 921 * Jobs that are executed by kernel doesn't have a corresponding xe_file 922 * and thus are not accounted. 923 */ 924 if (!q->xef) 925 return; 926 927 /* Synchronize with unbind while holding the xe file open */ 928 if (!drm_dev_enter(&xe->drm, &idx)) 929 return; 930 /* 931 * Only sample the first LRC. For parallel submission, all of them are 932 * scheduled together and we compensate that below by multiplying by 933 * width - this may introduce errors if that premise is not true and 934 * they don't exit 100% aligned. On the other hand, looping through 935 * the LRCs and reading them in different time could also introduce 936 * errors. 937 */ 938 lrc = q->lrc[0]; 939 new_ts = xe_lrc_update_timestamp(lrc, &old_ts); 940 q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; 941 942 drm_dev_exit(idx); 943 } 944 945 /** 946 * xe_exec_queue_kill - permanently stop all execution from an exec queue 947 * @q: The exec queue 948 * 949 * This function permanently stops all activity on an exec queue. If the queue 950 * is actively executing on the HW, it will be kicked off the engine; any 951 * pending jobs are discarded and all future submissions are rejected. 952 * This function is safe to call multiple times. 953 */ 954 void xe_exec_queue_kill(struct xe_exec_queue *q) 955 { 956 struct xe_exec_queue *eq = q, *next; 957 958 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 959 multi_gt_link) { 960 q->ops->kill(eq); 961 xe_vm_remove_compute_exec_queue(q->vm, eq); 962 } 963 964 q->ops->kill(q); 965 xe_vm_remove_compute_exec_queue(q->vm, q); 966 } 967 968 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 969 struct drm_file *file) 970 { 971 struct xe_device *xe = to_xe_device(dev); 972 struct xe_file *xef = to_xe_file(file); 973 struct drm_xe_exec_queue_destroy *args = data; 974 struct xe_exec_queue *q; 975 976 if (XE_IOCTL_DBG(xe, args->pad) || 977 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 978 return -EINVAL; 979 980 mutex_lock(&xef->exec_queue.lock); 981 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 982 if (q) 983 atomic_inc(&xef->exec_queue.pending_removal); 984 mutex_unlock(&xef->exec_queue.lock); 985 986 if (XE_IOCTL_DBG(xe, !q)) 987 return -ENOENT; 988 989 if (q->vm && q->hwe->hw_engine_group) 990 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 991 992 xe_exec_queue_kill(q); 993 994 trace_xe_exec_queue_close(q); 995 xe_exec_queue_put(q); 996 997 return 0; 998 } 999 1000 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 1001 struct xe_vm *vm) 1002 { 1003 if (q->flags & EXEC_QUEUE_FLAG_VM) { 1004 lockdep_assert_held(&vm->lock); 1005 } else { 1006 xe_vm_assert_held(vm); 1007 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); 1008 } 1009 } 1010 1011 /** 1012 * xe_exec_queue_last_fence_put() - Drop ref to last fence 1013 * @q: The exec queue 1014 * @vm: The VM the engine does a bind or exec for 1015 */ 1016 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 1017 { 1018 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1019 1020 xe_exec_queue_last_fence_put_unlocked(q); 1021 } 1022 1023 /** 1024 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 1025 * @q: The exec queue 1026 * 1027 * Only safe to be called from xe_exec_queue_destroy(). 1028 */ 1029 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 1030 { 1031 if (q->last_fence) { 1032 dma_fence_put(q->last_fence); 1033 q->last_fence = NULL; 1034 } 1035 } 1036 1037 /** 1038 * xe_exec_queue_last_fence_get() - Get last fence 1039 * @q: The exec queue 1040 * @vm: The VM the engine does a bind or exec for 1041 * 1042 * Get last fence, takes a ref 1043 * 1044 * Returns: last fence if not signaled, dma fence stub if signaled 1045 */ 1046 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 1047 struct xe_vm *vm) 1048 { 1049 struct dma_fence *fence; 1050 1051 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1052 1053 if (q->last_fence && 1054 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 1055 xe_exec_queue_last_fence_put(q, vm); 1056 1057 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 1058 dma_fence_get(fence); 1059 return fence; 1060 } 1061 1062 /** 1063 * xe_exec_queue_last_fence_get_for_resume() - Get last fence 1064 * @q: The exec queue 1065 * @vm: The VM the engine does a bind or exec for 1066 * 1067 * Get last fence, takes a ref. Only safe to be called in the context of 1068 * resuming the hw engine group's long-running exec queue, when the group 1069 * semaphore is held. 1070 * 1071 * Returns: last fence if not signaled, dma fence stub if signaled 1072 */ 1073 struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q, 1074 struct xe_vm *vm) 1075 { 1076 struct dma_fence *fence; 1077 1078 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); 1079 1080 if (q->last_fence && 1081 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 1082 xe_exec_queue_last_fence_put_unlocked(q); 1083 1084 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 1085 dma_fence_get(fence); 1086 return fence; 1087 } 1088 1089 /** 1090 * xe_exec_queue_last_fence_set() - Set last fence 1091 * @q: The exec queue 1092 * @vm: The VM the engine does a bind or exec for 1093 * @fence: The fence 1094 * 1095 * Set the last fence for the engine. Increases reference count for fence, when 1096 * closing engine xe_exec_queue_last_fence_put should be called. 1097 */ 1098 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 1099 struct dma_fence *fence) 1100 { 1101 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1102 1103 xe_exec_queue_last_fence_put(q, vm); 1104 q->last_fence = dma_fence_get(fence); 1105 } 1106 1107 /** 1108 * xe_exec_queue_last_fence_test_dep - Test last fence dependency of queue 1109 * @q: The exec queue 1110 * @vm: The VM the engine does a bind or exec for 1111 * 1112 * Returns: 1113 * -ETIME if there exists an unsignalled last fence dependency, zero otherwise. 1114 */ 1115 int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) 1116 { 1117 struct dma_fence *fence; 1118 int err = 0; 1119 1120 fence = xe_exec_queue_last_fence_get(q, vm); 1121 if (fence) { 1122 err = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 1123 0 : -ETIME; 1124 dma_fence_put(fence); 1125 } 1126 1127 return err; 1128 } 1129 1130 /** 1131 * xe_exec_queue_contexts_hwsp_rebase - Re-compute GGTT references 1132 * within all LRCs of a queue. 1133 * @q: the &xe_exec_queue struct instance containing target LRCs 1134 * @scratch: scratch buffer to be used as temporary storage 1135 * 1136 * Returns: zero on success, negative error code on failure 1137 */ 1138 int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch) 1139 { 1140 int i; 1141 int err = 0; 1142 1143 for (i = 0; i < q->width; ++i) { 1144 struct xe_lrc *lrc; 1145 1146 /* Pairs with WRITE_ONCE in __xe_exec_queue_init */ 1147 lrc = READ_ONCE(q->lrc[i]); 1148 if (!lrc) 1149 continue; 1150 1151 xe_lrc_update_memirq_regs_with_address(lrc, q->hwe, scratch); 1152 xe_lrc_update_hwctx_regs_with_address(lrc); 1153 err = xe_lrc_setup_wa_bb_with_scratch(lrc, q->hwe, scratch); 1154 if (err) 1155 break; 1156 } 1157 1158 return err; 1159 } 1160