1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_drv.h> 12 #include <drm/drm_file.h> 13 #include <uapi/drm/xe_drm.h> 14 15 #include "xe_dep_scheduler.h" 16 #include "xe_device.h" 17 #include "xe_gt.h" 18 #include "xe_hw_engine_class_sysfs.h" 19 #include "xe_hw_engine_group.h" 20 #include "xe_hw_fence.h" 21 #include "xe_irq.h" 22 #include "xe_lrc.h" 23 #include "xe_macros.h" 24 #include "xe_migrate.h" 25 #include "xe_pm.h" 26 #include "xe_ring_ops_types.h" 27 #include "xe_trace.h" 28 #include "xe_vm.h" 29 #include "xe_pxp.h" 30 31 enum xe_exec_queue_sched_prop { 32 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 33 XE_EXEC_QUEUE_TIMESLICE = 1, 34 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 35 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 36 }; 37 38 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 39 u64 extensions, int ext_number); 40 41 static void __xe_exec_queue_free(struct xe_exec_queue *q) 42 { 43 int i; 44 45 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) 46 if (q->tlb_inval[i].dep_scheduler) 47 xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); 48 49 if (xe_exec_queue_uses_pxp(q)) 50 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); 51 if (q->vm) 52 xe_vm_put(q->vm); 53 54 if (q->xef) 55 xe_file_put(q->xef); 56 57 kfree(q); 58 } 59 60 static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q) 61 { 62 struct xe_tile *tile = gt_to_tile(q->gt); 63 int i; 64 65 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) { 66 struct xe_dep_scheduler *dep_scheduler; 67 struct xe_gt *gt; 68 struct workqueue_struct *wq; 69 70 if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT) 71 gt = tile->primary_gt; 72 else 73 gt = tile->media_gt; 74 75 if (!gt) 76 continue; 77 78 wq = gt->tlb_inval.job_wq; 79 80 #define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */ 81 dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, 82 MAX_TLB_INVAL_JOBS); 83 if (IS_ERR(dep_scheduler)) 84 return PTR_ERR(dep_scheduler); 85 86 q->tlb_inval[i].dep_scheduler = dep_scheduler; 87 } 88 #undef MAX_TLB_INVAL_JOBS 89 90 return 0; 91 } 92 93 static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, 94 struct xe_vm *vm, 95 u32 logical_mask, 96 u16 width, struct xe_hw_engine *hwe, 97 u32 flags, u64 extensions) 98 { 99 struct xe_exec_queue *q; 100 struct xe_gt *gt = hwe->gt; 101 int err; 102 103 /* only kernel queues can be permanent */ 104 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 105 106 q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); 107 if (!q) 108 return ERR_PTR(-ENOMEM); 109 110 kref_init(&q->refcount); 111 q->flags = flags; 112 q->hwe = hwe; 113 q->gt = gt; 114 q->class = hwe->class; 115 q->width = width; 116 q->msix_vec = XE_IRQ_DEFAULT_MSIX; 117 q->logical_mask = logical_mask; 118 q->fence_irq = >->fence_irq[hwe->class]; 119 q->ring_ops = gt->ring_ops[hwe->class]; 120 q->ops = gt->exec_queue_ops; 121 INIT_LIST_HEAD(&q->lr.link); 122 INIT_LIST_HEAD(&q->multi_gt_link); 123 INIT_LIST_HEAD(&q->hw_engine_group_link); 124 INIT_LIST_HEAD(&q->pxp.link); 125 126 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 127 q->sched_props.preempt_timeout_us = 128 hwe->eclass->sched_props.preempt_timeout_us; 129 q->sched_props.job_timeout_ms = 130 hwe->eclass->sched_props.job_timeout_ms; 131 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && 132 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) 133 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; 134 else 135 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 136 137 if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) { 138 err = alloc_dep_schedulers(xe, q); 139 if (err) { 140 __xe_exec_queue_free(q); 141 return ERR_PTR(err); 142 } 143 } 144 145 if (vm) 146 q->vm = xe_vm_get(vm); 147 148 if (extensions) { 149 /* 150 * may set q->usm, must come before xe_lrc_create(), 151 * may overwrite q->sched_props, must come before q->ops->init() 152 */ 153 err = exec_queue_user_extensions(xe, q, extensions, 0); 154 if (err) { 155 __xe_exec_queue_free(q); 156 return ERR_PTR(err); 157 } 158 } 159 160 return q; 161 } 162 163 static int __xe_exec_queue_init(struct xe_exec_queue *q) 164 { 165 int i, err; 166 u32 flags = 0; 167 168 /* 169 * PXP workloads executing on RCS or CCS must run in isolation (i.e. no 170 * other workload can use the EUs at the same time). On MTL this is done 171 * by setting the RUNALONE bit in the LRC, while starting on Xe2 there 172 * is a dedicated bit for it. 173 */ 174 if (xe_exec_queue_uses_pxp(q) && 175 (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) { 176 if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20) 177 flags |= XE_LRC_CREATE_PXP; 178 else 179 flags |= XE_LRC_CREATE_RUNALONE; 180 } 181 182 for (i = 0; i < q->width; ++i) { 183 q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K, q->msix_vec, flags); 184 if (IS_ERR(q->lrc[i])) { 185 err = PTR_ERR(q->lrc[i]); 186 goto err_lrc; 187 } 188 } 189 190 err = q->ops->init(q); 191 if (err) 192 goto err_lrc; 193 194 return 0; 195 196 err_lrc: 197 for (i = i - 1; i >= 0; --i) 198 xe_lrc_put(q->lrc[i]); 199 return err; 200 } 201 202 static void __xe_exec_queue_fini(struct xe_exec_queue *q) 203 { 204 int i; 205 206 q->ops->fini(q); 207 208 for (i = 0; i < q->width; ++i) 209 xe_lrc_put(q->lrc[i]); 210 } 211 212 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 213 u32 logical_mask, u16 width, 214 struct xe_hw_engine *hwe, u32 flags, 215 u64 extensions) 216 { 217 struct xe_exec_queue *q; 218 int err; 219 220 /* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */ 221 xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0))); 222 223 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, 224 extensions); 225 if (IS_ERR(q)) 226 return q; 227 228 err = __xe_exec_queue_init(q); 229 if (err) 230 goto err_post_alloc; 231 232 /* 233 * We can only add the queue to the PXP list after the init is complete, 234 * because the PXP termination can call exec_queue_kill and that will 235 * go bad if the queue is only half-initialized. This means that we 236 * can't do it when we handle the PXP extension in __xe_exec_queue_alloc 237 * and we need to do it here instead. 238 */ 239 if (xe_exec_queue_uses_pxp(q)) { 240 err = xe_pxp_exec_queue_add(xe->pxp, q); 241 if (err) 242 goto err_post_init; 243 } 244 245 return q; 246 247 err_post_init: 248 __xe_exec_queue_fini(q); 249 err_post_alloc: 250 __xe_exec_queue_free(q); 251 return ERR_PTR(err); 252 } 253 ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO); 254 255 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 256 struct xe_vm *vm, 257 enum xe_engine_class class, 258 u32 flags, u64 extensions) 259 { 260 struct xe_hw_engine *hwe, *hwe0 = NULL; 261 enum xe_hw_engine_id id; 262 u32 logical_mask = 0; 263 264 for_each_hw_engine(hwe, gt, id) { 265 if (xe_hw_engine_is_reserved(hwe)) 266 continue; 267 268 if (hwe->class == class) { 269 logical_mask |= BIT(hwe->logical_instance); 270 if (!hwe0) 271 hwe0 = hwe; 272 } 273 } 274 275 if (!logical_mask) 276 return ERR_PTR(-ENODEV); 277 278 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); 279 } 280 281 /** 282 * xe_exec_queue_create_bind() - Create bind exec queue. 283 * @xe: Xe device. 284 * @tile: tile which bind exec queue belongs to. 285 * @flags: exec queue creation flags 286 * @extensions: exec queue creation extensions 287 * 288 * Normalize bind exec queue creation. Bind exec queue is tied to migration VM 289 * for access to physical memory required for page table programming. On a 290 * faulting devices the reserved copy engine instance must be used to avoid 291 * deadlocking (user binds cannot get stuck behind faults as kernel binds which 292 * resolve faults depend on user binds). On non-faulting devices any copy engine 293 * can be used. 294 * 295 * Returns exec queue on success, ERR_PTR on failure 296 */ 297 struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, 298 struct xe_tile *tile, 299 u32 flags, u64 extensions) 300 { 301 struct xe_gt *gt = tile->primary_gt; 302 struct xe_exec_queue *q; 303 struct xe_vm *migrate_vm; 304 305 migrate_vm = xe_migrate_get_vm(tile->migrate); 306 if (xe->info.has_usm) { 307 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, 308 XE_ENGINE_CLASS_COPY, 309 gt->usm.reserved_bcs_instance, 310 false); 311 312 if (!hwe) { 313 xe_vm_put(migrate_vm); 314 return ERR_PTR(-EINVAL); 315 } 316 317 q = xe_exec_queue_create(xe, migrate_vm, 318 BIT(hwe->logical_instance), 1, hwe, 319 flags, extensions); 320 } else { 321 q = xe_exec_queue_create_class(xe, gt, migrate_vm, 322 XE_ENGINE_CLASS_COPY, flags, 323 extensions); 324 } 325 xe_vm_put(migrate_vm); 326 327 return q; 328 } 329 ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO); 330 331 void xe_exec_queue_destroy(struct kref *ref) 332 { 333 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 334 struct xe_exec_queue *eq, *next; 335 336 if (xe_exec_queue_uses_pxp(q)) 337 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); 338 339 xe_exec_queue_last_fence_put_unlocked(q); 340 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 341 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 342 multi_gt_link) 343 xe_exec_queue_put(eq); 344 } 345 346 q->ops->destroy(q); 347 } 348 349 void xe_exec_queue_fini(struct xe_exec_queue *q) 350 { 351 /* 352 * Before releasing our ref to lrc and xef, accumulate our run ticks 353 * and wakeup any waiters. 354 */ 355 xe_exec_queue_update_run_ticks(q); 356 if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal)) 357 wake_up_var(&q->xef->exec_queue.pending_removal); 358 359 __xe_exec_queue_fini(q); 360 __xe_exec_queue_free(q); 361 } 362 363 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 364 { 365 switch (q->class) { 366 case XE_ENGINE_CLASS_RENDER: 367 snprintf(q->name, sizeof(q->name), "rcs%d", instance); 368 break; 369 case XE_ENGINE_CLASS_VIDEO_DECODE: 370 snprintf(q->name, sizeof(q->name), "vcs%d", instance); 371 break; 372 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 373 snprintf(q->name, sizeof(q->name), "vecs%d", instance); 374 break; 375 case XE_ENGINE_CLASS_COPY: 376 snprintf(q->name, sizeof(q->name), "bcs%d", instance); 377 break; 378 case XE_ENGINE_CLASS_COMPUTE: 379 snprintf(q->name, sizeof(q->name), "ccs%d", instance); 380 break; 381 case XE_ENGINE_CLASS_OTHER: 382 snprintf(q->name, sizeof(q->name), "gsccs%d", instance); 383 break; 384 default: 385 XE_WARN_ON(q->class); 386 } 387 } 388 389 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 390 { 391 struct xe_exec_queue *q; 392 393 mutex_lock(&xef->exec_queue.lock); 394 q = xa_load(&xef->exec_queue.xa, id); 395 if (q) 396 xe_exec_queue_get(q); 397 mutex_unlock(&xef->exec_queue.lock); 398 399 return q; 400 } 401 402 enum xe_exec_queue_priority 403 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 404 { 405 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 406 XE_EXEC_QUEUE_PRIORITY_NORMAL; 407 } 408 409 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 410 u64 value) 411 { 412 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 413 return -EINVAL; 414 415 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 416 return -EPERM; 417 418 q->sched_props.priority = value; 419 return 0; 420 } 421 422 static bool xe_exec_queue_enforce_schedule_limit(void) 423 { 424 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 425 return true; 426 #else 427 return !capable(CAP_SYS_NICE); 428 #endif 429 } 430 431 static void 432 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 433 enum xe_exec_queue_sched_prop prop, 434 u32 *min, u32 *max) 435 { 436 switch (prop) { 437 case XE_EXEC_QUEUE_JOB_TIMEOUT: 438 *min = eclass->sched_props.job_timeout_min; 439 *max = eclass->sched_props.job_timeout_max; 440 break; 441 case XE_EXEC_QUEUE_TIMESLICE: 442 *min = eclass->sched_props.timeslice_min; 443 *max = eclass->sched_props.timeslice_max; 444 break; 445 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 446 *min = eclass->sched_props.preempt_timeout_min; 447 *max = eclass->sched_props.preempt_timeout_max; 448 break; 449 default: 450 break; 451 } 452 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 453 if (capable(CAP_SYS_NICE)) { 454 switch (prop) { 455 case XE_EXEC_QUEUE_JOB_TIMEOUT: 456 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 457 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 458 break; 459 case XE_EXEC_QUEUE_TIMESLICE: 460 *min = XE_HW_ENGINE_TIMESLICE_MIN; 461 *max = XE_HW_ENGINE_TIMESLICE_MAX; 462 break; 463 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 464 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 465 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 466 break; 467 default: 468 break; 469 } 470 } 471 #endif 472 } 473 474 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 475 u64 value) 476 { 477 u32 min = 0, max = 0; 478 479 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 480 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 481 482 if (xe_exec_queue_enforce_schedule_limit() && 483 !xe_hw_engine_timeout_in_range(value, min, max)) 484 return -EINVAL; 485 486 q->sched_props.timeslice_us = value; 487 return 0; 488 } 489 490 static int 491 exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value) 492 { 493 if (value == DRM_XE_PXP_TYPE_NONE) 494 return 0; 495 496 /* we only support HWDRM sessions right now */ 497 if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM)) 498 return -EINVAL; 499 500 if (!xe_pxp_is_enabled(xe->pxp)) 501 return -ENODEV; 502 503 return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM); 504 } 505 506 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 507 struct xe_exec_queue *q, 508 u64 value); 509 510 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 511 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 512 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 513 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type, 514 }; 515 516 static int exec_queue_user_ext_set_property(struct xe_device *xe, 517 struct xe_exec_queue *q, 518 u64 extension) 519 { 520 u64 __user *address = u64_to_user_ptr(extension); 521 struct drm_xe_ext_set_property ext; 522 int err; 523 u32 idx; 524 525 err = copy_from_user(&ext, address, sizeof(ext)); 526 if (XE_IOCTL_DBG(xe, err)) 527 return -EFAULT; 528 529 if (XE_IOCTL_DBG(xe, ext.property >= 530 ARRAY_SIZE(exec_queue_set_property_funcs)) || 531 XE_IOCTL_DBG(xe, ext.pad) || 532 XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && 533 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE && 534 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE)) 535 return -EINVAL; 536 537 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 538 if (!exec_queue_set_property_funcs[idx]) 539 return -EINVAL; 540 541 return exec_queue_set_property_funcs[idx](xe, q, ext.value); 542 } 543 544 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 545 struct xe_exec_queue *q, 546 u64 extension); 547 548 static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { 549 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 550 }; 551 552 #define MAX_USER_EXTENSIONS 16 553 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 554 u64 extensions, int ext_number) 555 { 556 u64 __user *address = u64_to_user_ptr(extensions); 557 struct drm_xe_user_extension ext; 558 int err; 559 u32 idx; 560 561 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 562 return -E2BIG; 563 564 err = copy_from_user(&ext, address, sizeof(ext)); 565 if (XE_IOCTL_DBG(xe, err)) 566 return -EFAULT; 567 568 if (XE_IOCTL_DBG(xe, ext.pad) || 569 XE_IOCTL_DBG(xe, ext.name >= 570 ARRAY_SIZE(exec_queue_user_extension_funcs))) 571 return -EINVAL; 572 573 idx = array_index_nospec(ext.name, 574 ARRAY_SIZE(exec_queue_user_extension_funcs)); 575 err = exec_queue_user_extension_funcs[idx](xe, q, extensions); 576 if (XE_IOCTL_DBG(xe, err)) 577 return err; 578 579 if (ext.next_extension) 580 return exec_queue_user_extensions(xe, q, ext.next_extension, 581 ++ext_number); 582 583 return 0; 584 } 585 586 static u32 calc_validate_logical_mask(struct xe_device *xe, 587 struct drm_xe_engine_class_instance *eci, 588 u16 width, u16 num_placements) 589 { 590 int len = width * num_placements; 591 int i, j, n; 592 u16 class; 593 u16 gt_id; 594 u32 return_mask = 0, prev_mask; 595 596 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 597 len > 1)) 598 return 0; 599 600 for (i = 0; i < width; ++i) { 601 u32 current_mask = 0; 602 603 for (j = 0; j < num_placements; ++j) { 604 struct xe_hw_engine *hwe; 605 606 n = j * width + i; 607 608 hwe = xe_hw_engine_lookup(xe, eci[n]); 609 if (XE_IOCTL_DBG(xe, !hwe)) 610 return 0; 611 612 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 613 return 0; 614 615 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 616 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 617 return 0; 618 619 class = eci[n].engine_class; 620 gt_id = eci[n].gt_id; 621 622 if (width == 1 || !i) 623 return_mask |= BIT(eci[n].engine_instance); 624 current_mask |= BIT(eci[n].engine_instance); 625 } 626 627 /* Parallel submissions must be logically contiguous */ 628 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 629 return 0; 630 631 prev_mask = current_mask; 632 } 633 634 return return_mask; 635 } 636 637 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 638 struct drm_file *file) 639 { 640 struct xe_device *xe = to_xe_device(dev); 641 struct xe_file *xef = to_xe_file(file); 642 struct drm_xe_exec_queue_create *args = data; 643 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 644 struct drm_xe_engine_class_instance __user *user_eci = 645 u64_to_user_ptr(args->instances); 646 struct xe_hw_engine *hwe; 647 struct xe_vm *vm; 648 struct xe_tile *tile; 649 struct xe_exec_queue *q = NULL; 650 u32 logical_mask; 651 u32 flags = 0; 652 u32 id; 653 u32 len; 654 int err; 655 656 if (XE_IOCTL_DBG(xe, args->flags & ~DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) || 657 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 658 return -EINVAL; 659 660 len = args->width * args->num_placements; 661 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 662 return -EINVAL; 663 664 err = copy_from_user(eci, user_eci, 665 sizeof(struct drm_xe_engine_class_instance) * len); 666 if (XE_IOCTL_DBG(xe, err)) 667 return -EFAULT; 668 669 if (XE_IOCTL_DBG(xe, !xe_device_get_gt(xe, eci[0].gt_id))) 670 return -EINVAL; 671 672 if (args->flags & DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) 673 flags |= EXEC_QUEUE_FLAG_LOW_LATENCY; 674 675 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { 676 if (XE_IOCTL_DBG(xe, args->width != 1) || 677 XE_IOCTL_DBG(xe, args->num_placements != 1) || 678 XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 679 return -EINVAL; 680 681 for_each_tile(tile, xe, id) { 682 struct xe_exec_queue *new; 683 684 flags |= EXEC_QUEUE_FLAG_VM; 685 if (id) 686 flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; 687 688 new = xe_exec_queue_create_bind(xe, tile, flags, 689 args->extensions); 690 if (IS_ERR(new)) { 691 err = PTR_ERR(new); 692 if (q) 693 goto put_exec_queue; 694 return err; 695 } 696 if (id == 0) 697 q = new; 698 else 699 list_add_tail(&new->multi_gt_list, 700 &q->multi_gt_link); 701 } 702 } else { 703 logical_mask = calc_validate_logical_mask(xe, eci, 704 args->width, 705 args->num_placements); 706 if (XE_IOCTL_DBG(xe, !logical_mask)) 707 return -EINVAL; 708 709 hwe = xe_hw_engine_lookup(xe, eci[0]); 710 if (XE_IOCTL_DBG(xe, !hwe)) 711 return -EINVAL; 712 713 vm = xe_vm_lookup(xef, args->vm_id); 714 if (XE_IOCTL_DBG(xe, !vm)) 715 return -ENOENT; 716 717 err = down_read_interruptible(&vm->lock); 718 if (err) { 719 xe_vm_put(vm); 720 return err; 721 } 722 723 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 724 up_read(&vm->lock); 725 xe_vm_put(vm); 726 return -ENOENT; 727 } 728 729 q = xe_exec_queue_create(xe, vm, logical_mask, 730 args->width, hwe, flags, 731 args->extensions); 732 up_read(&vm->lock); 733 xe_vm_put(vm); 734 if (IS_ERR(q)) 735 return PTR_ERR(q); 736 737 if (xe_vm_in_preempt_fence_mode(vm)) { 738 q->lr.context = dma_fence_context_alloc(1); 739 740 err = xe_vm_add_compute_exec_queue(vm, q); 741 if (XE_IOCTL_DBG(xe, err)) 742 goto put_exec_queue; 743 } 744 745 if (q->vm && q->hwe->hw_engine_group) { 746 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); 747 if (err) 748 goto put_exec_queue; 749 } 750 } 751 752 q->xef = xe_file_get(xef); 753 754 /* user id alloc must always be last in ioctl to prevent UAF */ 755 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 756 if (err) 757 goto kill_exec_queue; 758 759 args->exec_queue_id = id; 760 761 return 0; 762 763 kill_exec_queue: 764 xe_exec_queue_kill(q); 765 put_exec_queue: 766 xe_exec_queue_put(q); 767 return err; 768 } 769 770 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 771 struct drm_file *file) 772 { 773 struct xe_device *xe = to_xe_device(dev); 774 struct xe_file *xef = to_xe_file(file); 775 struct drm_xe_exec_queue_get_property *args = data; 776 struct xe_exec_queue *q; 777 int ret; 778 779 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 780 return -EINVAL; 781 782 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 783 if (XE_IOCTL_DBG(xe, !q)) 784 return -ENOENT; 785 786 switch (args->property) { 787 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: 788 args->value = q->ops->reset_status(q); 789 ret = 0; 790 break; 791 default: 792 ret = -EINVAL; 793 } 794 795 xe_exec_queue_put(q); 796 797 return ret; 798 } 799 800 /** 801 * xe_exec_queue_lrc() - Get the LRC from exec queue. 802 * @q: The exec_queue. 803 * 804 * Retrieves the primary LRC for the exec queue. Note that this function 805 * returns only the first LRC instance, even when multiple parallel LRCs 806 * are configured. 807 * 808 * Return: Pointer to LRC on success, error on failure 809 */ 810 struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q) 811 { 812 return q->lrc[0]; 813 } 814 815 /** 816 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 817 * @q: The exec_queue 818 * 819 * Return: True if the exec_queue is long-running, false otherwise. 820 */ 821 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 822 { 823 return q->vm && xe_vm_in_lr_mode(q->vm) && 824 !(q->flags & EXEC_QUEUE_FLAG_VM); 825 } 826 827 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 828 { 829 return q->lrc[0]->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc[0]) - 1; 830 } 831 832 /** 833 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 834 * @q: The exec_queue 835 * 836 * Return: True if the exec_queue's ring is full, false otherwise. 837 */ 838 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 839 { 840 struct xe_lrc *lrc = q->lrc[0]; 841 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 842 843 return xe_exec_queue_num_job_inflight(q) >= max_job; 844 } 845 846 /** 847 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 848 * @q: The exec_queue 849 * 850 * FIXME: Need to determine what to use as the short-lived 851 * timeline lock for the exec_queues, so that the return value 852 * of this function becomes more than just an advisory 853 * snapshot in time. The timeline lock must protect the 854 * seqno from racing submissions on the same exec_queue. 855 * Typically vm->resv, but user-created timeline locks use the migrate vm 856 * and never grabs the migrate vm->resv so we have a race there. 857 * 858 * Return: True if the exec_queue is idle, false otherwise. 859 */ 860 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 861 { 862 if (xe_exec_queue_is_parallel(q)) { 863 int i; 864 865 for (i = 0; i < q->width; ++i) { 866 if (xe_lrc_seqno(q->lrc[i]) != 867 q->lrc[i]->fence_ctx.next_seqno - 1) 868 return false; 869 } 870 871 return true; 872 } 873 874 return xe_lrc_seqno(q->lrc[0]) == 875 q->lrc[0]->fence_ctx.next_seqno - 1; 876 } 877 878 /** 879 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue 880 * from hw 881 * @q: The exec queue 882 * 883 * Update the timestamp saved by HW for this exec queue and save run ticks 884 * calculated by using the delta from last update. 885 */ 886 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) 887 { 888 struct xe_device *xe = gt_to_xe(q->gt); 889 struct xe_lrc *lrc; 890 u64 old_ts, new_ts; 891 int idx; 892 893 /* 894 * Jobs that are executed by kernel doesn't have a corresponding xe_file 895 * and thus are not accounted. 896 */ 897 if (!q->xef) 898 return; 899 900 /* Synchronize with unbind while holding the xe file open */ 901 if (!drm_dev_enter(&xe->drm, &idx)) 902 return; 903 /* 904 * Only sample the first LRC. For parallel submission, all of them are 905 * scheduled together and we compensate that below by multiplying by 906 * width - this may introduce errors if that premise is not true and 907 * they don't exit 100% aligned. On the other hand, looping through 908 * the LRCs and reading them in different time could also introduce 909 * errors. 910 */ 911 lrc = q->lrc[0]; 912 new_ts = xe_lrc_update_timestamp(lrc, &old_ts); 913 q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; 914 915 drm_dev_exit(idx); 916 } 917 918 /** 919 * xe_exec_queue_kill - permanently stop all execution from an exec queue 920 * @q: The exec queue 921 * 922 * This function permanently stops all activity on an exec queue. If the queue 923 * is actively executing on the HW, it will be kicked off the engine; any 924 * pending jobs are discarded and all future submissions are rejected. 925 * This function is safe to call multiple times. 926 */ 927 void xe_exec_queue_kill(struct xe_exec_queue *q) 928 { 929 struct xe_exec_queue *eq = q, *next; 930 931 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 932 multi_gt_link) { 933 q->ops->kill(eq); 934 xe_vm_remove_compute_exec_queue(q->vm, eq); 935 } 936 937 q->ops->kill(q); 938 xe_vm_remove_compute_exec_queue(q->vm, q); 939 } 940 941 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 942 struct drm_file *file) 943 { 944 struct xe_device *xe = to_xe_device(dev); 945 struct xe_file *xef = to_xe_file(file); 946 struct drm_xe_exec_queue_destroy *args = data; 947 struct xe_exec_queue *q; 948 949 if (XE_IOCTL_DBG(xe, args->pad) || 950 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 951 return -EINVAL; 952 953 mutex_lock(&xef->exec_queue.lock); 954 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 955 if (q) 956 atomic_inc(&xef->exec_queue.pending_removal); 957 mutex_unlock(&xef->exec_queue.lock); 958 959 if (XE_IOCTL_DBG(xe, !q)) 960 return -ENOENT; 961 962 if (q->vm && q->hwe->hw_engine_group) 963 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 964 965 xe_exec_queue_kill(q); 966 967 trace_xe_exec_queue_close(q); 968 xe_exec_queue_put(q); 969 970 return 0; 971 } 972 973 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 974 struct xe_vm *vm) 975 { 976 if (q->flags & EXEC_QUEUE_FLAG_VM) { 977 lockdep_assert_held(&vm->lock); 978 } else { 979 xe_vm_assert_held(vm); 980 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); 981 } 982 } 983 984 /** 985 * xe_exec_queue_last_fence_put() - Drop ref to last fence 986 * @q: The exec queue 987 * @vm: The VM the engine does a bind or exec for 988 */ 989 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 990 { 991 xe_exec_queue_last_fence_lockdep_assert(q, vm); 992 993 xe_exec_queue_last_fence_put_unlocked(q); 994 } 995 996 /** 997 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 998 * @q: The exec queue 999 * 1000 * Only safe to be called from xe_exec_queue_destroy(). 1001 */ 1002 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 1003 { 1004 if (q->last_fence) { 1005 dma_fence_put(q->last_fence); 1006 q->last_fence = NULL; 1007 } 1008 } 1009 1010 /** 1011 * xe_exec_queue_last_fence_get() - Get last fence 1012 * @q: The exec queue 1013 * @vm: The VM the engine does a bind or exec for 1014 * 1015 * Get last fence, takes a ref 1016 * 1017 * Returns: last fence if not signaled, dma fence stub if signaled 1018 */ 1019 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 1020 struct xe_vm *vm) 1021 { 1022 struct dma_fence *fence; 1023 1024 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1025 1026 if (q->last_fence && 1027 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 1028 xe_exec_queue_last_fence_put(q, vm); 1029 1030 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 1031 dma_fence_get(fence); 1032 return fence; 1033 } 1034 1035 /** 1036 * xe_exec_queue_last_fence_get_for_resume() - Get last fence 1037 * @q: The exec queue 1038 * @vm: The VM the engine does a bind or exec for 1039 * 1040 * Get last fence, takes a ref. Only safe to be called in the context of 1041 * resuming the hw engine group's long-running exec queue, when the group 1042 * semaphore is held. 1043 * 1044 * Returns: last fence if not signaled, dma fence stub if signaled 1045 */ 1046 struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q, 1047 struct xe_vm *vm) 1048 { 1049 struct dma_fence *fence; 1050 1051 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); 1052 1053 if (q->last_fence && 1054 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 1055 xe_exec_queue_last_fence_put_unlocked(q); 1056 1057 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 1058 dma_fence_get(fence); 1059 return fence; 1060 } 1061 1062 /** 1063 * xe_exec_queue_last_fence_set() - Set last fence 1064 * @q: The exec queue 1065 * @vm: The VM the engine does a bind or exec for 1066 * @fence: The fence 1067 * 1068 * Set the last fence for the engine. Increases reference count for fence, when 1069 * closing engine xe_exec_queue_last_fence_put should be called. 1070 */ 1071 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 1072 struct dma_fence *fence) 1073 { 1074 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1075 1076 xe_exec_queue_last_fence_put(q, vm); 1077 q->last_fence = dma_fence_get(fence); 1078 } 1079 1080 /** 1081 * xe_exec_queue_last_fence_test_dep - Test last fence dependency of queue 1082 * @q: The exec queue 1083 * @vm: The VM the engine does a bind or exec for 1084 * 1085 * Returns: 1086 * -ETIME if there exists an unsignalled last fence dependency, zero otherwise. 1087 */ 1088 int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) 1089 { 1090 struct dma_fence *fence; 1091 int err = 0; 1092 1093 fence = xe_exec_queue_last_fence_get(q, vm); 1094 if (fence) { 1095 err = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 1096 0 : -ETIME; 1097 dma_fence_put(fence); 1098 } 1099 1100 return err; 1101 } 1102 1103 /** 1104 * xe_exec_queue_contexts_hwsp_rebase - Re-compute GGTT references 1105 * within all LRCs of a queue. 1106 * @q: the &xe_exec_queue struct instance containing target LRCs 1107 * @scratch: scratch buffer to be used as temporary storage 1108 * 1109 * Returns: zero on success, negative error code on failure 1110 */ 1111 int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch) 1112 { 1113 int i; 1114 int err = 0; 1115 1116 for (i = 0; i < q->width; ++i) { 1117 xe_lrc_update_memirq_regs_with_address(q->lrc[i], q->hwe, scratch); 1118 xe_lrc_update_hwctx_regs_with_address(q->lrc[i]); 1119 err = xe_lrc_setup_wa_bb_with_scratch(q->lrc[i], q->hwe, scratch); 1120 if (err) 1121 break; 1122 } 1123 1124 return err; 1125 } 1126 1127 /** 1128 * xe_exec_queue_jobs_ring_restore - Re-emit ring commands of requests pending on given queue. 1129 * @q: the &xe_exec_queue struct instance 1130 */ 1131 void xe_exec_queue_jobs_ring_restore(struct xe_exec_queue *q) 1132 { 1133 struct xe_gpu_scheduler *sched = &q->guc->sched; 1134 struct xe_sched_job *job; 1135 1136 /* 1137 * This routine is used within VF migration recovery. This means 1138 * using the lock here introduces a restriction: we cannot wait 1139 * for any GFX HW response while the lock is taken. 1140 */ 1141 spin_lock(&sched->base.job_list_lock); 1142 list_for_each_entry(job, &sched->base.pending_list, drm.list) { 1143 if (xe_sched_job_is_error(job)) 1144 continue; 1145 1146 q->ring_ops->emit_job(job); 1147 } 1148 spin_unlock(&sched->base.job_list_lock); 1149 } 1150