1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_drv.h> 12 #include <drm/drm_file.h> 13 #include <uapi/drm/xe_drm.h> 14 15 #include "xe_dep_scheduler.h" 16 #include "xe_device.h" 17 #include "xe_gt.h" 18 #include "xe_hw_engine_class_sysfs.h" 19 #include "xe_hw_engine_group.h" 20 #include "xe_hw_fence.h" 21 #include "xe_irq.h" 22 #include "xe_lrc.h" 23 #include "xe_macros.h" 24 #include "xe_migrate.h" 25 #include "xe_pm.h" 26 #include "xe_ring_ops_types.h" 27 #include "xe_trace.h" 28 #include "xe_vm.h" 29 #include "xe_pxp.h" 30 31 enum xe_exec_queue_sched_prop { 32 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 33 XE_EXEC_QUEUE_TIMESLICE = 1, 34 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 35 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 36 }; 37 38 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 39 u64 extensions, int ext_number); 40 41 static void __xe_exec_queue_free(struct xe_exec_queue *q) 42 { 43 int i; 44 45 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) 46 if (q->tlb_inval[i].dep_scheduler) 47 xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); 48 49 if (xe_exec_queue_uses_pxp(q)) 50 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); 51 if (q->vm) 52 xe_vm_put(q->vm); 53 54 if (q->xef) 55 xe_file_put(q->xef); 56 57 kfree(q); 58 } 59 60 static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q) 61 { 62 struct xe_tile *tile = gt_to_tile(q->gt); 63 int i; 64 65 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) { 66 struct xe_dep_scheduler *dep_scheduler; 67 struct xe_gt *gt; 68 struct workqueue_struct *wq; 69 70 if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT) 71 gt = tile->primary_gt; 72 else 73 gt = tile->media_gt; 74 75 if (!gt) 76 continue; 77 78 wq = gt->tlb_invalidation.job_wq; 79 80 #define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */ 81 dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, 82 MAX_TLB_INVAL_JOBS); 83 if (IS_ERR(dep_scheduler)) 84 return PTR_ERR(dep_scheduler); 85 86 q->tlb_inval[i].dep_scheduler = dep_scheduler; 87 } 88 #undef MAX_TLB_INVAL_JOBS 89 90 return 0; 91 } 92 93 static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, 94 struct xe_vm *vm, 95 u32 logical_mask, 96 u16 width, struct xe_hw_engine *hwe, 97 u32 flags, u64 extensions) 98 { 99 struct xe_exec_queue *q; 100 struct xe_gt *gt = hwe->gt; 101 int err; 102 103 /* only kernel queues can be permanent */ 104 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 105 106 q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); 107 if (!q) 108 return ERR_PTR(-ENOMEM); 109 110 kref_init(&q->refcount); 111 q->flags = flags; 112 q->hwe = hwe; 113 q->gt = gt; 114 q->class = hwe->class; 115 q->width = width; 116 q->msix_vec = XE_IRQ_DEFAULT_MSIX; 117 q->logical_mask = logical_mask; 118 q->fence_irq = >->fence_irq[hwe->class]; 119 q->ring_ops = gt->ring_ops[hwe->class]; 120 q->ops = gt->exec_queue_ops; 121 INIT_LIST_HEAD(&q->lr.link); 122 INIT_LIST_HEAD(&q->multi_gt_link); 123 INIT_LIST_HEAD(&q->hw_engine_group_link); 124 INIT_LIST_HEAD(&q->pxp.link); 125 126 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 127 q->sched_props.preempt_timeout_us = 128 hwe->eclass->sched_props.preempt_timeout_us; 129 q->sched_props.job_timeout_ms = 130 hwe->eclass->sched_props.job_timeout_ms; 131 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && 132 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) 133 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; 134 else 135 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 136 137 if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) { 138 err = alloc_dep_schedulers(xe, q); 139 if (err) { 140 __xe_exec_queue_free(q); 141 return ERR_PTR(err); 142 } 143 } 144 145 if (vm) 146 q->vm = xe_vm_get(vm); 147 148 if (extensions) { 149 /* 150 * may set q->usm, must come before xe_lrc_create(), 151 * may overwrite q->sched_props, must come before q->ops->init() 152 */ 153 err = exec_queue_user_extensions(xe, q, extensions, 0); 154 if (err) { 155 __xe_exec_queue_free(q); 156 return ERR_PTR(err); 157 } 158 } 159 160 return q; 161 } 162 163 static int __xe_exec_queue_init(struct xe_exec_queue *q) 164 { 165 int i, err; 166 u32 flags = 0; 167 168 /* 169 * PXP workloads executing on RCS or CCS must run in isolation (i.e. no 170 * other workload can use the EUs at the same time). On MTL this is done 171 * by setting the RUNALONE bit in the LRC, while starting on Xe2 there 172 * is a dedicated bit for it. 173 */ 174 if (xe_exec_queue_uses_pxp(q) && 175 (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) { 176 if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20) 177 flags |= XE_LRC_CREATE_PXP; 178 else 179 flags |= XE_LRC_CREATE_RUNALONE; 180 } 181 182 for (i = 0; i < q->width; ++i) { 183 q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K, q->msix_vec, flags); 184 if (IS_ERR(q->lrc[i])) { 185 err = PTR_ERR(q->lrc[i]); 186 goto err_lrc; 187 } 188 } 189 190 err = q->ops->init(q); 191 if (err) 192 goto err_lrc; 193 194 return 0; 195 196 err_lrc: 197 for (i = i - 1; i >= 0; --i) 198 xe_lrc_put(q->lrc[i]); 199 return err; 200 } 201 202 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 203 u32 logical_mask, u16 width, 204 struct xe_hw_engine *hwe, u32 flags, 205 u64 extensions) 206 { 207 struct xe_exec_queue *q; 208 int err; 209 210 /* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */ 211 xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0))); 212 213 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, 214 extensions); 215 if (IS_ERR(q)) 216 return q; 217 218 err = __xe_exec_queue_init(q); 219 if (err) 220 goto err_post_alloc; 221 222 /* 223 * We can only add the queue to the PXP list after the init is complete, 224 * because the PXP termination can call exec_queue_kill and that will 225 * go bad if the queue is only half-initialized. This means that we 226 * can't do it when we handle the PXP extension in __xe_exec_queue_alloc 227 * and we need to do it here instead. 228 */ 229 if (xe_exec_queue_uses_pxp(q)) { 230 err = xe_pxp_exec_queue_add(xe->pxp, q); 231 if (err) 232 goto err_post_alloc; 233 } 234 235 return q; 236 237 err_post_alloc: 238 __xe_exec_queue_free(q); 239 return ERR_PTR(err); 240 } 241 ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO); 242 243 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 244 struct xe_vm *vm, 245 enum xe_engine_class class, 246 u32 flags, u64 extensions) 247 { 248 struct xe_hw_engine *hwe, *hwe0 = NULL; 249 enum xe_hw_engine_id id; 250 u32 logical_mask = 0; 251 252 for_each_hw_engine(hwe, gt, id) { 253 if (xe_hw_engine_is_reserved(hwe)) 254 continue; 255 256 if (hwe->class == class) { 257 logical_mask |= BIT(hwe->logical_instance); 258 if (!hwe0) 259 hwe0 = hwe; 260 } 261 } 262 263 if (!logical_mask) 264 return ERR_PTR(-ENODEV); 265 266 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions); 267 } 268 269 /** 270 * xe_exec_queue_create_bind() - Create bind exec queue. 271 * @xe: Xe device. 272 * @tile: tile which bind exec queue belongs to. 273 * @flags: exec queue creation flags 274 * @extensions: exec queue creation extensions 275 * 276 * Normalize bind exec queue creation. Bind exec queue is tied to migration VM 277 * for access to physical memory required for page table programming. On a 278 * faulting devices the reserved copy engine instance must be used to avoid 279 * deadlocking (user binds cannot get stuck behind faults as kernel binds which 280 * resolve faults depend on user binds). On non-faulting devices any copy engine 281 * can be used. 282 * 283 * Returns exec queue on success, ERR_PTR on failure 284 */ 285 struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe, 286 struct xe_tile *tile, 287 u32 flags, u64 extensions) 288 { 289 struct xe_gt *gt = tile->primary_gt; 290 struct xe_exec_queue *q; 291 struct xe_vm *migrate_vm; 292 293 migrate_vm = xe_migrate_get_vm(tile->migrate); 294 if (xe->info.has_usm) { 295 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, 296 XE_ENGINE_CLASS_COPY, 297 gt->usm.reserved_bcs_instance, 298 false); 299 300 if (!hwe) { 301 xe_vm_put(migrate_vm); 302 return ERR_PTR(-EINVAL); 303 } 304 305 q = xe_exec_queue_create(xe, migrate_vm, 306 BIT(hwe->logical_instance), 1, hwe, 307 flags, extensions); 308 } else { 309 q = xe_exec_queue_create_class(xe, gt, migrate_vm, 310 XE_ENGINE_CLASS_COPY, flags, 311 extensions); 312 } 313 xe_vm_put(migrate_vm); 314 315 return q; 316 } 317 ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO); 318 319 void xe_exec_queue_destroy(struct kref *ref) 320 { 321 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 322 struct xe_exec_queue *eq, *next; 323 324 if (xe_exec_queue_uses_pxp(q)) 325 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); 326 327 xe_exec_queue_last_fence_put_unlocked(q); 328 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 329 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 330 multi_gt_link) 331 xe_exec_queue_put(eq); 332 } 333 334 q->ops->fini(q); 335 } 336 337 void xe_exec_queue_fini(struct xe_exec_queue *q) 338 { 339 int i; 340 341 /* 342 * Before releasing our ref to lrc and xef, accumulate our run ticks 343 * and wakeup any waiters. 344 */ 345 xe_exec_queue_update_run_ticks(q); 346 if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal)) 347 wake_up_var(&q->xef->exec_queue.pending_removal); 348 349 for (i = 0; i < q->width; ++i) 350 xe_lrc_put(q->lrc[i]); 351 352 __xe_exec_queue_free(q); 353 } 354 355 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 356 { 357 switch (q->class) { 358 case XE_ENGINE_CLASS_RENDER: 359 snprintf(q->name, sizeof(q->name), "rcs%d", instance); 360 break; 361 case XE_ENGINE_CLASS_VIDEO_DECODE: 362 snprintf(q->name, sizeof(q->name), "vcs%d", instance); 363 break; 364 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 365 snprintf(q->name, sizeof(q->name), "vecs%d", instance); 366 break; 367 case XE_ENGINE_CLASS_COPY: 368 snprintf(q->name, sizeof(q->name), "bcs%d", instance); 369 break; 370 case XE_ENGINE_CLASS_COMPUTE: 371 snprintf(q->name, sizeof(q->name), "ccs%d", instance); 372 break; 373 case XE_ENGINE_CLASS_OTHER: 374 snprintf(q->name, sizeof(q->name), "gsccs%d", instance); 375 break; 376 default: 377 XE_WARN_ON(q->class); 378 } 379 } 380 381 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 382 { 383 struct xe_exec_queue *q; 384 385 mutex_lock(&xef->exec_queue.lock); 386 q = xa_load(&xef->exec_queue.xa, id); 387 if (q) 388 xe_exec_queue_get(q); 389 mutex_unlock(&xef->exec_queue.lock); 390 391 return q; 392 } 393 394 enum xe_exec_queue_priority 395 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 396 { 397 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 398 XE_EXEC_QUEUE_PRIORITY_NORMAL; 399 } 400 401 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 402 u64 value) 403 { 404 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 405 return -EINVAL; 406 407 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 408 return -EPERM; 409 410 q->sched_props.priority = value; 411 return 0; 412 } 413 414 static bool xe_exec_queue_enforce_schedule_limit(void) 415 { 416 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 417 return true; 418 #else 419 return !capable(CAP_SYS_NICE); 420 #endif 421 } 422 423 static void 424 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 425 enum xe_exec_queue_sched_prop prop, 426 u32 *min, u32 *max) 427 { 428 switch (prop) { 429 case XE_EXEC_QUEUE_JOB_TIMEOUT: 430 *min = eclass->sched_props.job_timeout_min; 431 *max = eclass->sched_props.job_timeout_max; 432 break; 433 case XE_EXEC_QUEUE_TIMESLICE: 434 *min = eclass->sched_props.timeslice_min; 435 *max = eclass->sched_props.timeslice_max; 436 break; 437 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 438 *min = eclass->sched_props.preempt_timeout_min; 439 *max = eclass->sched_props.preempt_timeout_max; 440 break; 441 default: 442 break; 443 } 444 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 445 if (capable(CAP_SYS_NICE)) { 446 switch (prop) { 447 case XE_EXEC_QUEUE_JOB_TIMEOUT: 448 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 449 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 450 break; 451 case XE_EXEC_QUEUE_TIMESLICE: 452 *min = XE_HW_ENGINE_TIMESLICE_MIN; 453 *max = XE_HW_ENGINE_TIMESLICE_MAX; 454 break; 455 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 456 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 457 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 458 break; 459 default: 460 break; 461 } 462 } 463 #endif 464 } 465 466 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 467 u64 value) 468 { 469 u32 min = 0, max = 0; 470 471 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 472 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 473 474 if (xe_exec_queue_enforce_schedule_limit() && 475 !xe_hw_engine_timeout_in_range(value, min, max)) 476 return -EINVAL; 477 478 q->sched_props.timeslice_us = value; 479 return 0; 480 } 481 482 static int 483 exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value) 484 { 485 if (value == DRM_XE_PXP_TYPE_NONE) 486 return 0; 487 488 /* we only support HWDRM sessions right now */ 489 if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM)) 490 return -EINVAL; 491 492 if (!xe_pxp_is_enabled(xe->pxp)) 493 return -ENODEV; 494 495 return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM); 496 } 497 498 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 499 struct xe_exec_queue *q, 500 u64 value); 501 502 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 503 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 504 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 505 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type, 506 }; 507 508 static int exec_queue_user_ext_set_property(struct xe_device *xe, 509 struct xe_exec_queue *q, 510 u64 extension) 511 { 512 u64 __user *address = u64_to_user_ptr(extension); 513 struct drm_xe_ext_set_property ext; 514 int err; 515 u32 idx; 516 517 err = copy_from_user(&ext, address, sizeof(ext)); 518 if (XE_IOCTL_DBG(xe, err)) 519 return -EFAULT; 520 521 if (XE_IOCTL_DBG(xe, ext.property >= 522 ARRAY_SIZE(exec_queue_set_property_funcs)) || 523 XE_IOCTL_DBG(xe, ext.pad) || 524 XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && 525 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE && 526 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE)) 527 return -EINVAL; 528 529 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 530 if (!exec_queue_set_property_funcs[idx]) 531 return -EINVAL; 532 533 return exec_queue_set_property_funcs[idx](xe, q, ext.value); 534 } 535 536 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 537 struct xe_exec_queue *q, 538 u64 extension); 539 540 static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = { 541 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 542 }; 543 544 #define MAX_USER_EXTENSIONS 16 545 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 546 u64 extensions, int ext_number) 547 { 548 u64 __user *address = u64_to_user_ptr(extensions); 549 struct drm_xe_user_extension ext; 550 int err; 551 u32 idx; 552 553 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 554 return -E2BIG; 555 556 err = copy_from_user(&ext, address, sizeof(ext)); 557 if (XE_IOCTL_DBG(xe, err)) 558 return -EFAULT; 559 560 if (XE_IOCTL_DBG(xe, ext.pad) || 561 XE_IOCTL_DBG(xe, ext.name >= 562 ARRAY_SIZE(exec_queue_user_extension_funcs))) 563 return -EINVAL; 564 565 idx = array_index_nospec(ext.name, 566 ARRAY_SIZE(exec_queue_user_extension_funcs)); 567 err = exec_queue_user_extension_funcs[idx](xe, q, extensions); 568 if (XE_IOCTL_DBG(xe, err)) 569 return err; 570 571 if (ext.next_extension) 572 return exec_queue_user_extensions(xe, q, ext.next_extension, 573 ++ext_number); 574 575 return 0; 576 } 577 578 static u32 calc_validate_logical_mask(struct xe_device *xe, 579 struct drm_xe_engine_class_instance *eci, 580 u16 width, u16 num_placements) 581 { 582 int len = width * num_placements; 583 int i, j, n; 584 u16 class; 585 u16 gt_id; 586 u32 return_mask = 0, prev_mask; 587 588 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 589 len > 1)) 590 return 0; 591 592 for (i = 0; i < width; ++i) { 593 u32 current_mask = 0; 594 595 for (j = 0; j < num_placements; ++j) { 596 struct xe_hw_engine *hwe; 597 598 n = j * width + i; 599 600 hwe = xe_hw_engine_lookup(xe, eci[n]); 601 if (XE_IOCTL_DBG(xe, !hwe)) 602 return 0; 603 604 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 605 return 0; 606 607 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 608 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 609 return 0; 610 611 class = eci[n].engine_class; 612 gt_id = eci[n].gt_id; 613 614 if (width == 1 || !i) 615 return_mask |= BIT(eci[n].engine_instance); 616 current_mask |= BIT(eci[n].engine_instance); 617 } 618 619 /* Parallel submissions must be logically contiguous */ 620 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 621 return 0; 622 623 prev_mask = current_mask; 624 } 625 626 return return_mask; 627 } 628 629 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 630 struct drm_file *file) 631 { 632 struct xe_device *xe = to_xe_device(dev); 633 struct xe_file *xef = to_xe_file(file); 634 struct drm_xe_exec_queue_create *args = data; 635 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 636 struct drm_xe_engine_class_instance __user *user_eci = 637 u64_to_user_ptr(args->instances); 638 struct xe_hw_engine *hwe; 639 struct xe_vm *vm; 640 struct xe_tile *tile; 641 struct xe_exec_queue *q = NULL; 642 u32 logical_mask; 643 u32 flags = 0; 644 u32 id; 645 u32 len; 646 int err; 647 648 if (XE_IOCTL_DBG(xe, args->flags & ~DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) || 649 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 650 return -EINVAL; 651 652 len = args->width * args->num_placements; 653 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 654 return -EINVAL; 655 656 err = copy_from_user(eci, user_eci, 657 sizeof(struct drm_xe_engine_class_instance) * len); 658 if (XE_IOCTL_DBG(xe, err)) 659 return -EFAULT; 660 661 if (XE_IOCTL_DBG(xe, !xe_device_get_gt(xe, eci[0].gt_id))) 662 return -EINVAL; 663 664 if (args->flags & DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) 665 flags |= EXEC_QUEUE_FLAG_LOW_LATENCY; 666 667 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { 668 if (XE_IOCTL_DBG(xe, args->width != 1) || 669 XE_IOCTL_DBG(xe, args->num_placements != 1) || 670 XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 671 return -EINVAL; 672 673 for_each_tile(tile, xe, id) { 674 struct xe_exec_queue *new; 675 676 flags |= EXEC_QUEUE_FLAG_VM; 677 if (id) 678 flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; 679 680 new = xe_exec_queue_create_bind(xe, tile, flags, 681 args->extensions); 682 if (IS_ERR(new)) { 683 err = PTR_ERR(new); 684 if (q) 685 goto put_exec_queue; 686 return err; 687 } 688 if (id == 0) 689 q = new; 690 else 691 list_add_tail(&new->multi_gt_list, 692 &q->multi_gt_link); 693 } 694 } else { 695 logical_mask = calc_validate_logical_mask(xe, eci, 696 args->width, 697 args->num_placements); 698 if (XE_IOCTL_DBG(xe, !logical_mask)) 699 return -EINVAL; 700 701 hwe = xe_hw_engine_lookup(xe, eci[0]); 702 if (XE_IOCTL_DBG(xe, !hwe)) 703 return -EINVAL; 704 705 vm = xe_vm_lookup(xef, args->vm_id); 706 if (XE_IOCTL_DBG(xe, !vm)) 707 return -ENOENT; 708 709 err = down_read_interruptible(&vm->lock); 710 if (err) { 711 xe_vm_put(vm); 712 return err; 713 } 714 715 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 716 up_read(&vm->lock); 717 xe_vm_put(vm); 718 return -ENOENT; 719 } 720 721 q = xe_exec_queue_create(xe, vm, logical_mask, 722 args->width, hwe, flags, 723 args->extensions); 724 up_read(&vm->lock); 725 xe_vm_put(vm); 726 if (IS_ERR(q)) 727 return PTR_ERR(q); 728 729 if (xe_vm_in_preempt_fence_mode(vm)) { 730 q->lr.context = dma_fence_context_alloc(1); 731 732 err = xe_vm_add_compute_exec_queue(vm, q); 733 if (XE_IOCTL_DBG(xe, err)) 734 goto put_exec_queue; 735 } 736 737 if (q->vm && q->hwe->hw_engine_group) { 738 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); 739 if (err) 740 goto put_exec_queue; 741 } 742 } 743 744 q->xef = xe_file_get(xef); 745 746 /* user id alloc must always be last in ioctl to prevent UAF */ 747 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 748 if (err) 749 goto kill_exec_queue; 750 751 args->exec_queue_id = id; 752 753 return 0; 754 755 kill_exec_queue: 756 xe_exec_queue_kill(q); 757 put_exec_queue: 758 xe_exec_queue_put(q); 759 return err; 760 } 761 762 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 763 struct drm_file *file) 764 { 765 struct xe_device *xe = to_xe_device(dev); 766 struct xe_file *xef = to_xe_file(file); 767 struct drm_xe_exec_queue_get_property *args = data; 768 struct xe_exec_queue *q; 769 int ret; 770 771 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 772 return -EINVAL; 773 774 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 775 if (XE_IOCTL_DBG(xe, !q)) 776 return -ENOENT; 777 778 switch (args->property) { 779 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: 780 args->value = q->ops->reset_status(q); 781 ret = 0; 782 break; 783 default: 784 ret = -EINVAL; 785 } 786 787 xe_exec_queue_put(q); 788 789 return ret; 790 } 791 792 /** 793 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 794 * @q: The exec_queue 795 * 796 * Return: True if the exec_queue is long-running, false otherwise. 797 */ 798 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 799 { 800 return q->vm && xe_vm_in_lr_mode(q->vm) && 801 !(q->flags & EXEC_QUEUE_FLAG_VM); 802 } 803 804 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 805 { 806 return q->lrc[0]->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc[0]) - 1; 807 } 808 809 /** 810 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 811 * @q: The exec_queue 812 * 813 * Return: True if the exec_queue's ring is full, false otherwise. 814 */ 815 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 816 { 817 struct xe_lrc *lrc = q->lrc[0]; 818 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 819 820 return xe_exec_queue_num_job_inflight(q) >= max_job; 821 } 822 823 /** 824 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 825 * @q: The exec_queue 826 * 827 * FIXME: Need to determine what to use as the short-lived 828 * timeline lock for the exec_queues, so that the return value 829 * of this function becomes more than just an advisory 830 * snapshot in time. The timeline lock must protect the 831 * seqno from racing submissions on the same exec_queue. 832 * Typically vm->resv, but user-created timeline locks use the migrate vm 833 * and never grabs the migrate vm->resv so we have a race there. 834 * 835 * Return: True if the exec_queue is idle, false otherwise. 836 */ 837 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 838 { 839 if (xe_exec_queue_is_parallel(q)) { 840 int i; 841 842 for (i = 0; i < q->width; ++i) { 843 if (xe_lrc_seqno(q->lrc[i]) != 844 q->lrc[i]->fence_ctx.next_seqno - 1) 845 return false; 846 } 847 848 return true; 849 } 850 851 return xe_lrc_seqno(q->lrc[0]) == 852 q->lrc[0]->fence_ctx.next_seqno - 1; 853 } 854 855 /** 856 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue 857 * from hw 858 * @q: The exec queue 859 * 860 * Update the timestamp saved by HW for this exec queue and save run ticks 861 * calculated by using the delta from last update. 862 */ 863 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q) 864 { 865 struct xe_device *xe = gt_to_xe(q->gt); 866 struct xe_lrc *lrc; 867 u64 old_ts, new_ts; 868 int idx; 869 870 /* 871 * Jobs that are executed by kernel doesn't have a corresponding xe_file 872 * and thus are not accounted. 873 */ 874 if (!q->xef) 875 return; 876 877 /* Synchronize with unbind while holding the xe file open */ 878 if (!drm_dev_enter(&xe->drm, &idx)) 879 return; 880 /* 881 * Only sample the first LRC. For parallel submission, all of them are 882 * scheduled together and we compensate that below by multiplying by 883 * width - this may introduce errors if that premise is not true and 884 * they don't exit 100% aligned. On the other hand, looping through 885 * the LRCs and reading them in different time could also introduce 886 * errors. 887 */ 888 lrc = q->lrc[0]; 889 new_ts = xe_lrc_update_timestamp(lrc, &old_ts); 890 q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; 891 892 drm_dev_exit(idx); 893 } 894 895 /** 896 * xe_exec_queue_kill - permanently stop all execution from an exec queue 897 * @q: The exec queue 898 * 899 * This function permanently stops all activity on an exec queue. If the queue 900 * is actively executing on the HW, it will be kicked off the engine; any 901 * pending jobs are discarded and all future submissions are rejected. 902 * This function is safe to call multiple times. 903 */ 904 void xe_exec_queue_kill(struct xe_exec_queue *q) 905 { 906 struct xe_exec_queue *eq = q, *next; 907 908 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 909 multi_gt_link) { 910 q->ops->kill(eq); 911 xe_vm_remove_compute_exec_queue(q->vm, eq); 912 } 913 914 q->ops->kill(q); 915 xe_vm_remove_compute_exec_queue(q->vm, q); 916 } 917 918 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 919 struct drm_file *file) 920 { 921 struct xe_device *xe = to_xe_device(dev); 922 struct xe_file *xef = to_xe_file(file); 923 struct drm_xe_exec_queue_destroy *args = data; 924 struct xe_exec_queue *q; 925 926 if (XE_IOCTL_DBG(xe, args->pad) || 927 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 928 return -EINVAL; 929 930 mutex_lock(&xef->exec_queue.lock); 931 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 932 if (q) 933 atomic_inc(&xef->exec_queue.pending_removal); 934 mutex_unlock(&xef->exec_queue.lock); 935 936 if (XE_IOCTL_DBG(xe, !q)) 937 return -ENOENT; 938 939 if (q->vm && q->hwe->hw_engine_group) 940 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 941 942 xe_exec_queue_kill(q); 943 944 trace_xe_exec_queue_close(q); 945 xe_exec_queue_put(q); 946 947 return 0; 948 } 949 950 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 951 struct xe_vm *vm) 952 { 953 if (q->flags & EXEC_QUEUE_FLAG_VM) { 954 lockdep_assert_held(&vm->lock); 955 } else { 956 xe_vm_assert_held(vm); 957 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); 958 } 959 } 960 961 /** 962 * xe_exec_queue_last_fence_put() - Drop ref to last fence 963 * @q: The exec queue 964 * @vm: The VM the engine does a bind or exec for 965 */ 966 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 967 { 968 xe_exec_queue_last_fence_lockdep_assert(q, vm); 969 970 xe_exec_queue_last_fence_put_unlocked(q); 971 } 972 973 /** 974 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 975 * @q: The exec queue 976 * 977 * Only safe to be called from xe_exec_queue_destroy(). 978 */ 979 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 980 { 981 if (q->last_fence) { 982 dma_fence_put(q->last_fence); 983 q->last_fence = NULL; 984 } 985 } 986 987 /** 988 * xe_exec_queue_last_fence_get() - Get last fence 989 * @q: The exec queue 990 * @vm: The VM the engine does a bind or exec for 991 * 992 * Get last fence, takes a ref 993 * 994 * Returns: last fence if not signaled, dma fence stub if signaled 995 */ 996 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 997 struct xe_vm *vm) 998 { 999 struct dma_fence *fence; 1000 1001 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1002 1003 if (q->last_fence && 1004 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 1005 xe_exec_queue_last_fence_put(q, vm); 1006 1007 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 1008 dma_fence_get(fence); 1009 return fence; 1010 } 1011 1012 /** 1013 * xe_exec_queue_last_fence_get_for_resume() - Get last fence 1014 * @q: The exec queue 1015 * @vm: The VM the engine does a bind or exec for 1016 * 1017 * Get last fence, takes a ref. Only safe to be called in the context of 1018 * resuming the hw engine group's long-running exec queue, when the group 1019 * semaphore is held. 1020 * 1021 * Returns: last fence if not signaled, dma fence stub if signaled 1022 */ 1023 struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q, 1024 struct xe_vm *vm) 1025 { 1026 struct dma_fence *fence; 1027 1028 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); 1029 1030 if (q->last_fence && 1031 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 1032 xe_exec_queue_last_fence_put_unlocked(q); 1033 1034 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 1035 dma_fence_get(fence); 1036 return fence; 1037 } 1038 1039 /** 1040 * xe_exec_queue_last_fence_set() - Set last fence 1041 * @q: The exec queue 1042 * @vm: The VM the engine does a bind or exec for 1043 * @fence: The fence 1044 * 1045 * Set the last fence for the engine. Increases reference count for fence, when 1046 * closing engine xe_exec_queue_last_fence_put should be called. 1047 */ 1048 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 1049 struct dma_fence *fence) 1050 { 1051 xe_exec_queue_last_fence_lockdep_assert(q, vm); 1052 1053 xe_exec_queue_last_fence_put(q, vm); 1054 q->last_fence = dma_fence_get(fence); 1055 } 1056 1057 /** 1058 * xe_exec_queue_last_fence_test_dep - Test last fence dependency of queue 1059 * @q: The exec queue 1060 * @vm: The VM the engine does a bind or exec for 1061 * 1062 * Returns: 1063 * -ETIME if there exists an unsignalled last fence dependency, zero otherwise. 1064 */ 1065 int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm) 1066 { 1067 struct dma_fence *fence; 1068 int err = 0; 1069 1070 fence = xe_exec_queue_last_fence_get(q, vm); 1071 if (fence) { 1072 err = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 1073 0 : -ETIME; 1074 dma_fence_put(fence); 1075 } 1076 1077 return err; 1078 } 1079