1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_file.h> 12 #include <drm/xe_drm.h> 13 14 #include "xe_device.h" 15 #include "xe_gt.h" 16 #include "xe_hw_engine_class_sysfs.h" 17 #include "xe_hw_fence.h" 18 #include "xe_lrc.h" 19 #include "xe_macros.h" 20 #include "xe_migrate.h" 21 #include "xe_pm.h" 22 #include "xe_ring_ops_types.h" 23 #include "xe_trace.h" 24 #include "xe_vm.h" 25 26 enum xe_exec_queue_sched_prop { 27 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 28 XE_EXEC_QUEUE_TIMESLICE = 1, 29 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 30 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 31 }; 32 33 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 34 u64 extensions, int ext_number, bool create); 35 36 static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, 37 struct xe_vm *vm, 38 u32 logical_mask, 39 u16 width, struct xe_hw_engine *hwe, 40 u32 flags, u64 extensions) 41 { 42 struct xe_exec_queue *q; 43 struct xe_gt *gt = hwe->gt; 44 int err; 45 46 /* only kernel queues can be permanent */ 47 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 48 49 q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL); 50 if (!q) 51 return ERR_PTR(-ENOMEM); 52 53 kref_init(&q->refcount); 54 q->flags = flags; 55 q->hwe = hwe; 56 q->gt = gt; 57 q->class = hwe->class; 58 q->width = width; 59 q->logical_mask = logical_mask; 60 q->fence_irq = >->fence_irq[hwe->class]; 61 q->ring_ops = gt->ring_ops[hwe->class]; 62 q->ops = gt->exec_queue_ops; 63 INIT_LIST_HEAD(&q->compute.link); 64 INIT_LIST_HEAD(&q->multi_gt_link); 65 66 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 67 q->sched_props.preempt_timeout_us = 68 hwe->eclass->sched_props.preempt_timeout_us; 69 q->sched_props.job_timeout_ms = 70 hwe->eclass->sched_props.job_timeout_ms; 71 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && 72 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) 73 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; 74 else 75 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 76 77 if (extensions) { 78 /* 79 * may set q->usm, must come before xe_lrc_init(), 80 * may overwrite q->sched_props, must come before q->ops->init() 81 */ 82 err = exec_queue_user_extensions(xe, q, extensions, 0, true); 83 if (err) { 84 kfree(q); 85 return ERR_PTR(err); 86 } 87 } 88 89 if (vm) 90 q->vm = xe_vm_get(vm); 91 92 if (xe_exec_queue_is_parallel(q)) { 93 q->parallel.composite_fence_ctx = dma_fence_context_alloc(1); 94 q->parallel.composite_fence_seqno = XE_FENCE_INITIAL_SEQNO; 95 } 96 97 return q; 98 } 99 100 static void __xe_exec_queue_free(struct xe_exec_queue *q) 101 { 102 if (q->vm) 103 xe_vm_put(q->vm); 104 kfree(q); 105 } 106 107 static int __xe_exec_queue_init(struct xe_exec_queue *q) 108 { 109 struct xe_device *xe = gt_to_xe(q->gt); 110 int i, err; 111 112 for (i = 0; i < q->width; ++i) { 113 err = xe_lrc_init(q->lrc + i, q->hwe, q, q->vm, SZ_16K); 114 if (err) 115 goto err_lrc; 116 } 117 118 err = q->ops->init(q); 119 if (err) 120 goto err_lrc; 121 122 /* 123 * Normally the user vm holds an rpm ref to keep the device 124 * awake, and the context holds a ref for the vm, however for 125 * some engines we use the kernels migrate vm underneath which offers no 126 * such rpm ref, or we lack a vm. Make sure we keep a ref here, so we 127 * can perform GuC CT actions when needed. Caller is expected to have 128 * already grabbed the rpm ref outside any sensitive locks. 129 */ 130 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM || !q->vm)) 131 drm_WARN_ON(&xe->drm, !xe_device_mem_access_get_if_ongoing(xe)); 132 133 return 0; 134 135 err_lrc: 136 for (i = i - 1; i >= 0; --i) 137 xe_lrc_finish(q->lrc + i); 138 return err; 139 } 140 141 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 142 u32 logical_mask, u16 width, 143 struct xe_hw_engine *hwe, u32 flags, 144 u64 extensions) 145 { 146 struct xe_exec_queue *q; 147 int err; 148 149 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, 150 extensions); 151 if (IS_ERR(q)) 152 return q; 153 154 if (vm) { 155 err = xe_vm_lock(vm, true); 156 if (err) 157 goto err_post_alloc; 158 } 159 160 err = __xe_exec_queue_init(q); 161 if (vm) 162 xe_vm_unlock(vm); 163 if (err) 164 goto err_post_alloc; 165 166 return q; 167 168 err_post_alloc: 169 __xe_exec_queue_free(q); 170 return ERR_PTR(err); 171 } 172 173 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 174 struct xe_vm *vm, 175 enum xe_engine_class class, u32 flags) 176 { 177 struct xe_hw_engine *hwe, *hwe0 = NULL; 178 enum xe_hw_engine_id id; 179 u32 logical_mask = 0; 180 181 for_each_hw_engine(hwe, gt, id) { 182 if (xe_hw_engine_is_reserved(hwe)) 183 continue; 184 185 if (hwe->class == class) { 186 logical_mask |= BIT(hwe->logical_instance); 187 if (!hwe0) 188 hwe0 = hwe; 189 } 190 } 191 192 if (!logical_mask) 193 return ERR_PTR(-ENODEV); 194 195 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, 0); 196 } 197 198 void xe_exec_queue_destroy(struct kref *ref) 199 { 200 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 201 struct xe_exec_queue *eq, *next; 202 203 xe_exec_queue_last_fence_put_unlocked(q); 204 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 205 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 206 multi_gt_link) 207 xe_exec_queue_put(eq); 208 } 209 210 q->ops->fini(q); 211 } 212 213 void xe_exec_queue_fini(struct xe_exec_queue *q) 214 { 215 int i; 216 217 for (i = 0; i < q->width; ++i) 218 xe_lrc_finish(q->lrc + i); 219 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM || !q->vm)) 220 xe_device_mem_access_put(gt_to_xe(q->gt)); 221 __xe_exec_queue_free(q); 222 } 223 224 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 225 { 226 switch (q->class) { 227 case XE_ENGINE_CLASS_RENDER: 228 sprintf(q->name, "rcs%d", instance); 229 break; 230 case XE_ENGINE_CLASS_VIDEO_DECODE: 231 sprintf(q->name, "vcs%d", instance); 232 break; 233 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 234 sprintf(q->name, "vecs%d", instance); 235 break; 236 case XE_ENGINE_CLASS_COPY: 237 sprintf(q->name, "bcs%d", instance); 238 break; 239 case XE_ENGINE_CLASS_COMPUTE: 240 sprintf(q->name, "ccs%d", instance); 241 break; 242 case XE_ENGINE_CLASS_OTHER: 243 sprintf(q->name, "gsccs%d", instance); 244 break; 245 default: 246 XE_WARN_ON(q->class); 247 } 248 } 249 250 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 251 { 252 struct xe_exec_queue *q; 253 254 mutex_lock(&xef->exec_queue.lock); 255 q = xa_load(&xef->exec_queue.xa, id); 256 if (q) 257 xe_exec_queue_get(q); 258 mutex_unlock(&xef->exec_queue.lock); 259 260 return q; 261 } 262 263 enum xe_exec_queue_priority 264 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 265 { 266 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 267 XE_EXEC_QUEUE_PRIORITY_NORMAL; 268 } 269 270 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 271 u64 value, bool create) 272 { 273 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 274 return -EINVAL; 275 276 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 277 return -EPERM; 278 279 if (!create) 280 return q->ops->set_priority(q, value); 281 282 q->sched_props.priority = value; 283 return 0; 284 } 285 286 static bool xe_exec_queue_enforce_schedule_limit(void) 287 { 288 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 289 return true; 290 #else 291 return !capable(CAP_SYS_NICE); 292 #endif 293 } 294 295 static void 296 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 297 enum xe_exec_queue_sched_prop prop, 298 u32 *min, u32 *max) 299 { 300 switch (prop) { 301 case XE_EXEC_QUEUE_JOB_TIMEOUT: 302 *min = eclass->sched_props.job_timeout_min; 303 *max = eclass->sched_props.job_timeout_max; 304 break; 305 case XE_EXEC_QUEUE_TIMESLICE: 306 *min = eclass->sched_props.timeslice_min; 307 *max = eclass->sched_props.timeslice_max; 308 break; 309 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 310 *min = eclass->sched_props.preempt_timeout_min; 311 *max = eclass->sched_props.preempt_timeout_max; 312 break; 313 default: 314 break; 315 } 316 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 317 if (capable(CAP_SYS_NICE)) { 318 switch (prop) { 319 case XE_EXEC_QUEUE_JOB_TIMEOUT: 320 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 321 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 322 break; 323 case XE_EXEC_QUEUE_TIMESLICE: 324 *min = XE_HW_ENGINE_TIMESLICE_MIN; 325 *max = XE_HW_ENGINE_TIMESLICE_MAX; 326 break; 327 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 328 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 329 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 330 break; 331 default: 332 break; 333 } 334 } 335 #endif 336 } 337 338 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 339 u64 value, bool create) 340 { 341 u32 min = 0, max = 0; 342 343 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 344 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 345 346 if (xe_exec_queue_enforce_schedule_limit() && 347 !xe_hw_engine_timeout_in_range(value, min, max)) 348 return -EINVAL; 349 350 if (!create) 351 return q->ops->set_timeslice(q, value); 352 353 q->sched_props.timeslice_us = value; 354 return 0; 355 } 356 357 static int exec_queue_set_preemption_timeout(struct xe_device *xe, 358 struct xe_exec_queue *q, u64 value, 359 bool create) 360 { 361 u32 min = 0, max = 0; 362 363 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 364 XE_EXEC_QUEUE_PREEMPT_TIMEOUT, &min, &max); 365 366 if (xe_exec_queue_enforce_schedule_limit() && 367 !xe_hw_engine_timeout_in_range(value, min, max)) 368 return -EINVAL; 369 370 if (!create) 371 return q->ops->set_preempt_timeout(q, value); 372 373 q->sched_props.preempt_timeout_us = value; 374 return 0; 375 } 376 377 static int exec_queue_set_job_timeout(struct xe_device *xe, struct xe_exec_queue *q, 378 u64 value, bool create) 379 { 380 u32 min = 0, max = 0; 381 382 if (XE_IOCTL_DBG(xe, !create)) 383 return -EINVAL; 384 385 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 386 XE_EXEC_QUEUE_JOB_TIMEOUT, &min, &max); 387 388 if (xe_exec_queue_enforce_schedule_limit() && 389 !xe_hw_engine_timeout_in_range(value, min, max)) 390 return -EINVAL; 391 392 q->sched_props.job_timeout_ms = value; 393 394 return 0; 395 } 396 397 static int exec_queue_set_acc_trigger(struct xe_device *xe, struct xe_exec_queue *q, 398 u64 value, bool create) 399 { 400 if (XE_IOCTL_DBG(xe, !create)) 401 return -EINVAL; 402 403 if (XE_IOCTL_DBG(xe, !xe->info.has_usm)) 404 return -EINVAL; 405 406 q->usm.acc_trigger = value; 407 408 return 0; 409 } 410 411 static int exec_queue_set_acc_notify(struct xe_device *xe, struct xe_exec_queue *q, 412 u64 value, bool create) 413 { 414 if (XE_IOCTL_DBG(xe, !create)) 415 return -EINVAL; 416 417 if (XE_IOCTL_DBG(xe, !xe->info.has_usm)) 418 return -EINVAL; 419 420 q->usm.acc_notify = value; 421 422 return 0; 423 } 424 425 static int exec_queue_set_acc_granularity(struct xe_device *xe, struct xe_exec_queue *q, 426 u64 value, bool create) 427 { 428 if (XE_IOCTL_DBG(xe, !create)) 429 return -EINVAL; 430 431 if (XE_IOCTL_DBG(xe, !xe->info.has_usm)) 432 return -EINVAL; 433 434 if (value > DRM_XE_ACC_GRANULARITY_64M) 435 return -EINVAL; 436 437 q->usm.acc_granularity = value; 438 439 return 0; 440 } 441 442 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 443 struct xe_exec_queue *q, 444 u64 value, bool create); 445 446 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 447 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 448 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 449 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT] = exec_queue_set_preemption_timeout, 450 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT] = exec_queue_set_job_timeout, 451 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER] = exec_queue_set_acc_trigger, 452 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY] = exec_queue_set_acc_notify, 453 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY] = exec_queue_set_acc_granularity, 454 }; 455 456 static int exec_queue_user_ext_set_property(struct xe_device *xe, 457 struct xe_exec_queue *q, 458 u64 extension, 459 bool create) 460 { 461 u64 __user *address = u64_to_user_ptr(extension); 462 struct drm_xe_ext_set_property ext; 463 int err; 464 u32 idx; 465 466 err = __copy_from_user(&ext, address, sizeof(ext)); 467 if (XE_IOCTL_DBG(xe, err)) 468 return -EFAULT; 469 470 if (XE_IOCTL_DBG(xe, ext.property >= 471 ARRAY_SIZE(exec_queue_set_property_funcs)) || 472 XE_IOCTL_DBG(xe, ext.pad)) 473 return -EINVAL; 474 475 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 476 if (!exec_queue_set_property_funcs[idx]) 477 return -EINVAL; 478 479 return exec_queue_set_property_funcs[idx](xe, q, ext.value, create); 480 } 481 482 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 483 struct xe_exec_queue *q, 484 u64 extension, 485 bool create); 486 487 static const xe_exec_queue_set_property_fn exec_queue_user_extension_funcs[] = { 488 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 489 }; 490 491 #define MAX_USER_EXTENSIONS 16 492 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 493 u64 extensions, int ext_number, bool create) 494 { 495 u64 __user *address = u64_to_user_ptr(extensions); 496 struct drm_xe_user_extension ext; 497 int err; 498 u32 idx; 499 500 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 501 return -E2BIG; 502 503 err = __copy_from_user(&ext, address, sizeof(ext)); 504 if (XE_IOCTL_DBG(xe, err)) 505 return -EFAULT; 506 507 if (XE_IOCTL_DBG(xe, ext.pad) || 508 XE_IOCTL_DBG(xe, ext.name >= 509 ARRAY_SIZE(exec_queue_user_extension_funcs))) 510 return -EINVAL; 511 512 idx = array_index_nospec(ext.name, 513 ARRAY_SIZE(exec_queue_user_extension_funcs)); 514 err = exec_queue_user_extension_funcs[idx](xe, q, extensions, create); 515 if (XE_IOCTL_DBG(xe, err)) 516 return err; 517 518 if (ext.next_extension) 519 return exec_queue_user_extensions(xe, q, ext.next_extension, 520 ++ext_number, create); 521 522 return 0; 523 } 524 525 static const enum xe_engine_class user_to_xe_engine_class[] = { 526 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, 527 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, 528 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, 529 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, 530 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, 531 }; 532 533 static struct xe_hw_engine * 534 find_hw_engine(struct xe_device *xe, 535 struct drm_xe_engine_class_instance eci) 536 { 537 u32 idx; 538 539 if (eci.engine_class > ARRAY_SIZE(user_to_xe_engine_class)) 540 return NULL; 541 542 if (eci.gt_id >= xe->info.gt_count) 543 return NULL; 544 545 idx = array_index_nospec(eci.engine_class, 546 ARRAY_SIZE(user_to_xe_engine_class)); 547 548 return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id), 549 user_to_xe_engine_class[idx], 550 eci.engine_instance, true); 551 } 552 553 static u32 bind_exec_queue_logical_mask(struct xe_device *xe, struct xe_gt *gt, 554 struct drm_xe_engine_class_instance *eci, 555 u16 width, u16 num_placements) 556 { 557 struct xe_hw_engine *hwe; 558 enum xe_hw_engine_id id; 559 u32 logical_mask = 0; 560 561 if (XE_IOCTL_DBG(xe, width != 1)) 562 return 0; 563 if (XE_IOCTL_DBG(xe, num_placements != 1)) 564 return 0; 565 if (XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 566 return 0; 567 568 eci[0].engine_class = DRM_XE_ENGINE_CLASS_COPY; 569 570 for_each_hw_engine(hwe, gt, id) { 571 if (xe_hw_engine_is_reserved(hwe)) 572 continue; 573 574 if (hwe->class == 575 user_to_xe_engine_class[DRM_XE_ENGINE_CLASS_COPY]) 576 logical_mask |= BIT(hwe->logical_instance); 577 } 578 579 return logical_mask; 580 } 581 582 static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, 583 struct drm_xe_engine_class_instance *eci, 584 u16 width, u16 num_placements) 585 { 586 int len = width * num_placements; 587 int i, j, n; 588 u16 class; 589 u16 gt_id; 590 u32 return_mask = 0, prev_mask; 591 592 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 593 len > 1)) 594 return 0; 595 596 for (i = 0; i < width; ++i) { 597 u32 current_mask = 0; 598 599 for (j = 0; j < num_placements; ++j) { 600 struct xe_hw_engine *hwe; 601 602 n = j * width + i; 603 604 hwe = find_hw_engine(xe, eci[n]); 605 if (XE_IOCTL_DBG(xe, !hwe)) 606 return 0; 607 608 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 609 return 0; 610 611 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 612 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 613 return 0; 614 615 class = eci[n].engine_class; 616 gt_id = eci[n].gt_id; 617 618 if (width == 1 || !i) 619 return_mask |= BIT(eci[n].engine_instance); 620 current_mask |= BIT(eci[n].engine_instance); 621 } 622 623 /* Parallel submissions must be logically contiguous */ 624 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 625 return 0; 626 627 prev_mask = current_mask; 628 } 629 630 return return_mask; 631 } 632 633 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 634 struct drm_file *file) 635 { 636 struct xe_device *xe = to_xe_device(dev); 637 struct xe_file *xef = to_xe_file(file); 638 struct drm_xe_exec_queue_create *args = data; 639 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 640 struct drm_xe_engine_class_instance __user *user_eci = 641 u64_to_user_ptr(args->instances); 642 struct xe_hw_engine *hwe; 643 struct xe_vm *vm, *migrate_vm; 644 struct xe_gt *gt; 645 struct xe_exec_queue *q = NULL; 646 u32 logical_mask; 647 u32 id; 648 u32 len; 649 int err; 650 651 if (XE_IOCTL_DBG(xe, args->flags) || 652 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 653 return -EINVAL; 654 655 len = args->width * args->num_placements; 656 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 657 return -EINVAL; 658 659 err = __copy_from_user(eci, user_eci, 660 sizeof(struct drm_xe_engine_class_instance) * 661 len); 662 if (XE_IOCTL_DBG(xe, err)) 663 return -EFAULT; 664 665 if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) 666 return -EINVAL; 667 668 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { 669 for_each_gt(gt, xe, id) { 670 struct xe_exec_queue *new; 671 u32 flags; 672 673 if (xe_gt_is_media_type(gt)) 674 continue; 675 676 eci[0].gt_id = gt->info.id; 677 logical_mask = bind_exec_queue_logical_mask(xe, gt, eci, 678 args->width, 679 args->num_placements); 680 if (XE_IOCTL_DBG(xe, !logical_mask)) 681 return -EINVAL; 682 683 hwe = find_hw_engine(xe, eci[0]); 684 if (XE_IOCTL_DBG(xe, !hwe)) 685 return -EINVAL; 686 687 /* The migration vm doesn't hold rpm ref */ 688 xe_device_mem_access_get(xe); 689 690 flags = EXEC_QUEUE_FLAG_VM | (id ? EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD : 0); 691 692 migrate_vm = xe_migrate_get_vm(gt_to_tile(gt)->migrate); 693 new = xe_exec_queue_create(xe, migrate_vm, logical_mask, 694 args->width, hwe, flags, 695 args->extensions); 696 697 xe_device_mem_access_put(xe); /* now held by engine */ 698 699 xe_vm_put(migrate_vm); 700 if (IS_ERR(new)) { 701 err = PTR_ERR(new); 702 if (q) 703 goto put_exec_queue; 704 return err; 705 } 706 if (id == 0) 707 q = new; 708 else 709 list_add_tail(&new->multi_gt_list, 710 &q->multi_gt_link); 711 } 712 } else { 713 gt = xe_device_get_gt(xe, eci[0].gt_id); 714 logical_mask = calc_validate_logical_mask(xe, gt, eci, 715 args->width, 716 args->num_placements); 717 if (XE_IOCTL_DBG(xe, !logical_mask)) 718 return -EINVAL; 719 720 hwe = find_hw_engine(xe, eci[0]); 721 if (XE_IOCTL_DBG(xe, !hwe)) 722 return -EINVAL; 723 724 vm = xe_vm_lookup(xef, args->vm_id); 725 if (XE_IOCTL_DBG(xe, !vm)) 726 return -ENOENT; 727 728 err = down_read_interruptible(&vm->lock); 729 if (err) { 730 xe_vm_put(vm); 731 return err; 732 } 733 734 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 735 up_read(&vm->lock); 736 xe_vm_put(vm); 737 return -ENOENT; 738 } 739 740 q = xe_exec_queue_create(xe, vm, logical_mask, 741 args->width, hwe, 0, 742 args->extensions); 743 up_read(&vm->lock); 744 xe_vm_put(vm); 745 if (IS_ERR(q)) 746 return PTR_ERR(q); 747 748 if (xe_vm_in_preempt_fence_mode(vm)) { 749 q->compute.context = dma_fence_context_alloc(1); 750 spin_lock_init(&q->compute.lock); 751 752 err = xe_vm_add_compute_exec_queue(vm, q); 753 if (XE_IOCTL_DBG(xe, err)) 754 goto put_exec_queue; 755 } 756 } 757 758 mutex_lock(&xef->exec_queue.lock); 759 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 760 mutex_unlock(&xef->exec_queue.lock); 761 if (err) 762 goto kill_exec_queue; 763 764 args->exec_queue_id = id; 765 766 return 0; 767 768 kill_exec_queue: 769 xe_exec_queue_kill(q); 770 put_exec_queue: 771 xe_exec_queue_put(q); 772 return err; 773 } 774 775 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 776 struct drm_file *file) 777 { 778 struct xe_device *xe = to_xe_device(dev); 779 struct xe_file *xef = to_xe_file(file); 780 struct drm_xe_exec_queue_get_property *args = data; 781 struct xe_exec_queue *q; 782 int ret; 783 784 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 785 return -EINVAL; 786 787 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 788 if (XE_IOCTL_DBG(xe, !q)) 789 return -ENOENT; 790 791 switch (args->property) { 792 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: 793 args->value = !!(q->flags & EXEC_QUEUE_FLAG_BANNED); 794 ret = 0; 795 break; 796 default: 797 ret = -EINVAL; 798 } 799 800 xe_exec_queue_put(q); 801 802 return ret; 803 } 804 805 /** 806 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 807 * @q: The exec_queue 808 * 809 * Return: True if the exec_queue is long-running, false otherwise. 810 */ 811 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 812 { 813 return q->vm && xe_vm_in_lr_mode(q->vm) && 814 !(q->flags & EXEC_QUEUE_FLAG_VM); 815 } 816 817 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 818 { 819 return q->lrc->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc) - 1; 820 } 821 822 /** 823 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 824 * @q: The exec_queue 825 * 826 * Return: True if the exec_queue's ring is full, false otherwise. 827 */ 828 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 829 { 830 struct xe_lrc *lrc = q->lrc; 831 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 832 833 return xe_exec_queue_num_job_inflight(q) >= max_job; 834 } 835 836 /** 837 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 838 * @q: The exec_queue 839 * 840 * FIXME: Need to determine what to use as the short-lived 841 * timeline lock for the exec_queues, so that the return value 842 * of this function becomes more than just an advisory 843 * snapshot in time. The timeline lock must protect the 844 * seqno from racing submissions on the same exec_queue. 845 * Typically vm->resv, but user-created timeline locks use the migrate vm 846 * and never grabs the migrate vm->resv so we have a race there. 847 * 848 * Return: True if the exec_queue is idle, false otherwise. 849 */ 850 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 851 { 852 if (xe_exec_queue_is_parallel(q)) { 853 int i; 854 855 for (i = 0; i < q->width; ++i) { 856 if (xe_lrc_seqno(&q->lrc[i]) != 857 q->lrc[i].fence_ctx.next_seqno - 1) 858 return false; 859 } 860 861 return true; 862 } 863 864 return xe_lrc_seqno(&q->lrc[0]) == 865 q->lrc[0].fence_ctx.next_seqno - 1; 866 } 867 868 void xe_exec_queue_kill(struct xe_exec_queue *q) 869 { 870 struct xe_exec_queue *eq = q, *next; 871 872 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 873 multi_gt_link) { 874 q->ops->kill(eq); 875 xe_vm_remove_compute_exec_queue(q->vm, eq); 876 } 877 878 q->ops->kill(q); 879 xe_vm_remove_compute_exec_queue(q->vm, q); 880 } 881 882 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 883 struct drm_file *file) 884 { 885 struct xe_device *xe = to_xe_device(dev); 886 struct xe_file *xef = to_xe_file(file); 887 struct drm_xe_exec_queue_destroy *args = data; 888 struct xe_exec_queue *q; 889 890 if (XE_IOCTL_DBG(xe, args->pad) || 891 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 892 return -EINVAL; 893 894 mutex_lock(&xef->exec_queue.lock); 895 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 896 mutex_unlock(&xef->exec_queue.lock); 897 if (XE_IOCTL_DBG(xe, !q)) 898 return -ENOENT; 899 900 xe_exec_queue_kill(q); 901 902 trace_xe_exec_queue_close(q); 903 xe_exec_queue_put(q); 904 905 return 0; 906 } 907 908 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 909 struct xe_vm *vm) 910 { 911 if (q->flags & EXEC_QUEUE_FLAG_VM) 912 lockdep_assert_held(&vm->lock); 913 else 914 xe_vm_assert_held(vm); 915 } 916 917 /** 918 * xe_exec_queue_last_fence_put() - Drop ref to last fence 919 * @q: The exec queue 920 * @vm: The VM the engine does a bind or exec for 921 */ 922 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 923 { 924 xe_exec_queue_last_fence_lockdep_assert(q, vm); 925 926 if (q->last_fence) { 927 dma_fence_put(q->last_fence); 928 q->last_fence = NULL; 929 } 930 } 931 932 /** 933 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 934 * @q: The exec queue 935 * 936 * Only safe to be called from xe_exec_queue_destroy(). 937 */ 938 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 939 { 940 if (q->last_fence) { 941 dma_fence_put(q->last_fence); 942 q->last_fence = NULL; 943 } 944 } 945 946 /** 947 * xe_exec_queue_last_fence_get() - Get last fence 948 * @q: The exec queue 949 * @vm: The VM the engine does a bind or exec for 950 * 951 * Get last fence, takes a ref 952 * 953 * Returns: last fence if not signaled, dma fence stub if signaled 954 */ 955 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 956 struct xe_vm *vm) 957 { 958 struct dma_fence *fence; 959 960 xe_exec_queue_last_fence_lockdep_assert(q, vm); 961 962 if (q->last_fence && 963 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 964 xe_exec_queue_last_fence_put(q, vm); 965 966 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); 967 dma_fence_get(fence); 968 return fence; 969 } 970 971 /** 972 * xe_exec_queue_last_fence_set() - Set last fence 973 * @q: The exec queue 974 * @vm: The VM the engine does a bind or exec for 975 * @fence: The fence 976 * 977 * Set the last fence for the engine. Increases reference count for fence, when 978 * closing engine xe_exec_queue_last_fence_put should be called. 979 */ 980 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 981 struct dma_fence *fence) 982 { 983 xe_exec_queue_last_fence_lockdep_assert(q, vm); 984 985 xe_exec_queue_last_fence_put(q, vm); 986 q->last_fence = dma_fence_get(fence); 987 } 988