1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_file.h> 12 #include <drm/xe_drm.h> 13 14 #include "xe_device.h" 15 #include "xe_gt.h" 16 #include "xe_hw_engine_class_sysfs.h" 17 #include "xe_hw_fence.h" 18 #include "xe_lrc.h" 19 #include "xe_macros.h" 20 #include "xe_migrate.h" 21 #include "xe_pm.h" 22 #include "xe_ring_ops_types.h" 23 #include "xe_trace.h" 24 #include "xe_vm.h" 25 26 enum xe_exec_queue_sched_prop { 27 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 28 XE_EXEC_QUEUE_TIMESLICE = 1, 29 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 30 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 31 }; 32 33 static struct xe_exec_queue *__xe_exec_queue_create(struct xe_device *xe, 34 struct xe_vm *vm, 35 u32 logical_mask, 36 u16 width, struct xe_hw_engine *hwe, 37 u32 flags) 38 { 39 struct xe_exec_queue *q; 40 struct xe_gt *gt = hwe->gt; 41 int err; 42 int i; 43 44 /* only kernel queues can be permanent */ 45 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 46 47 q = kzalloc(sizeof(*q) + sizeof(struct xe_lrc) * width, GFP_KERNEL); 48 if (!q) 49 return ERR_PTR(-ENOMEM); 50 51 kref_init(&q->refcount); 52 q->flags = flags; 53 q->hwe = hwe; 54 q->gt = gt; 55 if (vm) 56 q->vm = xe_vm_get(vm); 57 q->class = hwe->class; 58 q->width = width; 59 q->logical_mask = logical_mask; 60 q->fence_irq = >->fence_irq[hwe->class]; 61 q->ring_ops = gt->ring_ops[hwe->class]; 62 q->ops = gt->exec_queue_ops; 63 INIT_LIST_HEAD(&q->persistent.link); 64 INIT_LIST_HEAD(&q->compute.link); 65 INIT_LIST_HEAD(&q->multi_gt_link); 66 67 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 68 q->sched_props.preempt_timeout_us = 69 hwe->eclass->sched_props.preempt_timeout_us; 70 71 if (xe_exec_queue_is_parallel(q)) { 72 q->parallel.composite_fence_ctx = dma_fence_context_alloc(1); 73 q->parallel.composite_fence_seqno = XE_FENCE_INITIAL_SEQNO; 74 } 75 if (q->flags & EXEC_QUEUE_FLAG_VM) { 76 q->bind.fence_ctx = dma_fence_context_alloc(1); 77 q->bind.fence_seqno = XE_FENCE_INITIAL_SEQNO; 78 } 79 80 for (i = 0; i < width; ++i) { 81 err = xe_lrc_init(q->lrc + i, hwe, q, vm, SZ_16K); 82 if (err) 83 goto err_lrc; 84 } 85 86 err = q->ops->init(q); 87 if (err) 88 goto err_lrc; 89 90 /* 91 * Normally the user vm holds an rpm ref to keep the device 92 * awake, and the context holds a ref for the vm, however for 93 * some engines we use the kernels migrate vm underneath which offers no 94 * such rpm ref, or we lack a vm. Make sure we keep a ref here, so we 95 * can perform GuC CT actions when needed. Caller is expected to have 96 * already grabbed the rpm ref outside any sensitive locks. 97 */ 98 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM || !vm)) 99 drm_WARN_ON(&xe->drm, !xe_device_mem_access_get_if_ongoing(xe)); 100 101 return q; 102 103 err_lrc: 104 for (i = i - 1; i >= 0; --i) 105 xe_lrc_finish(q->lrc + i); 106 kfree(q); 107 return ERR_PTR(err); 108 } 109 110 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 111 u32 logical_mask, u16 width, 112 struct xe_hw_engine *hwe, u32 flags) 113 { 114 struct xe_exec_queue *q; 115 int err; 116 117 if (vm) { 118 err = xe_vm_lock(vm, true); 119 if (err) 120 return ERR_PTR(err); 121 } 122 q = __xe_exec_queue_create(xe, vm, logical_mask, width, hwe, flags); 123 if (vm) 124 xe_vm_unlock(vm); 125 126 return q; 127 } 128 129 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 130 struct xe_vm *vm, 131 enum xe_engine_class class, u32 flags) 132 { 133 struct xe_hw_engine *hwe, *hwe0 = NULL; 134 enum xe_hw_engine_id id; 135 u32 logical_mask = 0; 136 137 for_each_hw_engine(hwe, gt, id) { 138 if (xe_hw_engine_is_reserved(hwe)) 139 continue; 140 141 if (hwe->class == class) { 142 logical_mask |= BIT(hwe->logical_instance); 143 if (!hwe0) 144 hwe0 = hwe; 145 } 146 } 147 148 if (!logical_mask) 149 return ERR_PTR(-ENODEV); 150 151 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags); 152 } 153 154 void xe_exec_queue_destroy(struct kref *ref) 155 { 156 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 157 struct xe_exec_queue *eq, *next; 158 159 xe_exec_queue_last_fence_put_unlocked(q); 160 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 161 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 162 multi_gt_link) 163 xe_exec_queue_put(eq); 164 } 165 166 q->ops->fini(q); 167 } 168 169 void xe_exec_queue_fini(struct xe_exec_queue *q) 170 { 171 int i; 172 173 for (i = 0; i < q->width; ++i) 174 xe_lrc_finish(q->lrc + i); 175 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM || !q->vm)) 176 xe_device_mem_access_put(gt_to_xe(q->gt)); 177 if (q->vm) 178 xe_vm_put(q->vm); 179 180 kfree(q); 181 } 182 183 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 184 { 185 switch (q->class) { 186 case XE_ENGINE_CLASS_RENDER: 187 sprintf(q->name, "rcs%d", instance); 188 break; 189 case XE_ENGINE_CLASS_VIDEO_DECODE: 190 sprintf(q->name, "vcs%d", instance); 191 break; 192 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 193 sprintf(q->name, "vecs%d", instance); 194 break; 195 case XE_ENGINE_CLASS_COPY: 196 sprintf(q->name, "bcs%d", instance); 197 break; 198 case XE_ENGINE_CLASS_COMPUTE: 199 sprintf(q->name, "ccs%d", instance); 200 break; 201 case XE_ENGINE_CLASS_OTHER: 202 sprintf(q->name, "gsccs%d", instance); 203 break; 204 default: 205 XE_WARN_ON(q->class); 206 } 207 } 208 209 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 210 { 211 struct xe_exec_queue *q; 212 213 mutex_lock(&xef->exec_queue.lock); 214 q = xa_load(&xef->exec_queue.xa, id); 215 if (q) 216 xe_exec_queue_get(q); 217 mutex_unlock(&xef->exec_queue.lock); 218 219 return q; 220 } 221 222 enum xe_exec_queue_priority 223 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 224 { 225 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 226 XE_EXEC_QUEUE_PRIORITY_NORMAL; 227 } 228 229 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 230 u64 value, bool create) 231 { 232 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 233 return -EINVAL; 234 235 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 236 return -EPERM; 237 238 return q->ops->set_priority(q, value); 239 } 240 241 static bool xe_exec_queue_enforce_schedule_limit(void) 242 { 243 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 244 return true; 245 #else 246 return !capable(CAP_SYS_NICE); 247 #endif 248 } 249 250 static void 251 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 252 enum xe_exec_queue_sched_prop prop, 253 u32 *min, u32 *max) 254 { 255 switch (prop) { 256 case XE_EXEC_QUEUE_JOB_TIMEOUT: 257 *min = eclass->sched_props.job_timeout_min; 258 *max = eclass->sched_props.job_timeout_max; 259 break; 260 case XE_EXEC_QUEUE_TIMESLICE: 261 *min = eclass->sched_props.timeslice_min; 262 *max = eclass->sched_props.timeslice_max; 263 break; 264 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 265 *min = eclass->sched_props.preempt_timeout_min; 266 *max = eclass->sched_props.preempt_timeout_max; 267 break; 268 default: 269 break; 270 } 271 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 272 if (capable(CAP_SYS_NICE)) { 273 switch (prop) { 274 case XE_EXEC_QUEUE_JOB_TIMEOUT: 275 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 276 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 277 break; 278 case XE_EXEC_QUEUE_TIMESLICE: 279 *min = XE_HW_ENGINE_TIMESLICE_MIN; 280 *max = XE_HW_ENGINE_TIMESLICE_MAX; 281 break; 282 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 283 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 284 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 285 break; 286 default: 287 break; 288 } 289 } 290 #endif 291 } 292 293 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 294 u64 value, bool create) 295 { 296 u32 min = 0, max = 0; 297 298 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 299 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 300 301 if (xe_exec_queue_enforce_schedule_limit() && 302 !xe_hw_engine_timeout_in_range(value, min, max)) 303 return -EINVAL; 304 305 return q->ops->set_timeslice(q, value); 306 } 307 308 static int exec_queue_set_preemption_timeout(struct xe_device *xe, 309 struct xe_exec_queue *q, u64 value, 310 bool create) 311 { 312 u32 min = 0, max = 0; 313 314 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 315 XE_EXEC_QUEUE_PREEMPT_TIMEOUT, &min, &max); 316 317 if (xe_exec_queue_enforce_schedule_limit() && 318 !xe_hw_engine_timeout_in_range(value, min, max)) 319 return -EINVAL; 320 321 return q->ops->set_preempt_timeout(q, value); 322 } 323 324 static int exec_queue_set_persistence(struct xe_device *xe, struct xe_exec_queue *q, 325 u64 value, bool create) 326 { 327 if (XE_IOCTL_DBG(xe, !create)) 328 return -EINVAL; 329 330 if (XE_IOCTL_DBG(xe, xe_vm_in_preempt_fence_mode(q->vm))) 331 return -EINVAL; 332 333 if (value) 334 q->flags |= EXEC_QUEUE_FLAG_PERSISTENT; 335 else 336 q->flags &= ~EXEC_QUEUE_FLAG_PERSISTENT; 337 338 return 0; 339 } 340 341 static int exec_queue_set_job_timeout(struct xe_device *xe, struct xe_exec_queue *q, 342 u64 value, bool create) 343 { 344 u32 min = 0, max = 0; 345 346 if (XE_IOCTL_DBG(xe, !create)) 347 return -EINVAL; 348 349 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 350 XE_EXEC_QUEUE_JOB_TIMEOUT, &min, &max); 351 352 if (xe_exec_queue_enforce_schedule_limit() && 353 !xe_hw_engine_timeout_in_range(value, min, max)) 354 return -EINVAL; 355 356 return q->ops->set_job_timeout(q, value); 357 } 358 359 static int exec_queue_set_acc_trigger(struct xe_device *xe, struct xe_exec_queue *q, 360 u64 value, bool create) 361 { 362 if (XE_IOCTL_DBG(xe, !create)) 363 return -EINVAL; 364 365 if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) 366 return -EINVAL; 367 368 q->usm.acc_trigger = value; 369 370 return 0; 371 } 372 373 static int exec_queue_set_acc_notify(struct xe_device *xe, struct xe_exec_queue *q, 374 u64 value, bool create) 375 { 376 if (XE_IOCTL_DBG(xe, !create)) 377 return -EINVAL; 378 379 if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) 380 return -EINVAL; 381 382 q->usm.acc_notify = value; 383 384 return 0; 385 } 386 387 static int exec_queue_set_acc_granularity(struct xe_device *xe, struct xe_exec_queue *q, 388 u64 value, bool create) 389 { 390 if (XE_IOCTL_DBG(xe, !create)) 391 return -EINVAL; 392 393 if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) 394 return -EINVAL; 395 396 if (value > DRM_XE_ACC_GRANULARITY_64M) 397 return -EINVAL; 398 399 q->usm.acc_granularity = value; 400 401 return 0; 402 } 403 404 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 405 struct xe_exec_queue *q, 406 u64 value, bool create); 407 408 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 409 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 410 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 411 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT] = exec_queue_set_preemption_timeout, 412 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE] = exec_queue_set_persistence, 413 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT] = exec_queue_set_job_timeout, 414 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER] = exec_queue_set_acc_trigger, 415 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY] = exec_queue_set_acc_notify, 416 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY] = exec_queue_set_acc_granularity, 417 }; 418 419 static int exec_queue_user_ext_set_property(struct xe_device *xe, 420 struct xe_exec_queue *q, 421 u64 extension, 422 bool create) 423 { 424 u64 __user *address = u64_to_user_ptr(extension); 425 struct drm_xe_ext_set_property ext; 426 int err; 427 u32 idx; 428 429 err = __copy_from_user(&ext, address, sizeof(ext)); 430 if (XE_IOCTL_DBG(xe, err)) 431 return -EFAULT; 432 433 if (XE_IOCTL_DBG(xe, ext.property >= 434 ARRAY_SIZE(exec_queue_set_property_funcs)) || 435 XE_IOCTL_DBG(xe, ext.pad)) 436 return -EINVAL; 437 438 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 439 return exec_queue_set_property_funcs[idx](xe, q, ext.value, create); 440 } 441 442 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 443 struct xe_exec_queue *q, 444 u64 extension, 445 bool create); 446 447 static const xe_exec_queue_set_property_fn exec_queue_user_extension_funcs[] = { 448 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 449 }; 450 451 #define MAX_USER_EXTENSIONS 16 452 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 453 u64 extensions, int ext_number, bool create) 454 { 455 u64 __user *address = u64_to_user_ptr(extensions); 456 struct xe_user_extension ext; 457 int err; 458 u32 idx; 459 460 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 461 return -E2BIG; 462 463 err = __copy_from_user(&ext, address, sizeof(ext)); 464 if (XE_IOCTL_DBG(xe, err)) 465 return -EFAULT; 466 467 if (XE_IOCTL_DBG(xe, ext.pad) || 468 XE_IOCTL_DBG(xe, ext.name >= 469 ARRAY_SIZE(exec_queue_user_extension_funcs))) 470 return -EINVAL; 471 472 idx = array_index_nospec(ext.name, 473 ARRAY_SIZE(exec_queue_user_extension_funcs)); 474 err = exec_queue_user_extension_funcs[idx](xe, q, extensions, create); 475 if (XE_IOCTL_DBG(xe, err)) 476 return err; 477 478 if (ext.next_extension) 479 return exec_queue_user_extensions(xe, q, ext.next_extension, 480 ++ext_number, create); 481 482 return 0; 483 } 484 485 static const enum xe_engine_class user_to_xe_engine_class[] = { 486 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, 487 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, 488 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, 489 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, 490 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, 491 }; 492 493 static struct xe_hw_engine * 494 find_hw_engine(struct xe_device *xe, 495 struct drm_xe_engine_class_instance eci) 496 { 497 u32 idx; 498 499 if (eci.engine_class > ARRAY_SIZE(user_to_xe_engine_class)) 500 return NULL; 501 502 if (eci.gt_id >= xe->info.gt_count) 503 return NULL; 504 505 idx = array_index_nospec(eci.engine_class, 506 ARRAY_SIZE(user_to_xe_engine_class)); 507 508 return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id), 509 user_to_xe_engine_class[idx], 510 eci.engine_instance, true); 511 } 512 513 static u32 bind_exec_queue_logical_mask(struct xe_device *xe, struct xe_gt *gt, 514 struct drm_xe_engine_class_instance *eci, 515 u16 width, u16 num_placements) 516 { 517 struct xe_hw_engine *hwe; 518 enum xe_hw_engine_id id; 519 u32 logical_mask = 0; 520 521 if (XE_IOCTL_DBG(xe, width != 1)) 522 return 0; 523 if (XE_IOCTL_DBG(xe, num_placements != 1)) 524 return 0; 525 if (XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 526 return 0; 527 528 eci[0].engine_class = DRM_XE_ENGINE_CLASS_COPY; 529 530 for_each_hw_engine(hwe, gt, id) { 531 if (xe_hw_engine_is_reserved(hwe)) 532 continue; 533 534 if (hwe->class == 535 user_to_xe_engine_class[DRM_XE_ENGINE_CLASS_COPY]) 536 logical_mask |= BIT(hwe->logical_instance); 537 } 538 539 return logical_mask; 540 } 541 542 static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, 543 struct drm_xe_engine_class_instance *eci, 544 u16 width, u16 num_placements) 545 { 546 int len = width * num_placements; 547 int i, j, n; 548 u16 class; 549 u16 gt_id; 550 u32 return_mask = 0, prev_mask; 551 552 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 553 len > 1)) 554 return 0; 555 556 for (i = 0; i < width; ++i) { 557 u32 current_mask = 0; 558 559 for (j = 0; j < num_placements; ++j) { 560 struct xe_hw_engine *hwe; 561 562 n = j * width + i; 563 564 hwe = find_hw_engine(xe, eci[n]); 565 if (XE_IOCTL_DBG(xe, !hwe)) 566 return 0; 567 568 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 569 return 0; 570 571 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 572 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 573 return 0; 574 575 class = eci[n].engine_class; 576 gt_id = eci[n].gt_id; 577 578 if (width == 1 || !i) 579 return_mask |= BIT(eci[n].engine_instance); 580 current_mask |= BIT(eci[n].engine_instance); 581 } 582 583 /* Parallel submissions must be logically contiguous */ 584 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 585 return 0; 586 587 prev_mask = current_mask; 588 } 589 590 return return_mask; 591 } 592 593 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 594 struct drm_file *file) 595 { 596 struct xe_device *xe = to_xe_device(dev); 597 struct xe_file *xef = to_xe_file(file); 598 struct drm_xe_exec_queue_create *args = data; 599 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 600 struct drm_xe_engine_class_instance __user *user_eci = 601 u64_to_user_ptr(args->instances); 602 struct xe_hw_engine *hwe; 603 struct xe_vm *vm, *migrate_vm; 604 struct xe_gt *gt; 605 struct xe_exec_queue *q = NULL; 606 u32 logical_mask; 607 u32 id; 608 u32 len; 609 int err; 610 611 if (XE_IOCTL_DBG(xe, args->flags) || 612 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 613 return -EINVAL; 614 615 len = args->width * args->num_placements; 616 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 617 return -EINVAL; 618 619 err = __copy_from_user(eci, user_eci, 620 sizeof(struct drm_xe_engine_class_instance) * 621 len); 622 if (XE_IOCTL_DBG(xe, err)) 623 return -EFAULT; 624 625 if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) 626 return -EINVAL; 627 628 if (eci[0].engine_class >= DRM_XE_ENGINE_CLASS_VM_BIND_ASYNC) { 629 bool sync = eci[0].engine_class == 630 DRM_XE_ENGINE_CLASS_VM_BIND_SYNC; 631 632 for_each_gt(gt, xe, id) { 633 struct xe_exec_queue *new; 634 635 if (xe_gt_is_media_type(gt)) 636 continue; 637 638 eci[0].gt_id = gt->info.id; 639 logical_mask = bind_exec_queue_logical_mask(xe, gt, eci, 640 args->width, 641 args->num_placements); 642 if (XE_IOCTL_DBG(xe, !logical_mask)) 643 return -EINVAL; 644 645 hwe = find_hw_engine(xe, eci[0]); 646 if (XE_IOCTL_DBG(xe, !hwe)) 647 return -EINVAL; 648 649 /* The migration vm doesn't hold rpm ref */ 650 xe_device_mem_access_get(xe); 651 652 migrate_vm = xe_migrate_get_vm(gt_to_tile(gt)->migrate); 653 new = xe_exec_queue_create(xe, migrate_vm, logical_mask, 654 args->width, hwe, 655 EXEC_QUEUE_FLAG_PERSISTENT | 656 EXEC_QUEUE_FLAG_VM | 657 (sync ? 0 : 658 EXEC_QUEUE_FLAG_VM_ASYNC) | 659 (id ? 660 EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD : 661 0)); 662 663 xe_device_mem_access_put(xe); /* now held by engine */ 664 665 xe_vm_put(migrate_vm); 666 if (IS_ERR(new)) { 667 err = PTR_ERR(new); 668 if (q) 669 goto put_exec_queue; 670 return err; 671 } 672 if (id == 0) 673 q = new; 674 else 675 list_add_tail(&new->multi_gt_list, 676 &q->multi_gt_link); 677 } 678 } else { 679 gt = xe_device_get_gt(xe, eci[0].gt_id); 680 logical_mask = calc_validate_logical_mask(xe, gt, eci, 681 args->width, 682 args->num_placements); 683 if (XE_IOCTL_DBG(xe, !logical_mask)) 684 return -EINVAL; 685 686 hwe = find_hw_engine(xe, eci[0]); 687 if (XE_IOCTL_DBG(xe, !hwe)) 688 return -EINVAL; 689 690 vm = xe_vm_lookup(xef, args->vm_id); 691 if (XE_IOCTL_DBG(xe, !vm)) 692 return -ENOENT; 693 694 err = down_read_interruptible(&vm->lock); 695 if (err) { 696 xe_vm_put(vm); 697 return err; 698 } 699 700 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 701 up_read(&vm->lock); 702 xe_vm_put(vm); 703 return -ENOENT; 704 } 705 706 q = xe_exec_queue_create(xe, vm, logical_mask, 707 args->width, hwe, 708 xe_vm_in_lr_mode(vm) ? 0 : 709 EXEC_QUEUE_FLAG_PERSISTENT); 710 up_read(&vm->lock); 711 xe_vm_put(vm); 712 if (IS_ERR(q)) 713 return PTR_ERR(q); 714 715 if (xe_vm_in_preempt_fence_mode(vm)) { 716 q->compute.context = dma_fence_context_alloc(1); 717 spin_lock_init(&q->compute.lock); 718 719 err = xe_vm_add_compute_exec_queue(vm, q); 720 if (XE_IOCTL_DBG(xe, err)) 721 goto put_exec_queue; 722 } 723 } 724 725 if (args->extensions) { 726 err = exec_queue_user_extensions(xe, q, args->extensions, 0, true); 727 if (XE_IOCTL_DBG(xe, err)) 728 goto kill_exec_queue; 729 } 730 731 q->persistent.xef = xef; 732 733 mutex_lock(&xef->exec_queue.lock); 734 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 735 mutex_unlock(&xef->exec_queue.lock); 736 if (err) 737 goto kill_exec_queue; 738 739 args->exec_queue_id = id; 740 741 return 0; 742 743 kill_exec_queue: 744 xe_exec_queue_kill(q); 745 put_exec_queue: 746 xe_exec_queue_put(q); 747 return err; 748 } 749 750 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 751 struct drm_file *file) 752 { 753 struct xe_device *xe = to_xe_device(dev); 754 struct xe_file *xef = to_xe_file(file); 755 struct drm_xe_exec_queue_get_property *args = data; 756 struct xe_exec_queue *q; 757 int ret; 758 759 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 760 return -EINVAL; 761 762 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 763 if (XE_IOCTL_DBG(xe, !q)) 764 return -ENOENT; 765 766 switch (args->property) { 767 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: 768 args->value = !!(q->flags & EXEC_QUEUE_FLAG_BANNED); 769 ret = 0; 770 break; 771 default: 772 ret = -EINVAL; 773 } 774 775 xe_exec_queue_put(q); 776 777 return ret; 778 } 779 780 /** 781 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 782 * @q: The exec_queue 783 * 784 * Return: True if the exec_queue is long-running, false otherwise. 785 */ 786 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 787 { 788 return q->vm && xe_vm_in_lr_mode(q->vm) && 789 !(q->flags & EXEC_QUEUE_FLAG_VM); 790 } 791 792 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 793 { 794 return q->lrc->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc) - 1; 795 } 796 797 /** 798 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 799 * @q: The exec_queue 800 * 801 * Return: True if the exec_queue's ring is full, false otherwise. 802 */ 803 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 804 { 805 struct xe_lrc *lrc = q->lrc; 806 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 807 808 return xe_exec_queue_num_job_inflight(q) >= max_job; 809 } 810 811 /** 812 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 813 * @q: The exec_queue 814 * 815 * FIXME: Need to determine what to use as the short-lived 816 * timeline lock for the exec_queues, so that the return value 817 * of this function becomes more than just an advisory 818 * snapshot in time. The timeline lock must protect the 819 * seqno from racing submissions on the same exec_queue. 820 * Typically vm->resv, but user-created timeline locks use the migrate vm 821 * and never grabs the migrate vm->resv so we have a race there. 822 * 823 * Return: True if the exec_queue is idle, false otherwise. 824 */ 825 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 826 { 827 if (xe_exec_queue_is_parallel(q)) { 828 int i; 829 830 for (i = 0; i < q->width; ++i) { 831 if (xe_lrc_seqno(&q->lrc[i]) != 832 q->lrc[i].fence_ctx.next_seqno - 1) 833 return false; 834 } 835 836 return true; 837 } 838 839 return xe_lrc_seqno(&q->lrc[0]) == 840 q->lrc[0].fence_ctx.next_seqno - 1; 841 } 842 843 void xe_exec_queue_kill(struct xe_exec_queue *q) 844 { 845 struct xe_exec_queue *eq = q, *next; 846 847 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 848 multi_gt_link) { 849 q->ops->kill(eq); 850 xe_vm_remove_compute_exec_queue(q->vm, eq); 851 } 852 853 q->ops->kill(q); 854 xe_vm_remove_compute_exec_queue(q->vm, q); 855 } 856 857 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 858 struct drm_file *file) 859 { 860 struct xe_device *xe = to_xe_device(dev); 861 struct xe_file *xef = to_xe_file(file); 862 struct drm_xe_exec_queue_destroy *args = data; 863 struct xe_exec_queue *q; 864 865 if (XE_IOCTL_DBG(xe, args->pad) || 866 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 867 return -EINVAL; 868 869 mutex_lock(&xef->exec_queue.lock); 870 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 871 mutex_unlock(&xef->exec_queue.lock); 872 if (XE_IOCTL_DBG(xe, !q)) 873 return -ENOENT; 874 875 if (!(q->flags & EXEC_QUEUE_FLAG_PERSISTENT)) 876 xe_exec_queue_kill(q); 877 else 878 xe_device_add_persistent_exec_queues(xe, q); 879 880 trace_xe_exec_queue_close(q); 881 xe_exec_queue_put(q); 882 883 return 0; 884 } 885 886 int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data, 887 struct drm_file *file) 888 { 889 struct xe_device *xe = to_xe_device(dev); 890 struct xe_file *xef = to_xe_file(file); 891 struct drm_xe_exec_queue_set_property *args = data; 892 struct xe_exec_queue *q; 893 int ret; 894 u32 idx; 895 896 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 897 return -EINVAL; 898 899 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 900 if (XE_IOCTL_DBG(xe, !q)) 901 return -ENOENT; 902 903 if (XE_IOCTL_DBG(xe, args->property >= 904 ARRAY_SIZE(exec_queue_set_property_funcs))) { 905 ret = -EINVAL; 906 goto out; 907 } 908 909 idx = array_index_nospec(args->property, 910 ARRAY_SIZE(exec_queue_set_property_funcs)); 911 ret = exec_queue_set_property_funcs[idx](xe, q, args->value, false); 912 if (XE_IOCTL_DBG(xe, ret)) 913 goto out; 914 915 if (args->extensions) 916 ret = exec_queue_user_extensions(xe, q, args->extensions, 0, 917 false); 918 out: 919 xe_exec_queue_put(q); 920 921 return ret; 922 } 923 924 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 925 struct xe_vm *vm) 926 { 927 lockdep_assert_held_write(&vm->lock); 928 } 929 930 /** 931 * xe_exec_queue_last_fence_put() - Drop ref to last fence 932 * @q: The exec queue 933 * @vm: The VM the engine does a bind or exec for 934 */ 935 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 936 { 937 xe_exec_queue_last_fence_lockdep_assert(q, vm); 938 939 if (q->last_fence) { 940 dma_fence_put(q->last_fence); 941 q->last_fence = NULL; 942 } 943 } 944 945 /** 946 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 947 * @q: The exec queue 948 * 949 * Only safe to be called from xe_exec_queue_destroy(). 950 */ 951 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 952 { 953 if (q->last_fence) { 954 dma_fence_put(q->last_fence); 955 q->last_fence = NULL; 956 } 957 } 958 959 /** 960 * xe_exec_queue_last_fence_get() - Get last fence 961 * @q: The exec queue 962 * @vm: The VM the engine does a bind or exec for 963 * 964 * Get last fence, does not take a ref 965 * 966 * Returns: last fence if not signaled, dma fence stub if signaled 967 */ 968 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 969 struct xe_vm *vm) 970 { 971 xe_exec_queue_last_fence_lockdep_assert(q, vm); 972 973 if (q->last_fence && 974 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 975 xe_exec_queue_last_fence_put(q, vm); 976 977 return q->last_fence ? q->last_fence : dma_fence_get_stub(); 978 } 979 980 /** 981 * xe_exec_queue_last_fence_set() - Set last fence 982 * @q: The exec queue 983 * @vm: The VM the engine does a bind or exec for 984 * @fence: The fence 985 * 986 * Set the last fence for the engine. Increases reference count for fence, when 987 * closing engine xe_exec_queue_last_fence_put should be called. 988 */ 989 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 990 struct dma_fence *fence) 991 { 992 xe_exec_queue_last_fence_lockdep_assert(q, vm); 993 994 xe_exec_queue_last_fence_put(q, vm); 995 q->last_fence = dma_fence_get(fence); 996 } 997