1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_file.h> 12 #include <drm/xe_drm.h> 13 14 #include "xe_device.h" 15 #include "xe_gt.h" 16 #include "xe_hw_engine_class_sysfs.h" 17 #include "xe_hw_fence.h" 18 #include "xe_lrc.h" 19 #include "xe_macros.h" 20 #include "xe_migrate.h" 21 #include "xe_pm.h" 22 #include "xe_ring_ops_types.h" 23 #include "xe_trace.h" 24 #include "xe_vm.h" 25 26 enum xe_exec_queue_sched_prop { 27 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 28 XE_EXEC_QUEUE_TIMESLICE = 1, 29 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 30 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 31 }; 32 33 static struct xe_exec_queue *__xe_exec_queue_create(struct xe_device *xe, 34 struct xe_vm *vm, 35 u32 logical_mask, 36 u16 width, struct xe_hw_engine *hwe, 37 u32 flags) 38 { 39 struct xe_exec_queue *q; 40 struct xe_gt *gt = hwe->gt; 41 int err; 42 int i; 43 44 /* only kernel queues can be permanent */ 45 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 46 47 q = kzalloc(sizeof(*q) + sizeof(struct xe_lrc) * width, GFP_KERNEL); 48 if (!q) 49 return ERR_PTR(-ENOMEM); 50 51 kref_init(&q->refcount); 52 q->flags = flags; 53 q->hwe = hwe; 54 q->gt = gt; 55 if (vm) 56 q->vm = xe_vm_get(vm); 57 q->class = hwe->class; 58 q->width = width; 59 q->logical_mask = logical_mask; 60 q->fence_irq = >->fence_irq[hwe->class]; 61 q->ring_ops = gt->ring_ops[hwe->class]; 62 q->ops = gt->exec_queue_ops; 63 INIT_LIST_HEAD(&q->persistent.link); 64 INIT_LIST_HEAD(&q->compute.link); 65 INIT_LIST_HEAD(&q->multi_gt_link); 66 67 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 68 q->sched_props.preempt_timeout_us = 69 hwe->eclass->sched_props.preempt_timeout_us; 70 71 if (xe_exec_queue_is_parallel(q)) { 72 q->parallel.composite_fence_ctx = dma_fence_context_alloc(1); 73 q->parallel.composite_fence_seqno = XE_FENCE_INITIAL_SEQNO; 74 } 75 if (q->flags & EXEC_QUEUE_FLAG_VM) { 76 q->bind.fence_ctx = dma_fence_context_alloc(1); 77 q->bind.fence_seqno = XE_FENCE_INITIAL_SEQNO; 78 } 79 80 for (i = 0; i < width; ++i) { 81 err = xe_lrc_init(q->lrc + i, hwe, q, vm, SZ_16K); 82 if (err) 83 goto err_lrc; 84 } 85 86 err = q->ops->init(q); 87 if (err) 88 goto err_lrc; 89 90 /* 91 * Normally the user vm holds an rpm ref to keep the device 92 * awake, and the context holds a ref for the vm, however for 93 * some engines we use the kernels migrate vm underneath which offers no 94 * such rpm ref, or we lack a vm. Make sure we keep a ref here, so we 95 * can perform GuC CT actions when needed. Caller is expected to have 96 * already grabbed the rpm ref outside any sensitive locks. 97 */ 98 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM || !vm)) 99 drm_WARN_ON(&xe->drm, !xe_device_mem_access_get_if_ongoing(xe)); 100 101 return q; 102 103 err_lrc: 104 for (i = i - 1; i >= 0; --i) 105 xe_lrc_finish(q->lrc + i); 106 kfree(q); 107 return ERR_PTR(err); 108 } 109 110 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 111 u32 logical_mask, u16 width, 112 struct xe_hw_engine *hwe, u32 flags) 113 { 114 struct xe_exec_queue *q; 115 int err; 116 117 if (vm) { 118 err = xe_vm_lock(vm, true); 119 if (err) 120 return ERR_PTR(err); 121 } 122 q = __xe_exec_queue_create(xe, vm, logical_mask, width, hwe, flags); 123 if (vm) 124 xe_vm_unlock(vm); 125 126 return q; 127 } 128 129 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 130 struct xe_vm *vm, 131 enum xe_engine_class class, u32 flags) 132 { 133 struct xe_hw_engine *hwe, *hwe0 = NULL; 134 enum xe_hw_engine_id id; 135 u32 logical_mask = 0; 136 137 for_each_hw_engine(hwe, gt, id) { 138 if (xe_hw_engine_is_reserved(hwe)) 139 continue; 140 141 if (hwe->class == class) { 142 logical_mask |= BIT(hwe->logical_instance); 143 if (!hwe0) 144 hwe0 = hwe; 145 } 146 } 147 148 if (!logical_mask) 149 return ERR_PTR(-ENODEV); 150 151 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags); 152 } 153 154 void xe_exec_queue_destroy(struct kref *ref) 155 { 156 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 157 struct xe_exec_queue *eq, *next; 158 159 xe_exec_queue_last_fence_put_unlocked(q); 160 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 161 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 162 multi_gt_link) 163 xe_exec_queue_put(eq); 164 } 165 166 q->ops->fini(q); 167 } 168 169 void xe_exec_queue_fini(struct xe_exec_queue *q) 170 { 171 int i; 172 173 for (i = 0; i < q->width; ++i) 174 xe_lrc_finish(q->lrc + i); 175 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM || !q->vm)) 176 xe_device_mem_access_put(gt_to_xe(q->gt)); 177 if (q->vm) 178 xe_vm_put(q->vm); 179 180 kfree(q); 181 } 182 183 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 184 { 185 switch (q->class) { 186 case XE_ENGINE_CLASS_RENDER: 187 sprintf(q->name, "rcs%d", instance); 188 break; 189 case XE_ENGINE_CLASS_VIDEO_DECODE: 190 sprintf(q->name, "vcs%d", instance); 191 break; 192 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 193 sprintf(q->name, "vecs%d", instance); 194 break; 195 case XE_ENGINE_CLASS_COPY: 196 sprintf(q->name, "bcs%d", instance); 197 break; 198 case XE_ENGINE_CLASS_COMPUTE: 199 sprintf(q->name, "ccs%d", instance); 200 break; 201 case XE_ENGINE_CLASS_OTHER: 202 sprintf(q->name, "gsccs%d", instance); 203 break; 204 default: 205 XE_WARN_ON(q->class); 206 } 207 } 208 209 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 210 { 211 struct xe_exec_queue *q; 212 213 mutex_lock(&xef->exec_queue.lock); 214 q = xa_load(&xef->exec_queue.xa, id); 215 if (q) 216 xe_exec_queue_get(q); 217 mutex_unlock(&xef->exec_queue.lock); 218 219 return q; 220 } 221 222 enum xe_exec_queue_priority 223 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 224 { 225 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 226 XE_EXEC_QUEUE_PRIORITY_NORMAL; 227 } 228 229 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 230 u64 value, bool create) 231 { 232 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 233 return -EINVAL; 234 235 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 236 return -EPERM; 237 238 return q->ops->set_priority(q, value); 239 } 240 241 static bool xe_exec_queue_enforce_schedule_limit(void) 242 { 243 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 244 return true; 245 #else 246 return !capable(CAP_SYS_NICE); 247 #endif 248 } 249 250 static void 251 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 252 enum xe_exec_queue_sched_prop prop, 253 u32 *min, u32 *max) 254 { 255 switch (prop) { 256 case XE_EXEC_QUEUE_JOB_TIMEOUT: 257 *min = eclass->sched_props.job_timeout_min; 258 *max = eclass->sched_props.job_timeout_max; 259 break; 260 case XE_EXEC_QUEUE_TIMESLICE: 261 *min = eclass->sched_props.timeslice_min; 262 *max = eclass->sched_props.timeslice_max; 263 break; 264 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 265 *min = eclass->sched_props.preempt_timeout_min; 266 *max = eclass->sched_props.preempt_timeout_max; 267 break; 268 default: 269 break; 270 } 271 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 272 if (capable(CAP_SYS_NICE)) { 273 switch (prop) { 274 case XE_EXEC_QUEUE_JOB_TIMEOUT: 275 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 276 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 277 break; 278 case XE_EXEC_QUEUE_TIMESLICE: 279 *min = XE_HW_ENGINE_TIMESLICE_MIN; 280 *max = XE_HW_ENGINE_TIMESLICE_MAX; 281 break; 282 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 283 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 284 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 285 break; 286 default: 287 break; 288 } 289 } 290 #endif 291 } 292 293 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 294 u64 value, bool create) 295 { 296 u32 min = 0, max = 0; 297 298 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 299 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 300 301 if (xe_exec_queue_enforce_schedule_limit() && 302 !xe_hw_engine_timeout_in_range(value, min, max)) 303 return -EINVAL; 304 305 return q->ops->set_timeslice(q, value); 306 } 307 308 static int exec_queue_set_preemption_timeout(struct xe_device *xe, 309 struct xe_exec_queue *q, u64 value, 310 bool create) 311 { 312 u32 min = 0, max = 0; 313 314 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 315 XE_EXEC_QUEUE_PREEMPT_TIMEOUT, &min, &max); 316 317 if (xe_exec_queue_enforce_schedule_limit() && 318 !xe_hw_engine_timeout_in_range(value, min, max)) 319 return -EINVAL; 320 321 return q->ops->set_preempt_timeout(q, value); 322 } 323 324 static int exec_queue_set_persistence(struct xe_device *xe, struct xe_exec_queue *q, 325 u64 value, bool create) 326 { 327 if (XE_IOCTL_DBG(xe, !create)) 328 return -EINVAL; 329 330 if (XE_IOCTL_DBG(xe, xe_vm_in_preempt_fence_mode(q->vm))) 331 return -EINVAL; 332 333 if (value) 334 q->flags |= EXEC_QUEUE_FLAG_PERSISTENT; 335 else 336 q->flags &= ~EXEC_QUEUE_FLAG_PERSISTENT; 337 338 return 0; 339 } 340 341 static int exec_queue_set_job_timeout(struct xe_device *xe, struct xe_exec_queue *q, 342 u64 value, bool create) 343 { 344 u32 min = 0, max = 0; 345 346 if (XE_IOCTL_DBG(xe, !create)) 347 return -EINVAL; 348 349 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 350 XE_EXEC_QUEUE_JOB_TIMEOUT, &min, &max); 351 352 if (xe_exec_queue_enforce_schedule_limit() && 353 !xe_hw_engine_timeout_in_range(value, min, max)) 354 return -EINVAL; 355 356 return q->ops->set_job_timeout(q, value); 357 } 358 359 static int exec_queue_set_acc_trigger(struct xe_device *xe, struct xe_exec_queue *q, 360 u64 value, bool create) 361 { 362 if (XE_IOCTL_DBG(xe, !create)) 363 return -EINVAL; 364 365 if (XE_IOCTL_DBG(xe, !xe->info.has_usm)) 366 return -EINVAL; 367 368 q->usm.acc_trigger = value; 369 370 return 0; 371 } 372 373 static int exec_queue_set_acc_notify(struct xe_device *xe, struct xe_exec_queue *q, 374 u64 value, bool create) 375 { 376 if (XE_IOCTL_DBG(xe, !create)) 377 return -EINVAL; 378 379 if (XE_IOCTL_DBG(xe, !xe->info.has_usm)) 380 return -EINVAL; 381 382 q->usm.acc_notify = value; 383 384 return 0; 385 } 386 387 static int exec_queue_set_acc_granularity(struct xe_device *xe, struct xe_exec_queue *q, 388 u64 value, bool create) 389 { 390 if (XE_IOCTL_DBG(xe, !create)) 391 return -EINVAL; 392 393 if (XE_IOCTL_DBG(xe, !xe->info.has_usm)) 394 return -EINVAL; 395 396 if (value > DRM_XE_ACC_GRANULARITY_64M) 397 return -EINVAL; 398 399 q->usm.acc_granularity = value; 400 401 return 0; 402 } 403 404 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 405 struct xe_exec_queue *q, 406 u64 value, bool create); 407 408 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 409 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 410 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 411 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT] = exec_queue_set_preemption_timeout, 412 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE] = exec_queue_set_persistence, 413 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT] = exec_queue_set_job_timeout, 414 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER] = exec_queue_set_acc_trigger, 415 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY] = exec_queue_set_acc_notify, 416 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY] = exec_queue_set_acc_granularity, 417 }; 418 419 static int exec_queue_user_ext_set_property(struct xe_device *xe, 420 struct xe_exec_queue *q, 421 u64 extension, 422 bool create) 423 { 424 u64 __user *address = u64_to_user_ptr(extension); 425 struct drm_xe_ext_set_property ext; 426 int err; 427 u32 idx; 428 429 err = __copy_from_user(&ext, address, sizeof(ext)); 430 if (XE_IOCTL_DBG(xe, err)) 431 return -EFAULT; 432 433 if (XE_IOCTL_DBG(xe, ext.property >= 434 ARRAY_SIZE(exec_queue_set_property_funcs)) || 435 XE_IOCTL_DBG(xe, ext.pad)) 436 return -EINVAL; 437 438 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 439 return exec_queue_set_property_funcs[idx](xe, q, ext.value, create); 440 } 441 442 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 443 struct xe_exec_queue *q, 444 u64 extension, 445 bool create); 446 447 static const xe_exec_queue_set_property_fn exec_queue_user_extension_funcs[] = { 448 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 449 }; 450 451 #define MAX_USER_EXTENSIONS 16 452 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 453 u64 extensions, int ext_number, bool create) 454 { 455 u64 __user *address = u64_to_user_ptr(extensions); 456 struct drm_xe_user_extension ext; 457 int err; 458 u32 idx; 459 460 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 461 return -E2BIG; 462 463 err = __copy_from_user(&ext, address, sizeof(ext)); 464 if (XE_IOCTL_DBG(xe, err)) 465 return -EFAULT; 466 467 if (XE_IOCTL_DBG(xe, ext.pad) || 468 XE_IOCTL_DBG(xe, ext.name >= 469 ARRAY_SIZE(exec_queue_user_extension_funcs))) 470 return -EINVAL; 471 472 idx = array_index_nospec(ext.name, 473 ARRAY_SIZE(exec_queue_user_extension_funcs)); 474 err = exec_queue_user_extension_funcs[idx](xe, q, extensions, create); 475 if (XE_IOCTL_DBG(xe, err)) 476 return err; 477 478 if (ext.next_extension) 479 return exec_queue_user_extensions(xe, q, ext.next_extension, 480 ++ext_number, create); 481 482 return 0; 483 } 484 485 static const enum xe_engine_class user_to_xe_engine_class[] = { 486 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, 487 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, 488 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, 489 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, 490 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, 491 }; 492 493 static struct xe_hw_engine * 494 find_hw_engine(struct xe_device *xe, 495 struct drm_xe_engine_class_instance eci) 496 { 497 u32 idx; 498 499 if (eci.engine_class > ARRAY_SIZE(user_to_xe_engine_class)) 500 return NULL; 501 502 if (eci.gt_id >= xe->info.gt_count) 503 return NULL; 504 505 idx = array_index_nospec(eci.engine_class, 506 ARRAY_SIZE(user_to_xe_engine_class)); 507 508 return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id), 509 user_to_xe_engine_class[idx], 510 eci.engine_instance, true); 511 } 512 513 static u32 bind_exec_queue_logical_mask(struct xe_device *xe, struct xe_gt *gt, 514 struct drm_xe_engine_class_instance *eci, 515 u16 width, u16 num_placements) 516 { 517 struct xe_hw_engine *hwe; 518 enum xe_hw_engine_id id; 519 u32 logical_mask = 0; 520 521 if (XE_IOCTL_DBG(xe, width != 1)) 522 return 0; 523 if (XE_IOCTL_DBG(xe, num_placements != 1)) 524 return 0; 525 if (XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 526 return 0; 527 528 eci[0].engine_class = DRM_XE_ENGINE_CLASS_COPY; 529 530 for_each_hw_engine(hwe, gt, id) { 531 if (xe_hw_engine_is_reserved(hwe)) 532 continue; 533 534 if (hwe->class == 535 user_to_xe_engine_class[DRM_XE_ENGINE_CLASS_COPY]) 536 logical_mask |= BIT(hwe->logical_instance); 537 } 538 539 return logical_mask; 540 } 541 542 static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, 543 struct drm_xe_engine_class_instance *eci, 544 u16 width, u16 num_placements) 545 { 546 int len = width * num_placements; 547 int i, j, n; 548 u16 class; 549 u16 gt_id; 550 u32 return_mask = 0, prev_mask; 551 552 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 553 len > 1)) 554 return 0; 555 556 for (i = 0; i < width; ++i) { 557 u32 current_mask = 0; 558 559 for (j = 0; j < num_placements; ++j) { 560 struct xe_hw_engine *hwe; 561 562 n = j * width + i; 563 564 hwe = find_hw_engine(xe, eci[n]); 565 if (XE_IOCTL_DBG(xe, !hwe)) 566 return 0; 567 568 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 569 return 0; 570 571 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 572 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 573 return 0; 574 575 class = eci[n].engine_class; 576 gt_id = eci[n].gt_id; 577 578 if (width == 1 || !i) 579 return_mask |= BIT(eci[n].engine_instance); 580 current_mask |= BIT(eci[n].engine_instance); 581 } 582 583 /* Parallel submissions must be logically contiguous */ 584 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 585 return 0; 586 587 prev_mask = current_mask; 588 } 589 590 return return_mask; 591 } 592 593 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 594 struct drm_file *file) 595 { 596 struct xe_device *xe = to_xe_device(dev); 597 struct xe_file *xef = to_xe_file(file); 598 struct drm_xe_exec_queue_create *args = data; 599 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 600 struct drm_xe_engine_class_instance __user *user_eci = 601 u64_to_user_ptr(args->instances); 602 struct xe_hw_engine *hwe; 603 struct xe_vm *vm, *migrate_vm; 604 struct xe_gt *gt; 605 struct xe_exec_queue *q = NULL; 606 u32 logical_mask; 607 u32 id; 608 u32 len; 609 int err; 610 611 if (XE_IOCTL_DBG(xe, args->flags) || 612 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 613 return -EINVAL; 614 615 len = args->width * args->num_placements; 616 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 617 return -EINVAL; 618 619 err = __copy_from_user(eci, user_eci, 620 sizeof(struct drm_xe_engine_class_instance) * 621 len); 622 if (XE_IOCTL_DBG(xe, err)) 623 return -EFAULT; 624 625 if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) 626 return -EINVAL; 627 628 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { 629 for_each_gt(gt, xe, id) { 630 struct xe_exec_queue *new; 631 632 if (xe_gt_is_media_type(gt)) 633 continue; 634 635 eci[0].gt_id = gt->info.id; 636 logical_mask = bind_exec_queue_logical_mask(xe, gt, eci, 637 args->width, 638 args->num_placements); 639 if (XE_IOCTL_DBG(xe, !logical_mask)) 640 return -EINVAL; 641 642 hwe = find_hw_engine(xe, eci[0]); 643 if (XE_IOCTL_DBG(xe, !hwe)) 644 return -EINVAL; 645 646 /* The migration vm doesn't hold rpm ref */ 647 xe_device_mem_access_get(xe); 648 649 migrate_vm = xe_migrate_get_vm(gt_to_tile(gt)->migrate); 650 new = xe_exec_queue_create(xe, migrate_vm, logical_mask, 651 args->width, hwe, 652 EXEC_QUEUE_FLAG_PERSISTENT | 653 EXEC_QUEUE_FLAG_VM | 654 (id ? 655 EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD : 656 0)); 657 658 xe_device_mem_access_put(xe); /* now held by engine */ 659 660 xe_vm_put(migrate_vm); 661 if (IS_ERR(new)) { 662 err = PTR_ERR(new); 663 if (q) 664 goto put_exec_queue; 665 return err; 666 } 667 if (id == 0) 668 q = new; 669 else 670 list_add_tail(&new->multi_gt_list, 671 &q->multi_gt_link); 672 } 673 } else { 674 gt = xe_device_get_gt(xe, eci[0].gt_id); 675 logical_mask = calc_validate_logical_mask(xe, gt, eci, 676 args->width, 677 args->num_placements); 678 if (XE_IOCTL_DBG(xe, !logical_mask)) 679 return -EINVAL; 680 681 hwe = find_hw_engine(xe, eci[0]); 682 if (XE_IOCTL_DBG(xe, !hwe)) 683 return -EINVAL; 684 685 vm = xe_vm_lookup(xef, args->vm_id); 686 if (XE_IOCTL_DBG(xe, !vm)) 687 return -ENOENT; 688 689 err = down_read_interruptible(&vm->lock); 690 if (err) { 691 xe_vm_put(vm); 692 return err; 693 } 694 695 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 696 up_read(&vm->lock); 697 xe_vm_put(vm); 698 return -ENOENT; 699 } 700 701 q = xe_exec_queue_create(xe, vm, logical_mask, 702 args->width, hwe, 703 xe_vm_in_lr_mode(vm) ? 0 : 704 EXEC_QUEUE_FLAG_PERSISTENT); 705 up_read(&vm->lock); 706 xe_vm_put(vm); 707 if (IS_ERR(q)) 708 return PTR_ERR(q); 709 710 if (xe_vm_in_preempt_fence_mode(vm)) { 711 q->compute.context = dma_fence_context_alloc(1); 712 spin_lock_init(&q->compute.lock); 713 714 err = xe_vm_add_compute_exec_queue(vm, q); 715 if (XE_IOCTL_DBG(xe, err)) 716 goto put_exec_queue; 717 } 718 } 719 720 if (args->extensions) { 721 err = exec_queue_user_extensions(xe, q, args->extensions, 0, true); 722 if (XE_IOCTL_DBG(xe, err)) 723 goto kill_exec_queue; 724 } 725 726 q->persistent.xef = xef; 727 728 mutex_lock(&xef->exec_queue.lock); 729 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 730 mutex_unlock(&xef->exec_queue.lock); 731 if (err) 732 goto kill_exec_queue; 733 734 args->exec_queue_id = id; 735 736 return 0; 737 738 kill_exec_queue: 739 xe_exec_queue_kill(q); 740 put_exec_queue: 741 xe_exec_queue_put(q); 742 return err; 743 } 744 745 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 746 struct drm_file *file) 747 { 748 struct xe_device *xe = to_xe_device(dev); 749 struct xe_file *xef = to_xe_file(file); 750 struct drm_xe_exec_queue_get_property *args = data; 751 struct xe_exec_queue *q; 752 int ret; 753 754 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 755 return -EINVAL; 756 757 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 758 if (XE_IOCTL_DBG(xe, !q)) 759 return -ENOENT; 760 761 switch (args->property) { 762 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: 763 args->value = !!(q->flags & EXEC_QUEUE_FLAG_BANNED); 764 ret = 0; 765 break; 766 default: 767 ret = -EINVAL; 768 } 769 770 xe_exec_queue_put(q); 771 772 return ret; 773 } 774 775 /** 776 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 777 * @q: The exec_queue 778 * 779 * Return: True if the exec_queue is long-running, false otherwise. 780 */ 781 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 782 { 783 return q->vm && xe_vm_in_lr_mode(q->vm) && 784 !(q->flags & EXEC_QUEUE_FLAG_VM); 785 } 786 787 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 788 { 789 return q->lrc->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc) - 1; 790 } 791 792 /** 793 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 794 * @q: The exec_queue 795 * 796 * Return: True if the exec_queue's ring is full, false otherwise. 797 */ 798 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 799 { 800 struct xe_lrc *lrc = q->lrc; 801 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 802 803 return xe_exec_queue_num_job_inflight(q) >= max_job; 804 } 805 806 /** 807 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 808 * @q: The exec_queue 809 * 810 * FIXME: Need to determine what to use as the short-lived 811 * timeline lock for the exec_queues, so that the return value 812 * of this function becomes more than just an advisory 813 * snapshot in time. The timeline lock must protect the 814 * seqno from racing submissions on the same exec_queue. 815 * Typically vm->resv, but user-created timeline locks use the migrate vm 816 * and never grabs the migrate vm->resv so we have a race there. 817 * 818 * Return: True if the exec_queue is idle, false otherwise. 819 */ 820 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 821 { 822 if (xe_exec_queue_is_parallel(q)) { 823 int i; 824 825 for (i = 0; i < q->width; ++i) { 826 if (xe_lrc_seqno(&q->lrc[i]) != 827 q->lrc[i].fence_ctx.next_seqno - 1) 828 return false; 829 } 830 831 return true; 832 } 833 834 return xe_lrc_seqno(&q->lrc[0]) == 835 q->lrc[0].fence_ctx.next_seqno - 1; 836 } 837 838 void xe_exec_queue_kill(struct xe_exec_queue *q) 839 { 840 struct xe_exec_queue *eq = q, *next; 841 842 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 843 multi_gt_link) { 844 q->ops->kill(eq); 845 xe_vm_remove_compute_exec_queue(q->vm, eq); 846 } 847 848 q->ops->kill(q); 849 xe_vm_remove_compute_exec_queue(q->vm, q); 850 } 851 852 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 853 struct drm_file *file) 854 { 855 struct xe_device *xe = to_xe_device(dev); 856 struct xe_file *xef = to_xe_file(file); 857 struct drm_xe_exec_queue_destroy *args = data; 858 struct xe_exec_queue *q; 859 860 if (XE_IOCTL_DBG(xe, args->pad) || 861 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 862 return -EINVAL; 863 864 mutex_lock(&xef->exec_queue.lock); 865 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 866 mutex_unlock(&xef->exec_queue.lock); 867 if (XE_IOCTL_DBG(xe, !q)) 868 return -ENOENT; 869 870 if (!(q->flags & EXEC_QUEUE_FLAG_PERSISTENT)) 871 xe_exec_queue_kill(q); 872 else 873 xe_device_add_persistent_exec_queues(xe, q); 874 875 trace_xe_exec_queue_close(q); 876 xe_exec_queue_put(q); 877 878 return 0; 879 } 880 881 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 882 struct xe_vm *vm) 883 { 884 if (q->flags & EXEC_QUEUE_FLAG_VM) 885 lockdep_assert_held(&vm->lock); 886 else 887 xe_vm_assert_held(vm); 888 } 889 890 /** 891 * xe_exec_queue_last_fence_put() - Drop ref to last fence 892 * @q: The exec queue 893 * @vm: The VM the engine does a bind or exec for 894 */ 895 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 896 { 897 xe_exec_queue_last_fence_lockdep_assert(q, vm); 898 899 if (q->last_fence) { 900 dma_fence_put(q->last_fence); 901 q->last_fence = NULL; 902 } 903 } 904 905 /** 906 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 907 * @q: The exec queue 908 * 909 * Only safe to be called from xe_exec_queue_destroy(). 910 */ 911 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 912 { 913 if (q->last_fence) { 914 dma_fence_put(q->last_fence); 915 q->last_fence = NULL; 916 } 917 } 918 919 /** 920 * xe_exec_queue_last_fence_get() - Get last fence 921 * @q: The exec queue 922 * @vm: The VM the engine does a bind or exec for 923 * 924 * Get last fence, does not take a ref 925 * 926 * Returns: last fence if not signaled, dma fence stub if signaled 927 */ 928 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 929 struct xe_vm *vm) 930 { 931 xe_exec_queue_last_fence_lockdep_assert(q, vm); 932 933 if (q->last_fence && 934 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 935 xe_exec_queue_last_fence_put(q, vm); 936 937 return q->last_fence ? q->last_fence : dma_fence_get_stub(); 938 } 939 940 /** 941 * xe_exec_queue_last_fence_set() - Set last fence 942 * @q: The exec queue 943 * @vm: The VM the engine does a bind or exec for 944 * @fence: The fence 945 * 946 * Set the last fence for the engine. Increases reference count for fence, when 947 * closing engine xe_exec_queue_last_fence_put should be called. 948 */ 949 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 950 struct dma_fence *fence) 951 { 952 xe_exec_queue_last_fence_lockdep_assert(q, vm); 953 954 xe_exec_queue_last_fence_put(q, vm); 955 q->last_fence = dma_fence_get(fence); 956 } 957