1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_file.h> 12 #include <drm/xe_drm.h> 13 14 #include "xe_device.h" 15 #include "xe_gt.h" 16 #include "xe_hw_engine_class_sysfs.h" 17 #include "xe_hw_fence.h" 18 #include "xe_lrc.h" 19 #include "xe_macros.h" 20 #include "xe_migrate.h" 21 #include "xe_pm.h" 22 #include "xe_ring_ops_types.h" 23 #include "xe_trace.h" 24 #include "xe_vm.h" 25 26 enum xe_exec_queue_sched_prop { 27 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 28 XE_EXEC_QUEUE_TIMESLICE = 1, 29 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 30 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 31 }; 32 33 static struct xe_exec_queue *__xe_exec_queue_create(struct xe_device *xe, 34 struct xe_vm *vm, 35 u32 logical_mask, 36 u16 width, struct xe_hw_engine *hwe, 37 u32 flags) 38 { 39 struct xe_exec_queue *q; 40 struct xe_gt *gt = hwe->gt; 41 int err; 42 int i; 43 44 /* only kernel queues can be permanent */ 45 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 46 47 q = kzalloc(sizeof(*q) + sizeof(struct xe_lrc) * width, GFP_KERNEL); 48 if (!q) 49 return ERR_PTR(-ENOMEM); 50 51 kref_init(&q->refcount); 52 q->flags = flags; 53 q->hwe = hwe; 54 q->gt = gt; 55 if (vm) 56 q->vm = xe_vm_get(vm); 57 q->class = hwe->class; 58 q->width = width; 59 q->logical_mask = logical_mask; 60 q->fence_irq = >->fence_irq[hwe->class]; 61 q->ring_ops = gt->ring_ops[hwe->class]; 62 q->ops = gt->exec_queue_ops; 63 INIT_LIST_HEAD(&q->persistent.link); 64 INIT_LIST_HEAD(&q->compute.link); 65 INIT_LIST_HEAD(&q->multi_gt_link); 66 67 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 68 q->sched_props.preempt_timeout_us = 69 hwe->eclass->sched_props.preempt_timeout_us; 70 71 if (xe_exec_queue_is_parallel(q)) { 72 q->parallel.composite_fence_ctx = dma_fence_context_alloc(1); 73 q->parallel.composite_fence_seqno = XE_FENCE_INITIAL_SEQNO; 74 } 75 if (q->flags & EXEC_QUEUE_FLAG_VM) { 76 q->bind.fence_ctx = dma_fence_context_alloc(1); 77 q->bind.fence_seqno = XE_FENCE_INITIAL_SEQNO; 78 } 79 80 for (i = 0; i < width; ++i) { 81 err = xe_lrc_init(q->lrc + i, hwe, q, vm, SZ_16K); 82 if (err) 83 goto err_lrc; 84 } 85 86 err = q->ops->init(q); 87 if (err) 88 goto err_lrc; 89 90 /* 91 * Normally the user vm holds an rpm ref to keep the device 92 * awake, and the context holds a ref for the vm, however for 93 * some engines we use the kernels migrate vm underneath which 94 * offers no such rpm ref. Make sure we keep a ref here, so we 95 * can perform GuC CT actions when needed. Caller is expected to 96 * have already grabbed the rpm ref outside any sensitive locks. 97 */ 98 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM)) 99 drm_WARN_ON(&xe->drm, !xe_device_mem_access_get_if_ongoing(xe)); 100 101 return q; 102 103 err_lrc: 104 for (i = i - 1; i >= 0; --i) 105 xe_lrc_finish(q->lrc + i); 106 kfree(q); 107 return ERR_PTR(err); 108 } 109 110 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 111 u32 logical_mask, u16 width, 112 struct xe_hw_engine *hwe, u32 flags) 113 { 114 struct xe_exec_queue *q; 115 int err; 116 117 if (vm) { 118 err = xe_vm_lock(vm, true); 119 if (err) 120 return ERR_PTR(err); 121 } 122 q = __xe_exec_queue_create(xe, vm, logical_mask, width, hwe, flags); 123 if (vm) 124 xe_vm_unlock(vm); 125 126 return q; 127 } 128 129 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 130 struct xe_vm *vm, 131 enum xe_engine_class class, u32 flags) 132 { 133 struct xe_hw_engine *hwe, *hwe0 = NULL; 134 enum xe_hw_engine_id id; 135 u32 logical_mask = 0; 136 137 for_each_hw_engine(hwe, gt, id) { 138 if (xe_hw_engine_is_reserved(hwe)) 139 continue; 140 141 if (hwe->class == class) { 142 logical_mask |= BIT(hwe->logical_instance); 143 if (!hwe0) 144 hwe0 = hwe; 145 } 146 } 147 148 if (!logical_mask) 149 return ERR_PTR(-ENODEV); 150 151 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags); 152 } 153 154 void xe_exec_queue_destroy(struct kref *ref) 155 { 156 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 157 struct xe_exec_queue *eq, *next; 158 159 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 160 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 161 multi_gt_link) 162 xe_exec_queue_put(eq); 163 } 164 165 q->ops->fini(q); 166 } 167 168 void xe_exec_queue_fini(struct xe_exec_queue *q) 169 { 170 int i; 171 172 for (i = 0; i < q->width; ++i) 173 xe_lrc_finish(q->lrc + i); 174 if (q->vm) 175 xe_vm_put(q->vm); 176 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM)) 177 xe_device_mem_access_put(gt_to_xe(q->gt)); 178 179 kfree(q); 180 } 181 182 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 183 { 184 switch (q->class) { 185 case XE_ENGINE_CLASS_RENDER: 186 sprintf(q->name, "rcs%d", instance); 187 break; 188 case XE_ENGINE_CLASS_VIDEO_DECODE: 189 sprintf(q->name, "vcs%d", instance); 190 break; 191 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 192 sprintf(q->name, "vecs%d", instance); 193 break; 194 case XE_ENGINE_CLASS_COPY: 195 sprintf(q->name, "bcs%d", instance); 196 break; 197 case XE_ENGINE_CLASS_COMPUTE: 198 sprintf(q->name, "ccs%d", instance); 199 break; 200 case XE_ENGINE_CLASS_OTHER: 201 sprintf(q->name, "gsccs%d", instance); 202 break; 203 default: 204 XE_WARN_ON(q->class); 205 } 206 } 207 208 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 209 { 210 struct xe_exec_queue *q; 211 212 mutex_lock(&xef->exec_queue.lock); 213 q = xa_load(&xef->exec_queue.xa, id); 214 if (q) 215 xe_exec_queue_get(q); 216 mutex_unlock(&xef->exec_queue.lock); 217 218 return q; 219 } 220 221 enum xe_exec_queue_priority 222 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 223 { 224 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 225 XE_EXEC_QUEUE_PRIORITY_NORMAL; 226 } 227 228 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 229 u64 value, bool create) 230 { 231 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 232 return -EINVAL; 233 234 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 235 return -EPERM; 236 237 return q->ops->set_priority(q, value); 238 } 239 240 static bool xe_exec_queue_enforce_schedule_limit(void) 241 { 242 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 243 return true; 244 #else 245 return !capable(CAP_SYS_NICE); 246 #endif 247 } 248 249 static void 250 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 251 enum xe_exec_queue_sched_prop prop, 252 u32 *min, u32 *max) 253 { 254 switch (prop) { 255 case XE_EXEC_QUEUE_JOB_TIMEOUT: 256 *min = eclass->sched_props.job_timeout_min; 257 *max = eclass->sched_props.job_timeout_max; 258 break; 259 case XE_EXEC_QUEUE_TIMESLICE: 260 *min = eclass->sched_props.timeslice_min; 261 *max = eclass->sched_props.timeslice_max; 262 break; 263 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 264 *min = eclass->sched_props.preempt_timeout_min; 265 *max = eclass->sched_props.preempt_timeout_max; 266 break; 267 default: 268 break; 269 } 270 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 271 if (capable(CAP_SYS_NICE)) { 272 switch (prop) { 273 case XE_EXEC_QUEUE_JOB_TIMEOUT: 274 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 275 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 276 break; 277 case XE_EXEC_QUEUE_TIMESLICE: 278 *min = XE_HW_ENGINE_TIMESLICE_MIN; 279 *max = XE_HW_ENGINE_TIMESLICE_MAX; 280 break; 281 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 282 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 283 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 284 break; 285 default: 286 break; 287 } 288 } 289 #endif 290 } 291 292 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 293 u64 value, bool create) 294 { 295 u32 min = 0, max = 0; 296 297 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 298 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 299 300 if (xe_exec_queue_enforce_schedule_limit() && 301 !xe_hw_engine_timeout_in_range(value, min, max)) 302 return -EINVAL; 303 304 return q->ops->set_timeslice(q, value); 305 } 306 307 static int exec_queue_set_preemption_timeout(struct xe_device *xe, 308 struct xe_exec_queue *q, u64 value, 309 bool create) 310 { 311 u32 min = 0, max = 0; 312 313 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 314 XE_EXEC_QUEUE_PREEMPT_TIMEOUT, &min, &max); 315 316 if (xe_exec_queue_enforce_schedule_limit() && 317 !xe_hw_engine_timeout_in_range(value, min, max)) 318 return -EINVAL; 319 320 return q->ops->set_preempt_timeout(q, value); 321 } 322 323 static int exec_queue_set_compute_mode(struct xe_device *xe, struct xe_exec_queue *q, 324 u64 value, bool create) 325 { 326 if (XE_IOCTL_DBG(xe, !create)) 327 return -EINVAL; 328 329 if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_COMPUTE_MODE)) 330 return -EINVAL; 331 332 if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) 333 return -EINVAL; 334 335 if (value) { 336 struct xe_vm *vm = q->vm; 337 int err; 338 339 if (XE_IOCTL_DBG(xe, xe_vm_in_fault_mode(vm))) 340 return -EOPNOTSUPP; 341 342 if (XE_IOCTL_DBG(xe, !xe_vm_in_compute_mode(vm))) 343 return -EOPNOTSUPP; 344 345 if (XE_IOCTL_DBG(xe, q->width != 1)) 346 return -EINVAL; 347 348 q->compute.context = dma_fence_context_alloc(1); 349 spin_lock_init(&q->compute.lock); 350 351 err = xe_vm_add_compute_exec_queue(vm, q); 352 if (XE_IOCTL_DBG(xe, err)) 353 return err; 354 355 q->flags |= EXEC_QUEUE_FLAG_COMPUTE_MODE; 356 q->flags &= ~EXEC_QUEUE_FLAG_PERSISTENT; 357 } 358 359 return 0; 360 } 361 362 static int exec_queue_set_persistence(struct xe_device *xe, struct xe_exec_queue *q, 363 u64 value, bool create) 364 { 365 if (XE_IOCTL_DBG(xe, !create)) 366 return -EINVAL; 367 368 if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_COMPUTE_MODE)) 369 return -EINVAL; 370 371 if (value) 372 q->flags |= EXEC_QUEUE_FLAG_PERSISTENT; 373 else 374 q->flags &= ~EXEC_QUEUE_FLAG_PERSISTENT; 375 376 return 0; 377 } 378 379 static int exec_queue_set_job_timeout(struct xe_device *xe, struct xe_exec_queue *q, 380 u64 value, bool create) 381 { 382 u32 min = 0, max = 0; 383 384 if (XE_IOCTL_DBG(xe, !create)) 385 return -EINVAL; 386 387 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 388 XE_EXEC_QUEUE_JOB_TIMEOUT, &min, &max); 389 390 if (xe_exec_queue_enforce_schedule_limit() && 391 !xe_hw_engine_timeout_in_range(value, min, max)) 392 return -EINVAL; 393 394 return q->ops->set_job_timeout(q, value); 395 } 396 397 static int exec_queue_set_acc_trigger(struct xe_device *xe, struct xe_exec_queue *q, 398 u64 value, bool create) 399 { 400 if (XE_IOCTL_DBG(xe, !create)) 401 return -EINVAL; 402 403 if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) 404 return -EINVAL; 405 406 q->usm.acc_trigger = value; 407 408 return 0; 409 } 410 411 static int exec_queue_set_acc_notify(struct xe_device *xe, struct xe_exec_queue *q, 412 u64 value, bool create) 413 { 414 if (XE_IOCTL_DBG(xe, !create)) 415 return -EINVAL; 416 417 if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) 418 return -EINVAL; 419 420 q->usm.acc_notify = value; 421 422 return 0; 423 } 424 425 static int exec_queue_set_acc_granularity(struct xe_device *xe, struct xe_exec_queue *q, 426 u64 value, bool create) 427 { 428 if (XE_IOCTL_DBG(xe, !create)) 429 return -EINVAL; 430 431 if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) 432 return -EINVAL; 433 434 q->usm.acc_granularity = value; 435 436 return 0; 437 } 438 439 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 440 struct xe_exec_queue *q, 441 u64 value, bool create); 442 443 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 444 [XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 445 [XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 446 [XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT] = exec_queue_set_preemption_timeout, 447 [XE_EXEC_QUEUE_SET_PROPERTY_COMPUTE_MODE] = exec_queue_set_compute_mode, 448 [XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE] = exec_queue_set_persistence, 449 [XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT] = exec_queue_set_job_timeout, 450 [XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER] = exec_queue_set_acc_trigger, 451 [XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY] = exec_queue_set_acc_notify, 452 [XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY] = exec_queue_set_acc_granularity, 453 }; 454 455 static int exec_queue_user_ext_set_property(struct xe_device *xe, 456 struct xe_exec_queue *q, 457 u64 extension, 458 bool create) 459 { 460 u64 __user *address = u64_to_user_ptr(extension); 461 struct drm_xe_ext_exec_queue_set_property ext; 462 int err; 463 u32 idx; 464 465 err = __copy_from_user(&ext, address, sizeof(ext)); 466 if (XE_IOCTL_DBG(xe, err)) 467 return -EFAULT; 468 469 if (XE_IOCTL_DBG(xe, ext.property >= 470 ARRAY_SIZE(exec_queue_set_property_funcs)) || 471 XE_IOCTL_DBG(xe, ext.pad)) 472 return -EINVAL; 473 474 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 475 return exec_queue_set_property_funcs[idx](xe, q, ext.value, create); 476 } 477 478 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 479 struct xe_exec_queue *q, 480 u64 extension, 481 bool create); 482 483 static const xe_exec_queue_set_property_fn exec_queue_user_extension_funcs[] = { 484 [XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 485 }; 486 487 #define MAX_USER_EXTENSIONS 16 488 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 489 u64 extensions, int ext_number, bool create) 490 { 491 u64 __user *address = u64_to_user_ptr(extensions); 492 struct xe_user_extension ext; 493 int err; 494 u32 idx; 495 496 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 497 return -E2BIG; 498 499 err = __copy_from_user(&ext, address, sizeof(ext)); 500 if (XE_IOCTL_DBG(xe, err)) 501 return -EFAULT; 502 503 if (XE_IOCTL_DBG(xe, ext.pad) || 504 XE_IOCTL_DBG(xe, ext.name >= 505 ARRAY_SIZE(exec_queue_user_extension_funcs))) 506 return -EINVAL; 507 508 idx = array_index_nospec(ext.name, 509 ARRAY_SIZE(exec_queue_user_extension_funcs)); 510 err = exec_queue_user_extension_funcs[idx](xe, q, extensions, create); 511 if (XE_IOCTL_DBG(xe, err)) 512 return err; 513 514 if (ext.next_extension) 515 return exec_queue_user_extensions(xe, q, ext.next_extension, 516 ++ext_number, create); 517 518 return 0; 519 } 520 521 static const enum xe_engine_class user_to_xe_engine_class[] = { 522 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, 523 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, 524 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, 525 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, 526 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, 527 }; 528 529 static struct xe_hw_engine * 530 find_hw_engine(struct xe_device *xe, 531 struct drm_xe_engine_class_instance eci) 532 { 533 u32 idx; 534 535 if (eci.engine_class > ARRAY_SIZE(user_to_xe_engine_class)) 536 return NULL; 537 538 if (eci.gt_id >= xe->info.gt_count) 539 return NULL; 540 541 idx = array_index_nospec(eci.engine_class, 542 ARRAY_SIZE(user_to_xe_engine_class)); 543 544 return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id), 545 user_to_xe_engine_class[idx], 546 eci.engine_instance, true); 547 } 548 549 static u32 bind_exec_queue_logical_mask(struct xe_device *xe, struct xe_gt *gt, 550 struct drm_xe_engine_class_instance *eci, 551 u16 width, u16 num_placements) 552 { 553 struct xe_hw_engine *hwe; 554 enum xe_hw_engine_id id; 555 u32 logical_mask = 0; 556 557 if (XE_IOCTL_DBG(xe, width != 1)) 558 return 0; 559 if (XE_IOCTL_DBG(xe, num_placements != 1)) 560 return 0; 561 if (XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 562 return 0; 563 564 eci[0].engine_class = DRM_XE_ENGINE_CLASS_COPY; 565 566 for_each_hw_engine(hwe, gt, id) { 567 if (xe_hw_engine_is_reserved(hwe)) 568 continue; 569 570 if (hwe->class == 571 user_to_xe_engine_class[DRM_XE_ENGINE_CLASS_COPY]) 572 logical_mask |= BIT(hwe->logical_instance); 573 } 574 575 return logical_mask; 576 } 577 578 static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, 579 struct drm_xe_engine_class_instance *eci, 580 u16 width, u16 num_placements) 581 { 582 int len = width * num_placements; 583 int i, j, n; 584 u16 class; 585 u16 gt_id; 586 u32 return_mask = 0, prev_mask; 587 588 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 589 len > 1)) 590 return 0; 591 592 for (i = 0; i < width; ++i) { 593 u32 current_mask = 0; 594 595 for (j = 0; j < num_placements; ++j) { 596 struct xe_hw_engine *hwe; 597 598 n = j * width + i; 599 600 hwe = find_hw_engine(xe, eci[n]); 601 if (XE_IOCTL_DBG(xe, !hwe)) 602 return 0; 603 604 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 605 return 0; 606 607 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 608 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 609 return 0; 610 611 class = eci[n].engine_class; 612 gt_id = eci[n].gt_id; 613 614 if (width == 1 || !i) 615 return_mask |= BIT(eci[n].engine_instance); 616 current_mask |= BIT(eci[n].engine_instance); 617 } 618 619 /* Parallel submissions must be logically contiguous */ 620 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 621 return 0; 622 623 prev_mask = current_mask; 624 } 625 626 return return_mask; 627 } 628 629 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 630 struct drm_file *file) 631 { 632 struct xe_device *xe = to_xe_device(dev); 633 struct xe_file *xef = to_xe_file(file); 634 struct drm_xe_exec_queue_create *args = data; 635 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 636 struct drm_xe_engine_class_instance __user *user_eci = 637 u64_to_user_ptr(args->instances); 638 struct xe_hw_engine *hwe; 639 struct xe_vm *vm, *migrate_vm; 640 struct xe_gt *gt; 641 struct xe_exec_queue *q = NULL; 642 u32 logical_mask; 643 u32 id; 644 u32 len; 645 int err; 646 647 if (XE_IOCTL_DBG(xe, args->flags) || 648 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 649 return -EINVAL; 650 651 len = args->width * args->num_placements; 652 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 653 return -EINVAL; 654 655 err = __copy_from_user(eci, user_eci, 656 sizeof(struct drm_xe_engine_class_instance) * 657 len); 658 if (XE_IOCTL_DBG(xe, err)) 659 return -EFAULT; 660 661 if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) 662 return -EINVAL; 663 664 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { 665 for_each_gt(gt, xe, id) { 666 struct xe_exec_queue *new; 667 668 if (xe_gt_is_media_type(gt)) 669 continue; 670 671 eci[0].gt_id = gt->info.id; 672 logical_mask = bind_exec_queue_logical_mask(xe, gt, eci, 673 args->width, 674 args->num_placements); 675 if (XE_IOCTL_DBG(xe, !logical_mask)) 676 return -EINVAL; 677 678 hwe = find_hw_engine(xe, eci[0]); 679 if (XE_IOCTL_DBG(xe, !hwe)) 680 return -EINVAL; 681 682 /* The migration vm doesn't hold rpm ref */ 683 xe_device_mem_access_get(xe); 684 685 migrate_vm = xe_migrate_get_vm(gt_to_tile(gt)->migrate); 686 new = xe_exec_queue_create(xe, migrate_vm, logical_mask, 687 args->width, hwe, 688 EXEC_QUEUE_FLAG_PERSISTENT | 689 EXEC_QUEUE_FLAG_VM | 690 (id ? 691 EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD : 692 0)); 693 694 xe_device_mem_access_put(xe); /* now held by engine */ 695 696 xe_vm_put(migrate_vm); 697 if (IS_ERR(new)) { 698 err = PTR_ERR(new); 699 if (q) 700 goto put_exec_queue; 701 return err; 702 } 703 if (id == 0) 704 q = new; 705 else 706 list_add_tail(&new->multi_gt_list, 707 &q->multi_gt_link); 708 } 709 } else { 710 gt = xe_device_get_gt(xe, eci[0].gt_id); 711 logical_mask = calc_validate_logical_mask(xe, gt, eci, 712 args->width, 713 args->num_placements); 714 if (XE_IOCTL_DBG(xe, !logical_mask)) 715 return -EINVAL; 716 717 hwe = find_hw_engine(xe, eci[0]); 718 if (XE_IOCTL_DBG(xe, !hwe)) 719 return -EINVAL; 720 721 vm = xe_vm_lookup(xef, args->vm_id); 722 if (XE_IOCTL_DBG(xe, !vm)) 723 return -ENOENT; 724 725 err = down_read_interruptible(&vm->lock); 726 if (err) { 727 xe_vm_put(vm); 728 return err; 729 } 730 731 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 732 up_read(&vm->lock); 733 xe_vm_put(vm); 734 return -ENOENT; 735 } 736 737 q = xe_exec_queue_create(xe, vm, logical_mask, 738 args->width, hwe, 739 xe_vm_no_dma_fences(vm) ? 0 : 740 EXEC_QUEUE_FLAG_PERSISTENT); 741 up_read(&vm->lock); 742 xe_vm_put(vm); 743 if (IS_ERR(q)) 744 return PTR_ERR(q); 745 } 746 747 if (args->extensions) { 748 err = exec_queue_user_extensions(xe, q, args->extensions, 0, true); 749 if (XE_IOCTL_DBG(xe, err)) 750 goto put_exec_queue; 751 } 752 753 if (XE_IOCTL_DBG(xe, q->vm && xe_vm_in_compute_mode(q->vm) != 754 !!(q->flags & EXEC_QUEUE_FLAG_COMPUTE_MODE))) { 755 err = -EOPNOTSUPP; 756 goto put_exec_queue; 757 } 758 759 q->persistent.xef = xef; 760 761 mutex_lock(&xef->exec_queue.lock); 762 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 763 mutex_unlock(&xef->exec_queue.lock); 764 if (err) 765 goto put_exec_queue; 766 767 args->exec_queue_id = id; 768 769 return 0; 770 771 put_exec_queue: 772 xe_exec_queue_kill(q); 773 xe_exec_queue_put(q); 774 return err; 775 } 776 777 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 778 struct drm_file *file) 779 { 780 struct xe_device *xe = to_xe_device(dev); 781 struct xe_file *xef = to_xe_file(file); 782 struct drm_xe_exec_queue_get_property *args = data; 783 struct xe_exec_queue *q; 784 int ret; 785 786 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 787 return -EINVAL; 788 789 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 790 if (XE_IOCTL_DBG(xe, !q)) 791 return -ENOENT; 792 793 switch (args->property) { 794 case XE_EXEC_QUEUE_GET_PROPERTY_BAN: 795 args->value = !!(q->flags & EXEC_QUEUE_FLAG_BANNED); 796 ret = 0; 797 break; 798 default: 799 ret = -EINVAL; 800 } 801 802 xe_exec_queue_put(q); 803 804 return ret; 805 } 806 807 static void exec_queue_kill_compute(struct xe_exec_queue *q) 808 { 809 if (!xe_vm_in_compute_mode(q->vm)) 810 return; 811 812 down_write(&q->vm->lock); 813 list_del(&q->compute.link); 814 --q->vm->preempt.num_exec_queues; 815 if (q->compute.pfence) { 816 dma_fence_enable_sw_signaling(q->compute.pfence); 817 dma_fence_put(q->compute.pfence); 818 q->compute.pfence = NULL; 819 } 820 up_write(&q->vm->lock); 821 } 822 823 /** 824 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 825 * @q: The exec_queue 826 * 827 * Return: True if the exec_queue is long-running, false otherwise. 828 */ 829 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 830 { 831 return q->vm && xe_vm_no_dma_fences(q->vm) && 832 !(q->flags & EXEC_QUEUE_FLAG_VM); 833 } 834 835 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 836 { 837 return q->lrc->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc) - 1; 838 } 839 840 /** 841 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 842 * @q: The exec_queue 843 * 844 * Return: True if the exec_queue's ring is full, false otherwise. 845 */ 846 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 847 { 848 struct xe_lrc *lrc = q->lrc; 849 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 850 851 return xe_exec_queue_num_job_inflight(q) >= max_job; 852 } 853 854 /** 855 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 856 * @q: The exec_queue 857 * 858 * FIXME: Need to determine what to use as the short-lived 859 * timeline lock for the exec_queues, so that the return value 860 * of this function becomes more than just an advisory 861 * snapshot in time. The timeline lock must protect the 862 * seqno from racing submissions on the same exec_queue. 863 * Typically vm->resv, but user-created timeline locks use the migrate vm 864 * and never grabs the migrate vm->resv so we have a race there. 865 * 866 * Return: True if the exec_queue is idle, false otherwise. 867 */ 868 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 869 { 870 if (XE_WARN_ON(xe_exec_queue_is_parallel(q))) 871 return false; 872 873 return xe_lrc_seqno(&q->lrc[0]) == 874 q->lrc[0].fence_ctx.next_seqno - 1; 875 } 876 877 void xe_exec_queue_kill(struct xe_exec_queue *q) 878 { 879 struct xe_exec_queue *eq = q, *next; 880 881 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 882 multi_gt_link) { 883 q->ops->kill(eq); 884 exec_queue_kill_compute(eq); 885 } 886 887 q->ops->kill(q); 888 exec_queue_kill_compute(q); 889 } 890 891 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 892 struct drm_file *file) 893 { 894 struct xe_device *xe = to_xe_device(dev); 895 struct xe_file *xef = to_xe_file(file); 896 struct drm_xe_exec_queue_destroy *args = data; 897 struct xe_exec_queue *q; 898 899 if (XE_IOCTL_DBG(xe, args->pad) || 900 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 901 return -EINVAL; 902 903 mutex_lock(&xef->exec_queue.lock); 904 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 905 mutex_unlock(&xef->exec_queue.lock); 906 if (XE_IOCTL_DBG(xe, !q)) 907 return -ENOENT; 908 909 if (!(q->flags & EXEC_QUEUE_FLAG_PERSISTENT)) 910 xe_exec_queue_kill(q); 911 else 912 xe_device_add_persistent_exec_queues(xe, q); 913 914 trace_xe_exec_queue_close(q); 915 xe_exec_queue_put(q); 916 917 return 0; 918 } 919 920 int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data, 921 struct drm_file *file) 922 { 923 struct xe_device *xe = to_xe_device(dev); 924 struct xe_file *xef = to_xe_file(file); 925 struct drm_xe_exec_queue_set_property *args = data; 926 struct xe_exec_queue *q; 927 int ret; 928 u32 idx; 929 930 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 931 return -EINVAL; 932 933 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 934 if (XE_IOCTL_DBG(xe, !q)) 935 return -ENOENT; 936 937 if (XE_IOCTL_DBG(xe, args->property >= 938 ARRAY_SIZE(exec_queue_set_property_funcs))) { 939 ret = -EINVAL; 940 goto out; 941 } 942 943 idx = array_index_nospec(args->property, 944 ARRAY_SIZE(exec_queue_set_property_funcs)); 945 ret = exec_queue_set_property_funcs[idx](xe, q, args->value, false); 946 if (XE_IOCTL_DBG(xe, ret)) 947 goto out; 948 949 if (args->extensions) 950 ret = exec_queue_user_extensions(xe, q, args->extensions, 0, 951 false); 952 out: 953 xe_exec_queue_put(q); 954 955 return ret; 956 } 957