1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_exec_queue.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_device.h> 11 #include <drm/drm_file.h> 12 #include <drm/xe_drm.h> 13 14 #include "xe_device.h" 15 #include "xe_gt.h" 16 #include "xe_hw_engine_class_sysfs.h" 17 #include "xe_hw_fence.h" 18 #include "xe_lrc.h" 19 #include "xe_macros.h" 20 #include "xe_migrate.h" 21 #include "xe_pm.h" 22 #include "xe_ring_ops_types.h" 23 #include "xe_trace.h" 24 #include "xe_vm.h" 25 26 enum xe_exec_queue_sched_prop { 27 XE_EXEC_QUEUE_JOB_TIMEOUT = 0, 28 XE_EXEC_QUEUE_TIMESLICE = 1, 29 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, 30 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, 31 }; 32 33 static struct xe_exec_queue *__xe_exec_queue_create(struct xe_device *xe, 34 struct xe_vm *vm, 35 u32 logical_mask, 36 u16 width, struct xe_hw_engine *hwe, 37 u32 flags) 38 { 39 struct xe_exec_queue *q; 40 struct xe_gt *gt = hwe->gt; 41 int err; 42 int i; 43 44 /* only kernel queues can be permanent */ 45 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 46 47 q = kzalloc(sizeof(*q) + sizeof(struct xe_lrc) * width, GFP_KERNEL); 48 if (!q) 49 return ERR_PTR(-ENOMEM); 50 51 kref_init(&q->refcount); 52 q->flags = flags; 53 q->hwe = hwe; 54 q->gt = gt; 55 if (vm) 56 q->vm = xe_vm_get(vm); 57 q->class = hwe->class; 58 q->width = width; 59 q->logical_mask = logical_mask; 60 q->fence_irq = >->fence_irq[hwe->class]; 61 q->ring_ops = gt->ring_ops[hwe->class]; 62 q->ops = gt->exec_queue_ops; 63 INIT_LIST_HEAD(&q->persistent.link); 64 INIT_LIST_HEAD(&q->compute.link); 65 INIT_LIST_HEAD(&q->multi_gt_link); 66 67 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; 68 q->sched_props.preempt_timeout_us = 69 hwe->eclass->sched_props.preempt_timeout_us; 70 71 if (xe_exec_queue_is_parallel(q)) { 72 q->parallel.composite_fence_ctx = dma_fence_context_alloc(1); 73 q->parallel.composite_fence_seqno = XE_FENCE_INITIAL_SEQNO; 74 } 75 if (q->flags & EXEC_QUEUE_FLAG_VM) { 76 q->bind.fence_ctx = dma_fence_context_alloc(1); 77 q->bind.fence_seqno = XE_FENCE_INITIAL_SEQNO; 78 } 79 80 for (i = 0; i < width; ++i) { 81 err = xe_lrc_init(q->lrc + i, hwe, q, vm, SZ_16K); 82 if (err) 83 goto err_lrc; 84 } 85 86 err = q->ops->init(q); 87 if (err) 88 goto err_lrc; 89 90 /* 91 * Normally the user vm holds an rpm ref to keep the device 92 * awake, and the context holds a ref for the vm, however for 93 * some engines we use the kernels migrate vm underneath which 94 * offers no such rpm ref. Make sure we keep a ref here, so we 95 * can perform GuC CT actions when needed. Caller is expected to 96 * have already grabbed the rpm ref outside any sensitive locks. 97 */ 98 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM)) 99 drm_WARN_ON(&xe->drm, !xe_device_mem_access_get_if_ongoing(xe)); 100 101 return q; 102 103 err_lrc: 104 for (i = i - 1; i >= 0; --i) 105 xe_lrc_finish(q->lrc + i); 106 kfree(q); 107 return ERR_PTR(err); 108 } 109 110 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, 111 u32 logical_mask, u16 width, 112 struct xe_hw_engine *hwe, u32 flags) 113 { 114 struct xe_exec_queue *q; 115 int err; 116 117 if (vm) { 118 err = xe_vm_lock(vm, true); 119 if (err) 120 return ERR_PTR(err); 121 } 122 q = __xe_exec_queue_create(xe, vm, logical_mask, width, hwe, flags); 123 if (vm) 124 xe_vm_unlock(vm); 125 126 return q; 127 } 128 129 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, 130 struct xe_vm *vm, 131 enum xe_engine_class class, u32 flags) 132 { 133 struct xe_hw_engine *hwe, *hwe0 = NULL; 134 enum xe_hw_engine_id id; 135 u32 logical_mask = 0; 136 137 for_each_hw_engine(hwe, gt, id) { 138 if (xe_hw_engine_is_reserved(hwe)) 139 continue; 140 141 if (hwe->class == class) { 142 logical_mask |= BIT(hwe->logical_instance); 143 if (!hwe0) 144 hwe0 = hwe; 145 } 146 } 147 148 if (!logical_mask) 149 return ERR_PTR(-ENODEV); 150 151 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags); 152 } 153 154 void xe_exec_queue_destroy(struct kref *ref) 155 { 156 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); 157 struct xe_exec_queue *eq, *next; 158 159 xe_exec_queue_last_fence_put_unlocked(q); 160 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { 161 list_for_each_entry_safe(eq, next, &q->multi_gt_list, 162 multi_gt_link) 163 xe_exec_queue_put(eq); 164 } 165 166 q->ops->fini(q); 167 } 168 169 void xe_exec_queue_fini(struct xe_exec_queue *q) 170 { 171 int i; 172 173 for (i = 0; i < q->width; ++i) 174 xe_lrc_finish(q->lrc + i); 175 if (q->vm) 176 xe_vm_put(q->vm); 177 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM)) 178 xe_device_mem_access_put(gt_to_xe(q->gt)); 179 180 kfree(q); 181 } 182 183 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance) 184 { 185 switch (q->class) { 186 case XE_ENGINE_CLASS_RENDER: 187 sprintf(q->name, "rcs%d", instance); 188 break; 189 case XE_ENGINE_CLASS_VIDEO_DECODE: 190 sprintf(q->name, "vcs%d", instance); 191 break; 192 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 193 sprintf(q->name, "vecs%d", instance); 194 break; 195 case XE_ENGINE_CLASS_COPY: 196 sprintf(q->name, "bcs%d", instance); 197 break; 198 case XE_ENGINE_CLASS_COMPUTE: 199 sprintf(q->name, "ccs%d", instance); 200 break; 201 case XE_ENGINE_CLASS_OTHER: 202 sprintf(q->name, "gsccs%d", instance); 203 break; 204 default: 205 XE_WARN_ON(q->class); 206 } 207 } 208 209 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) 210 { 211 struct xe_exec_queue *q; 212 213 mutex_lock(&xef->exec_queue.lock); 214 q = xa_load(&xef->exec_queue.xa, id); 215 if (q) 216 xe_exec_queue_get(q); 217 mutex_unlock(&xef->exec_queue.lock); 218 219 return q; 220 } 221 222 enum xe_exec_queue_priority 223 xe_exec_queue_device_get_max_priority(struct xe_device *xe) 224 { 225 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : 226 XE_EXEC_QUEUE_PRIORITY_NORMAL; 227 } 228 229 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, 230 u64 value, bool create) 231 { 232 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) 233 return -EINVAL; 234 235 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) 236 return -EPERM; 237 238 return q->ops->set_priority(q, value); 239 } 240 241 static bool xe_exec_queue_enforce_schedule_limit(void) 242 { 243 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 244 return true; 245 #else 246 return !capable(CAP_SYS_NICE); 247 #endif 248 } 249 250 static void 251 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass, 252 enum xe_exec_queue_sched_prop prop, 253 u32 *min, u32 *max) 254 { 255 switch (prop) { 256 case XE_EXEC_QUEUE_JOB_TIMEOUT: 257 *min = eclass->sched_props.job_timeout_min; 258 *max = eclass->sched_props.job_timeout_max; 259 break; 260 case XE_EXEC_QUEUE_TIMESLICE: 261 *min = eclass->sched_props.timeslice_min; 262 *max = eclass->sched_props.timeslice_max; 263 break; 264 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 265 *min = eclass->sched_props.preempt_timeout_min; 266 *max = eclass->sched_props.preempt_timeout_max; 267 break; 268 default: 269 break; 270 } 271 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT) 272 if (capable(CAP_SYS_NICE)) { 273 switch (prop) { 274 case XE_EXEC_QUEUE_JOB_TIMEOUT: 275 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 276 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 277 break; 278 case XE_EXEC_QUEUE_TIMESLICE: 279 *min = XE_HW_ENGINE_TIMESLICE_MIN; 280 *max = XE_HW_ENGINE_TIMESLICE_MAX; 281 break; 282 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT: 283 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 284 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 285 break; 286 default: 287 break; 288 } 289 } 290 #endif 291 } 292 293 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, 294 u64 value, bool create) 295 { 296 u32 min = 0, max = 0; 297 298 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 299 XE_EXEC_QUEUE_TIMESLICE, &min, &max); 300 301 if (xe_exec_queue_enforce_schedule_limit() && 302 !xe_hw_engine_timeout_in_range(value, min, max)) 303 return -EINVAL; 304 305 return q->ops->set_timeslice(q, value); 306 } 307 308 static int exec_queue_set_preemption_timeout(struct xe_device *xe, 309 struct xe_exec_queue *q, u64 value, 310 bool create) 311 { 312 u32 min = 0, max = 0; 313 314 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 315 XE_EXEC_QUEUE_PREEMPT_TIMEOUT, &min, &max); 316 317 if (xe_exec_queue_enforce_schedule_limit() && 318 !xe_hw_engine_timeout_in_range(value, min, max)) 319 return -EINVAL; 320 321 return q->ops->set_preempt_timeout(q, value); 322 } 323 324 static int exec_queue_set_persistence(struct xe_device *xe, struct xe_exec_queue *q, 325 u64 value, bool create) 326 { 327 if (XE_IOCTL_DBG(xe, !create)) 328 return -EINVAL; 329 330 if (XE_IOCTL_DBG(xe, xe_vm_in_compute_mode(q->vm))) 331 return -EINVAL; 332 333 if (value) 334 q->flags |= EXEC_QUEUE_FLAG_PERSISTENT; 335 else 336 q->flags &= ~EXEC_QUEUE_FLAG_PERSISTENT; 337 338 return 0; 339 } 340 341 static int exec_queue_set_job_timeout(struct xe_device *xe, struct xe_exec_queue *q, 342 u64 value, bool create) 343 { 344 u32 min = 0, max = 0; 345 346 if (XE_IOCTL_DBG(xe, !create)) 347 return -EINVAL; 348 349 xe_exec_queue_get_prop_minmax(q->hwe->eclass, 350 XE_EXEC_QUEUE_JOB_TIMEOUT, &min, &max); 351 352 if (xe_exec_queue_enforce_schedule_limit() && 353 !xe_hw_engine_timeout_in_range(value, min, max)) 354 return -EINVAL; 355 356 return q->ops->set_job_timeout(q, value); 357 } 358 359 static int exec_queue_set_acc_trigger(struct xe_device *xe, struct xe_exec_queue *q, 360 u64 value, bool create) 361 { 362 if (XE_IOCTL_DBG(xe, !create)) 363 return -EINVAL; 364 365 if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) 366 return -EINVAL; 367 368 q->usm.acc_trigger = value; 369 370 return 0; 371 } 372 373 static int exec_queue_set_acc_notify(struct xe_device *xe, struct xe_exec_queue *q, 374 u64 value, bool create) 375 { 376 if (XE_IOCTL_DBG(xe, !create)) 377 return -EINVAL; 378 379 if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) 380 return -EINVAL; 381 382 q->usm.acc_notify = value; 383 384 return 0; 385 } 386 387 static int exec_queue_set_acc_granularity(struct xe_device *xe, struct xe_exec_queue *q, 388 u64 value, bool create) 389 { 390 if (XE_IOCTL_DBG(xe, !create)) 391 return -EINVAL; 392 393 if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) 394 return -EINVAL; 395 396 q->usm.acc_granularity = value; 397 398 return 0; 399 } 400 401 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, 402 struct xe_exec_queue *q, 403 u64 value, bool create); 404 405 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { 406 [XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, 407 [XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, 408 [XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT] = exec_queue_set_preemption_timeout, 409 [XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE] = exec_queue_set_persistence, 410 [XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT] = exec_queue_set_job_timeout, 411 [XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER] = exec_queue_set_acc_trigger, 412 [XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY] = exec_queue_set_acc_notify, 413 [XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY] = exec_queue_set_acc_granularity, 414 }; 415 416 static int exec_queue_user_ext_set_property(struct xe_device *xe, 417 struct xe_exec_queue *q, 418 u64 extension, 419 bool create) 420 { 421 u64 __user *address = u64_to_user_ptr(extension); 422 struct drm_xe_ext_set_property ext; 423 int err; 424 u32 idx; 425 426 err = __copy_from_user(&ext, address, sizeof(ext)); 427 if (XE_IOCTL_DBG(xe, err)) 428 return -EFAULT; 429 430 if (XE_IOCTL_DBG(xe, ext.property >= 431 ARRAY_SIZE(exec_queue_set_property_funcs)) || 432 XE_IOCTL_DBG(xe, ext.pad)) 433 return -EINVAL; 434 435 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); 436 return exec_queue_set_property_funcs[idx](xe, q, ext.value, create); 437 } 438 439 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, 440 struct xe_exec_queue *q, 441 u64 extension, 442 bool create); 443 444 static const xe_exec_queue_set_property_fn exec_queue_user_extension_funcs[] = { 445 [XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, 446 }; 447 448 #define MAX_USER_EXTENSIONS 16 449 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, 450 u64 extensions, int ext_number, bool create) 451 { 452 u64 __user *address = u64_to_user_ptr(extensions); 453 struct xe_user_extension ext; 454 int err; 455 u32 idx; 456 457 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) 458 return -E2BIG; 459 460 err = __copy_from_user(&ext, address, sizeof(ext)); 461 if (XE_IOCTL_DBG(xe, err)) 462 return -EFAULT; 463 464 if (XE_IOCTL_DBG(xe, ext.pad) || 465 XE_IOCTL_DBG(xe, ext.name >= 466 ARRAY_SIZE(exec_queue_user_extension_funcs))) 467 return -EINVAL; 468 469 idx = array_index_nospec(ext.name, 470 ARRAY_SIZE(exec_queue_user_extension_funcs)); 471 err = exec_queue_user_extension_funcs[idx](xe, q, extensions, create); 472 if (XE_IOCTL_DBG(xe, err)) 473 return err; 474 475 if (ext.next_extension) 476 return exec_queue_user_extensions(xe, q, ext.next_extension, 477 ++ext_number, create); 478 479 return 0; 480 } 481 482 static const enum xe_engine_class user_to_xe_engine_class[] = { 483 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, 484 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, 485 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, 486 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, 487 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, 488 }; 489 490 static struct xe_hw_engine * 491 find_hw_engine(struct xe_device *xe, 492 struct drm_xe_engine_class_instance eci) 493 { 494 u32 idx; 495 496 if (eci.engine_class > ARRAY_SIZE(user_to_xe_engine_class)) 497 return NULL; 498 499 if (eci.gt_id >= xe->info.gt_count) 500 return NULL; 501 502 idx = array_index_nospec(eci.engine_class, 503 ARRAY_SIZE(user_to_xe_engine_class)); 504 505 return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id), 506 user_to_xe_engine_class[idx], 507 eci.engine_instance, true); 508 } 509 510 static u32 bind_exec_queue_logical_mask(struct xe_device *xe, struct xe_gt *gt, 511 struct drm_xe_engine_class_instance *eci, 512 u16 width, u16 num_placements) 513 { 514 struct xe_hw_engine *hwe; 515 enum xe_hw_engine_id id; 516 u32 logical_mask = 0; 517 518 if (XE_IOCTL_DBG(xe, width != 1)) 519 return 0; 520 if (XE_IOCTL_DBG(xe, num_placements != 1)) 521 return 0; 522 if (XE_IOCTL_DBG(xe, eci[0].engine_instance != 0)) 523 return 0; 524 525 eci[0].engine_class = DRM_XE_ENGINE_CLASS_COPY; 526 527 for_each_hw_engine(hwe, gt, id) { 528 if (xe_hw_engine_is_reserved(hwe)) 529 continue; 530 531 if (hwe->class == 532 user_to_xe_engine_class[DRM_XE_ENGINE_CLASS_COPY]) 533 logical_mask |= BIT(hwe->logical_instance); 534 } 535 536 return logical_mask; 537 } 538 539 static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, 540 struct drm_xe_engine_class_instance *eci, 541 u16 width, u16 num_placements) 542 { 543 int len = width * num_placements; 544 int i, j, n; 545 u16 class; 546 u16 gt_id; 547 u32 return_mask = 0, prev_mask; 548 549 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) && 550 len > 1)) 551 return 0; 552 553 for (i = 0; i < width; ++i) { 554 u32 current_mask = 0; 555 556 for (j = 0; j < num_placements; ++j) { 557 struct xe_hw_engine *hwe; 558 559 n = j * width + i; 560 561 hwe = find_hw_engine(xe, eci[n]); 562 if (XE_IOCTL_DBG(xe, !hwe)) 563 return 0; 564 565 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe))) 566 return 0; 567 568 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) || 569 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class)) 570 return 0; 571 572 class = eci[n].engine_class; 573 gt_id = eci[n].gt_id; 574 575 if (width == 1 || !i) 576 return_mask |= BIT(eci[n].engine_instance); 577 current_mask |= BIT(eci[n].engine_instance); 578 } 579 580 /* Parallel submissions must be logically contiguous */ 581 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1)) 582 return 0; 583 584 prev_mask = current_mask; 585 } 586 587 return return_mask; 588 } 589 590 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, 591 struct drm_file *file) 592 { 593 struct xe_device *xe = to_xe_device(dev); 594 struct xe_file *xef = to_xe_file(file); 595 struct drm_xe_exec_queue_create *args = data; 596 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; 597 struct drm_xe_engine_class_instance __user *user_eci = 598 u64_to_user_ptr(args->instances); 599 struct xe_hw_engine *hwe; 600 struct xe_vm *vm, *migrate_vm; 601 struct xe_gt *gt; 602 struct xe_exec_queue *q = NULL; 603 u32 logical_mask; 604 u32 id; 605 u32 len; 606 int err; 607 608 if (XE_IOCTL_DBG(xe, args->flags) || 609 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 610 return -EINVAL; 611 612 len = args->width * args->num_placements; 613 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) 614 return -EINVAL; 615 616 err = __copy_from_user(eci, user_eci, 617 sizeof(struct drm_xe_engine_class_instance) * 618 len); 619 if (XE_IOCTL_DBG(xe, err)) 620 return -EFAULT; 621 622 if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) 623 return -EINVAL; 624 625 if (eci[0].engine_class >= DRM_XE_ENGINE_CLASS_VM_BIND_ASYNC) { 626 bool sync = eci[0].engine_class == 627 DRM_XE_ENGINE_CLASS_VM_BIND_SYNC; 628 629 for_each_gt(gt, xe, id) { 630 struct xe_exec_queue *new; 631 632 if (xe_gt_is_media_type(gt)) 633 continue; 634 635 eci[0].gt_id = gt->info.id; 636 logical_mask = bind_exec_queue_logical_mask(xe, gt, eci, 637 args->width, 638 args->num_placements); 639 if (XE_IOCTL_DBG(xe, !logical_mask)) 640 return -EINVAL; 641 642 hwe = find_hw_engine(xe, eci[0]); 643 if (XE_IOCTL_DBG(xe, !hwe)) 644 return -EINVAL; 645 646 /* The migration vm doesn't hold rpm ref */ 647 xe_device_mem_access_get(xe); 648 649 migrate_vm = xe_migrate_get_vm(gt_to_tile(gt)->migrate); 650 new = xe_exec_queue_create(xe, migrate_vm, logical_mask, 651 args->width, hwe, 652 EXEC_QUEUE_FLAG_PERSISTENT | 653 EXEC_QUEUE_FLAG_VM | 654 (sync ? 0 : 655 EXEC_QUEUE_FLAG_VM_ASYNC) | 656 (id ? 657 EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD : 658 0)); 659 660 xe_device_mem_access_put(xe); /* now held by engine */ 661 662 xe_vm_put(migrate_vm); 663 if (IS_ERR(new)) { 664 err = PTR_ERR(new); 665 if (q) 666 goto put_exec_queue; 667 return err; 668 } 669 if (id == 0) 670 q = new; 671 else 672 list_add_tail(&new->multi_gt_list, 673 &q->multi_gt_link); 674 } 675 } else { 676 gt = xe_device_get_gt(xe, eci[0].gt_id); 677 logical_mask = calc_validate_logical_mask(xe, gt, eci, 678 args->width, 679 args->num_placements); 680 if (XE_IOCTL_DBG(xe, !logical_mask)) 681 return -EINVAL; 682 683 hwe = find_hw_engine(xe, eci[0]); 684 if (XE_IOCTL_DBG(xe, !hwe)) 685 return -EINVAL; 686 687 vm = xe_vm_lookup(xef, args->vm_id); 688 if (XE_IOCTL_DBG(xe, !vm)) 689 return -ENOENT; 690 691 err = down_read_interruptible(&vm->lock); 692 if (err) { 693 xe_vm_put(vm); 694 return err; 695 } 696 697 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 698 up_read(&vm->lock); 699 xe_vm_put(vm); 700 return -ENOENT; 701 } 702 703 q = xe_exec_queue_create(xe, vm, logical_mask, 704 args->width, hwe, 705 xe_vm_no_dma_fences(vm) ? 0 : 706 EXEC_QUEUE_FLAG_PERSISTENT); 707 up_read(&vm->lock); 708 xe_vm_put(vm); 709 if (IS_ERR(q)) 710 return PTR_ERR(q); 711 712 if (xe_vm_in_compute_mode(vm)) { 713 q->compute.context = dma_fence_context_alloc(1); 714 spin_lock_init(&q->compute.lock); 715 716 err = xe_vm_add_compute_exec_queue(vm, q); 717 if (XE_IOCTL_DBG(xe, err)) 718 goto put_exec_queue; 719 } 720 } 721 722 if (args->extensions) { 723 err = exec_queue_user_extensions(xe, q, args->extensions, 0, true); 724 if (XE_IOCTL_DBG(xe, err)) 725 goto kill_exec_queue; 726 } 727 728 q->persistent.xef = xef; 729 730 mutex_lock(&xef->exec_queue.lock); 731 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); 732 mutex_unlock(&xef->exec_queue.lock); 733 if (err) 734 goto kill_exec_queue; 735 736 args->exec_queue_id = id; 737 738 return 0; 739 740 kill_exec_queue: 741 xe_exec_queue_kill(q); 742 put_exec_queue: 743 xe_exec_queue_put(q); 744 return err; 745 } 746 747 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, 748 struct drm_file *file) 749 { 750 struct xe_device *xe = to_xe_device(dev); 751 struct xe_file *xef = to_xe_file(file); 752 struct drm_xe_exec_queue_get_property *args = data; 753 struct xe_exec_queue *q; 754 int ret; 755 756 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 757 return -EINVAL; 758 759 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 760 if (XE_IOCTL_DBG(xe, !q)) 761 return -ENOENT; 762 763 switch (args->property) { 764 case XE_EXEC_QUEUE_GET_PROPERTY_BAN: 765 args->value = !!(q->flags & EXEC_QUEUE_FLAG_BANNED); 766 ret = 0; 767 break; 768 default: 769 ret = -EINVAL; 770 } 771 772 xe_exec_queue_put(q); 773 774 return ret; 775 } 776 777 /** 778 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running 779 * @q: The exec_queue 780 * 781 * Return: True if the exec_queue is long-running, false otherwise. 782 */ 783 bool xe_exec_queue_is_lr(struct xe_exec_queue *q) 784 { 785 return q->vm && xe_vm_no_dma_fences(q->vm) && 786 !(q->flags & EXEC_QUEUE_FLAG_VM); 787 } 788 789 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) 790 { 791 return q->lrc->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc) - 1; 792 } 793 794 /** 795 * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full 796 * @q: The exec_queue 797 * 798 * Return: True if the exec_queue's ring is full, false otherwise. 799 */ 800 bool xe_exec_queue_ring_full(struct xe_exec_queue *q) 801 { 802 struct xe_lrc *lrc = q->lrc; 803 s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; 804 805 return xe_exec_queue_num_job_inflight(q) >= max_job; 806 } 807 808 /** 809 * xe_exec_queue_is_idle() - Whether an exec_queue is idle. 810 * @q: The exec_queue 811 * 812 * FIXME: Need to determine what to use as the short-lived 813 * timeline lock for the exec_queues, so that the return value 814 * of this function becomes more than just an advisory 815 * snapshot in time. The timeline lock must protect the 816 * seqno from racing submissions on the same exec_queue. 817 * Typically vm->resv, but user-created timeline locks use the migrate vm 818 * and never grabs the migrate vm->resv so we have a race there. 819 * 820 * Return: True if the exec_queue is idle, false otherwise. 821 */ 822 bool xe_exec_queue_is_idle(struct xe_exec_queue *q) 823 { 824 if (xe_exec_queue_is_parallel(q)) { 825 int i; 826 827 for (i = 0; i < q->width; ++i) { 828 if (xe_lrc_seqno(&q->lrc[i]) != 829 q->lrc[i].fence_ctx.next_seqno - 1) 830 return false; 831 } 832 833 return true; 834 } 835 836 return xe_lrc_seqno(&q->lrc[0]) == 837 q->lrc[0].fence_ctx.next_seqno - 1; 838 } 839 840 void xe_exec_queue_kill(struct xe_exec_queue *q) 841 { 842 struct xe_exec_queue *eq = q, *next; 843 844 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, 845 multi_gt_link) { 846 q->ops->kill(eq); 847 xe_vm_remove_compute_exec_queue(q->vm, eq); 848 } 849 850 q->ops->kill(q); 851 xe_vm_remove_compute_exec_queue(q->vm, q); 852 } 853 854 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, 855 struct drm_file *file) 856 { 857 struct xe_device *xe = to_xe_device(dev); 858 struct xe_file *xef = to_xe_file(file); 859 struct drm_xe_exec_queue_destroy *args = data; 860 struct xe_exec_queue *q; 861 862 if (XE_IOCTL_DBG(xe, args->pad) || 863 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 864 return -EINVAL; 865 866 mutex_lock(&xef->exec_queue.lock); 867 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); 868 mutex_unlock(&xef->exec_queue.lock); 869 if (XE_IOCTL_DBG(xe, !q)) 870 return -ENOENT; 871 872 if (!(q->flags & EXEC_QUEUE_FLAG_PERSISTENT)) 873 xe_exec_queue_kill(q); 874 else 875 xe_device_add_persistent_exec_queues(xe, q); 876 877 trace_xe_exec_queue_close(q); 878 xe_exec_queue_put(q); 879 880 return 0; 881 } 882 883 int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data, 884 struct drm_file *file) 885 { 886 struct xe_device *xe = to_xe_device(dev); 887 struct xe_file *xef = to_xe_file(file); 888 struct drm_xe_exec_queue_set_property *args = data; 889 struct xe_exec_queue *q; 890 int ret; 891 u32 idx; 892 893 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 894 return -EINVAL; 895 896 q = xe_exec_queue_lookup(xef, args->exec_queue_id); 897 if (XE_IOCTL_DBG(xe, !q)) 898 return -ENOENT; 899 900 if (XE_IOCTL_DBG(xe, args->property >= 901 ARRAY_SIZE(exec_queue_set_property_funcs))) { 902 ret = -EINVAL; 903 goto out; 904 } 905 906 idx = array_index_nospec(args->property, 907 ARRAY_SIZE(exec_queue_set_property_funcs)); 908 ret = exec_queue_set_property_funcs[idx](xe, q, args->value, false); 909 if (XE_IOCTL_DBG(xe, ret)) 910 goto out; 911 912 if (args->extensions) 913 ret = exec_queue_user_extensions(xe, q, args->extensions, 0, 914 false); 915 out: 916 xe_exec_queue_put(q); 917 918 return ret; 919 } 920 921 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, 922 struct xe_vm *vm) 923 { 924 lockdep_assert_held_write(&vm->lock); 925 } 926 927 /** 928 * xe_exec_queue_last_fence_put() - Drop ref to last fence 929 * @q: The exec queue 930 * @vm: The VM the engine does a bind or exec for 931 */ 932 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm) 933 { 934 xe_exec_queue_last_fence_lockdep_assert(q, vm); 935 936 if (q->last_fence) { 937 dma_fence_put(q->last_fence); 938 q->last_fence = NULL; 939 } 940 } 941 942 /** 943 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked 944 * @q: The exec queue 945 * 946 * Only safe to be called from xe_exec_queue_destroy(). 947 */ 948 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q) 949 { 950 if (q->last_fence) { 951 dma_fence_put(q->last_fence); 952 q->last_fence = NULL; 953 } 954 } 955 956 /** 957 * xe_exec_queue_last_fence_get() - Get last fence 958 * @q: The exec queue 959 * @vm: The VM the engine does a bind or exec for 960 * 961 * Get last fence, does not take a ref 962 * 963 * Returns: last fence if not signaled, dma fence stub if signaled 964 */ 965 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q, 966 struct xe_vm *vm) 967 { 968 xe_exec_queue_last_fence_lockdep_assert(q, vm); 969 970 if (q->last_fence && 971 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) 972 xe_exec_queue_last_fence_put(q, vm); 973 974 return q->last_fence ? q->last_fence : dma_fence_get_stub(); 975 } 976 977 /** 978 * xe_exec_queue_last_fence_set() - Set last fence 979 * @q: The exec queue 980 * @vm: The VM the engine does a bind or exec for 981 * @fence: The fence 982 * 983 * Set the last fence for the engine. Increases reference count for fence, when 984 * closing engine xe_exec_queue_last_fence_put should be called. 985 */ 986 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm, 987 struct dma_fence *fence) 988 { 989 xe_exec_queue_last_fence_lockdep_assert(q, vm); 990 991 xe_exec_queue_last_fence_put(q, vm); 992 q->last_fence = dma_fence_get(fence); 993 } 994