1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_gt_pagefault.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/circ_buf.h> 10 11 #include <drm/drm_exec.h> 12 #include <drm/drm_managed.h> 13 #include <drm/ttm/ttm_execbuf_util.h> 14 15 #include "abi/guc_actions_abi.h" 16 #include "xe_bo.h" 17 #include "xe_gt.h" 18 #include "xe_gt_tlb_invalidation.h" 19 #include "xe_guc.h" 20 #include "xe_guc_ct.h" 21 #include "xe_migrate.h" 22 #include "xe_trace.h" 23 #include "xe_vm.h" 24 25 struct pagefault { 26 u64 page_addr; 27 u32 asid; 28 u16 pdata; 29 u8 vfid; 30 u8 access_type; 31 u8 fault_type; 32 u8 fault_level; 33 u8 engine_class; 34 u8 engine_instance; 35 u8 fault_unsuccessful; 36 bool trva_fault; 37 }; 38 39 enum access_type { 40 ACCESS_TYPE_READ = 0, 41 ACCESS_TYPE_WRITE = 1, 42 ACCESS_TYPE_ATOMIC = 2, 43 ACCESS_TYPE_RESERVED = 3, 44 }; 45 46 enum fault_type { 47 NOT_PRESENT = 0, 48 WRITE_ACCESS_VIOLATION = 1, 49 ATOMIC_ACCESS_VIOLATION = 2, 50 }; 51 52 struct acc { 53 u64 va_range_base; 54 u32 asid; 55 u32 sub_granularity; 56 u8 granularity; 57 u8 vfid; 58 u8 access_type; 59 u8 engine_class; 60 u8 engine_instance; 61 }; 62 63 static bool access_is_atomic(enum access_type access_type) 64 { 65 return access_type == ACCESS_TYPE_ATOMIC; 66 } 67 68 static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma) 69 { 70 return BIT(tile->id) & vma->tile_present && 71 !(BIT(tile->id) & vma->tile_invalidated); 72 } 73 74 static bool vma_matches(struct xe_vma *vma, u64 page_addr) 75 { 76 if (page_addr > xe_vma_end(vma) - 1 || 77 page_addr + SZ_4K - 1 < xe_vma_start(vma)) 78 return false; 79 80 return true; 81 } 82 83 static struct xe_vma *lookup_vma(struct xe_vm *vm, u64 page_addr) 84 { 85 struct xe_vma *vma = NULL; 86 87 if (vm->usm.last_fault_vma) { /* Fast lookup */ 88 if (vma_matches(vm->usm.last_fault_vma, page_addr)) 89 vma = vm->usm.last_fault_vma; 90 } 91 if (!vma) 92 vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K); 93 94 return vma; 95 } 96 97 static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma, 98 bool atomic, unsigned int id) 99 { 100 struct xe_bo *bo = xe_vma_bo(vma); 101 struct xe_vm *vm = xe_vma_vm(vma); 102 int err; 103 104 err = xe_vm_lock_vma(exec, vma); 105 if (err) 106 return err; 107 108 if (atomic && IS_DGFX(vm->xe)) { 109 if (xe_vma_is_userptr(vma)) { 110 err = -EACCES; 111 return err; 112 } 113 114 /* Migrate to VRAM, move should invalidate the VMA first */ 115 err = xe_bo_migrate(bo, XE_PL_VRAM0 + id); 116 if (err) 117 return err; 118 } else if (bo) { 119 /* Create backing store if needed */ 120 err = xe_bo_validate(bo, vm, true); 121 if (err) 122 return err; 123 } 124 125 return 0; 126 } 127 128 static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf) 129 { 130 struct xe_device *xe = gt_to_xe(gt); 131 struct xe_tile *tile = gt_to_tile(gt); 132 struct drm_exec exec; 133 struct xe_vm *vm; 134 struct xe_vma *vma = NULL; 135 struct dma_fence *fence; 136 bool write_locked; 137 int ret = 0; 138 bool atomic; 139 140 /* SW isn't expected to handle TRTT faults */ 141 if (pf->trva_fault) 142 return -EFAULT; 143 144 /* ASID to VM */ 145 mutex_lock(&xe->usm.lock); 146 vm = xa_load(&xe->usm.asid_to_vm, pf->asid); 147 if (vm && xe_vm_in_fault_mode(vm)) 148 xe_vm_get(vm); 149 else 150 vm = NULL; 151 mutex_unlock(&xe->usm.lock); 152 if (!vm) 153 return -EINVAL; 154 155 retry_userptr: 156 /* 157 * TODO: Avoid exclusive lock if VM doesn't have userptrs, or 158 * start out read-locked? 159 */ 160 down_write(&vm->lock); 161 write_locked = true; 162 vma = lookup_vma(vm, pf->page_addr); 163 if (!vma) { 164 ret = -EINVAL; 165 goto unlock_vm; 166 } 167 168 if (!xe_vma_is_userptr(vma) || 169 !xe_vma_userptr_check_repin(to_userptr_vma(vma))) { 170 downgrade_write(&vm->lock); 171 write_locked = false; 172 } 173 174 trace_xe_vma_pagefault(vma); 175 176 atomic = access_is_atomic(pf->access_type); 177 178 /* Check if VMA is valid */ 179 if (vma_is_valid(tile, vma) && !atomic) 180 goto unlock_vm; 181 182 /* TODO: Validate fault */ 183 184 if (xe_vma_is_userptr(vma) && write_locked) { 185 struct xe_userptr_vma *uvma = to_userptr_vma(vma); 186 187 spin_lock(&vm->userptr.invalidated_lock); 188 list_del_init(&uvma->userptr.invalidate_link); 189 spin_unlock(&vm->userptr.invalidated_lock); 190 191 ret = xe_vma_userptr_pin_pages(uvma); 192 if (ret) 193 goto unlock_vm; 194 195 downgrade_write(&vm->lock); 196 write_locked = false; 197 } 198 199 /* Lock VM and BOs dma-resv */ 200 drm_exec_init(&exec, 0, 0); 201 drm_exec_until_all_locked(&exec) { 202 ret = xe_pf_begin(&exec, vma, atomic, tile->id); 203 drm_exec_retry_on_contention(&exec); 204 if (ret) 205 goto unlock_dma_resv; 206 207 /* Bind VMA only to the GT that has faulted */ 208 trace_xe_vma_pf_bind(vma); 209 fence = xe_vma_rebind(vm, vma, BIT(tile->id)); 210 if (IS_ERR(fence)) { 211 ret = PTR_ERR(fence); 212 goto unlock_dma_resv; 213 } 214 } 215 216 /* 217 * XXX: Should we drop the lock before waiting? This only helps if doing 218 * GPU binds which is currently only done if we have to wait for more 219 * than 10ms on a move. 220 */ 221 dma_fence_wait(fence, false); 222 dma_fence_put(fence); 223 224 if (xe_vma_is_userptr(vma)) 225 ret = xe_vma_userptr_check_repin(to_userptr_vma(vma)); 226 vma->tile_invalidated &= ~BIT(tile->id); 227 228 unlock_dma_resv: 229 drm_exec_fini(&exec); 230 unlock_vm: 231 if (!ret) 232 vm->usm.last_fault_vma = vma; 233 if (write_locked) 234 up_write(&vm->lock); 235 else 236 up_read(&vm->lock); 237 if (ret == -EAGAIN) 238 goto retry_userptr; 239 240 if (!ret) { 241 ret = xe_gt_tlb_invalidation_vma(gt, NULL, vma); 242 if (ret >= 0) 243 ret = 0; 244 } 245 xe_vm_put(vm); 246 247 return ret; 248 } 249 250 static int send_pagefault_reply(struct xe_guc *guc, 251 struct xe_guc_pagefault_reply *reply) 252 { 253 u32 action[] = { 254 XE_GUC_ACTION_PAGE_FAULT_RES_DESC, 255 reply->dw0, 256 reply->dw1, 257 }; 258 259 return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); 260 } 261 262 static void print_pagefault(struct xe_device *xe, struct pagefault *pf) 263 { 264 drm_dbg(&xe->drm, "\n\tASID: %d\n" 265 "\tVFID: %d\n" 266 "\tPDATA: 0x%04x\n" 267 "\tFaulted Address: 0x%08x%08x\n" 268 "\tFaultType: %d\n" 269 "\tAccessType: %d\n" 270 "\tFaultLevel: %d\n" 271 "\tEngineClass: %d\n" 272 "\tEngineInstance: %d\n", 273 pf->asid, pf->vfid, pf->pdata, upper_32_bits(pf->page_addr), 274 lower_32_bits(pf->page_addr), 275 pf->fault_type, pf->access_type, pf->fault_level, 276 pf->engine_class, pf->engine_instance); 277 } 278 279 #define PF_MSG_LEN_DW 4 280 281 static bool get_pagefault(struct pf_queue *pf_queue, struct pagefault *pf) 282 { 283 const struct xe_guc_pagefault_desc *desc; 284 bool ret = false; 285 286 spin_lock_irq(&pf_queue->lock); 287 if (pf_queue->tail != pf_queue->head) { 288 desc = (const struct xe_guc_pagefault_desc *) 289 (pf_queue->data + pf_queue->tail); 290 291 pf->fault_level = FIELD_GET(PFD_FAULT_LEVEL, desc->dw0); 292 pf->trva_fault = FIELD_GET(XE2_PFD_TRVA_FAULT, desc->dw0); 293 pf->engine_class = FIELD_GET(PFD_ENG_CLASS, desc->dw0); 294 pf->engine_instance = FIELD_GET(PFD_ENG_INSTANCE, desc->dw0); 295 pf->pdata = FIELD_GET(PFD_PDATA_HI, desc->dw1) << 296 PFD_PDATA_HI_SHIFT; 297 pf->pdata |= FIELD_GET(PFD_PDATA_LO, desc->dw0); 298 pf->asid = FIELD_GET(PFD_ASID, desc->dw1); 299 pf->vfid = FIELD_GET(PFD_VFID, desc->dw2); 300 pf->access_type = FIELD_GET(PFD_ACCESS_TYPE, desc->dw2); 301 pf->fault_type = FIELD_GET(PFD_FAULT_TYPE, desc->dw2); 302 pf->page_addr = (u64)(FIELD_GET(PFD_VIRTUAL_ADDR_HI, desc->dw3)) << 303 PFD_VIRTUAL_ADDR_HI_SHIFT; 304 pf->page_addr |= FIELD_GET(PFD_VIRTUAL_ADDR_LO, desc->dw2) << 305 PFD_VIRTUAL_ADDR_LO_SHIFT; 306 307 pf_queue->tail = (pf_queue->tail + PF_MSG_LEN_DW) % 308 PF_QUEUE_NUM_DW; 309 ret = true; 310 } 311 spin_unlock_irq(&pf_queue->lock); 312 313 return ret; 314 } 315 316 static bool pf_queue_full(struct pf_queue *pf_queue) 317 { 318 lockdep_assert_held(&pf_queue->lock); 319 320 return CIRC_SPACE(pf_queue->head, pf_queue->tail, PF_QUEUE_NUM_DW) <= 321 PF_MSG_LEN_DW; 322 } 323 324 int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len) 325 { 326 struct xe_gt *gt = guc_to_gt(guc); 327 struct xe_device *xe = gt_to_xe(gt); 328 struct pf_queue *pf_queue; 329 unsigned long flags; 330 u32 asid; 331 bool full; 332 333 /* 334 * The below logic doesn't work unless PF_QUEUE_NUM_DW % PF_MSG_LEN_DW == 0 335 */ 336 BUILD_BUG_ON(PF_QUEUE_NUM_DW % PF_MSG_LEN_DW); 337 338 if (unlikely(len != PF_MSG_LEN_DW)) 339 return -EPROTO; 340 341 asid = FIELD_GET(PFD_ASID, msg[1]); 342 pf_queue = gt->usm.pf_queue + (asid % NUM_PF_QUEUE); 343 344 spin_lock_irqsave(&pf_queue->lock, flags); 345 full = pf_queue_full(pf_queue); 346 if (!full) { 347 memcpy(pf_queue->data + pf_queue->head, msg, len * sizeof(u32)); 348 pf_queue->head = (pf_queue->head + len) % PF_QUEUE_NUM_DW; 349 queue_work(gt->usm.pf_wq, &pf_queue->worker); 350 } else { 351 drm_warn(&xe->drm, "PF Queue full, shouldn't be possible"); 352 } 353 spin_unlock_irqrestore(&pf_queue->lock, flags); 354 355 return full ? -ENOSPC : 0; 356 } 357 358 #define USM_QUEUE_MAX_RUNTIME_MS 20 359 360 static void pf_queue_work_func(struct work_struct *w) 361 { 362 struct pf_queue *pf_queue = container_of(w, struct pf_queue, worker); 363 struct xe_gt *gt = pf_queue->gt; 364 struct xe_device *xe = gt_to_xe(gt); 365 struct xe_guc_pagefault_reply reply = {}; 366 struct pagefault pf = {}; 367 unsigned long threshold; 368 int ret; 369 370 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS); 371 372 while (get_pagefault(pf_queue, &pf)) { 373 ret = handle_pagefault(gt, &pf); 374 if (unlikely(ret)) { 375 print_pagefault(xe, &pf); 376 pf.fault_unsuccessful = 1; 377 drm_dbg(&xe->drm, "Fault response: Unsuccessful %d\n", ret); 378 } 379 380 reply.dw0 = FIELD_PREP(PFR_VALID, 1) | 381 FIELD_PREP(PFR_SUCCESS, pf.fault_unsuccessful) | 382 FIELD_PREP(PFR_REPLY, PFR_ACCESS) | 383 FIELD_PREP(PFR_DESC_TYPE, FAULT_RESPONSE_DESC) | 384 FIELD_PREP(PFR_ASID, pf.asid); 385 386 reply.dw1 = FIELD_PREP(PFR_VFID, pf.vfid) | 387 FIELD_PREP(PFR_ENG_INSTANCE, pf.engine_instance) | 388 FIELD_PREP(PFR_ENG_CLASS, pf.engine_class) | 389 FIELD_PREP(PFR_PDATA, pf.pdata); 390 391 send_pagefault_reply(>->uc.guc, &reply); 392 393 if (time_after(jiffies, threshold) && 394 pf_queue->tail != pf_queue->head) { 395 queue_work(gt->usm.pf_wq, w); 396 break; 397 } 398 } 399 } 400 401 static void acc_queue_work_func(struct work_struct *w); 402 403 int xe_gt_pagefault_init(struct xe_gt *gt) 404 { 405 struct xe_device *xe = gt_to_xe(gt); 406 int i; 407 408 if (!xe->info.has_usm) 409 return 0; 410 411 for (i = 0; i < NUM_PF_QUEUE; ++i) { 412 gt->usm.pf_queue[i].gt = gt; 413 spin_lock_init(>->usm.pf_queue[i].lock); 414 INIT_WORK(>->usm.pf_queue[i].worker, pf_queue_work_func); 415 } 416 for (i = 0; i < NUM_ACC_QUEUE; ++i) { 417 gt->usm.acc_queue[i].gt = gt; 418 spin_lock_init(>->usm.acc_queue[i].lock); 419 INIT_WORK(>->usm.acc_queue[i].worker, acc_queue_work_func); 420 } 421 422 gt->usm.pf_wq = alloc_workqueue("xe_gt_page_fault_work_queue", 423 WQ_UNBOUND | WQ_HIGHPRI, NUM_PF_QUEUE); 424 if (!gt->usm.pf_wq) 425 return -ENOMEM; 426 427 gt->usm.acc_wq = alloc_workqueue("xe_gt_access_counter_work_queue", 428 WQ_UNBOUND | WQ_HIGHPRI, 429 NUM_ACC_QUEUE); 430 if (!gt->usm.acc_wq) 431 return -ENOMEM; 432 433 return 0; 434 } 435 436 void xe_gt_pagefault_reset(struct xe_gt *gt) 437 { 438 struct xe_device *xe = gt_to_xe(gt); 439 int i; 440 441 if (!xe->info.has_usm) 442 return; 443 444 for (i = 0; i < NUM_PF_QUEUE; ++i) { 445 spin_lock_irq(>->usm.pf_queue[i].lock); 446 gt->usm.pf_queue[i].head = 0; 447 gt->usm.pf_queue[i].tail = 0; 448 spin_unlock_irq(>->usm.pf_queue[i].lock); 449 } 450 451 for (i = 0; i < NUM_ACC_QUEUE; ++i) { 452 spin_lock(>->usm.acc_queue[i].lock); 453 gt->usm.acc_queue[i].head = 0; 454 gt->usm.acc_queue[i].tail = 0; 455 spin_unlock(>->usm.acc_queue[i].lock); 456 } 457 } 458 459 static int granularity_in_byte(int val) 460 { 461 switch (val) { 462 case 0: 463 return SZ_128K; 464 case 1: 465 return SZ_2M; 466 case 2: 467 return SZ_16M; 468 case 3: 469 return SZ_64M; 470 default: 471 return 0; 472 } 473 } 474 475 static int sub_granularity_in_byte(int val) 476 { 477 return (granularity_in_byte(val) / 32); 478 } 479 480 static void print_acc(struct xe_device *xe, struct acc *acc) 481 { 482 drm_warn(&xe->drm, "Access counter request:\n" 483 "\tType: %s\n" 484 "\tASID: %d\n" 485 "\tVFID: %d\n" 486 "\tEngine: %d:%d\n" 487 "\tGranularity: 0x%x KB Region/ %d KB sub-granularity\n" 488 "\tSub_Granularity Vector: 0x%08x\n" 489 "\tVA Range base: 0x%016llx\n", 490 acc->access_type ? "AC_NTFY_VAL" : "AC_TRIG_VAL", 491 acc->asid, acc->vfid, acc->engine_class, acc->engine_instance, 492 granularity_in_byte(acc->granularity) / SZ_1K, 493 sub_granularity_in_byte(acc->granularity) / SZ_1K, 494 acc->sub_granularity, acc->va_range_base); 495 } 496 497 static struct xe_vma *get_acc_vma(struct xe_vm *vm, struct acc *acc) 498 { 499 u64 page_va = acc->va_range_base + (ffs(acc->sub_granularity) - 1) * 500 sub_granularity_in_byte(acc->granularity); 501 502 return xe_vm_find_overlapping_vma(vm, page_va, SZ_4K); 503 } 504 505 static int handle_acc(struct xe_gt *gt, struct acc *acc) 506 { 507 struct xe_device *xe = gt_to_xe(gt); 508 struct xe_tile *tile = gt_to_tile(gt); 509 struct drm_exec exec; 510 struct xe_vm *vm; 511 struct xe_vma *vma; 512 int ret = 0; 513 514 /* We only support ACC_TRIGGER at the moment */ 515 if (acc->access_type != ACC_TRIGGER) 516 return -EINVAL; 517 518 /* ASID to VM */ 519 mutex_lock(&xe->usm.lock); 520 vm = xa_load(&xe->usm.asid_to_vm, acc->asid); 521 if (vm) 522 xe_vm_get(vm); 523 mutex_unlock(&xe->usm.lock); 524 if (!vm || !xe_vm_in_fault_mode(vm)) 525 return -EINVAL; 526 527 down_read(&vm->lock); 528 529 /* Lookup VMA */ 530 vma = get_acc_vma(vm, acc); 531 if (!vma) { 532 ret = -EINVAL; 533 goto unlock_vm; 534 } 535 536 trace_xe_vma_acc(vma); 537 538 /* Userptr or null can't be migrated, nothing to do */ 539 if (xe_vma_has_no_bo(vma)) 540 goto unlock_vm; 541 542 /* Lock VM and BOs dma-resv */ 543 drm_exec_init(&exec, 0, 0); 544 drm_exec_until_all_locked(&exec) { 545 ret = xe_pf_begin(&exec, vma, true, tile->id); 546 drm_exec_retry_on_contention(&exec); 547 if (ret) 548 break; 549 } 550 551 drm_exec_fini(&exec); 552 unlock_vm: 553 up_read(&vm->lock); 554 xe_vm_put(vm); 555 556 return ret; 557 } 558 559 #define make_u64(hi__, low__) ((u64)(hi__) << 32 | (u64)(low__)) 560 561 #define ACC_MSG_LEN_DW 4 562 563 static bool get_acc(struct acc_queue *acc_queue, struct acc *acc) 564 { 565 const struct xe_guc_acc_desc *desc; 566 bool ret = false; 567 568 spin_lock(&acc_queue->lock); 569 if (acc_queue->tail != acc_queue->head) { 570 desc = (const struct xe_guc_acc_desc *) 571 (acc_queue->data + acc_queue->tail); 572 573 acc->granularity = FIELD_GET(ACC_GRANULARITY, desc->dw2); 574 acc->sub_granularity = FIELD_GET(ACC_SUBG_HI, desc->dw1) << 31 | 575 FIELD_GET(ACC_SUBG_LO, desc->dw0); 576 acc->engine_class = FIELD_GET(ACC_ENG_CLASS, desc->dw1); 577 acc->engine_instance = FIELD_GET(ACC_ENG_INSTANCE, desc->dw1); 578 acc->asid = FIELD_GET(ACC_ASID, desc->dw1); 579 acc->vfid = FIELD_GET(ACC_VFID, desc->dw2); 580 acc->access_type = FIELD_GET(ACC_TYPE, desc->dw0); 581 acc->va_range_base = make_u64(desc->dw3 & ACC_VIRTUAL_ADDR_RANGE_HI, 582 desc->dw2 & ACC_VIRTUAL_ADDR_RANGE_LO); 583 584 acc_queue->tail = (acc_queue->tail + ACC_MSG_LEN_DW) % 585 ACC_QUEUE_NUM_DW; 586 ret = true; 587 } 588 spin_unlock(&acc_queue->lock); 589 590 return ret; 591 } 592 593 static void acc_queue_work_func(struct work_struct *w) 594 { 595 struct acc_queue *acc_queue = container_of(w, struct acc_queue, worker); 596 struct xe_gt *gt = acc_queue->gt; 597 struct xe_device *xe = gt_to_xe(gt); 598 struct acc acc = {}; 599 unsigned long threshold; 600 int ret; 601 602 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS); 603 604 while (get_acc(acc_queue, &acc)) { 605 ret = handle_acc(gt, &acc); 606 if (unlikely(ret)) { 607 print_acc(xe, &acc); 608 drm_warn(&xe->drm, "ACC: Unsuccessful %d\n", ret); 609 } 610 611 if (time_after(jiffies, threshold) && 612 acc_queue->tail != acc_queue->head) { 613 queue_work(gt->usm.acc_wq, w); 614 break; 615 } 616 } 617 } 618 619 static bool acc_queue_full(struct acc_queue *acc_queue) 620 { 621 lockdep_assert_held(&acc_queue->lock); 622 623 return CIRC_SPACE(acc_queue->head, acc_queue->tail, ACC_QUEUE_NUM_DW) <= 624 ACC_MSG_LEN_DW; 625 } 626 627 int xe_guc_access_counter_notify_handler(struct xe_guc *guc, u32 *msg, u32 len) 628 { 629 struct xe_gt *gt = guc_to_gt(guc); 630 struct acc_queue *acc_queue; 631 u32 asid; 632 bool full; 633 634 /* 635 * The below logic doesn't work unless ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW == 0 636 */ 637 BUILD_BUG_ON(ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW); 638 639 if (unlikely(len != ACC_MSG_LEN_DW)) 640 return -EPROTO; 641 642 asid = FIELD_GET(ACC_ASID, msg[1]); 643 acc_queue = >->usm.acc_queue[asid % NUM_ACC_QUEUE]; 644 645 spin_lock(&acc_queue->lock); 646 full = acc_queue_full(acc_queue); 647 if (!full) { 648 memcpy(acc_queue->data + acc_queue->head, msg, 649 len * sizeof(u32)); 650 acc_queue->head = (acc_queue->head + len) % ACC_QUEUE_NUM_DW; 651 queue_work(gt->usm.acc_wq, &acc_queue->worker); 652 } else { 653 drm_warn(>_to_xe(gt)->drm, "ACC Queue full, dropping ACC"); 654 } 655 spin_unlock(&acc_queue->lock); 656 657 return full ? -ENOSPC : 0; 658 } 659