1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_gt_pagefault.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/circ_buf.h> 10 11 #include <drm/drm_exec.h> 12 #include <drm/drm_managed.h> 13 #include <drm/ttm/ttm_execbuf_util.h> 14 15 #include "abi/guc_actions_abi.h" 16 #include "xe_bo.h" 17 #include "xe_gt.h" 18 #include "xe_gt_tlb_invalidation.h" 19 #include "xe_guc.h" 20 #include "xe_guc_ct.h" 21 #include "xe_migrate.h" 22 #include "xe_trace_bo.h" 23 #include "xe_vm.h" 24 25 struct pagefault { 26 u64 page_addr; 27 u32 asid; 28 u16 pdata; 29 u8 vfid; 30 u8 access_type; 31 u8 fault_type; 32 u8 fault_level; 33 u8 engine_class; 34 u8 engine_instance; 35 u8 fault_unsuccessful; 36 bool trva_fault; 37 }; 38 39 enum access_type { 40 ACCESS_TYPE_READ = 0, 41 ACCESS_TYPE_WRITE = 1, 42 ACCESS_TYPE_ATOMIC = 2, 43 ACCESS_TYPE_RESERVED = 3, 44 }; 45 46 enum fault_type { 47 NOT_PRESENT = 0, 48 WRITE_ACCESS_VIOLATION = 1, 49 ATOMIC_ACCESS_VIOLATION = 2, 50 }; 51 52 struct acc { 53 u64 va_range_base; 54 u32 asid; 55 u32 sub_granularity; 56 u8 granularity; 57 u8 vfid; 58 u8 access_type; 59 u8 engine_class; 60 u8 engine_instance; 61 }; 62 63 static bool access_is_atomic(enum access_type access_type) 64 { 65 return access_type == ACCESS_TYPE_ATOMIC; 66 } 67 68 static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma) 69 { 70 return BIT(tile->id) & vma->tile_present && 71 !(BIT(tile->id) & vma->tile_invalidated); 72 } 73 74 static bool vma_matches(struct xe_vma *vma, u64 page_addr) 75 { 76 if (page_addr > xe_vma_end(vma) - 1 || 77 page_addr + SZ_4K - 1 < xe_vma_start(vma)) 78 return false; 79 80 return true; 81 } 82 83 static struct xe_vma *lookup_vma(struct xe_vm *vm, u64 page_addr) 84 { 85 struct xe_vma *vma = NULL; 86 87 if (vm->usm.last_fault_vma) { /* Fast lookup */ 88 if (vma_matches(vm->usm.last_fault_vma, page_addr)) 89 vma = vm->usm.last_fault_vma; 90 } 91 if (!vma) 92 vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K); 93 94 return vma; 95 } 96 97 static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma, 98 bool atomic, unsigned int id) 99 { 100 struct xe_bo *bo = xe_vma_bo(vma); 101 struct xe_vm *vm = xe_vma_vm(vma); 102 int err; 103 104 err = xe_vm_lock_vma(exec, vma); 105 if (err) 106 return err; 107 108 if (atomic && IS_DGFX(vm->xe)) { 109 if (xe_vma_is_userptr(vma)) { 110 err = -EACCES; 111 return err; 112 } 113 114 /* Migrate to VRAM, move should invalidate the VMA first */ 115 err = xe_bo_migrate(bo, XE_PL_VRAM0 + id); 116 if (err) 117 return err; 118 } else if (bo) { 119 /* Create backing store if needed */ 120 err = xe_bo_validate(bo, vm, true); 121 if (err) 122 return err; 123 } 124 125 return 0; 126 } 127 128 static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf, 129 struct xe_vma *vma) 130 { 131 struct xe_vm *vm = xe_vma_vm(vma); 132 struct drm_exec exec; 133 struct dma_fence *fence; 134 ktime_t end = 0; 135 int err; 136 bool atomic; 137 138 trace_xe_vma_pagefault(vma); 139 atomic = access_is_atomic(pf->access_type); 140 141 /* Check if VMA is valid */ 142 if (vma_is_valid(tile, vma) && !atomic) 143 return 0; 144 145 retry_userptr: 146 if (xe_vma_is_userptr(vma) && 147 xe_vma_userptr_check_repin(to_userptr_vma(vma))) { 148 struct xe_userptr_vma *uvma = to_userptr_vma(vma); 149 150 err = xe_vma_userptr_pin_pages(uvma); 151 if (err) 152 return err; 153 } 154 155 /* Lock VM and BOs dma-resv */ 156 drm_exec_init(&exec, 0, 0); 157 drm_exec_until_all_locked(&exec) { 158 err = xe_pf_begin(&exec, vma, atomic, tile->id); 159 drm_exec_retry_on_contention(&exec); 160 if (xe_vm_validate_should_retry(&exec, err, &end)) 161 err = -EAGAIN; 162 if (err) 163 goto unlock_dma_resv; 164 165 /* Bind VMA only to the GT that has faulted */ 166 trace_xe_vma_pf_bind(vma); 167 fence = xe_vma_rebind(vm, vma, BIT(tile->id)); 168 if (IS_ERR(fence)) { 169 err = PTR_ERR(fence); 170 if (xe_vm_validate_should_retry(&exec, err, &end)) 171 err = -EAGAIN; 172 goto unlock_dma_resv; 173 } 174 } 175 176 dma_fence_wait(fence, false); 177 dma_fence_put(fence); 178 vma->tile_invalidated &= ~BIT(tile->id); 179 180 unlock_dma_resv: 181 drm_exec_fini(&exec); 182 if (err == -EAGAIN) 183 goto retry_userptr; 184 185 return err; 186 } 187 188 static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf) 189 { 190 struct xe_device *xe = gt_to_xe(gt); 191 struct xe_tile *tile = gt_to_tile(gt); 192 struct xe_vm *vm; 193 struct xe_vma *vma = NULL; 194 int err; 195 196 /* SW isn't expected to handle TRTT faults */ 197 if (pf->trva_fault) 198 return -EFAULT; 199 200 /* ASID to VM */ 201 mutex_lock(&xe->usm.lock); 202 vm = xa_load(&xe->usm.asid_to_vm, pf->asid); 203 if (vm && xe_vm_in_fault_mode(vm)) 204 xe_vm_get(vm); 205 else 206 vm = NULL; 207 mutex_unlock(&xe->usm.lock); 208 if (!vm) 209 return -EINVAL; 210 211 /* 212 * TODO: Change to read lock? Using write lock for simplicity. 213 */ 214 down_write(&vm->lock); 215 216 if (xe_vm_is_closed(vm)) { 217 err = -ENOENT; 218 goto unlock_vm; 219 } 220 221 vma = lookup_vma(vm, pf->page_addr); 222 if (!vma) { 223 err = -EINVAL; 224 goto unlock_vm; 225 } 226 227 err = handle_vma_pagefault(tile, pf, vma); 228 229 unlock_vm: 230 if (!err) 231 vm->usm.last_fault_vma = vma; 232 up_write(&vm->lock); 233 xe_vm_put(vm); 234 235 return err; 236 } 237 238 static int send_pagefault_reply(struct xe_guc *guc, 239 struct xe_guc_pagefault_reply *reply) 240 { 241 u32 action[] = { 242 XE_GUC_ACTION_PAGE_FAULT_RES_DESC, 243 reply->dw0, 244 reply->dw1, 245 }; 246 247 return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); 248 } 249 250 static void print_pagefault(struct xe_device *xe, struct pagefault *pf) 251 { 252 drm_dbg(&xe->drm, "\n\tASID: %d\n" 253 "\tVFID: %d\n" 254 "\tPDATA: 0x%04x\n" 255 "\tFaulted Address: 0x%08x%08x\n" 256 "\tFaultType: %d\n" 257 "\tAccessType: %d\n" 258 "\tFaultLevel: %d\n" 259 "\tEngineClass: %d\n" 260 "\tEngineInstance: %d\n", 261 pf->asid, pf->vfid, pf->pdata, upper_32_bits(pf->page_addr), 262 lower_32_bits(pf->page_addr), 263 pf->fault_type, pf->access_type, pf->fault_level, 264 pf->engine_class, pf->engine_instance); 265 } 266 267 #define PF_MSG_LEN_DW 4 268 269 static bool get_pagefault(struct pf_queue *pf_queue, struct pagefault *pf) 270 { 271 const struct xe_guc_pagefault_desc *desc; 272 bool ret = false; 273 274 spin_lock_irq(&pf_queue->lock); 275 if (pf_queue->tail != pf_queue->head) { 276 desc = (const struct xe_guc_pagefault_desc *) 277 (pf_queue->data + pf_queue->tail); 278 279 pf->fault_level = FIELD_GET(PFD_FAULT_LEVEL, desc->dw0); 280 pf->trva_fault = FIELD_GET(XE2_PFD_TRVA_FAULT, desc->dw0); 281 pf->engine_class = FIELD_GET(PFD_ENG_CLASS, desc->dw0); 282 pf->engine_instance = FIELD_GET(PFD_ENG_INSTANCE, desc->dw0); 283 pf->pdata = FIELD_GET(PFD_PDATA_HI, desc->dw1) << 284 PFD_PDATA_HI_SHIFT; 285 pf->pdata |= FIELD_GET(PFD_PDATA_LO, desc->dw0); 286 pf->asid = FIELD_GET(PFD_ASID, desc->dw1); 287 pf->vfid = FIELD_GET(PFD_VFID, desc->dw2); 288 pf->access_type = FIELD_GET(PFD_ACCESS_TYPE, desc->dw2); 289 pf->fault_type = FIELD_GET(PFD_FAULT_TYPE, desc->dw2); 290 pf->page_addr = (u64)(FIELD_GET(PFD_VIRTUAL_ADDR_HI, desc->dw3)) << 291 PFD_VIRTUAL_ADDR_HI_SHIFT; 292 pf->page_addr |= FIELD_GET(PFD_VIRTUAL_ADDR_LO, desc->dw2) << 293 PFD_VIRTUAL_ADDR_LO_SHIFT; 294 295 pf_queue->tail = (pf_queue->tail + PF_MSG_LEN_DW) % 296 pf_queue->num_dw; 297 ret = true; 298 } 299 spin_unlock_irq(&pf_queue->lock); 300 301 return ret; 302 } 303 304 static bool pf_queue_full(struct pf_queue *pf_queue) 305 { 306 lockdep_assert_held(&pf_queue->lock); 307 308 return CIRC_SPACE(pf_queue->head, pf_queue->tail, 309 pf_queue->num_dw) <= 310 PF_MSG_LEN_DW; 311 } 312 313 int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len) 314 { 315 struct xe_gt *gt = guc_to_gt(guc); 316 struct xe_device *xe = gt_to_xe(gt); 317 struct pf_queue *pf_queue; 318 unsigned long flags; 319 u32 asid; 320 bool full; 321 322 if (unlikely(len != PF_MSG_LEN_DW)) 323 return -EPROTO; 324 325 asid = FIELD_GET(PFD_ASID, msg[1]); 326 pf_queue = gt->usm.pf_queue + (asid % NUM_PF_QUEUE); 327 328 /* 329 * The below logic doesn't work unless PF_QUEUE_NUM_DW % PF_MSG_LEN_DW == 0 330 */ 331 xe_gt_assert(gt, !(pf_queue->num_dw % PF_MSG_LEN_DW)); 332 333 spin_lock_irqsave(&pf_queue->lock, flags); 334 full = pf_queue_full(pf_queue); 335 if (!full) { 336 memcpy(pf_queue->data + pf_queue->head, msg, len * sizeof(u32)); 337 pf_queue->head = (pf_queue->head + len) % 338 pf_queue->num_dw; 339 queue_work(gt->usm.pf_wq, &pf_queue->worker); 340 } else { 341 drm_warn(&xe->drm, "PF Queue full, shouldn't be possible"); 342 } 343 spin_unlock_irqrestore(&pf_queue->lock, flags); 344 345 return full ? -ENOSPC : 0; 346 } 347 348 #define USM_QUEUE_MAX_RUNTIME_MS 20 349 350 static void pf_queue_work_func(struct work_struct *w) 351 { 352 struct pf_queue *pf_queue = container_of(w, struct pf_queue, worker); 353 struct xe_gt *gt = pf_queue->gt; 354 struct xe_device *xe = gt_to_xe(gt); 355 struct xe_guc_pagefault_reply reply = {}; 356 struct pagefault pf = {}; 357 unsigned long threshold; 358 int ret; 359 360 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS); 361 362 while (get_pagefault(pf_queue, &pf)) { 363 ret = handle_pagefault(gt, &pf); 364 if (unlikely(ret)) { 365 print_pagefault(xe, &pf); 366 pf.fault_unsuccessful = 1; 367 drm_dbg(&xe->drm, "Fault response: Unsuccessful %d\n", ret); 368 } 369 370 reply.dw0 = FIELD_PREP(PFR_VALID, 1) | 371 FIELD_PREP(PFR_SUCCESS, pf.fault_unsuccessful) | 372 FIELD_PREP(PFR_REPLY, PFR_ACCESS) | 373 FIELD_PREP(PFR_DESC_TYPE, FAULT_RESPONSE_DESC) | 374 FIELD_PREP(PFR_ASID, pf.asid); 375 376 reply.dw1 = FIELD_PREP(PFR_VFID, pf.vfid) | 377 FIELD_PREP(PFR_ENG_INSTANCE, pf.engine_instance) | 378 FIELD_PREP(PFR_ENG_CLASS, pf.engine_class) | 379 FIELD_PREP(PFR_PDATA, pf.pdata); 380 381 send_pagefault_reply(>->uc.guc, &reply); 382 383 if (time_after(jiffies, threshold) && 384 pf_queue->tail != pf_queue->head) { 385 queue_work(gt->usm.pf_wq, w); 386 break; 387 } 388 } 389 } 390 391 static void acc_queue_work_func(struct work_struct *w); 392 393 static void pagefault_fini(void *arg) 394 { 395 struct xe_gt *gt = arg; 396 struct xe_device *xe = gt_to_xe(gt); 397 398 if (!xe->info.has_usm) 399 return; 400 401 destroy_workqueue(gt->usm.acc_wq); 402 destroy_workqueue(gt->usm.pf_wq); 403 } 404 405 static int xe_alloc_pf_queue(struct xe_gt *gt, struct pf_queue *pf_queue) 406 { 407 struct xe_device *xe = gt_to_xe(gt); 408 xe_dss_mask_t all_dss; 409 int num_dss, num_eus; 410 411 bitmap_or(all_dss, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask, 412 XE_MAX_DSS_FUSE_BITS); 413 414 num_dss = bitmap_weight(all_dss, XE_MAX_DSS_FUSE_BITS); 415 num_eus = bitmap_weight(gt->fuse_topo.eu_mask_per_dss, 416 XE_MAX_EU_FUSE_BITS) * num_dss; 417 418 /* user can issue separate page faults per EU and per CS */ 419 pf_queue->num_dw = 420 (num_eus + XE_NUM_HW_ENGINES) * PF_MSG_LEN_DW; 421 422 pf_queue->gt = gt; 423 pf_queue->data = devm_kcalloc(xe->drm.dev, pf_queue->num_dw, 424 sizeof(u32), GFP_KERNEL); 425 if (!pf_queue->data) 426 return -ENOMEM; 427 428 spin_lock_init(&pf_queue->lock); 429 INIT_WORK(&pf_queue->worker, pf_queue_work_func); 430 431 return 0; 432 } 433 434 int xe_gt_pagefault_init(struct xe_gt *gt) 435 { 436 struct xe_device *xe = gt_to_xe(gt); 437 int i, ret = 0; 438 439 if (!xe->info.has_usm) 440 return 0; 441 442 for (i = 0; i < NUM_PF_QUEUE; ++i) { 443 ret = xe_alloc_pf_queue(gt, >->usm.pf_queue[i]); 444 if (ret) 445 return ret; 446 } 447 for (i = 0; i < NUM_ACC_QUEUE; ++i) { 448 gt->usm.acc_queue[i].gt = gt; 449 spin_lock_init(>->usm.acc_queue[i].lock); 450 INIT_WORK(>->usm.acc_queue[i].worker, acc_queue_work_func); 451 } 452 453 gt->usm.pf_wq = alloc_workqueue("xe_gt_page_fault_work_queue", 454 WQ_UNBOUND | WQ_HIGHPRI, NUM_PF_QUEUE); 455 if (!gt->usm.pf_wq) 456 return -ENOMEM; 457 458 gt->usm.acc_wq = alloc_workqueue("xe_gt_access_counter_work_queue", 459 WQ_UNBOUND | WQ_HIGHPRI, 460 NUM_ACC_QUEUE); 461 if (!gt->usm.acc_wq) { 462 destroy_workqueue(gt->usm.pf_wq); 463 return -ENOMEM; 464 } 465 466 return devm_add_action_or_reset(xe->drm.dev, pagefault_fini, gt); 467 } 468 469 void xe_gt_pagefault_reset(struct xe_gt *gt) 470 { 471 struct xe_device *xe = gt_to_xe(gt); 472 int i; 473 474 if (!xe->info.has_usm) 475 return; 476 477 for (i = 0; i < NUM_PF_QUEUE; ++i) { 478 spin_lock_irq(>->usm.pf_queue[i].lock); 479 gt->usm.pf_queue[i].head = 0; 480 gt->usm.pf_queue[i].tail = 0; 481 spin_unlock_irq(>->usm.pf_queue[i].lock); 482 } 483 484 for (i = 0; i < NUM_ACC_QUEUE; ++i) { 485 spin_lock(>->usm.acc_queue[i].lock); 486 gt->usm.acc_queue[i].head = 0; 487 gt->usm.acc_queue[i].tail = 0; 488 spin_unlock(>->usm.acc_queue[i].lock); 489 } 490 } 491 492 static int granularity_in_byte(int val) 493 { 494 switch (val) { 495 case 0: 496 return SZ_128K; 497 case 1: 498 return SZ_2M; 499 case 2: 500 return SZ_16M; 501 case 3: 502 return SZ_64M; 503 default: 504 return 0; 505 } 506 } 507 508 static int sub_granularity_in_byte(int val) 509 { 510 return (granularity_in_byte(val) / 32); 511 } 512 513 static void print_acc(struct xe_device *xe, struct acc *acc) 514 { 515 drm_warn(&xe->drm, "Access counter request:\n" 516 "\tType: %s\n" 517 "\tASID: %d\n" 518 "\tVFID: %d\n" 519 "\tEngine: %d:%d\n" 520 "\tGranularity: 0x%x KB Region/ %d KB sub-granularity\n" 521 "\tSub_Granularity Vector: 0x%08x\n" 522 "\tVA Range base: 0x%016llx\n", 523 acc->access_type ? "AC_NTFY_VAL" : "AC_TRIG_VAL", 524 acc->asid, acc->vfid, acc->engine_class, acc->engine_instance, 525 granularity_in_byte(acc->granularity) / SZ_1K, 526 sub_granularity_in_byte(acc->granularity) / SZ_1K, 527 acc->sub_granularity, acc->va_range_base); 528 } 529 530 static struct xe_vma *get_acc_vma(struct xe_vm *vm, struct acc *acc) 531 { 532 u64 page_va = acc->va_range_base + (ffs(acc->sub_granularity) - 1) * 533 sub_granularity_in_byte(acc->granularity); 534 535 return xe_vm_find_overlapping_vma(vm, page_va, SZ_4K); 536 } 537 538 static int handle_acc(struct xe_gt *gt, struct acc *acc) 539 { 540 struct xe_device *xe = gt_to_xe(gt); 541 struct xe_tile *tile = gt_to_tile(gt); 542 struct drm_exec exec; 543 struct xe_vm *vm; 544 struct xe_vma *vma; 545 int ret = 0; 546 547 /* We only support ACC_TRIGGER at the moment */ 548 if (acc->access_type != ACC_TRIGGER) 549 return -EINVAL; 550 551 /* ASID to VM */ 552 mutex_lock(&xe->usm.lock); 553 vm = xa_load(&xe->usm.asid_to_vm, acc->asid); 554 if (vm) 555 xe_vm_get(vm); 556 mutex_unlock(&xe->usm.lock); 557 if (!vm || !xe_vm_in_fault_mode(vm)) 558 return -EINVAL; 559 560 down_read(&vm->lock); 561 562 /* Lookup VMA */ 563 vma = get_acc_vma(vm, acc); 564 if (!vma) { 565 ret = -EINVAL; 566 goto unlock_vm; 567 } 568 569 trace_xe_vma_acc(vma); 570 571 /* Userptr or null can't be migrated, nothing to do */ 572 if (xe_vma_has_no_bo(vma)) 573 goto unlock_vm; 574 575 /* Lock VM and BOs dma-resv */ 576 drm_exec_init(&exec, 0, 0); 577 drm_exec_until_all_locked(&exec) { 578 ret = xe_pf_begin(&exec, vma, true, tile->id); 579 drm_exec_retry_on_contention(&exec); 580 if (ret) 581 break; 582 } 583 584 drm_exec_fini(&exec); 585 unlock_vm: 586 up_read(&vm->lock); 587 xe_vm_put(vm); 588 589 return ret; 590 } 591 592 #define make_u64(hi__, low__) ((u64)(hi__) << 32 | (u64)(low__)) 593 594 #define ACC_MSG_LEN_DW 4 595 596 static bool get_acc(struct acc_queue *acc_queue, struct acc *acc) 597 { 598 const struct xe_guc_acc_desc *desc; 599 bool ret = false; 600 601 spin_lock(&acc_queue->lock); 602 if (acc_queue->tail != acc_queue->head) { 603 desc = (const struct xe_guc_acc_desc *) 604 (acc_queue->data + acc_queue->tail); 605 606 acc->granularity = FIELD_GET(ACC_GRANULARITY, desc->dw2); 607 acc->sub_granularity = FIELD_GET(ACC_SUBG_HI, desc->dw1) << 31 | 608 FIELD_GET(ACC_SUBG_LO, desc->dw0); 609 acc->engine_class = FIELD_GET(ACC_ENG_CLASS, desc->dw1); 610 acc->engine_instance = FIELD_GET(ACC_ENG_INSTANCE, desc->dw1); 611 acc->asid = FIELD_GET(ACC_ASID, desc->dw1); 612 acc->vfid = FIELD_GET(ACC_VFID, desc->dw2); 613 acc->access_type = FIELD_GET(ACC_TYPE, desc->dw0); 614 acc->va_range_base = make_u64(desc->dw3 & ACC_VIRTUAL_ADDR_RANGE_HI, 615 desc->dw2 & ACC_VIRTUAL_ADDR_RANGE_LO); 616 617 acc_queue->tail = (acc_queue->tail + ACC_MSG_LEN_DW) % 618 ACC_QUEUE_NUM_DW; 619 ret = true; 620 } 621 spin_unlock(&acc_queue->lock); 622 623 return ret; 624 } 625 626 static void acc_queue_work_func(struct work_struct *w) 627 { 628 struct acc_queue *acc_queue = container_of(w, struct acc_queue, worker); 629 struct xe_gt *gt = acc_queue->gt; 630 struct xe_device *xe = gt_to_xe(gt); 631 struct acc acc = {}; 632 unsigned long threshold; 633 int ret; 634 635 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS); 636 637 while (get_acc(acc_queue, &acc)) { 638 ret = handle_acc(gt, &acc); 639 if (unlikely(ret)) { 640 print_acc(xe, &acc); 641 drm_warn(&xe->drm, "ACC: Unsuccessful %d\n", ret); 642 } 643 644 if (time_after(jiffies, threshold) && 645 acc_queue->tail != acc_queue->head) { 646 queue_work(gt->usm.acc_wq, w); 647 break; 648 } 649 } 650 } 651 652 static bool acc_queue_full(struct acc_queue *acc_queue) 653 { 654 lockdep_assert_held(&acc_queue->lock); 655 656 return CIRC_SPACE(acc_queue->head, acc_queue->tail, ACC_QUEUE_NUM_DW) <= 657 ACC_MSG_LEN_DW; 658 } 659 660 int xe_guc_access_counter_notify_handler(struct xe_guc *guc, u32 *msg, u32 len) 661 { 662 struct xe_gt *gt = guc_to_gt(guc); 663 struct acc_queue *acc_queue; 664 u32 asid; 665 bool full; 666 667 /* 668 * The below logic doesn't work unless ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW == 0 669 */ 670 BUILD_BUG_ON(ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW); 671 672 if (unlikely(len != ACC_MSG_LEN_DW)) 673 return -EPROTO; 674 675 asid = FIELD_GET(ACC_ASID, msg[1]); 676 acc_queue = >->usm.acc_queue[asid % NUM_ACC_QUEUE]; 677 678 spin_lock(&acc_queue->lock); 679 full = acc_queue_full(acc_queue); 680 if (!full) { 681 memcpy(acc_queue->data + acc_queue->head, msg, 682 len * sizeof(u32)); 683 acc_queue->head = (acc_queue->head + len) % ACC_QUEUE_NUM_DW; 684 queue_work(gt->usm.acc_wq, &acc_queue->worker); 685 } else { 686 drm_warn(>_to_xe(gt)->drm, "ACC Queue full, dropping ACC"); 687 } 688 spin_unlock(&acc_queue->lock); 689 690 return full ? -ENOSPC : 0; 691 } 692