1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2026, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_managed.h> 8 #include <drm/drm_print.h> 9 #include <linux/firmware.h> 10 #include <linux/sizes.h> 11 12 #include "aie.h" 13 #include "aie4_msg_priv.h" 14 #include "aie4_pci.h" 15 #include "amdxdna_mailbox.h" 16 #include "amdxdna_mailbox_helper.h" 17 #include "amdxdna_pci_drv.h" 18 19 #define NO_IOHUB 0 20 #define PSP_NOTIFY_INTR 0xD007BE11 21 #define AIE4_TOTAL_COLUMN 3 22 23 /* 24 * The management mailbox channel is allocated by firmware. 25 * The related register and ring buffer information is on SRAM BAR. 26 * This struct is the register layout. 27 */ 28 struct mailbox_info { 29 __u32 valid; 30 __u32 protocol_major; 31 __u32 protocol_minor; 32 __u32 x2i_tail_offset; 33 __u32 x2i_head_offset; 34 __u32 x2i_buffer_addr; 35 __u32 x2i_buffer_size; 36 __u32 i2x_tail_offset; 37 __u32 i2x_head_offset; 38 __u32 i2x_buffer_addr; 39 __u32 i2x_buffer_size; 40 __u32 i2x_msi_idx; 41 __u32 reserved[4]; 42 }; 43 44 static int aie4_fw_is_alive(struct amdxdna_dev *xdna) 45 { 46 const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv; 47 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 48 u32 __iomem *src; 49 u32 fw_is_valid; 50 int ret; 51 52 src = ndev->rbuf_base + npriv->mbox_info_off; 53 54 ret = readx_poll_timeout(readl, src + offsetof(struct mailbox_info, valid), 55 fw_is_valid, (fw_is_valid == 0x1), 56 AIE_INTERVAL, AIE_TIMEOUT); 57 if (ret) 58 XDNA_ERR(xdna, "fw_is_valid=%d after %d ms", 59 fw_is_valid, DIV_ROUND_CLOSEST(AIE_TIMEOUT, 1000000)); 60 61 return ret; 62 } 63 64 static void aie4_read_mbox_info(struct amdxdna_dev *xdna, 65 struct mailbox_info *mbox_info) 66 { 67 const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv; 68 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 69 u32 *dst = (u32 *)mbox_info; 70 u32 __iomem *src; 71 int i; 72 73 src = ndev->rbuf_base + npriv->mbox_info_off; 74 75 for (i = 0; i < sizeof(*mbox_info) / sizeof(u32); i++) 76 dst[i] = readl(&src[i]); 77 } 78 79 static int aie4_mailbox_info(struct amdxdna_dev *xdna, 80 struct mailbox_info *mbox_info) 81 { 82 int ret; 83 84 ret = aie4_fw_is_alive(xdna); 85 if (ret) 86 return ret; 87 88 aie4_read_mbox_info(xdna, mbox_info); 89 90 ret = aie_check_protocol(&xdna->dev_handle->aie, 91 mbox_info->protocol_major, 92 mbox_info->protocol_minor); 93 if (ret) 94 XDNA_ERR(xdna, "mailbox major.minor %d.%d is not supported", 95 mbox_info->protocol_major, mbox_info->protocol_minor); 96 97 return ret; 98 } 99 100 static void aie4_mailbox_fini(struct amdxdna_dev_hdl *ndev) 101 { 102 struct amdxdna_dev *xdna = ndev->aie.xdna; 103 104 aie_destroy_chann(&ndev->aie, &ndev->aie.mgmt_chann); 105 drmm_kfree(&xdna->ddev, ndev->mbox); 106 ndev->mbox = NULL; 107 } 108 109 static int aie4_irq_init(struct amdxdna_dev *xdna) 110 { 111 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 112 int ret, nvec; 113 114 nvec = pci_msix_vec_count(pdev); 115 XDNA_DBG(xdna, "irq vectors:%d", nvec); 116 if (nvec <= 0) { 117 XDNA_ERR(xdna, "does not get number of interrupt vector"); 118 return -EINVAL; 119 } 120 121 ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 122 if (ret < 0) { 123 XDNA_ERR(xdna, "failed to alloc irq vector, ret: %d", ret); 124 return ret; 125 } 126 127 return 0; 128 } 129 130 static int aie4_mailbox_start(struct amdxdna_dev *xdna, 131 struct mailbox_info *mbi) 132 { 133 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 134 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 135 const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv; 136 struct xdna_mailbox_chann_res *i2x; 137 struct xdna_mailbox_chann_res *x2i; 138 int mgmt_mb_irq; 139 int ret; 140 141 struct xdna_mailbox_res mbox_res = { 142 .ringbuf_base = ndev->rbuf_base, 143 .ringbuf_size = pci_resource_len(pdev, npriv->mbox_rbuf_bar), 144 .mbox_base = ndev->mbox_base, 145 .mbox_size = pci_resource_len(pdev, npriv->mbox_bar), 146 .name = "xdna_aie4_mailbox", 147 }; 148 149 i2x = &ndev->aie.mgmt_i2x; 150 x2i = &ndev->aie.mgmt_x2i; 151 152 x2i->mb_head_ptr_reg = mbi->x2i_head_offset; 153 x2i->mb_tail_ptr_reg = mbi->x2i_tail_offset; 154 x2i->rb_start_addr = mbi->x2i_buffer_addr; 155 x2i->rb_size = mbi->x2i_buffer_size; 156 157 i2x->rb_start_addr = mbi->i2x_buffer_addr; 158 i2x->rb_size = mbi->i2x_buffer_size; 159 i2x->mb_head_ptr_reg = mbi->i2x_head_offset; 160 i2x->mb_tail_ptr_reg = mbi->i2x_tail_offset; 161 162 ndev->aie.mgmt_chan_idx = mbi->i2x_msi_idx; 163 aie_dump_mgmt_chann_debug(&ndev->aie); 164 165 ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res); 166 if (!ndev->mbox) { 167 XDNA_ERR(xdna, "failed to create mailbox device"); 168 return -ENODEV; 169 } 170 171 ndev->aie.mgmt_chann = xdna_mailbox_alloc_channel(ndev->mbox); 172 if (!ndev->aie.mgmt_chann) { 173 XDNA_ERR(xdna, "failed to alloc mailbox channel"); 174 return -ENODEV; 175 } 176 177 mgmt_mb_irq = pci_irq_vector(pdev, ndev->aie.mgmt_chan_idx); 178 if (mgmt_mb_irq < 0) { 179 XDNA_ERR(xdna, "failed to alloc irq vector, return %d", mgmt_mb_irq); 180 ret = mgmt_mb_irq; 181 goto free_channel; 182 } 183 184 ret = xdna_mailbox_start_channel(ndev->aie.mgmt_chann, 185 &ndev->aie.mgmt_x2i, 186 &ndev->aie.mgmt_i2x, 187 NO_IOHUB, 188 mgmt_mb_irq); 189 if (ret) { 190 XDNA_ERR(xdna, "failed to start management mailbox channel"); 191 ret = -EINVAL; 192 goto free_channel; 193 } 194 195 XDNA_DBG(xdna, "Mailbox management channel created"); 196 return 0; 197 198 free_channel: 199 xdna_mailbox_free_channel(ndev->aie.mgmt_chann); 200 ndev->aie.mgmt_chann = NULL; 201 return ret; 202 } 203 204 static int aie4_mailbox_init(struct amdxdna_dev_hdl *ndev) 205 { 206 struct amdxdna_dev *xdna = ndev->aie.xdna; 207 struct mailbox_info mbox_info; 208 int ret; 209 210 ret = aie4_mailbox_info(xdna, &mbox_info); 211 if (ret) 212 return ret; 213 214 return aie4_mailbox_start(xdna, &mbox_info); 215 } 216 217 static void aie4_fw_stop(struct amdxdna_dev_hdl *ndev) 218 { 219 aie_psp_stop(ndev->aie.psp_hdl); 220 aie_smu_fini(ndev->aie.smu_hdl); 221 } 222 223 static int aie4_fw_start(struct amdxdna_dev_hdl *ndev) 224 { 225 int ret; 226 227 ret = aie_smu_init(ndev->aie.smu_hdl); 228 if (ret) { 229 XDNA_ERR(ndev->aie.xdna, "failed to init smu, ret %d", ret); 230 return ret; 231 } 232 233 ret = aie_psp_start(ndev->aie.psp_hdl); 234 if (ret) { 235 XDNA_ERR(ndev->aie.xdna, "failed to start psp, ret %d", ret); 236 aie_smu_fini(ndev->aie.smu_hdl); 237 } 238 239 return ret; 240 } 241 242 static int aie4_partition_init(struct amdxdna_dev_hdl *ndev) 243 { 244 DECLARE_AIE_MSG(aie4_msg_create_partition, AIE4_MSG_OP_CREATE_PARTITION); 245 struct amdxdna_dev *xdna = ndev->aie.xdna; 246 int ret; 247 248 req.partition_col_start = 0; 249 req.partition_col_count = AIE4_TOTAL_COLUMN; 250 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 251 if (ret) { 252 XDNA_ERR(xdna, "partition init failed: %d", ret); 253 return ret; 254 } 255 256 ndev->partition_id = resp.partition_id; 257 return 0; 258 } 259 260 static void aie4_partition_fini(struct amdxdna_dev_hdl *ndev) 261 { 262 DECLARE_AIE_MSG(aie4_msg_destroy_partition, AIE4_MSG_OP_DESTROY_PARTITION); 263 struct amdxdna_dev *xdna = ndev->aie.xdna; 264 int ret; 265 266 req.partition_id = ndev->partition_id; 267 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 268 if (ret) 269 XDNA_ERR(xdna, "partition fini failed: %d", ret); 270 } 271 272 static int aie4_query(struct amdxdna_dev_hdl *ndev) 273 { 274 return aie4_query_aie_metadata(ndev, &ndev->aie.metadata); 275 } 276 277 static int aie4_pf_hw_start(struct amdxdna_dev_hdl *ndev) 278 { 279 int ret; 280 281 ret = aie4_fw_start(ndev); 282 if (ret) 283 return ret; 284 285 ret = aie4_mailbox_init(ndev); 286 if (ret) 287 goto stop_fw; 288 289 ret = aie4_attach_work_buffer(ndev); 290 if (ret) 291 goto mbox_fini; 292 293 return 0; 294 295 mbox_fini: 296 aie4_mailbox_fini(ndev); 297 stop_fw: 298 aie4_fw_stop(ndev); 299 300 return ret; 301 } 302 303 static void aie4_pf_hw_stop(struct amdxdna_dev_hdl *ndev) 304 { 305 struct amdxdna_dev *xdna = ndev->aie.xdna; 306 307 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 308 309 aie4_suspend_fw(ndev); 310 aie4_mailbox_fini(ndev); 311 aie4_fw_stop(ndev); 312 } 313 314 static int aie4_vf_hw_start(struct amdxdna_dev_hdl *ndev) 315 { 316 int ret; 317 318 ret = aie4_mailbox_init(ndev); 319 if (ret) 320 return ret; 321 322 ret = aie4_query(ndev); 323 if (ret) 324 goto mailbox_fini; 325 326 ret = aie4_partition_init(ndev); 327 if (ret) 328 goto mailbox_fini; 329 330 return 0; 331 332 mailbox_fini: 333 aie4_mailbox_fini(ndev); 334 return ret; 335 } 336 337 static void aie4_vf_hw_stop(struct amdxdna_dev_hdl *ndev) 338 { 339 struct amdxdna_dev *xdna = ndev->aie.xdna; 340 341 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 342 343 aie4_partition_fini(ndev); 344 aie4_mailbox_fini(ndev); 345 } 346 347 static int aie4_request_firmware(struct amdxdna_dev_hdl *ndev, 348 const struct firmware **npufw, 349 const struct firmware **certfw) 350 { 351 struct amdxdna_dev *xdna = ndev->aie.xdna; 352 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 353 char fw_name[128]; 354 int ret; 355 356 ret = snprintf(fw_name, sizeof(fw_name), "amdnpu/%04x_%02x/%s", 357 pdev->device, pdev->revision, ndev->priv->npufw_path); 358 if (ret >= sizeof(fw_name)) { 359 XDNA_ERR(xdna, "npu firmware path is truncated"); 360 return -EINVAL; 361 } 362 363 ret = request_firmware(npufw, fw_name, &pdev->dev); 364 if (ret) { 365 XDNA_ERR(xdna, "failed to request_firmware %s, ret %d", fw_name, ret); 366 return ret; 367 } 368 369 ret = snprintf(fw_name, sizeof(fw_name), "amdnpu/%04x_%02x/%s", 370 pdev->device, pdev->revision, ndev->priv->certfw_path); 371 if (ret >= sizeof(fw_name)) { 372 XDNA_ERR(xdna, "cert firmware path is truncated"); 373 ret = -EINVAL; 374 goto release_npufw; 375 } 376 377 ret = request_firmware(certfw, fw_name, &pdev->dev); 378 if (ret) { 379 XDNA_ERR(xdna, "failed to request_firmware %s, ret %d", fw_name, ret); 380 goto release_npufw; 381 } 382 383 return 0; 384 385 release_npufw: 386 release_firmware(*npufw); 387 388 return ret; 389 } 390 391 static void aie4_release_firmware(struct amdxdna_dev_hdl *ndev, 392 const struct firmware *npufw, 393 const struct firmware *certfw) 394 { 395 release_firmware(certfw); 396 release_firmware(npufw); 397 } 398 399 static int aie4_prepare_firmware(struct amdxdna_dev_hdl *ndev, 400 const struct firmware *npufw, 401 const struct firmware *certfw, 402 void __iomem *tbl[PCI_NUM_RESOURCES]) 403 { 404 struct amdxdna_dev *xdna = ndev->aie.xdna; 405 struct psp_config psp_conf; 406 struct smu_config smu_conf; 407 int i; 408 409 psp_conf.fw_size = npufw->size; 410 psp_conf.fw_buf = npufw->data; 411 psp_conf.certfw_size = certfw->size; 412 psp_conf.certfw_buf = certfw->data; 413 psp_conf.arg2_mask = ~0; 414 psp_conf.notify_val = PSP_NOTIFY_INTR; 415 for (i = 0; i < PSP_MAX_REGS; i++) 416 psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i); 417 ndev->aie.psp_hdl = aiem_psp_create(&xdna->ddev, &psp_conf); 418 if (!ndev->aie.psp_hdl) { 419 XDNA_ERR(xdna, "failed to create psp"); 420 return -ENOMEM; 421 } 422 423 for (i = 0; i < SMU_MAX_REGS; i++) 424 smu_conf.smu_regs[i] = tbl[SMU_REG_BAR(ndev, i)] + SMU_REG_OFF(ndev, i); 425 ndev->aie.smu_hdl = aiem_smu_create(&xdna->ddev, &smu_conf); 426 if (!ndev->aie.smu_hdl) { 427 XDNA_ERR(xdna, "failed to create smu"); 428 return -ENOMEM; 429 } 430 431 return 0; 432 } 433 434 static int aie4_load_fw(struct amdxdna_dev_hdl *ndev, 435 void __iomem *tbl[PCI_NUM_RESOURCES]) 436 { 437 const struct firmware *npufw, *certfw; 438 int ret; 439 440 if (!ndev->priv->npufw_path && !ndev->priv->certfw_path) 441 return 0; 442 443 ret = aie4_request_firmware(ndev, &npufw, &certfw); 444 if (ret) 445 return ret; 446 447 ret = aie4_prepare_firmware(ndev, npufw, certfw, tbl); 448 aie4_release_firmware(ndev, npufw, certfw); 449 450 return ret; 451 } 452 453 static int aie4m_pcidev_init(struct amdxdna_dev *xdna) 454 { 455 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 456 struct amdxdna_dev_hdl *ndev; 457 void __iomem *tbl[PCI_NUM_RESOURCES] = {0}; 458 unsigned long bars = 0; 459 int ret, i; 460 461 ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL); 462 if (!ndev) 463 return -ENOMEM; 464 465 ndev->priv = xdna->dev_info->dev_priv; 466 ndev->aie.xdna = xdna; 467 xdna->dev_handle = ndev; 468 469 xa_init_flags(&ndev->cert_comp_xa, XA_FLAGS_ALLOC); 470 mutex_init(&ndev->cert_comp_lock); 471 472 /* Enable managed PCI device */ 473 ret = pcim_enable_device(pdev); 474 if (ret) { 475 XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret); 476 return ret; 477 } 478 479 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 480 if (ret) { 481 XDNA_ERR(xdna, "failed to set DMA mask to 64:%d", ret); 482 return ret; 483 } 484 485 for (i = 0; i < PSP_MAX_REGS; i++) 486 set_bit(PSP_REG_BAR(ndev, i), &bars); 487 for (i = 0; i < SMU_MAX_REGS; i++) 488 set_bit(SMU_REG_BAR(ndev, i), &bars); 489 set_bit(xdna->dev_info->mbox_bar, &bars); 490 set_bit(xdna->dev_info->sram_bar, &bars); 491 492 for (i = 0; i < PCI_NUM_RESOURCES; i++) { 493 if (!test_bit(i, &bars)) 494 continue; 495 tbl[i] = pcim_iomap(pdev, i, 0); 496 if (!tbl[i]) { 497 XDNA_ERR(xdna, "map bar %d failed", i); 498 return -ENOMEM; 499 } 500 } 501 502 ndev->mbox_base = tbl[xdna->dev_info->mbox_bar]; 503 ndev->rbuf_base = tbl[xdna->dev_info->sram_bar]; 504 505 pci_set_master(pdev); 506 507 ret = aie4_load_fw(ndev, tbl); 508 if (ret) 509 return ret; 510 511 ret = aie4_irq_init(xdna); 512 if (ret) 513 return ret; 514 515 amdxdna_vbnv_init(xdna); 516 XDNA_DBG(xdna, "init finished"); 517 518 return 0; 519 } 520 521 static int aie4_doorbell_mmap(struct amdxdna_client *client, struct vm_area_struct *vma) 522 { 523 struct amdxdna_dev *xdna = client->xdna; 524 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 525 const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv; 526 phys_addr_t res_start; 527 unsigned long pfn; 528 int ret; 529 530 if (!aie4_hwctx_valid_doorbell(client, vma->vm_pgoff)) { 531 XDNA_ERR(xdna, "Invalid doorbell page offset 0x%lx", vma->vm_pgoff); 532 return -EINVAL; 533 } 534 535 if (vma_pages(vma) != 1) { 536 XDNA_ERR(xdna, "can only map one page, got %ld", vma_pages(vma)); 537 return -EINVAL; 538 } 539 540 res_start = pci_resource_start(pdev, xdna->dev_info->doorbell_bar) + npriv->doorbell_off; 541 pfn = PHYS_PFN(res_start) + vma->vm_pgoff; 542 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 543 vm_flags_set(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP); 544 ret = io_remap_pfn_range(vma, vma->vm_start, 545 pfn, 546 PAGE_SIZE, 547 vma->vm_page_prot); 548 549 XDNA_DBG(xdna, "doorbell ret %d", ret); 550 return ret; 551 } 552 553 static int aie4_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args) 554 { 555 struct amdxdna_dev *xdna = client->xdna; 556 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 557 int ret; 558 559 switch (args->param) { 560 case DRM_AMDXDNA_QUERY_AIE_METADATA: 561 ret = amdxdna_get_metadata(&ndev->aie, client, args); 562 break; 563 default: 564 XDNA_ERR(xdna, "Not supported request parameter %u", args->param); 565 ret = -EOPNOTSUPP; 566 } 567 568 XDNA_DBG(xdna, "Got param %d", args->param); 569 570 return ret; 571 } 572 573 static int aie4_alloc_work_buffer(struct amdxdna_dev_hdl *ndev) 574 { 575 struct amdxdna_dev *xdna = ndev->aie.xdna; 576 u32 buf_size = AIE4_WORK_BUFFER_MIN_SIZE; 577 578 ndev->work_buf = amdxdna_alloc_msg_buffer(xdna, &buf_size, 579 &ndev->work_buf_addr); 580 if (IS_ERR(ndev->work_buf)) { 581 int ret = PTR_ERR(ndev->work_buf); 582 583 XDNA_ERR(xdna, "Failed to alloc work buffer, size 0x%x", 584 AIE4_WORK_BUFFER_MIN_SIZE); 585 ndev->work_buf = NULL; 586 return ret; 587 } 588 589 ndev->work_buf_size = buf_size; 590 XDNA_DBG(xdna, "Work buffer allocated: size 0x%x", buf_size); 591 592 return 0; 593 } 594 595 static void aie4_free_work_buffer(struct amdxdna_dev_hdl *ndev) 596 { 597 struct amdxdna_dev *xdna = ndev->aie.xdna; 598 599 if (!ndev->work_buf) 600 return; 601 602 amdxdna_free_msg_buffer(xdna, ndev->work_buf_size, ndev->work_buf, 603 ndev->work_buf_addr); 604 ndev->work_buf = NULL; 605 } 606 607 static int aie4_pf_init(struct amdxdna_dev *xdna) 608 { 609 int ret; 610 611 ret = aie4m_pcidev_init(xdna); 612 if (ret) 613 return ret; 614 615 ret = aie4_alloc_work_buffer(xdna->dev_handle); 616 if (ret) 617 return ret; 618 619 ret = aie4_pf_hw_start(xdna->dev_handle); 620 if (ret) 621 goto free_work_buf; 622 623 return 0; 624 625 free_work_buf: 626 aie4_free_work_buffer(xdna->dev_handle); 627 return ret; 628 } 629 630 static int aie4_vf_init(struct amdxdna_dev *xdna) 631 { 632 int ret; 633 634 ret = aie4m_pcidev_init(xdna); 635 if (ret) 636 return ret; 637 638 return aie4_vf_hw_start(xdna->dev_handle); 639 } 640 641 static void aie4_pf_fini(struct amdxdna_dev *xdna) 642 { 643 aie4_sriov_stop(xdna->dev_handle); 644 aie4_pf_hw_stop(xdna->dev_handle); 645 aie4_free_work_buffer(xdna->dev_handle); 646 } 647 648 static void aie4_vf_fini(struct amdxdna_dev *xdna) 649 { 650 aie4_vf_hw_stop(xdna->dev_handle); 651 } 652 653 const struct amdxdna_dev_ops aie4_pf_ops = { 654 .init = aie4_pf_init, 655 .fini = aie4_pf_fini, 656 .sriov_configure = aie4_sriov_configure, 657 }; 658 659 const struct amdxdna_dev_ops aie4_vf_ops = { 660 .init = aie4_vf_init, 661 .fini = aie4_vf_fini, 662 .hwctx_init = aie4_hwctx_init, 663 .hwctx_fini = aie4_hwctx_fini, 664 .mmap = aie4_doorbell_mmap, 665 .cmd_wait = aie4_cmd_wait, 666 .get_aie_info = aie4_get_info, 667 }; 668