1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_gem_shmem_helper.h> 10 #include <drm/drm_managed.h> 11 #include <drm/drm_print.h> 12 #include <drm/gpu_scheduler.h> 13 #include <linux/amd-pmf-io.h> 14 #include <linux/cleanup.h> 15 #include <linux/errno.h> 16 #include <linux/firmware.h> 17 #include <linux/iommu.h> 18 #include <linux/iopoll.h> 19 #include <linux/pci.h> 20 #include <linux/xarray.h> 21 #include <asm/hypervisor.h> 22 23 #include "aie2_msg_priv.h" 24 #include "aie2_pci.h" 25 #include "aie2_solver.h" 26 #include "amdxdna_ctx.h" 27 #include "amdxdna_gem.h" 28 #include "amdxdna_mailbox.h" 29 #include "amdxdna_pci_drv.h" 30 #include "amdxdna_pm.h" 31 32 static int aie2_max_col = XRS_MAX_COL; 33 module_param(aie2_max_col, uint, 0600); 34 MODULE_PARM_DESC(aie2_max_col, "Maximum column could be used"); 35 36 static char *npu_fw[] = { 37 "npu_7.sbin", 38 "npu.sbin" 39 }; 40 41 /* 42 * The management mailbox channel is allocated by firmware. 43 * The related register and ring buffer information is on SRAM BAR. 44 * This struct is the register layout. 45 */ 46 #define MGMT_MBOX_MAGIC 0x55504e5f /* _NPU */ 47 struct mgmt_mbox_chann_info { 48 __u32 x2i_tail; 49 __u32 x2i_head; 50 __u32 x2i_buf; 51 __u32 x2i_buf_sz; 52 __u32 i2x_tail; 53 __u32 i2x_head; 54 __u32 i2x_buf; 55 __u32 i2x_buf_sz; 56 __u32 magic; 57 __u32 msi_id; 58 __u32 prot_major; 59 __u32 prot_minor; 60 __u32 rsvd[4]; 61 }; 62 63 static int aie2_check_protocol(struct amdxdna_dev_hdl *ndev, u32 fw_major, u32 fw_minor) 64 { 65 const struct aie2_fw_feature_tbl *feature; 66 bool found = false; 67 68 for (feature = ndev->priv->fw_feature_tbl; feature->major; feature++) { 69 if (feature->major != fw_major) 70 continue; 71 if (fw_minor < feature->min_minor) 72 continue; 73 if (feature->max_minor > 0 && fw_minor > feature->max_minor) 74 continue; 75 76 ndev->feature_mask |= feature->features; 77 78 /* firmware version matches one of the driver support entry */ 79 found = true; 80 } 81 82 return found ? 0 : -EOPNOTSUPP; 83 } 84 85 static void aie2_dump_chann_info_debug(struct amdxdna_dev_hdl *ndev) 86 { 87 struct amdxdna_dev *xdna = ndev->xdna; 88 89 XDNA_DBG(xdna, "i2x tail 0x%x", ndev->mgmt_i2x.mb_tail_ptr_reg); 90 XDNA_DBG(xdna, "i2x head 0x%x", ndev->mgmt_i2x.mb_head_ptr_reg); 91 XDNA_DBG(xdna, "i2x ringbuf 0x%x", ndev->mgmt_i2x.rb_start_addr); 92 XDNA_DBG(xdna, "i2x rsize 0x%x", ndev->mgmt_i2x.rb_size); 93 XDNA_DBG(xdna, "x2i tail 0x%x", ndev->mgmt_x2i.mb_tail_ptr_reg); 94 XDNA_DBG(xdna, "x2i head 0x%x", ndev->mgmt_x2i.mb_head_ptr_reg); 95 XDNA_DBG(xdna, "x2i ringbuf 0x%x", ndev->mgmt_x2i.rb_start_addr); 96 XDNA_DBG(xdna, "x2i rsize 0x%x", ndev->mgmt_x2i.rb_size); 97 XDNA_DBG(xdna, "x2i chann index 0x%x", ndev->mgmt_chan_idx); 98 XDNA_DBG(xdna, "mailbox protocol major 0x%x", ndev->mgmt_prot_major); 99 XDNA_DBG(xdna, "mailbox protocol minor 0x%x", ndev->mgmt_prot_minor); 100 } 101 102 static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev) 103 { 104 struct mgmt_mbox_chann_info info_regs; 105 struct xdna_mailbox_chann_res *i2x; 106 struct xdna_mailbox_chann_res *x2i; 107 u32 addr, off; 108 u32 *reg; 109 int ret; 110 int i; 111 112 /* 113 * Once firmware is alive, it will write management channel 114 * information in SRAM BAR and write the address of that information 115 * at FW_ALIVE_OFF offset in SRMA BAR. 116 * 117 * Read a non-zero value from FW_ALIVE_OFF implies that firmware 118 * is alive. 119 */ 120 ret = readx_poll_timeout(readl, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF), 121 addr, addr, AIE2_INTERVAL, AIE2_TIMEOUT); 122 if (ret || !addr) 123 return -ETIME; 124 125 off = AIE2_SRAM_OFF(ndev, addr); 126 reg = (u32 *)&info_regs; 127 for (i = 0; i < sizeof(info_regs) / sizeof(u32); i++) 128 reg[i] = readl(ndev->sram_base + off + i * sizeof(u32)); 129 130 if (info_regs.magic != MGMT_MBOX_MAGIC) { 131 XDNA_ERR(ndev->xdna, "Invalid mbox magic 0x%x", info_regs.magic); 132 ret = -EINVAL; 133 goto done; 134 } 135 136 i2x = &ndev->mgmt_i2x; 137 x2i = &ndev->mgmt_x2i; 138 139 i2x->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_head); 140 i2x->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_tail); 141 i2x->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.i2x_buf); 142 i2x->rb_size = info_regs.i2x_buf_sz; 143 144 x2i->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_head); 145 x2i->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_tail); 146 x2i->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.x2i_buf); 147 x2i->rb_size = info_regs.x2i_buf_sz; 148 149 ndev->mgmt_chan_idx = info_regs.msi_id; 150 ndev->mgmt_prot_major = info_regs.prot_major; 151 ndev->mgmt_prot_minor = info_regs.prot_minor; 152 153 ret = aie2_check_protocol(ndev, ndev->mgmt_prot_major, ndev->mgmt_prot_minor); 154 155 done: 156 aie2_dump_chann_info_debug(ndev); 157 158 /* Must clear address at FW_ALIVE_OFF */ 159 writel(0, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF)); 160 161 return ret; 162 } 163 164 int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev, 165 enum rt_config_category category, u32 *val) 166 { 167 const struct rt_config *cfg; 168 u32 value; 169 int ret; 170 171 for (cfg = ndev->priv->rt_config; cfg->type; cfg++) { 172 if (cfg->category != category) 173 continue; 174 175 if (cfg->feature_mask && 176 bitmap_subset(&cfg->feature_mask, &ndev->feature_mask, AIE2_FEATURE_MAX)) 177 continue; 178 179 value = val ? *val : cfg->value; 180 ret = aie2_set_runtime_cfg(ndev, cfg->type, value); 181 if (ret) { 182 XDNA_ERR(ndev->xdna, "Set type %d value %d failed", 183 cfg->type, value); 184 return ret; 185 } 186 } 187 188 return 0; 189 } 190 191 static int aie2_xdna_reset(struct amdxdna_dev_hdl *ndev) 192 { 193 int ret; 194 195 ret = aie2_suspend_fw(ndev); 196 if (ret) { 197 XDNA_ERR(ndev->xdna, "Suspend firmware failed"); 198 return ret; 199 } 200 201 ret = aie2_resume_fw(ndev); 202 if (ret) { 203 XDNA_ERR(ndev->xdna, "Resume firmware failed"); 204 return ret; 205 } 206 207 return 0; 208 } 209 210 static int aie2_mgmt_fw_init(struct amdxdna_dev_hdl *ndev) 211 { 212 int ret; 213 214 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_INIT, NULL); 215 if (ret) { 216 XDNA_ERR(ndev->xdna, "Runtime config failed"); 217 return ret; 218 } 219 220 ret = aie2_assign_mgmt_pasid(ndev, 0); 221 if (ret) { 222 XDNA_ERR(ndev->xdna, "Can not assign PASID"); 223 return ret; 224 } 225 226 ret = aie2_xdna_reset(ndev); 227 if (ret) { 228 XDNA_ERR(ndev->xdna, "Reset firmware failed"); 229 return ret; 230 } 231 232 return 0; 233 } 234 235 static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev) 236 { 237 int ret; 238 239 ret = aie2_query_firmware_version(ndev, &ndev->xdna->fw_ver); 240 if (ret) { 241 XDNA_ERR(ndev->xdna, "query firmware version failed"); 242 return ret; 243 } 244 245 ret = aie2_query_aie_version(ndev, &ndev->version); 246 if (ret) { 247 XDNA_ERR(ndev->xdna, "Query AIE version failed"); 248 return ret; 249 } 250 251 ret = aie2_query_aie_metadata(ndev, &ndev->metadata); 252 if (ret) { 253 XDNA_ERR(ndev->xdna, "Query AIE metadata failed"); 254 return ret; 255 } 256 257 ndev->total_col = min(aie2_max_col, ndev->metadata.cols); 258 259 return 0; 260 } 261 262 static void aie2_mgmt_fw_fini(struct amdxdna_dev_hdl *ndev) 263 { 264 if (aie2_suspend_fw(ndev)) 265 XDNA_ERR(ndev->xdna, "Suspend_fw failed"); 266 XDNA_DBG(ndev->xdna, "Firmware suspended"); 267 } 268 269 static int aie2_xrs_load(void *cb_arg, struct xrs_action_load *action) 270 { 271 struct amdxdna_hwctx *hwctx = cb_arg; 272 struct amdxdna_dev *xdna; 273 int ret; 274 275 xdna = hwctx->client->xdna; 276 277 hwctx->start_col = action->part.start_col; 278 hwctx->num_col = action->part.ncols; 279 ret = aie2_create_context(xdna->dev_handle, hwctx); 280 if (ret) 281 XDNA_ERR(xdna, "create context failed, ret %d", ret); 282 283 return ret; 284 } 285 286 static int aie2_xrs_unload(void *cb_arg) 287 { 288 struct amdxdna_hwctx *hwctx = cb_arg; 289 struct amdxdna_dev *xdna; 290 int ret; 291 292 xdna = hwctx->client->xdna; 293 294 ret = aie2_destroy_context(xdna->dev_handle, hwctx); 295 if (ret) 296 XDNA_ERR(xdna, "destroy context failed, ret %d", ret); 297 298 return ret; 299 } 300 301 static int aie2_xrs_set_dft_dpm_level(struct drm_device *ddev, u32 dpm_level) 302 { 303 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 304 struct amdxdna_dev_hdl *ndev; 305 306 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 307 308 ndev = xdna->dev_handle; 309 ndev->dft_dpm_level = dpm_level; 310 if (ndev->pw_mode != POWER_MODE_DEFAULT || ndev->dpm_level == dpm_level) 311 return 0; 312 313 return aie2_pm_set_dpm(ndev, dpm_level); 314 } 315 316 static struct xrs_action_ops aie2_xrs_actions = { 317 .load = aie2_xrs_load, 318 .unload = aie2_xrs_unload, 319 .set_dft_dpm_level = aie2_xrs_set_dft_dpm_level, 320 }; 321 322 static void aie2_hw_stop(struct amdxdna_dev *xdna) 323 { 324 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 325 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 326 327 if (ndev->dev_status <= AIE2_DEV_INIT) { 328 XDNA_ERR(xdna, "device is already stopped"); 329 return; 330 } 331 332 aie2_runtime_cfg(ndev, AIE2_RT_CFG_CLK_GATING, NULL); 333 aie2_mgmt_fw_fini(ndev); 334 aie2_destroy_mgmt_chann(ndev); 335 drmm_kfree(&xdna->ddev, ndev->mbox); 336 ndev->mbox = NULL; 337 aie2_psp_stop(ndev->psp_hdl); 338 aie2_smu_fini(ndev); 339 aie2_error_async_events_free(ndev); 340 pci_disable_device(pdev); 341 342 ndev->dev_status = AIE2_DEV_INIT; 343 } 344 345 static int aie2_hw_start(struct amdxdna_dev *xdna) 346 { 347 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 348 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 349 struct xdna_mailbox_res mbox_res; 350 u32 xdna_mailbox_intr_reg; 351 int mgmt_mb_irq, ret; 352 353 if (ndev->dev_status >= AIE2_DEV_START) { 354 XDNA_INFO(xdna, "device is already started"); 355 return 0; 356 } 357 358 ret = pci_enable_device(pdev); 359 if (ret) { 360 XDNA_ERR(xdna, "failed to enable device, ret %d", ret); 361 return ret; 362 } 363 pci_set_master(pdev); 364 365 mbox_res.ringbuf_base = ndev->sram_base; 366 mbox_res.ringbuf_size = pci_resource_len(pdev, xdna->dev_info->sram_bar); 367 mbox_res.mbox_base = ndev->mbox_base; 368 mbox_res.mbox_size = MBOX_SIZE(ndev); 369 mbox_res.name = "xdna_mailbox"; 370 ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res); 371 if (!ndev->mbox) { 372 XDNA_ERR(xdna, "failed to create mailbox device"); 373 ret = -ENODEV; 374 goto disable_dev; 375 } 376 377 ndev->mgmt_chann = xdna_mailbox_alloc_channel(ndev->mbox); 378 if (!ndev->mgmt_chann) { 379 XDNA_ERR(xdna, "failed to alloc channel"); 380 ret = -ENODEV; 381 goto disable_dev; 382 } 383 384 ret = aie2_smu_init(ndev); 385 if (ret) { 386 XDNA_ERR(xdna, "failed to init smu, ret %d", ret); 387 goto free_channel; 388 } 389 390 ret = aie2_psp_start(ndev->psp_hdl); 391 if (ret) { 392 XDNA_ERR(xdna, "failed to start psp, ret %d", ret); 393 goto fini_smu; 394 } 395 396 ret = aie2_get_mgmt_chann_info(ndev); 397 if (ret) { 398 XDNA_ERR(xdna, "firmware is not alive"); 399 goto stop_psp; 400 } 401 402 mgmt_mb_irq = pci_irq_vector(pdev, ndev->mgmt_chan_idx); 403 if (mgmt_mb_irq < 0) { 404 ret = mgmt_mb_irq; 405 XDNA_ERR(xdna, "failed to alloc irq vector, ret %d", ret); 406 goto stop_psp; 407 } 408 409 xdna_mailbox_intr_reg = ndev->mgmt_i2x.mb_head_ptr_reg + 4; 410 ret = xdna_mailbox_start_channel(ndev->mgmt_chann, 411 &ndev->mgmt_x2i, 412 &ndev->mgmt_i2x, 413 xdna_mailbox_intr_reg, 414 mgmt_mb_irq); 415 if (ret) { 416 XDNA_ERR(xdna, "failed to start management mailbox channel"); 417 ret = -EINVAL; 418 goto stop_psp; 419 } 420 421 ret = aie2_mgmt_fw_init(ndev); 422 if (ret) { 423 XDNA_ERR(xdna, "initial mgmt firmware failed, ret %d", ret); 424 goto stop_fw; 425 } 426 427 ret = aie2_pm_init(ndev); 428 if (ret) { 429 XDNA_ERR(xdna, "failed to init pm, ret %d", ret); 430 goto stop_fw; 431 } 432 433 ret = aie2_mgmt_fw_query(ndev); 434 if (ret) { 435 XDNA_ERR(xdna, "failed to query fw, ret %d", ret); 436 goto stop_fw; 437 } 438 439 ret = aie2_error_async_events_alloc(ndev); 440 if (ret) { 441 XDNA_ERR(xdna, "Allocate async events failed, ret %d", ret); 442 goto stop_fw; 443 } 444 445 ndev->dev_status = AIE2_DEV_START; 446 447 return 0; 448 449 stop_fw: 450 aie2_suspend_fw(ndev); 451 xdna_mailbox_stop_channel(ndev->mgmt_chann); 452 stop_psp: 453 aie2_psp_stop(ndev->psp_hdl); 454 fini_smu: 455 aie2_smu_fini(ndev); 456 free_channel: 457 xdna_mailbox_free_channel(ndev->mgmt_chann); 458 ndev->mgmt_chann = NULL; 459 disable_dev: 460 pci_disable_device(pdev); 461 462 return ret; 463 } 464 465 static int aie2_hw_suspend(struct amdxdna_dev *xdna) 466 { 467 struct amdxdna_client *client; 468 469 list_for_each_entry(client, &xdna->client_list, node) 470 aie2_hwctx_suspend(client); 471 472 aie2_hw_stop(xdna); 473 474 return 0; 475 } 476 477 static int aie2_hw_resume(struct amdxdna_dev *xdna) 478 { 479 struct amdxdna_client *client; 480 int ret; 481 482 ret = aie2_hw_start(xdna); 483 if (ret) { 484 XDNA_ERR(xdna, "Start hardware failed, %d", ret); 485 return ret; 486 } 487 488 list_for_each_entry(client, &xdna->client_list, node) { 489 ret = aie2_hwctx_resume(client); 490 if (ret) 491 break; 492 } 493 494 return ret; 495 } 496 497 static int aie2_init(struct amdxdna_dev *xdna) 498 { 499 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 500 void __iomem *tbl[PCI_NUM_RESOURCES] = {0}; 501 struct init_config xrs_cfg = { 0 }; 502 struct amdxdna_dev_hdl *ndev; 503 struct psp_config psp_conf; 504 const struct firmware *fw; 505 unsigned long bars = 0; 506 char *fw_full_path; 507 int i, nvec, ret; 508 509 if (!hypervisor_is_type(X86_HYPER_NATIVE)) { 510 XDNA_ERR(xdna, "Running under hypervisor not supported"); 511 return -EINVAL; 512 } 513 514 if (!xdna->group) { 515 XDNA_ERR(xdna, "Running without IOMMU not supported"); 516 return -EINVAL; 517 } 518 519 ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL); 520 if (!ndev) 521 return -ENOMEM; 522 523 ndev->priv = xdna->dev_info->dev_priv; 524 ndev->xdna = xdna; 525 526 for (i = 0; i < ARRAY_SIZE(npu_fw); i++) { 527 fw_full_path = kasprintf(GFP_KERNEL, "%s%s", ndev->priv->fw_path, npu_fw[i]); 528 if (!fw_full_path) 529 return -ENOMEM; 530 531 ret = firmware_request_nowarn(&fw, fw_full_path, &pdev->dev); 532 kfree(fw_full_path); 533 if (!ret) { 534 XDNA_INFO(xdna, "Load firmware %s%s", ndev->priv->fw_path, npu_fw[i]); 535 break; 536 } 537 } 538 539 if (ret) { 540 XDNA_ERR(xdna, "failed to request_firmware %s, ret %d", 541 ndev->priv->fw_path, ret); 542 return ret; 543 } 544 545 ret = pcim_enable_device(pdev); 546 if (ret) { 547 XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret); 548 goto release_fw; 549 } 550 551 for (i = 0; i < PSP_MAX_REGS; i++) 552 set_bit(PSP_REG_BAR(ndev, i), &bars); 553 554 set_bit(xdna->dev_info->sram_bar, &bars); 555 set_bit(xdna->dev_info->smu_bar, &bars); 556 set_bit(xdna->dev_info->mbox_bar, &bars); 557 558 for (i = 0; i < PCI_NUM_RESOURCES; i++) { 559 if (!test_bit(i, &bars)) 560 continue; 561 tbl[i] = pcim_iomap(pdev, i, 0); 562 if (!tbl[i]) { 563 XDNA_ERR(xdna, "map bar %d failed", i); 564 ret = -ENOMEM; 565 goto release_fw; 566 } 567 } 568 569 ndev->sram_base = tbl[xdna->dev_info->sram_bar]; 570 ndev->smu_base = tbl[xdna->dev_info->smu_bar]; 571 ndev->mbox_base = tbl[xdna->dev_info->mbox_bar]; 572 573 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 574 if (ret) { 575 XDNA_ERR(xdna, "Failed to set DMA mask: %d", ret); 576 goto release_fw; 577 } 578 579 nvec = pci_msix_vec_count(pdev); 580 if (nvec <= 0) { 581 XDNA_ERR(xdna, "does not get number of interrupt vector"); 582 ret = -EINVAL; 583 goto release_fw; 584 } 585 586 ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 587 if (ret < 0) { 588 XDNA_ERR(xdna, "failed to alloc irq vectors, ret %d", ret); 589 goto release_fw; 590 } 591 592 psp_conf.fw_size = fw->size; 593 psp_conf.fw_buf = fw->data; 594 for (i = 0; i < PSP_MAX_REGS; i++) 595 psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i); 596 ndev->psp_hdl = aie2m_psp_create(&xdna->ddev, &psp_conf); 597 if (!ndev->psp_hdl) { 598 XDNA_ERR(xdna, "failed to create psp"); 599 ret = -ENOMEM; 600 goto release_fw; 601 } 602 xdna->dev_handle = ndev; 603 604 ret = aie2_hw_start(xdna); 605 if (ret) { 606 XDNA_ERR(xdna, "start npu failed, ret %d", ret); 607 goto release_fw; 608 } 609 610 xrs_cfg.clk_list.num_levels = ndev->max_dpm_level + 1; 611 for (i = 0; i < xrs_cfg.clk_list.num_levels; i++) 612 xrs_cfg.clk_list.cu_clk_list[i] = ndev->priv->dpm_clk_tbl[i].hclk; 613 xrs_cfg.sys_eff_factor = 1; 614 xrs_cfg.ddev = &xdna->ddev; 615 xrs_cfg.actions = &aie2_xrs_actions; 616 xrs_cfg.total_col = ndev->total_col; 617 618 xdna->xrs_hdl = xrsm_init(&xrs_cfg); 619 if (!xdna->xrs_hdl) { 620 XDNA_ERR(xdna, "Initialize resolver failed"); 621 ret = -EINVAL; 622 goto stop_hw; 623 } 624 625 release_firmware(fw); 626 aie2_msg_init(ndev); 627 amdxdna_pm_init(xdna); 628 return 0; 629 630 stop_hw: 631 aie2_hw_stop(xdna); 632 release_fw: 633 release_firmware(fw); 634 635 return ret; 636 } 637 638 static void aie2_fini(struct amdxdna_dev *xdna) 639 { 640 amdxdna_pm_fini(xdna); 641 aie2_hw_stop(xdna); 642 } 643 644 static int aie2_get_aie_status(struct amdxdna_client *client, 645 struct amdxdna_drm_get_info *args) 646 { 647 struct amdxdna_drm_query_aie_status status; 648 struct amdxdna_dev *xdna = client->xdna; 649 struct amdxdna_dev_hdl *ndev; 650 int ret; 651 652 ndev = xdna->dev_handle; 653 if (copy_from_user(&status, u64_to_user_ptr(args->buffer), sizeof(status))) { 654 XDNA_ERR(xdna, "Failed to copy AIE request into kernel"); 655 return -EFAULT; 656 } 657 658 if (ndev->metadata.cols * ndev->metadata.size < status.buffer_size) { 659 XDNA_ERR(xdna, "Invalid buffer size. Given Size: %u. Need Size: %u.", 660 status.buffer_size, ndev->metadata.cols * ndev->metadata.size); 661 return -EINVAL; 662 } 663 664 ret = aie2_query_status(ndev, u64_to_user_ptr(status.buffer), 665 status.buffer_size, &status.cols_filled); 666 if (ret) { 667 XDNA_ERR(xdna, "Failed to get AIE status info. Ret: %d", ret); 668 return ret; 669 } 670 671 if (copy_to_user(u64_to_user_ptr(args->buffer), &status, sizeof(status))) { 672 XDNA_ERR(xdna, "Failed to copy AIE request info to user space"); 673 return -EFAULT; 674 } 675 676 return 0; 677 } 678 679 static int aie2_get_aie_metadata(struct amdxdna_client *client, 680 struct amdxdna_drm_get_info *args) 681 { 682 struct amdxdna_drm_query_aie_metadata *meta; 683 struct amdxdna_dev *xdna = client->xdna; 684 struct amdxdna_dev_hdl *ndev; 685 int ret = 0; 686 687 ndev = xdna->dev_handle; 688 meta = kzalloc_obj(*meta); 689 if (!meta) 690 return -ENOMEM; 691 692 meta->col_size = ndev->metadata.size; 693 meta->cols = ndev->metadata.cols; 694 meta->rows = ndev->metadata.rows; 695 696 meta->version.major = ndev->metadata.version.major; 697 meta->version.minor = ndev->metadata.version.minor; 698 699 meta->core.row_count = ndev->metadata.core.row_count; 700 meta->core.row_start = ndev->metadata.core.row_start; 701 meta->core.dma_channel_count = ndev->metadata.core.dma_channel_count; 702 meta->core.lock_count = ndev->metadata.core.lock_count; 703 meta->core.event_reg_count = ndev->metadata.core.event_reg_count; 704 705 meta->mem.row_count = ndev->metadata.mem.row_count; 706 meta->mem.row_start = ndev->metadata.mem.row_start; 707 meta->mem.dma_channel_count = ndev->metadata.mem.dma_channel_count; 708 meta->mem.lock_count = ndev->metadata.mem.lock_count; 709 meta->mem.event_reg_count = ndev->metadata.mem.event_reg_count; 710 711 meta->shim.row_count = ndev->metadata.shim.row_count; 712 meta->shim.row_start = ndev->metadata.shim.row_start; 713 meta->shim.dma_channel_count = ndev->metadata.shim.dma_channel_count; 714 meta->shim.lock_count = ndev->metadata.shim.lock_count; 715 meta->shim.event_reg_count = ndev->metadata.shim.event_reg_count; 716 717 if (copy_to_user(u64_to_user_ptr(args->buffer), meta, sizeof(*meta))) 718 ret = -EFAULT; 719 720 kfree(meta); 721 return ret; 722 } 723 724 static int aie2_get_aie_version(struct amdxdna_client *client, 725 struct amdxdna_drm_get_info *args) 726 { 727 struct amdxdna_drm_query_aie_version version; 728 struct amdxdna_dev *xdna = client->xdna; 729 struct amdxdna_dev_hdl *ndev; 730 731 ndev = xdna->dev_handle; 732 version.major = ndev->version.major; 733 version.minor = ndev->version.minor; 734 735 if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version))) 736 return -EFAULT; 737 738 return 0; 739 } 740 741 static int aie2_get_firmware_version(struct amdxdna_client *client, 742 struct amdxdna_drm_get_info *args) 743 { 744 struct amdxdna_drm_query_firmware_version version; 745 struct amdxdna_dev *xdna = client->xdna; 746 747 version.major = xdna->fw_ver.major; 748 version.minor = xdna->fw_ver.minor; 749 version.patch = xdna->fw_ver.sub; 750 version.build = xdna->fw_ver.build; 751 752 if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version))) 753 return -EFAULT; 754 755 return 0; 756 } 757 758 static int aie2_get_power_mode(struct amdxdna_client *client, 759 struct amdxdna_drm_get_info *args) 760 { 761 struct amdxdna_drm_get_power_mode mode = {}; 762 struct amdxdna_dev *xdna = client->xdna; 763 struct amdxdna_dev_hdl *ndev; 764 765 ndev = xdna->dev_handle; 766 mode.power_mode = ndev->pw_mode; 767 768 if (copy_to_user(u64_to_user_ptr(args->buffer), &mode, sizeof(mode))) 769 return -EFAULT; 770 771 return 0; 772 } 773 774 static int aie2_get_clock_metadata(struct amdxdna_client *client, 775 struct amdxdna_drm_get_info *args) 776 { 777 struct amdxdna_drm_query_clock_metadata *clock; 778 struct amdxdna_dev *xdna = client->xdna; 779 struct amdxdna_dev_hdl *ndev; 780 int ret = 0; 781 782 ndev = xdna->dev_handle; 783 clock = kzalloc_obj(*clock); 784 if (!clock) 785 return -ENOMEM; 786 787 snprintf(clock->mp_npu_clock.name, sizeof(clock->mp_npu_clock.name), 788 "MP-NPU Clock"); 789 clock->mp_npu_clock.freq_mhz = ndev->npuclk_freq; 790 snprintf(clock->h_clock.name, sizeof(clock->h_clock.name), "H Clock"); 791 clock->h_clock.freq_mhz = ndev->hclk_freq; 792 793 if (copy_to_user(u64_to_user_ptr(args->buffer), clock, sizeof(*clock))) 794 ret = -EFAULT; 795 796 kfree(clock); 797 return ret; 798 } 799 800 static int aie2_get_sensors(struct amdxdna_client *client, 801 struct amdxdna_drm_get_info *args) 802 { 803 struct amdxdna_dev_hdl *ndev = client->xdna->dev_handle; 804 struct amdxdna_drm_query_sensor sensor = {}; 805 struct amd_pmf_npu_metrics npu_metrics; 806 u32 sensors_count = 0, i; 807 int ret; 808 809 ret = AIE2_GET_PMF_NPU_METRICS(&npu_metrics); 810 if (ret) 811 return ret; 812 813 sensor.type = AMDXDNA_SENSOR_TYPE_POWER; 814 sensor.input = npu_metrics.npu_power; 815 sensor.unitm = -3; 816 scnprintf(sensor.label, sizeof(sensor.label), "Total Power"); 817 scnprintf(sensor.units, sizeof(sensor.units), "mW"); 818 819 if (copy_to_user(u64_to_user_ptr(args->buffer), &sensor, sizeof(sensor))) 820 return -EFAULT; 821 822 sensors_count++; 823 if (args->buffer_size <= sensors_count * sizeof(sensor)) 824 goto out; 825 826 for (i = 0; i < min_t(u32, ndev->total_col, 8); i++) { 827 memset(&sensor, 0, sizeof(sensor)); 828 sensor.input = npu_metrics.npu_busy[i]; 829 sensor.type = AMDXDNA_SENSOR_TYPE_COLUMN_UTILIZATION; 830 sensor.unitm = 0; 831 scnprintf(sensor.label, sizeof(sensor.label), "Column %d Utilization", i); 832 scnprintf(sensor.units, sizeof(sensor.units), "%%"); 833 834 if (copy_to_user(u64_to_user_ptr(args->buffer) + sensors_count * sizeof(sensor), 835 &sensor, sizeof(sensor))) 836 return -EFAULT; 837 838 sensors_count++; 839 if (args->buffer_size <= sensors_count * sizeof(sensor)) 840 goto out; 841 } 842 843 out: 844 args->buffer_size = sensors_count * sizeof(sensor); 845 846 return 0; 847 } 848 849 static int aie2_hwctx_status_cb(struct amdxdna_hwctx *hwctx, void *arg) 850 { 851 struct amdxdna_drm_hwctx_entry *tmp __free(kfree) = NULL; 852 struct amdxdna_drm_get_array *array_args = arg; 853 struct amdxdna_drm_hwctx_entry __user *buf; 854 struct app_health_report report; 855 struct amdxdna_dev_hdl *ndev; 856 u32 size; 857 int ret; 858 859 if (!array_args->num_element) 860 return -EINVAL; 861 862 tmp = kzalloc_obj(*tmp); 863 if (!tmp) 864 return -ENOMEM; 865 866 tmp->pid = hwctx->client->pid; 867 tmp->context_id = hwctx->id; 868 tmp->start_col = hwctx->start_col; 869 tmp->num_col = hwctx->num_col; 870 tmp->command_submissions = hwctx->priv->seq; 871 tmp->command_completions = hwctx->priv->completed; 872 tmp->pasid = hwctx->client->pasid; 873 tmp->heap_usage = hwctx->client->heap_usage; 874 tmp->priority = hwctx->qos.priority; 875 tmp->gops = hwctx->qos.gops; 876 tmp->fps = hwctx->qos.fps; 877 tmp->dma_bandwidth = hwctx->qos.dma_bandwidth; 878 tmp->latency = hwctx->qos.latency; 879 tmp->frame_exec_time = hwctx->qos.frame_exec_time; 880 tmp->state = AMDXDNA_HWCTX_STATE_ACTIVE; 881 ndev = hwctx->client->xdna->dev_handle; 882 ret = aie2_query_app_health(ndev, hwctx->fw_ctx_id, &report); 883 if (!ret) { 884 /* Fill in app health report fields */ 885 tmp->txn_op_idx = report.txn_op_id; 886 tmp->ctx_pc = report.ctx_pc; 887 tmp->fatal_error_type = report.fatal_info.fatal_type; 888 tmp->fatal_error_exception_type = report.fatal_info.exception_type; 889 tmp->fatal_error_exception_pc = report.fatal_info.exception_pc; 890 tmp->fatal_error_app_module = report.fatal_info.app_module; 891 } 892 893 buf = u64_to_user_ptr(array_args->buffer); 894 size = min(sizeof(*tmp), array_args->element_size); 895 896 if (copy_to_user(buf, tmp, size)) 897 return -EFAULT; 898 899 array_args->buffer += size; 900 array_args->num_element--; 901 902 return 0; 903 } 904 905 static int aie2_get_hwctx_status(struct amdxdna_client *client, 906 struct amdxdna_drm_get_info *args) 907 { 908 struct amdxdna_drm_get_array array_args; 909 struct amdxdna_dev *xdna = client->xdna; 910 struct amdxdna_client *tmp_client; 911 int ret; 912 913 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 914 915 array_args.element_size = sizeof(struct amdxdna_drm_query_hwctx); 916 array_args.buffer = args->buffer; 917 array_args.num_element = args->buffer_size / array_args.element_size; 918 list_for_each_entry(tmp_client, &xdna->client_list, node) { 919 ret = amdxdna_hwctx_walk(tmp_client, &array_args, 920 aie2_hwctx_status_cb); 921 if (ret) 922 break; 923 } 924 925 args->buffer_size -= (u32)(array_args.buffer - args->buffer); 926 return 0; 927 } 928 929 static int aie2_query_resource_info(struct amdxdna_client *client, 930 struct amdxdna_drm_get_info *args) 931 { 932 struct amdxdna_drm_get_resource_info res_info; 933 const struct amdxdna_dev_priv *priv; 934 struct amdxdna_dev_hdl *ndev; 935 struct amdxdna_dev *xdna; 936 937 xdna = client->xdna; 938 ndev = xdna->dev_handle; 939 priv = ndev->priv; 940 941 res_info.npu_clk_max = priv->dpm_clk_tbl[ndev->max_dpm_level].hclk; 942 res_info.npu_tops_max = ndev->max_tops; 943 res_info.npu_task_max = priv->hwctx_limit; 944 res_info.npu_tops_curr = ndev->curr_tops; 945 res_info.npu_task_curr = ndev->hwctx_num; 946 947 if (copy_to_user(u64_to_user_ptr(args->buffer), &res_info, sizeof(res_info))) 948 return -EFAULT; 949 950 return 0; 951 } 952 953 static int aie2_fill_hwctx_map(struct amdxdna_hwctx *hwctx, void *arg) 954 { 955 struct amdxdna_dev *xdna = hwctx->client->xdna; 956 u32 *map = arg; 957 958 if (hwctx->fw_ctx_id >= xdna->dev_handle->priv->hwctx_limit) { 959 XDNA_ERR(xdna, "Invalid fw ctx id %d/%d ", hwctx->fw_ctx_id, 960 xdna->dev_handle->priv->hwctx_limit); 961 return -EINVAL; 962 } 963 964 map[hwctx->fw_ctx_id] = hwctx->id; 965 return 0; 966 } 967 968 static int aie2_get_telemetry(struct amdxdna_client *client, 969 struct amdxdna_drm_get_info *args) 970 { 971 struct amdxdna_drm_query_telemetry_header *header __free(kfree) = NULL; 972 u32 telemetry_data_sz, header_sz, elem_num; 973 struct amdxdna_dev *xdna = client->xdna; 974 struct amdxdna_client *tmp_client; 975 int ret; 976 977 elem_num = xdna->dev_handle->priv->hwctx_limit; 978 header_sz = struct_size(header, map, elem_num); 979 if (args->buffer_size <= header_sz) { 980 XDNA_ERR(xdna, "Invalid buffer size"); 981 return -EINVAL; 982 } 983 984 telemetry_data_sz = args->buffer_size - header_sz; 985 if (telemetry_data_sz > SZ_4M) { 986 XDNA_ERR(xdna, "Buffer size is too big, %d", telemetry_data_sz); 987 return -EINVAL; 988 } 989 990 header = kzalloc(header_sz, GFP_KERNEL); 991 if (!header) 992 return -ENOMEM; 993 994 if (copy_from_user(header, u64_to_user_ptr(args->buffer), sizeof(*header))) { 995 XDNA_ERR(xdna, "Failed to copy telemetry header from user"); 996 return -EFAULT; 997 } 998 999 header->map_num_elements = elem_num; 1000 list_for_each_entry(tmp_client, &xdna->client_list, node) { 1001 ret = amdxdna_hwctx_walk(tmp_client, &header->map, 1002 aie2_fill_hwctx_map); 1003 if (ret) 1004 return ret; 1005 } 1006 1007 ret = aie2_query_telemetry(xdna->dev_handle, 1008 u64_to_user_ptr(args->buffer + header_sz), 1009 telemetry_data_sz, header); 1010 if (ret) { 1011 XDNA_ERR(xdna, "Query telemetry failed ret %d", ret); 1012 return ret; 1013 } 1014 1015 if (copy_to_user(u64_to_user_ptr(args->buffer), header, header_sz)) { 1016 XDNA_ERR(xdna, "Copy header failed"); 1017 return -EFAULT; 1018 } 1019 1020 return 0; 1021 } 1022 1023 static int aie2_get_preempt_state(struct amdxdna_client *client, 1024 struct amdxdna_drm_get_info *args) 1025 { 1026 struct amdxdna_drm_attribute_state state = {}; 1027 struct amdxdna_dev *xdna = client->xdna; 1028 struct amdxdna_dev_hdl *ndev; 1029 1030 ndev = xdna->dev_handle; 1031 if (args->param == DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE) 1032 state.state = ndev->force_preempt_enabled; 1033 else if (args->param == DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE) 1034 state.state = ndev->frame_boundary_preempt; 1035 1036 if (copy_to_user(u64_to_user_ptr(args->buffer), &state, sizeof(state))) 1037 return -EFAULT; 1038 1039 return 0; 1040 } 1041 1042 static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args) 1043 { 1044 struct amdxdna_dev *xdna = client->xdna; 1045 int ret, idx; 1046 1047 if (!drm_dev_enter(&xdna->ddev, &idx)) 1048 return -ENODEV; 1049 1050 ret = amdxdna_pm_resume_get_locked(xdna); 1051 if (ret) 1052 goto dev_exit; 1053 1054 switch (args->param) { 1055 case DRM_AMDXDNA_QUERY_AIE_STATUS: 1056 ret = aie2_get_aie_status(client, args); 1057 break; 1058 case DRM_AMDXDNA_QUERY_AIE_METADATA: 1059 ret = aie2_get_aie_metadata(client, args); 1060 break; 1061 case DRM_AMDXDNA_QUERY_AIE_VERSION: 1062 ret = aie2_get_aie_version(client, args); 1063 break; 1064 case DRM_AMDXDNA_QUERY_CLOCK_METADATA: 1065 ret = aie2_get_clock_metadata(client, args); 1066 break; 1067 case DRM_AMDXDNA_QUERY_SENSORS: 1068 ret = aie2_get_sensors(client, args); 1069 break; 1070 case DRM_AMDXDNA_QUERY_HW_CONTEXTS: 1071 ret = aie2_get_hwctx_status(client, args); 1072 break; 1073 case DRM_AMDXDNA_QUERY_FIRMWARE_VERSION: 1074 ret = aie2_get_firmware_version(client, args); 1075 break; 1076 case DRM_AMDXDNA_GET_POWER_MODE: 1077 ret = aie2_get_power_mode(client, args); 1078 break; 1079 case DRM_AMDXDNA_QUERY_TELEMETRY: 1080 ret = aie2_get_telemetry(client, args); 1081 break; 1082 case DRM_AMDXDNA_QUERY_RESOURCE_INFO: 1083 ret = aie2_query_resource_info(client, args); 1084 break; 1085 case DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE: 1086 case DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE: 1087 ret = aie2_get_preempt_state(client, args); 1088 break; 1089 default: 1090 XDNA_ERR(xdna, "Not supported request parameter %u", args->param); 1091 ret = -EOPNOTSUPP; 1092 } 1093 1094 amdxdna_pm_suspend_put(xdna); 1095 XDNA_DBG(xdna, "Got param %d", args->param); 1096 1097 dev_exit: 1098 drm_dev_exit(idx); 1099 return ret; 1100 } 1101 1102 static int aie2_query_ctx_status_array(struct amdxdna_client *client, 1103 struct amdxdna_drm_get_array *args) 1104 { 1105 struct amdxdna_drm_get_array array_args; 1106 struct amdxdna_dev *xdna = client->xdna; 1107 struct amdxdna_client *tmp_client; 1108 int ret; 1109 1110 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 1111 1112 if (args->element_size > SZ_4K || args->num_element > SZ_1K) { 1113 XDNA_DBG(xdna, "Invalid element size %d or number of element %d", 1114 args->element_size, args->num_element); 1115 return -EINVAL; 1116 } 1117 1118 array_args.element_size = min(args->element_size, 1119 sizeof(struct amdxdna_drm_hwctx_entry)); 1120 array_args.buffer = args->buffer; 1121 array_args.num_element = args->num_element * args->element_size / 1122 array_args.element_size; 1123 list_for_each_entry(tmp_client, &xdna->client_list, node) { 1124 ret = amdxdna_hwctx_walk(tmp_client, &array_args, 1125 aie2_hwctx_status_cb); 1126 if (ret) 1127 break; 1128 } 1129 1130 args->element_size = array_args.element_size; 1131 args->num_element = (u32)((array_args.buffer - args->buffer) / 1132 args->element_size); 1133 1134 return 0; 1135 } 1136 1137 static int aie2_get_array(struct amdxdna_client *client, 1138 struct amdxdna_drm_get_array *args) 1139 { 1140 struct amdxdna_dev *xdna = client->xdna; 1141 int ret, idx; 1142 1143 if (!drm_dev_enter(&xdna->ddev, &idx)) 1144 return -ENODEV; 1145 1146 ret = amdxdna_pm_resume_get_locked(xdna); 1147 if (ret) 1148 goto dev_exit; 1149 1150 switch (args->param) { 1151 case DRM_AMDXDNA_HW_CONTEXT_ALL: 1152 ret = aie2_query_ctx_status_array(client, args); 1153 break; 1154 case DRM_AMDXDNA_HW_LAST_ASYNC_ERR: 1155 ret = aie2_get_array_async_error(xdna->dev_handle, args); 1156 break; 1157 case DRM_AMDXDNA_BO_USAGE: 1158 ret = amdxdna_drm_get_bo_usage(&xdna->ddev, args); 1159 break; 1160 default: 1161 XDNA_ERR(xdna, "Not supported request parameter %u", args->param); 1162 ret = -EOPNOTSUPP; 1163 } 1164 1165 amdxdna_pm_suspend_put(xdna); 1166 XDNA_DBG(xdna, "Got param %d", args->param); 1167 1168 dev_exit: 1169 drm_dev_exit(idx); 1170 return ret; 1171 } 1172 1173 static int aie2_set_power_mode(struct amdxdna_client *client, 1174 struct amdxdna_drm_set_state *args) 1175 { 1176 struct amdxdna_drm_set_power_mode power_state; 1177 enum amdxdna_power_mode_type power_mode; 1178 struct amdxdna_dev *xdna = client->xdna; 1179 1180 if (copy_from_user(&power_state, u64_to_user_ptr(args->buffer), 1181 sizeof(power_state))) { 1182 XDNA_ERR(xdna, "Failed to copy power mode request into kernel"); 1183 return -EFAULT; 1184 } 1185 1186 if (XDNA_MBZ_DBG(xdna, power_state.pad, sizeof(power_state.pad))) 1187 return -EINVAL; 1188 1189 power_mode = power_state.power_mode; 1190 if (power_mode > POWER_MODE_TURBO) { 1191 XDNA_ERR(xdna, "Invalid power mode %d", power_mode); 1192 return -EINVAL; 1193 } 1194 1195 return aie2_pm_set_mode(xdna->dev_handle, power_mode); 1196 } 1197 1198 static int aie2_set_preempt_state(struct amdxdna_client *client, 1199 struct amdxdna_drm_set_state *args) 1200 { 1201 struct amdxdna_dev_hdl *ndev = client->xdna->dev_handle; 1202 struct amdxdna_drm_attribute_state state; 1203 u32 val; 1204 int ret; 1205 1206 if (copy_from_user(&state, u64_to_user_ptr(args->buffer), sizeof(state))) 1207 return -EFAULT; 1208 1209 if (state.state > 1) 1210 return -EINVAL; 1211 1212 if (XDNA_MBZ_DBG(client->xdna, state.pad, sizeof(state.pad))) 1213 return -EINVAL; 1214 1215 if (args->param == DRM_AMDXDNA_SET_FORCE_PREEMPT) { 1216 ndev->force_preempt_enabled = state.state; 1217 } else if (args->param == DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT) { 1218 val = state.state; 1219 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT, 1220 &val); 1221 if (ret) 1222 return ret; 1223 1224 ndev->frame_boundary_preempt = state.state; 1225 } 1226 1227 return 0; 1228 } 1229 1230 static int aie2_set_state(struct amdxdna_client *client, 1231 struct amdxdna_drm_set_state *args) 1232 { 1233 struct amdxdna_dev *xdna = client->xdna; 1234 int ret, idx; 1235 1236 if (!drm_dev_enter(&xdna->ddev, &idx)) 1237 return -ENODEV; 1238 1239 ret = amdxdna_pm_resume_get_locked(xdna); 1240 if (ret) 1241 goto dev_exit; 1242 1243 switch (args->param) { 1244 case DRM_AMDXDNA_SET_POWER_MODE: 1245 ret = aie2_set_power_mode(client, args); 1246 break; 1247 case DRM_AMDXDNA_SET_FORCE_PREEMPT: 1248 case DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT: 1249 ret = aie2_set_preempt_state(client, args); 1250 break; 1251 default: 1252 XDNA_ERR(xdna, "Not supported request parameter %u", args->param); 1253 ret = -EOPNOTSUPP; 1254 break; 1255 } 1256 1257 amdxdna_pm_suspend_put(xdna); 1258 dev_exit: 1259 drm_dev_exit(idx); 1260 return ret; 1261 } 1262 1263 const struct amdxdna_dev_ops aie2_ops = { 1264 .init = aie2_init, 1265 .fini = aie2_fini, 1266 .resume = aie2_hw_resume, 1267 .suspend = aie2_hw_suspend, 1268 .get_aie_info = aie2_get_info, 1269 .set_aie_state = aie2_set_state, 1270 .hwctx_init = aie2_hwctx_init, 1271 .hwctx_fini = aie2_hwctx_fini, 1272 .hwctx_config = aie2_hwctx_config, 1273 .hwctx_sync_debug_bo = aie2_hwctx_sync_debug_bo, 1274 .cmd_submit = aie2_cmd_submit, 1275 .hmm_invalidate = aie2_hmm_invalidate, 1276 .get_array = aie2_get_array, 1277 }; 1278