1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_gem_shmem_helper.h> 10 #include <drm/drm_managed.h> 11 #include <drm/drm_print.h> 12 #include <drm/gpu_scheduler.h> 13 #include <linux/cleanup.h> 14 #include <linux/errno.h> 15 #include <linux/firmware.h> 16 #include <linux/iommu.h> 17 #include <linux/iopoll.h> 18 #include <linux/pci.h> 19 #include <linux/xarray.h> 20 #include <asm/hypervisor.h> 21 22 #include "aie2_msg_priv.h" 23 #include "aie2_pci.h" 24 #include "aie2_solver.h" 25 #include "amdxdna_ctx.h" 26 #include "amdxdna_gem.h" 27 #include "amdxdna_mailbox.h" 28 #include "amdxdna_pci_drv.h" 29 #include "amdxdna_pm.h" 30 31 static int aie2_max_col = XRS_MAX_COL; 32 module_param(aie2_max_col, uint, 0600); 33 MODULE_PARM_DESC(aie2_max_col, "Maximum column could be used"); 34 35 /* 36 * The management mailbox channel is allocated by firmware. 37 * The related register and ring buffer information is on SRAM BAR. 38 * This struct is the register layout. 39 */ 40 #define MGMT_MBOX_MAGIC 0x55504e5f /* _NPU */ 41 struct mgmt_mbox_chann_info { 42 __u32 x2i_tail; 43 __u32 x2i_head; 44 __u32 x2i_buf; 45 __u32 x2i_buf_sz; 46 __u32 i2x_tail; 47 __u32 i2x_head; 48 __u32 i2x_buf; 49 __u32 i2x_buf_sz; 50 __u32 magic; 51 __u32 msi_id; 52 __u32 prot_major; 53 __u32 prot_minor; 54 __u32 rsvd[4]; 55 }; 56 57 static int aie2_check_protocol(struct amdxdna_dev_hdl *ndev, u32 fw_major, u32 fw_minor) 58 { 59 const struct aie2_fw_feature_tbl *feature; 60 bool found = false; 61 62 for (feature = ndev->priv->fw_feature_tbl; feature->major; feature++) { 63 if (feature->major != fw_major) 64 continue; 65 if (fw_minor < feature->min_minor) 66 continue; 67 if (feature->max_minor > 0 && fw_minor > feature->max_minor) 68 continue; 69 70 ndev->feature_mask |= feature->features; 71 72 /* firmware version matches one of the driver support entry */ 73 found = true; 74 } 75 76 return found ? 0 : -EOPNOTSUPP; 77 } 78 79 static void aie2_dump_chann_info_debug(struct amdxdna_dev_hdl *ndev) 80 { 81 struct amdxdna_dev *xdna = ndev->xdna; 82 83 XDNA_DBG(xdna, "i2x tail 0x%x", ndev->mgmt_i2x.mb_tail_ptr_reg); 84 XDNA_DBG(xdna, "i2x head 0x%x", ndev->mgmt_i2x.mb_head_ptr_reg); 85 XDNA_DBG(xdna, "i2x ringbuf 0x%x", ndev->mgmt_i2x.rb_start_addr); 86 XDNA_DBG(xdna, "i2x rsize 0x%x", ndev->mgmt_i2x.rb_size); 87 XDNA_DBG(xdna, "x2i tail 0x%x", ndev->mgmt_x2i.mb_tail_ptr_reg); 88 XDNA_DBG(xdna, "x2i head 0x%x", ndev->mgmt_x2i.mb_head_ptr_reg); 89 XDNA_DBG(xdna, "x2i ringbuf 0x%x", ndev->mgmt_x2i.rb_start_addr); 90 XDNA_DBG(xdna, "x2i rsize 0x%x", ndev->mgmt_x2i.rb_size); 91 XDNA_DBG(xdna, "x2i chann index 0x%x", ndev->mgmt_chan_idx); 92 XDNA_DBG(xdna, "mailbox protocol major 0x%x", ndev->mgmt_prot_major); 93 XDNA_DBG(xdna, "mailbox protocol minor 0x%x", ndev->mgmt_prot_minor); 94 } 95 96 static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev) 97 { 98 struct mgmt_mbox_chann_info info_regs; 99 struct xdna_mailbox_chann_res *i2x; 100 struct xdna_mailbox_chann_res *x2i; 101 u32 addr, off; 102 u32 *reg; 103 int ret; 104 int i; 105 106 /* 107 * Once firmware is alive, it will write management channel 108 * information in SRAM BAR and write the address of that information 109 * at FW_ALIVE_OFF offset in SRMA BAR. 110 * 111 * Read a non-zero value from FW_ALIVE_OFF implies that firmware 112 * is alive. 113 */ 114 ret = readx_poll_timeout(readl, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF), 115 addr, addr, AIE2_INTERVAL, AIE2_TIMEOUT); 116 if (ret || !addr) 117 return -ETIME; 118 119 off = AIE2_SRAM_OFF(ndev, addr); 120 reg = (u32 *)&info_regs; 121 for (i = 0; i < sizeof(info_regs) / sizeof(u32); i++) 122 reg[i] = readl(ndev->sram_base + off + i * sizeof(u32)); 123 124 if (info_regs.magic != MGMT_MBOX_MAGIC) { 125 XDNA_ERR(ndev->xdna, "Invalid mbox magic 0x%x", info_regs.magic); 126 ret = -EINVAL; 127 goto done; 128 } 129 130 i2x = &ndev->mgmt_i2x; 131 x2i = &ndev->mgmt_x2i; 132 133 i2x->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_head); 134 i2x->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_tail); 135 i2x->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.i2x_buf); 136 i2x->rb_size = info_regs.i2x_buf_sz; 137 138 x2i->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_head); 139 x2i->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_tail); 140 x2i->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.x2i_buf); 141 x2i->rb_size = info_regs.x2i_buf_sz; 142 143 ndev->mgmt_chan_idx = info_regs.msi_id; 144 ndev->mgmt_prot_major = info_regs.prot_major; 145 ndev->mgmt_prot_minor = info_regs.prot_minor; 146 147 ret = aie2_check_protocol(ndev, ndev->mgmt_prot_major, ndev->mgmt_prot_minor); 148 149 done: 150 aie2_dump_chann_info_debug(ndev); 151 152 /* Must clear address at FW_ALIVE_OFF */ 153 writel(0, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF)); 154 155 return ret; 156 } 157 158 int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev, 159 enum rt_config_category category, u32 *val) 160 { 161 const struct rt_config *cfg; 162 u32 value; 163 int ret; 164 165 for (cfg = ndev->priv->rt_config; cfg->type; cfg++) { 166 if (cfg->category != category) 167 continue; 168 169 if (cfg->feature_mask && 170 bitmap_subset(&cfg->feature_mask, &ndev->feature_mask, AIE2_FEATURE_MAX)) 171 continue; 172 173 value = val ? *val : cfg->value; 174 ret = aie2_set_runtime_cfg(ndev, cfg->type, value); 175 if (ret) { 176 XDNA_ERR(ndev->xdna, "Set type %d value %d failed", 177 cfg->type, value); 178 return ret; 179 } 180 } 181 182 return 0; 183 } 184 185 static int aie2_xdna_reset(struct amdxdna_dev_hdl *ndev) 186 { 187 int ret; 188 189 ret = aie2_suspend_fw(ndev); 190 if (ret) { 191 XDNA_ERR(ndev->xdna, "Suspend firmware failed"); 192 return ret; 193 } 194 195 ret = aie2_resume_fw(ndev); 196 if (ret) { 197 XDNA_ERR(ndev->xdna, "Resume firmware failed"); 198 return ret; 199 } 200 201 return 0; 202 } 203 204 static int aie2_mgmt_fw_init(struct amdxdna_dev_hdl *ndev) 205 { 206 int ret; 207 208 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_INIT, NULL); 209 if (ret) { 210 XDNA_ERR(ndev->xdna, "Runtime config failed"); 211 return ret; 212 } 213 214 ret = aie2_assign_mgmt_pasid(ndev, 0); 215 if (ret) { 216 XDNA_ERR(ndev->xdna, "Can not assign PASID"); 217 return ret; 218 } 219 220 ret = aie2_xdna_reset(ndev); 221 if (ret) { 222 XDNA_ERR(ndev->xdna, "Reset firmware failed"); 223 return ret; 224 } 225 226 return 0; 227 } 228 229 static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev) 230 { 231 int ret; 232 233 ret = aie2_query_firmware_version(ndev, &ndev->xdna->fw_ver); 234 if (ret) { 235 XDNA_ERR(ndev->xdna, "query firmware version failed"); 236 return ret; 237 } 238 239 ret = aie2_query_aie_version(ndev, &ndev->version); 240 if (ret) { 241 XDNA_ERR(ndev->xdna, "Query AIE version failed"); 242 return ret; 243 } 244 245 ret = aie2_query_aie_metadata(ndev, &ndev->metadata); 246 if (ret) { 247 XDNA_ERR(ndev->xdna, "Query AIE metadata failed"); 248 return ret; 249 } 250 251 ndev->total_col = min(aie2_max_col, ndev->metadata.cols); 252 253 return 0; 254 } 255 256 static void aie2_mgmt_fw_fini(struct amdxdna_dev_hdl *ndev) 257 { 258 if (aie2_suspend_fw(ndev)) 259 XDNA_ERR(ndev->xdna, "Suspend_fw failed"); 260 XDNA_DBG(ndev->xdna, "Firmware suspended"); 261 } 262 263 static int aie2_xrs_load(void *cb_arg, struct xrs_action_load *action) 264 { 265 struct amdxdna_hwctx *hwctx = cb_arg; 266 struct amdxdna_dev *xdna; 267 int ret; 268 269 xdna = hwctx->client->xdna; 270 271 hwctx->start_col = action->part.start_col; 272 hwctx->num_col = action->part.ncols; 273 ret = aie2_create_context(xdna->dev_handle, hwctx); 274 if (ret) 275 XDNA_ERR(xdna, "create context failed, ret %d", ret); 276 277 return ret; 278 } 279 280 static int aie2_xrs_unload(void *cb_arg) 281 { 282 struct amdxdna_hwctx *hwctx = cb_arg; 283 struct amdxdna_dev *xdna; 284 int ret; 285 286 xdna = hwctx->client->xdna; 287 288 ret = aie2_destroy_context(xdna->dev_handle, hwctx); 289 if (ret) 290 XDNA_ERR(xdna, "destroy context failed, ret %d", ret); 291 292 return ret; 293 } 294 295 static int aie2_xrs_set_dft_dpm_level(struct drm_device *ddev, u32 dpm_level) 296 { 297 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 298 struct amdxdna_dev_hdl *ndev; 299 300 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 301 302 ndev = xdna->dev_handle; 303 ndev->dft_dpm_level = dpm_level; 304 if (ndev->pw_mode != POWER_MODE_DEFAULT || ndev->dpm_level == dpm_level) 305 return 0; 306 307 return aie2_pm_set_dpm(ndev, dpm_level); 308 } 309 310 static struct xrs_action_ops aie2_xrs_actions = { 311 .load = aie2_xrs_load, 312 .unload = aie2_xrs_unload, 313 .set_dft_dpm_level = aie2_xrs_set_dft_dpm_level, 314 }; 315 316 static void aie2_hw_stop(struct amdxdna_dev *xdna) 317 { 318 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 319 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 320 321 if (ndev->dev_status <= AIE2_DEV_INIT) { 322 XDNA_ERR(xdna, "device is already stopped"); 323 return; 324 } 325 326 aie2_mgmt_fw_fini(ndev); 327 xdna_mailbox_stop_channel(ndev->mgmt_chann); 328 xdna_mailbox_destroy_channel(ndev->mgmt_chann); 329 ndev->mgmt_chann = NULL; 330 drmm_kfree(&xdna->ddev, ndev->mbox); 331 ndev->mbox = NULL; 332 aie2_psp_stop(ndev->psp_hdl); 333 aie2_smu_fini(ndev); 334 aie2_error_async_events_free(ndev); 335 pci_disable_device(pdev); 336 337 ndev->dev_status = AIE2_DEV_INIT; 338 } 339 340 static int aie2_hw_start(struct amdxdna_dev *xdna) 341 { 342 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 343 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 344 struct xdna_mailbox_res mbox_res; 345 u32 xdna_mailbox_intr_reg; 346 int mgmt_mb_irq, ret; 347 348 if (ndev->dev_status >= AIE2_DEV_START) { 349 XDNA_INFO(xdna, "device is already started"); 350 return 0; 351 } 352 353 ret = pci_enable_device(pdev); 354 if (ret) { 355 XDNA_ERR(xdna, "failed to enable device, ret %d", ret); 356 return ret; 357 } 358 pci_set_master(pdev); 359 360 ret = aie2_smu_init(ndev); 361 if (ret) { 362 XDNA_ERR(xdna, "failed to init smu, ret %d", ret); 363 goto disable_dev; 364 } 365 366 ret = aie2_psp_start(ndev->psp_hdl); 367 if (ret) { 368 XDNA_ERR(xdna, "failed to start psp, ret %d", ret); 369 goto fini_smu; 370 } 371 372 ret = aie2_get_mgmt_chann_info(ndev); 373 if (ret) { 374 XDNA_ERR(xdna, "firmware is not alive"); 375 goto stop_psp; 376 } 377 378 mbox_res.ringbuf_base = ndev->sram_base; 379 mbox_res.ringbuf_size = pci_resource_len(pdev, xdna->dev_info->sram_bar); 380 mbox_res.mbox_base = ndev->mbox_base; 381 mbox_res.mbox_size = MBOX_SIZE(ndev); 382 mbox_res.name = "xdna_mailbox"; 383 ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res); 384 if (!ndev->mbox) { 385 XDNA_ERR(xdna, "failed to create mailbox device"); 386 ret = -ENODEV; 387 goto stop_psp; 388 } 389 390 mgmt_mb_irq = pci_irq_vector(pdev, ndev->mgmt_chan_idx); 391 if (mgmt_mb_irq < 0) { 392 ret = mgmt_mb_irq; 393 XDNA_ERR(xdna, "failed to alloc irq vector, ret %d", ret); 394 goto stop_psp; 395 } 396 397 xdna_mailbox_intr_reg = ndev->mgmt_i2x.mb_head_ptr_reg + 4; 398 ndev->mgmt_chann = xdna_mailbox_create_channel(ndev->mbox, 399 &ndev->mgmt_x2i, 400 &ndev->mgmt_i2x, 401 xdna_mailbox_intr_reg, 402 mgmt_mb_irq); 403 if (!ndev->mgmt_chann) { 404 XDNA_ERR(xdna, "failed to create management mailbox channel"); 405 ret = -EINVAL; 406 goto stop_psp; 407 } 408 409 ret = aie2_pm_init(ndev); 410 if (ret) { 411 XDNA_ERR(xdna, "failed to init pm, ret %d", ret); 412 goto destroy_mgmt_chann; 413 } 414 415 ret = aie2_mgmt_fw_init(ndev); 416 if (ret) { 417 XDNA_ERR(xdna, "initial mgmt firmware failed, ret %d", ret); 418 goto destroy_mgmt_chann; 419 } 420 421 ret = aie2_mgmt_fw_query(ndev); 422 if (ret) { 423 XDNA_ERR(xdna, "failed to query fw, ret %d", ret); 424 goto destroy_mgmt_chann; 425 } 426 427 ret = aie2_error_async_events_alloc(ndev); 428 if (ret) { 429 XDNA_ERR(xdna, "Allocate async events failed, ret %d", ret); 430 goto destroy_mgmt_chann; 431 } 432 433 ndev->dev_status = AIE2_DEV_START; 434 435 return 0; 436 437 destroy_mgmt_chann: 438 xdna_mailbox_stop_channel(ndev->mgmt_chann); 439 xdna_mailbox_destroy_channel(ndev->mgmt_chann); 440 stop_psp: 441 aie2_psp_stop(ndev->psp_hdl); 442 fini_smu: 443 aie2_smu_fini(ndev); 444 disable_dev: 445 pci_disable_device(pdev); 446 447 return ret; 448 } 449 450 static int aie2_hw_suspend(struct amdxdna_dev *xdna) 451 { 452 struct amdxdna_client *client; 453 454 guard(mutex)(&xdna->dev_lock); 455 list_for_each_entry(client, &xdna->client_list, node) 456 aie2_hwctx_suspend(client); 457 458 aie2_hw_stop(xdna); 459 460 return 0; 461 } 462 463 static int aie2_hw_resume(struct amdxdna_dev *xdna) 464 { 465 struct amdxdna_client *client; 466 int ret; 467 468 ret = aie2_hw_start(xdna); 469 if (ret) { 470 XDNA_ERR(xdna, "Start hardware failed, %d", ret); 471 return ret; 472 } 473 474 list_for_each_entry(client, &xdna->client_list, node) { 475 ret = aie2_hwctx_resume(client); 476 if (ret) 477 break; 478 } 479 480 return ret; 481 } 482 483 static int aie2_init(struct amdxdna_dev *xdna) 484 { 485 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev); 486 void __iomem *tbl[PCI_NUM_RESOURCES] = {0}; 487 struct init_config xrs_cfg = { 0 }; 488 struct amdxdna_dev_hdl *ndev; 489 struct psp_config psp_conf; 490 const struct firmware *fw; 491 unsigned long bars = 0; 492 int i, nvec, ret; 493 494 if (!hypervisor_is_type(X86_HYPER_NATIVE)) { 495 XDNA_ERR(xdna, "Running under hypervisor not supported"); 496 return -EINVAL; 497 } 498 499 ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL); 500 if (!ndev) 501 return -ENOMEM; 502 503 ndev->priv = xdna->dev_info->dev_priv; 504 ndev->xdna = xdna; 505 506 ret = request_firmware(&fw, ndev->priv->fw_path, &pdev->dev); 507 if (ret) { 508 XDNA_ERR(xdna, "failed to request_firmware %s, ret %d", 509 ndev->priv->fw_path, ret); 510 return ret; 511 } 512 513 ret = pcim_enable_device(pdev); 514 if (ret) { 515 XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret); 516 goto release_fw; 517 } 518 519 for (i = 0; i < PSP_MAX_REGS; i++) 520 set_bit(PSP_REG_BAR(ndev, i), &bars); 521 522 set_bit(xdna->dev_info->sram_bar, &bars); 523 set_bit(xdna->dev_info->smu_bar, &bars); 524 set_bit(xdna->dev_info->mbox_bar, &bars); 525 526 for (i = 0; i < PCI_NUM_RESOURCES; i++) { 527 if (!test_bit(i, &bars)) 528 continue; 529 tbl[i] = pcim_iomap(pdev, i, 0); 530 if (!tbl[i]) { 531 XDNA_ERR(xdna, "map bar %d failed", i); 532 ret = -ENOMEM; 533 goto release_fw; 534 } 535 } 536 537 ndev->sram_base = tbl[xdna->dev_info->sram_bar]; 538 ndev->smu_base = tbl[xdna->dev_info->smu_bar]; 539 ndev->mbox_base = tbl[xdna->dev_info->mbox_bar]; 540 541 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 542 if (ret) { 543 XDNA_ERR(xdna, "Failed to set DMA mask: %d", ret); 544 goto release_fw; 545 } 546 547 nvec = pci_msix_vec_count(pdev); 548 if (nvec <= 0) { 549 XDNA_ERR(xdna, "does not get number of interrupt vector"); 550 ret = -EINVAL; 551 goto release_fw; 552 } 553 554 ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); 555 if (ret < 0) { 556 XDNA_ERR(xdna, "failed to alloc irq vectors, ret %d", ret); 557 goto release_fw; 558 } 559 560 psp_conf.fw_size = fw->size; 561 psp_conf.fw_buf = fw->data; 562 for (i = 0; i < PSP_MAX_REGS; i++) 563 psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i); 564 ndev->psp_hdl = aie2m_psp_create(&xdna->ddev, &psp_conf); 565 if (!ndev->psp_hdl) { 566 XDNA_ERR(xdna, "failed to create psp"); 567 ret = -ENOMEM; 568 goto release_fw; 569 } 570 xdna->dev_handle = ndev; 571 572 ret = aie2_hw_start(xdna); 573 if (ret) { 574 XDNA_ERR(xdna, "start npu failed, ret %d", ret); 575 goto release_fw; 576 } 577 578 xrs_cfg.clk_list.num_levels = ndev->max_dpm_level + 1; 579 for (i = 0; i < xrs_cfg.clk_list.num_levels; i++) 580 xrs_cfg.clk_list.cu_clk_list[i] = ndev->priv->dpm_clk_tbl[i].hclk; 581 xrs_cfg.sys_eff_factor = 1; 582 xrs_cfg.ddev = &xdna->ddev; 583 xrs_cfg.actions = &aie2_xrs_actions; 584 xrs_cfg.total_col = ndev->total_col; 585 586 xdna->xrs_hdl = xrsm_init(&xrs_cfg); 587 if (!xdna->xrs_hdl) { 588 XDNA_ERR(xdna, "Initialize resolver failed"); 589 ret = -EINVAL; 590 goto stop_hw; 591 } 592 593 release_firmware(fw); 594 aie2_msg_init(ndev); 595 amdxdna_pm_init(xdna); 596 return 0; 597 598 stop_hw: 599 aie2_hw_stop(xdna); 600 release_fw: 601 release_firmware(fw); 602 603 return ret; 604 } 605 606 static void aie2_fini(struct amdxdna_dev *xdna) 607 { 608 amdxdna_pm_fini(xdna); 609 aie2_hw_stop(xdna); 610 } 611 612 static int aie2_get_aie_status(struct amdxdna_client *client, 613 struct amdxdna_drm_get_info *args) 614 { 615 struct amdxdna_drm_query_aie_status status; 616 struct amdxdna_dev *xdna = client->xdna; 617 struct amdxdna_dev_hdl *ndev; 618 int ret; 619 620 ndev = xdna->dev_handle; 621 if (copy_from_user(&status, u64_to_user_ptr(args->buffer), sizeof(status))) { 622 XDNA_ERR(xdna, "Failed to copy AIE request into kernel"); 623 return -EFAULT; 624 } 625 626 if (ndev->metadata.cols * ndev->metadata.size < status.buffer_size) { 627 XDNA_ERR(xdna, "Invalid buffer size. Given Size: %u. Need Size: %u.", 628 status.buffer_size, ndev->metadata.cols * ndev->metadata.size); 629 return -EINVAL; 630 } 631 632 ret = aie2_query_status(ndev, u64_to_user_ptr(status.buffer), 633 status.buffer_size, &status.cols_filled); 634 if (ret) { 635 XDNA_ERR(xdna, "Failed to get AIE status info. Ret: %d", ret); 636 return ret; 637 } 638 639 if (copy_to_user(u64_to_user_ptr(args->buffer), &status, sizeof(status))) { 640 XDNA_ERR(xdna, "Failed to copy AIE request info to user space"); 641 return -EFAULT; 642 } 643 644 return 0; 645 } 646 647 static int aie2_get_aie_metadata(struct amdxdna_client *client, 648 struct amdxdna_drm_get_info *args) 649 { 650 struct amdxdna_drm_query_aie_metadata *meta; 651 struct amdxdna_dev *xdna = client->xdna; 652 struct amdxdna_dev_hdl *ndev; 653 int ret = 0; 654 655 ndev = xdna->dev_handle; 656 meta = kzalloc_obj(*meta); 657 if (!meta) 658 return -ENOMEM; 659 660 meta->col_size = ndev->metadata.size; 661 meta->cols = ndev->metadata.cols; 662 meta->rows = ndev->metadata.rows; 663 664 meta->version.major = ndev->metadata.version.major; 665 meta->version.minor = ndev->metadata.version.minor; 666 667 meta->core.row_count = ndev->metadata.core.row_count; 668 meta->core.row_start = ndev->metadata.core.row_start; 669 meta->core.dma_channel_count = ndev->metadata.core.dma_channel_count; 670 meta->core.lock_count = ndev->metadata.core.lock_count; 671 meta->core.event_reg_count = ndev->metadata.core.event_reg_count; 672 673 meta->mem.row_count = ndev->metadata.mem.row_count; 674 meta->mem.row_start = ndev->metadata.mem.row_start; 675 meta->mem.dma_channel_count = ndev->metadata.mem.dma_channel_count; 676 meta->mem.lock_count = ndev->metadata.mem.lock_count; 677 meta->mem.event_reg_count = ndev->metadata.mem.event_reg_count; 678 679 meta->shim.row_count = ndev->metadata.shim.row_count; 680 meta->shim.row_start = ndev->metadata.shim.row_start; 681 meta->shim.dma_channel_count = ndev->metadata.shim.dma_channel_count; 682 meta->shim.lock_count = ndev->metadata.shim.lock_count; 683 meta->shim.event_reg_count = ndev->metadata.shim.event_reg_count; 684 685 if (copy_to_user(u64_to_user_ptr(args->buffer), meta, sizeof(*meta))) 686 ret = -EFAULT; 687 688 kfree(meta); 689 return ret; 690 } 691 692 static int aie2_get_aie_version(struct amdxdna_client *client, 693 struct amdxdna_drm_get_info *args) 694 { 695 struct amdxdna_drm_query_aie_version version; 696 struct amdxdna_dev *xdna = client->xdna; 697 struct amdxdna_dev_hdl *ndev; 698 699 ndev = xdna->dev_handle; 700 version.major = ndev->version.major; 701 version.minor = ndev->version.minor; 702 703 if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version))) 704 return -EFAULT; 705 706 return 0; 707 } 708 709 static int aie2_get_firmware_version(struct amdxdna_client *client, 710 struct amdxdna_drm_get_info *args) 711 { 712 struct amdxdna_drm_query_firmware_version version; 713 struct amdxdna_dev *xdna = client->xdna; 714 715 version.major = xdna->fw_ver.major; 716 version.minor = xdna->fw_ver.minor; 717 version.patch = xdna->fw_ver.sub; 718 version.build = xdna->fw_ver.build; 719 720 if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version))) 721 return -EFAULT; 722 723 return 0; 724 } 725 726 static int aie2_get_power_mode(struct amdxdna_client *client, 727 struct amdxdna_drm_get_info *args) 728 { 729 struct amdxdna_drm_get_power_mode mode = {}; 730 struct amdxdna_dev *xdna = client->xdna; 731 struct amdxdna_dev_hdl *ndev; 732 733 ndev = xdna->dev_handle; 734 mode.power_mode = ndev->pw_mode; 735 736 if (copy_to_user(u64_to_user_ptr(args->buffer), &mode, sizeof(mode))) 737 return -EFAULT; 738 739 return 0; 740 } 741 742 static int aie2_get_clock_metadata(struct amdxdna_client *client, 743 struct amdxdna_drm_get_info *args) 744 { 745 struct amdxdna_drm_query_clock_metadata *clock; 746 struct amdxdna_dev *xdna = client->xdna; 747 struct amdxdna_dev_hdl *ndev; 748 int ret = 0; 749 750 ndev = xdna->dev_handle; 751 clock = kzalloc_obj(*clock); 752 if (!clock) 753 return -ENOMEM; 754 755 snprintf(clock->mp_npu_clock.name, sizeof(clock->mp_npu_clock.name), 756 "MP-NPU Clock"); 757 clock->mp_npu_clock.freq_mhz = ndev->npuclk_freq; 758 snprintf(clock->h_clock.name, sizeof(clock->h_clock.name), "H Clock"); 759 clock->h_clock.freq_mhz = ndev->hclk_freq; 760 761 if (copy_to_user(u64_to_user_ptr(args->buffer), clock, sizeof(*clock))) 762 ret = -EFAULT; 763 764 kfree(clock); 765 return ret; 766 } 767 768 static int aie2_hwctx_status_cb(struct amdxdna_hwctx *hwctx, void *arg) 769 { 770 struct amdxdna_drm_hwctx_entry *tmp __free(kfree) = NULL; 771 struct amdxdna_drm_get_array *array_args = arg; 772 struct amdxdna_drm_hwctx_entry __user *buf; 773 u32 size; 774 775 if (!array_args->num_element) 776 return -EINVAL; 777 778 tmp = kzalloc_obj(*tmp); 779 if (!tmp) 780 return -ENOMEM; 781 782 tmp->pid = hwctx->client->pid; 783 tmp->context_id = hwctx->id; 784 tmp->start_col = hwctx->start_col; 785 tmp->num_col = hwctx->num_col; 786 tmp->command_submissions = hwctx->priv->seq; 787 tmp->command_completions = hwctx->priv->completed; 788 tmp->pasid = hwctx->client->pasid; 789 tmp->priority = hwctx->qos.priority; 790 tmp->gops = hwctx->qos.gops; 791 tmp->fps = hwctx->qos.fps; 792 tmp->dma_bandwidth = hwctx->qos.dma_bandwidth; 793 tmp->latency = hwctx->qos.latency; 794 tmp->frame_exec_time = hwctx->qos.frame_exec_time; 795 tmp->state = AMDXDNA_HWCTX_STATE_ACTIVE; 796 797 buf = u64_to_user_ptr(array_args->buffer); 798 size = min(sizeof(*tmp), array_args->element_size); 799 800 if (copy_to_user(buf, tmp, size)) 801 return -EFAULT; 802 803 array_args->buffer += size; 804 array_args->num_element--; 805 806 return 0; 807 } 808 809 static int aie2_get_hwctx_status(struct amdxdna_client *client, 810 struct amdxdna_drm_get_info *args) 811 { 812 struct amdxdna_drm_get_array array_args; 813 struct amdxdna_dev *xdna = client->xdna; 814 struct amdxdna_client *tmp_client; 815 int ret; 816 817 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 818 819 array_args.element_size = sizeof(struct amdxdna_drm_query_hwctx); 820 array_args.buffer = args->buffer; 821 array_args.num_element = args->buffer_size / array_args.element_size; 822 list_for_each_entry(tmp_client, &xdna->client_list, node) { 823 ret = amdxdna_hwctx_walk(tmp_client, &array_args, 824 aie2_hwctx_status_cb); 825 if (ret) 826 break; 827 } 828 829 args->buffer_size -= (u32)(array_args.buffer - args->buffer); 830 return 0; 831 } 832 833 static int aie2_query_resource_info(struct amdxdna_client *client, 834 struct amdxdna_drm_get_info *args) 835 { 836 struct amdxdna_drm_get_resource_info res_info; 837 const struct amdxdna_dev_priv *priv; 838 struct amdxdna_dev_hdl *ndev; 839 struct amdxdna_dev *xdna; 840 841 xdna = client->xdna; 842 ndev = xdna->dev_handle; 843 priv = ndev->priv; 844 845 res_info.npu_clk_max = priv->dpm_clk_tbl[ndev->max_dpm_level].hclk; 846 res_info.npu_tops_max = ndev->max_tops; 847 res_info.npu_task_max = priv->hwctx_limit; 848 res_info.npu_tops_curr = ndev->curr_tops; 849 res_info.npu_task_curr = ndev->hwctx_num; 850 851 if (copy_to_user(u64_to_user_ptr(args->buffer), &res_info, sizeof(res_info))) 852 return -EFAULT; 853 854 return 0; 855 } 856 857 static int aie2_fill_hwctx_map(struct amdxdna_hwctx *hwctx, void *arg) 858 { 859 struct amdxdna_dev *xdna = hwctx->client->xdna; 860 u32 *map = arg; 861 862 if (hwctx->fw_ctx_id >= xdna->dev_handle->priv->hwctx_limit) { 863 XDNA_ERR(xdna, "Invalid fw ctx id %d/%d ", hwctx->fw_ctx_id, 864 xdna->dev_handle->priv->hwctx_limit); 865 return -EINVAL; 866 } 867 868 map[hwctx->fw_ctx_id] = hwctx->id; 869 return 0; 870 } 871 872 static int aie2_get_telemetry(struct amdxdna_client *client, 873 struct amdxdna_drm_get_info *args) 874 { 875 struct amdxdna_drm_query_telemetry_header *header __free(kfree) = NULL; 876 u32 telemetry_data_sz, header_sz, elem_num; 877 struct amdxdna_dev *xdna = client->xdna; 878 struct amdxdna_client *tmp_client; 879 int ret; 880 881 elem_num = xdna->dev_handle->priv->hwctx_limit; 882 header_sz = struct_size(header, map, elem_num); 883 if (args->buffer_size <= header_sz) { 884 XDNA_ERR(xdna, "Invalid buffer size"); 885 return -EINVAL; 886 } 887 888 telemetry_data_sz = args->buffer_size - header_sz; 889 if (telemetry_data_sz > SZ_4M) { 890 XDNA_ERR(xdna, "Buffer size is too big, %d", telemetry_data_sz); 891 return -EINVAL; 892 } 893 894 header = kzalloc(header_sz, GFP_KERNEL); 895 if (!header) 896 return -ENOMEM; 897 898 if (copy_from_user(header, u64_to_user_ptr(args->buffer), sizeof(*header))) { 899 XDNA_ERR(xdna, "Failed to copy telemetry header from user"); 900 return -EFAULT; 901 } 902 903 header->map_num_elements = elem_num; 904 list_for_each_entry(tmp_client, &xdna->client_list, node) { 905 ret = amdxdna_hwctx_walk(tmp_client, &header->map, 906 aie2_fill_hwctx_map); 907 if (ret) 908 return ret; 909 } 910 911 ret = aie2_query_telemetry(xdna->dev_handle, 912 u64_to_user_ptr(args->buffer + header_sz), 913 telemetry_data_sz, header); 914 if (ret) { 915 XDNA_ERR(xdna, "Query telemetry failed ret %d", ret); 916 return ret; 917 } 918 919 if (copy_to_user(u64_to_user_ptr(args->buffer), header, header_sz)) { 920 XDNA_ERR(xdna, "Copy header failed"); 921 return -EFAULT; 922 } 923 924 return 0; 925 } 926 927 static int aie2_get_preempt_state(struct amdxdna_client *client, 928 struct amdxdna_drm_get_info *args) 929 { 930 struct amdxdna_drm_attribute_state state = {}; 931 struct amdxdna_dev *xdna = client->xdna; 932 struct amdxdna_dev_hdl *ndev; 933 934 ndev = xdna->dev_handle; 935 if (args->param == DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE) 936 state.state = ndev->force_preempt_enabled; 937 else if (args->param == DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE) 938 state.state = ndev->frame_boundary_preempt; 939 940 if (copy_to_user(u64_to_user_ptr(args->buffer), &state, sizeof(state))) 941 return -EFAULT; 942 943 return 0; 944 } 945 946 static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args) 947 { 948 struct amdxdna_dev *xdna = client->xdna; 949 int ret, idx; 950 951 if (!drm_dev_enter(&xdna->ddev, &idx)) 952 return -ENODEV; 953 954 ret = amdxdna_pm_resume_get(xdna); 955 if (ret) 956 goto dev_exit; 957 958 switch (args->param) { 959 case DRM_AMDXDNA_QUERY_AIE_STATUS: 960 ret = aie2_get_aie_status(client, args); 961 break; 962 case DRM_AMDXDNA_QUERY_AIE_METADATA: 963 ret = aie2_get_aie_metadata(client, args); 964 break; 965 case DRM_AMDXDNA_QUERY_AIE_VERSION: 966 ret = aie2_get_aie_version(client, args); 967 break; 968 case DRM_AMDXDNA_QUERY_CLOCK_METADATA: 969 ret = aie2_get_clock_metadata(client, args); 970 break; 971 case DRM_AMDXDNA_QUERY_HW_CONTEXTS: 972 ret = aie2_get_hwctx_status(client, args); 973 break; 974 case DRM_AMDXDNA_QUERY_FIRMWARE_VERSION: 975 ret = aie2_get_firmware_version(client, args); 976 break; 977 case DRM_AMDXDNA_GET_POWER_MODE: 978 ret = aie2_get_power_mode(client, args); 979 break; 980 case DRM_AMDXDNA_QUERY_TELEMETRY: 981 ret = aie2_get_telemetry(client, args); 982 break; 983 case DRM_AMDXDNA_QUERY_RESOURCE_INFO: 984 ret = aie2_query_resource_info(client, args); 985 break; 986 case DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE: 987 case DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE: 988 ret = aie2_get_preempt_state(client, args); 989 break; 990 default: 991 XDNA_ERR(xdna, "Not supported request parameter %u", args->param); 992 ret = -EOPNOTSUPP; 993 } 994 995 amdxdna_pm_suspend_put(xdna); 996 XDNA_DBG(xdna, "Got param %d", args->param); 997 998 dev_exit: 999 drm_dev_exit(idx); 1000 return ret; 1001 } 1002 1003 static int aie2_query_ctx_status_array(struct amdxdna_client *client, 1004 struct amdxdna_drm_get_array *args) 1005 { 1006 struct amdxdna_drm_get_array array_args; 1007 struct amdxdna_dev *xdna = client->xdna; 1008 struct amdxdna_client *tmp_client; 1009 int ret; 1010 1011 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 1012 1013 if (args->element_size > SZ_4K || args->num_element > SZ_1K) { 1014 XDNA_DBG(xdna, "Invalid element size %d or number of element %d", 1015 args->element_size, args->num_element); 1016 return -EINVAL; 1017 } 1018 1019 array_args.element_size = min(args->element_size, 1020 sizeof(struct amdxdna_drm_hwctx_entry)); 1021 array_args.buffer = args->buffer; 1022 array_args.num_element = args->num_element * args->element_size / 1023 array_args.element_size; 1024 list_for_each_entry(tmp_client, &xdna->client_list, node) { 1025 ret = amdxdna_hwctx_walk(tmp_client, &array_args, 1026 aie2_hwctx_status_cb); 1027 if (ret) 1028 break; 1029 } 1030 1031 args->element_size = array_args.element_size; 1032 args->num_element = (u32)((array_args.buffer - args->buffer) / 1033 args->element_size); 1034 1035 return 0; 1036 } 1037 1038 static int aie2_get_array(struct amdxdna_client *client, 1039 struct amdxdna_drm_get_array *args) 1040 { 1041 struct amdxdna_dev *xdna = client->xdna; 1042 int ret, idx; 1043 1044 if (!drm_dev_enter(&xdna->ddev, &idx)) 1045 return -ENODEV; 1046 1047 ret = amdxdna_pm_resume_get(xdna); 1048 if (ret) 1049 goto dev_exit; 1050 1051 switch (args->param) { 1052 case DRM_AMDXDNA_HW_CONTEXT_ALL: 1053 ret = aie2_query_ctx_status_array(client, args); 1054 break; 1055 case DRM_AMDXDNA_HW_LAST_ASYNC_ERR: 1056 ret = aie2_get_array_async_error(xdna->dev_handle, args); 1057 break; 1058 default: 1059 XDNA_ERR(xdna, "Not supported request parameter %u", args->param); 1060 ret = -EOPNOTSUPP; 1061 } 1062 1063 amdxdna_pm_suspend_put(xdna); 1064 XDNA_DBG(xdna, "Got param %d", args->param); 1065 1066 dev_exit: 1067 drm_dev_exit(idx); 1068 return ret; 1069 } 1070 1071 static int aie2_set_power_mode(struct amdxdna_client *client, 1072 struct amdxdna_drm_set_state *args) 1073 { 1074 struct amdxdna_drm_set_power_mode power_state; 1075 enum amdxdna_power_mode_type power_mode; 1076 struct amdxdna_dev *xdna = client->xdna; 1077 1078 if (copy_from_user(&power_state, u64_to_user_ptr(args->buffer), 1079 sizeof(power_state))) { 1080 XDNA_ERR(xdna, "Failed to copy power mode request into kernel"); 1081 return -EFAULT; 1082 } 1083 1084 if (XDNA_MBZ_DBG(xdna, power_state.pad, sizeof(power_state.pad))) 1085 return -EINVAL; 1086 1087 power_mode = power_state.power_mode; 1088 if (power_mode > POWER_MODE_TURBO) { 1089 XDNA_ERR(xdna, "Invalid power mode %d", power_mode); 1090 return -EINVAL; 1091 } 1092 1093 return aie2_pm_set_mode(xdna->dev_handle, power_mode); 1094 } 1095 1096 static int aie2_set_preempt_state(struct amdxdna_client *client, 1097 struct amdxdna_drm_set_state *args) 1098 { 1099 struct amdxdna_dev_hdl *ndev = client->xdna->dev_handle; 1100 struct amdxdna_drm_attribute_state state; 1101 u32 val; 1102 int ret; 1103 1104 if (copy_from_user(&state, u64_to_user_ptr(args->buffer), sizeof(state))) 1105 return -EFAULT; 1106 1107 if (state.state > 1) 1108 return -EINVAL; 1109 1110 if (XDNA_MBZ_DBG(client->xdna, state.pad, sizeof(state.pad))) 1111 return -EINVAL; 1112 1113 if (args->param == DRM_AMDXDNA_SET_FORCE_PREEMPT) { 1114 ndev->force_preempt_enabled = state.state; 1115 } else if (args->param == DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT) { 1116 val = state.state; 1117 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT, 1118 &val); 1119 if (ret) 1120 return ret; 1121 1122 ndev->frame_boundary_preempt = state.state; 1123 } 1124 1125 return 0; 1126 } 1127 1128 static int aie2_set_state(struct amdxdna_client *client, 1129 struct amdxdna_drm_set_state *args) 1130 { 1131 struct amdxdna_dev *xdna = client->xdna; 1132 int ret, idx; 1133 1134 if (!drm_dev_enter(&xdna->ddev, &idx)) 1135 return -ENODEV; 1136 1137 ret = amdxdna_pm_resume_get(xdna); 1138 if (ret) 1139 goto dev_exit; 1140 1141 switch (args->param) { 1142 case DRM_AMDXDNA_SET_POWER_MODE: 1143 ret = aie2_set_power_mode(client, args); 1144 break; 1145 case DRM_AMDXDNA_SET_FORCE_PREEMPT: 1146 case DRM_AMDXDNA_SET_FRAME_BOUNDARY_PREEMPT: 1147 ret = aie2_set_preempt_state(client, args); 1148 break; 1149 default: 1150 XDNA_ERR(xdna, "Not supported request parameter %u", args->param); 1151 ret = -EOPNOTSUPP; 1152 break; 1153 } 1154 1155 amdxdna_pm_suspend_put(xdna); 1156 dev_exit: 1157 drm_dev_exit(idx); 1158 return ret; 1159 } 1160 1161 const struct amdxdna_dev_ops aie2_ops = { 1162 .init = aie2_init, 1163 .fini = aie2_fini, 1164 .resume = aie2_hw_resume, 1165 .suspend = aie2_hw_suspend, 1166 .get_aie_info = aie2_get_info, 1167 .set_aie_state = aie2_set_state, 1168 .hwctx_init = aie2_hwctx_init, 1169 .hwctx_fini = aie2_hwctx_fini, 1170 .hwctx_config = aie2_hwctx_config, 1171 .hwctx_sync_debug_bo = aie2_hwctx_sync_debug_bo, 1172 .cmd_submit = aie2_cmd_submit, 1173 .hmm_invalidate = aie2_hmm_invalidate, 1174 .get_array = aie2_get_array, 1175 }; 1176