1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_cache.h> 8 #include <drm/drm_device.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/drm_print.h> 12 #include <drm/gpu_scheduler.h> 13 #include <linux/bitfield.h> 14 #include <linux/errno.h> 15 #include <linux/pci.h> 16 #include <linux/types.h> 17 #include <linux/xarray.h> 18 19 #include "aie.h" 20 #include "aie2_msg_priv.h" 21 #include "aie2_pci.h" 22 #include "amdxdna_ctx.h" 23 #include "amdxdna_gem.h" 24 #include "amdxdna_mailbox.h" 25 #include "amdxdna_mailbox_helper.h" 26 #include "amdxdna_pci_drv.h" 27 28 #define EXEC_MSG_OPS(xdna) ((xdna)->dev_handle->exec_msg_ops) 29 30 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev) 31 { 32 DECLARE_AIE_MSG(suspend, MSG_OP_SUSPEND); 33 int ret; 34 35 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 36 if (ret) { 37 XDNA_ERR(ndev->aie.xdna, "Failed to suspend fw, ret %d", ret); 38 return ret; 39 } 40 41 return aie_psp_waitmode_poll(ndev->aie.psp_hdl); 42 } 43 44 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev) 45 { 46 DECLARE_AIE_MSG(suspend, MSG_OP_RESUME); 47 48 return aie_send_mgmt_msg_wait(&ndev->aie, &msg); 49 } 50 51 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value) 52 { 53 DECLARE_AIE_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG); 54 int ret; 55 56 req.type = type; 57 req.value = value; 58 59 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 60 if (ret) { 61 XDNA_ERR(ndev->aie.xdna, "Failed to set runtime config, ret %d", ret); 62 return ret; 63 } 64 65 return 0; 66 } 67 68 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value) 69 { 70 DECLARE_AIE_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG); 71 int ret; 72 73 req.type = type; 74 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 75 if (ret) { 76 XDNA_ERR(ndev->aie.xdna, "Failed to get runtime config, ret %d", ret); 77 return ret; 78 } 79 80 *value = resp.value; 81 return 0; 82 } 83 84 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid) 85 { 86 DECLARE_AIE_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID); 87 88 req.pasid = pasid; 89 90 return aie_send_mgmt_msg_wait(&ndev->aie, &msg); 91 } 92 93 int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, 94 struct amdxdna_drm_query_aie_version *version) 95 { 96 DECLARE_AIE_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION); 97 struct amdxdna_dev *xdna = ndev->aie.xdna; 98 int ret; 99 100 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 101 if (ret) 102 return ret; 103 104 XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed", 105 resp.major, resp.minor); 106 107 version->major = resp.major; 108 version->minor = resp.minor; 109 110 return 0; 111 } 112 113 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, 114 struct amdxdna_drm_query_aie_metadata *metadata) 115 { 116 DECLARE_AIE_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO); 117 int ret; 118 119 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 120 if (ret) 121 return ret; 122 123 metadata->col_size = resp.info.size; 124 metadata->cols = resp.info.cols; 125 metadata->rows = resp.info.rows; 126 127 metadata->version.major = resp.info.major; 128 metadata->version.minor = resp.info.minor; 129 130 metadata->core.row_count = resp.info.core_rows; 131 metadata->core.row_start = resp.info.core_row_start; 132 metadata->core.dma_channel_count = resp.info.core_dma_channels; 133 metadata->core.lock_count = resp.info.core_locks; 134 metadata->core.event_reg_count = resp.info.core_events; 135 136 metadata->mem.row_count = resp.info.mem_rows; 137 metadata->mem.row_start = resp.info.mem_row_start; 138 metadata->mem.dma_channel_count = resp.info.mem_dma_channels; 139 metadata->mem.lock_count = resp.info.mem_locks; 140 metadata->mem.event_reg_count = resp.info.mem_events; 141 142 metadata->shim.row_count = resp.info.shim_rows; 143 metadata->shim.row_start = resp.info.shim_row_start; 144 metadata->shim.dma_channel_count = resp.info.shim_dma_channels; 145 metadata->shim.lock_count = resp.info.shim_locks; 146 metadata->shim.event_reg_count = resp.info.shim_events; 147 148 return 0; 149 } 150 151 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, 152 struct amdxdna_fw_ver *fw_ver) 153 { 154 DECLARE_AIE_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION); 155 int ret; 156 157 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 158 if (ret) 159 return ret; 160 161 fw_ver->major = resp.major; 162 fw_ver->minor = resp.minor; 163 fw_ver->sub = resp.sub; 164 fw_ver->build = resp.build; 165 166 return 0; 167 } 168 169 static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id) 170 { 171 DECLARE_AIE_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); 172 struct amdxdna_dev *xdna = ndev->aie.xdna; 173 int ret; 174 175 req.context_id = id; 176 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 177 if (ret && ret != -ENODEV) 178 XDNA_WARN(xdna, "Destroy context failed, ret %d", ret); 179 else if (ret == -ENODEV) 180 XDNA_DBG(xdna, "Destroy context: device already stopped"); 181 182 return ret; 183 } 184 185 static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev, 186 struct amdxdna_hwctx *hwctx) 187 { 188 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_PREEMPT)) 189 return PRIORITY_HIGH; 190 191 switch (hwctx->qos.priority) { 192 case AMDXDNA_QOS_REALTIME_PRIORITY: 193 return PRIORITY_REALTIME; 194 case AMDXDNA_QOS_HIGH_PRIORITY: 195 return PRIORITY_HIGH; 196 case AMDXDNA_QOS_NORMAL_PRIORITY: 197 return PRIORITY_NORMAL; 198 case AMDXDNA_QOS_LOW_PRIORITY: 199 return PRIORITY_LOW; 200 default: 201 return PRIORITY_HIGH; 202 } 203 } 204 205 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 206 { 207 DECLARE_AIE_MSG(create_ctx, MSG_OP_CREATE_CONTEXT); 208 struct amdxdna_dev *xdna = ndev->aie.xdna; 209 struct xdna_mailbox_chann_res x2i; 210 struct xdna_mailbox_chann_res i2x; 211 struct cq_pair *cq_pair; 212 u32 intr_reg; 213 int ret; 214 215 req.aie_type = 1; 216 req.start_col = hwctx->start_col; 217 req.num_col = hwctx->num_col; 218 req.num_unused_col = hwctx->num_unused_col; 219 req.num_cq_pairs_requested = 1; 220 req.pasid = amdxdna_pasid_on(hwctx->client) ? hwctx->client->pasid : 0; 221 req.context_priority = aie2_get_context_priority(ndev, hwctx); 222 223 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 224 if (ret) 225 return ret; 226 227 hwctx->fw_ctx_id = resp.context_id; 228 if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1)) 229 return -EINVAL; 230 231 if (ndev->force_preempt_enabled) { 232 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id); 233 if (ret) { 234 XDNA_ERR(xdna, "failed to enable force preempt %d", ret); 235 goto del_ctx_req; 236 } 237 } 238 239 cq_pair = &resp.cq_pair[0]; 240 x2i.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.head_addr); 241 x2i.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.tail_addr); 242 x2i.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->x2i_q.buf_addr); 243 x2i.rb_size = cq_pair->x2i_q.buf_size; 244 245 i2x.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.head_addr); 246 i2x.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.tail_addr); 247 i2x.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->i2x_q.buf_addr); 248 i2x.rb_size = cq_pair->i2x_q.buf_size; 249 250 ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id); 251 if (ret == -EINVAL) { 252 XDNA_ERR(xdna, "Alloc IRQ failed %d", ret); 253 goto del_ctx_req; 254 } 255 256 intr_reg = i2x.mb_head_ptr_reg + 4; 257 hwctx->priv->mbox_chann = xdna_mailbox_alloc_channel(ndev->mbox); 258 if (!hwctx->priv->mbox_chann) { 259 XDNA_ERR(xdna, "Not able to create channel"); 260 ret = -EINVAL; 261 goto del_ctx_req; 262 } 263 264 ret = xdna_mailbox_start_channel(hwctx->priv->mbox_chann, &x2i, &i2x, 265 intr_reg, ret); 266 if (ret) { 267 XDNA_ERR(xdna, "Not able to create channel"); 268 ret = -EINVAL; 269 goto free_channel; 270 } 271 ndev->hwctx_num++; 272 273 XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id); 274 XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid); 275 276 return 0; 277 278 free_channel: 279 xdna_mailbox_free_channel(hwctx->priv->mbox_chann); 280 del_ctx_req: 281 aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); 282 return ret; 283 } 284 285 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 286 { 287 struct amdxdna_dev *xdna = ndev->aie.xdna; 288 int ret; 289 290 if (!hwctx->priv->mbox_chann) 291 return 0; 292 293 xdna_mailbox_stop_channel(hwctx->priv->mbox_chann); 294 ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); 295 xdna_mailbox_free_channel(hwctx->priv->mbox_chann); 296 XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id); 297 hwctx->priv->mbox_chann = NULL; 298 hwctx->fw_ctx_id = -1; 299 ndev->hwctx_num--; 300 301 return ret; 302 } 303 304 static int aie2_send_host_buf_msgs(struct amdxdna_dev_hdl *ndev, u32 context_id, 305 u64 addr, u64 size, u32 initial_opcode) 306 { 307 DECLARE_AIE_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER); 308 struct amdxdna_dev *xdna = ndev->aie.xdna; 309 size_t chunk_size; 310 int ret; 311 312 chunk_size = xdna->dev_info->dev_mem_size; 313 if (!size || !IS_ALIGNED(size, chunk_size)) { 314 XDNA_ERR(xdna, "Invalid size 0x%llx for chunk 0x%lx", 315 size, chunk_size); 316 return -EINVAL; 317 } 318 319 msg.opcode = initial_opcode; 320 do { 321 req.context_id = context_id; 322 req.buf_addr = addr; 323 req.buf_size = chunk_size; 324 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 325 if (ret) { 326 XDNA_ERR(xdna, "fw ctx %d addr 0x%llx size 0x%lx", 327 context_id, addr, chunk_size); 328 return ret; 329 } 330 331 XDNA_DBG(xdna, "fw ctx %d host buf op 0x%x addr 0x%llx size 0x%lx", 332 context_id, msg.opcode, addr, chunk_size); 333 334 addr += chunk_size; 335 size -= chunk_size; 336 msg.opcode = MSG_OP_ADD_HOST_BUFFER; 337 } while (size); 338 339 return 0; 340 } 341 342 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size) 343 { 344 return aie2_send_host_buf_msgs(ndev, context_id, addr, size, 345 MSG_OP_MAP_HOST_BUFFER); 346 } 347 348 int aie2_add_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size) 349 { 350 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_ADD_HOST_BUFFER)) 351 return -EOPNOTSUPP; 352 353 return aie2_send_host_buf_msgs(ndev, context_id, addr, size, 354 MSG_OP_ADD_HOST_BUFFER); 355 } 356 357 static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg) 358 { 359 u32 *bitmap = arg; 360 361 *bitmap |= GENMASK(hwctx->start_col + hwctx->num_col - 1, hwctx->start_col); 362 363 return 0; 364 } 365 366 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, 367 u32 size, u32 *cols_filled) 368 { 369 DECLARE_AIE_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS); 370 struct amdxdna_dev *xdna = ndev->aie.xdna; 371 u32 buf_sz, aie_bitmap = 0; 372 struct amdxdna_client *client; 373 dma_addr_t dma_addr; 374 u8 *buff_addr; 375 int ret; 376 377 buf_sz = ndev->aie.metadata.cols * ndev->aie.metadata.col_size; 378 buff_addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr); 379 if (IS_ERR(buff_addr)) 380 return PTR_ERR(buff_addr); 381 382 /* Go through each hardware context and mark the AIE columns that are active */ 383 list_for_each_entry(client, &xdna->client_list, node) 384 amdxdna_hwctx_walk(client, &aie_bitmap, amdxdna_hwctx_col_map); 385 386 *cols_filled = 0; 387 req.dump_buff_addr = dma_addr; 388 req.dump_buff_size = buf_sz; 389 req.num_cols = hweight32(aie_bitmap); 390 req.aie_bitmap = aie_bitmap; 391 392 drm_clflush_virt_range(buff_addr, req.dump_buff_size); /* device can access */ 393 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 394 if (ret) { 395 XDNA_ERR(xdna, "Error during NPU query, status %d", ret); 396 goto fail; 397 } 398 399 XDNA_DBG(xdna, "Query NPU status completed"); 400 401 if (buf_sz < resp.size) { 402 ret = -EINVAL; 403 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size); 404 goto fail; 405 } 406 407 size = min(size, resp.size); 408 if (copy_to_user(buf, buff_addr, size)) { 409 ret = -EFAULT; 410 XDNA_ERR(xdna, "Failed to copy NPU status to user space"); 411 goto fail; 412 } 413 414 *cols_filled = aie_bitmap; 415 416 fail: 417 amdxdna_free_msg_buffer(xdna, buf_sz, buff_addr, dma_addr); 418 return ret; 419 } 420 421 int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, 422 char __user *buf, u32 size, 423 struct amdxdna_drm_query_telemetry_header *header) 424 { 425 DECLARE_AIE_MSG(get_telemetry, MSG_OP_GET_TELEMETRY); 426 struct amdxdna_dev *xdna = ndev->aie.xdna; 427 dma_addr_t dma_addr; 428 u32 buf_sz; 429 u8 *addr; 430 int ret; 431 432 if (header->type >= MAX_TELEMETRY_TYPE) 433 return -EINVAL; 434 435 buf_sz = min(size, SZ_4M); 436 addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr); 437 if (IS_ERR(addr)) 438 return PTR_ERR(addr); 439 440 req.buf_addr = dma_addr; 441 req.buf_size = buf_sz; 442 req.type = header->type; 443 444 drm_clflush_virt_range(addr, req.buf_size); /* device can access */ 445 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 446 if (ret) { 447 XDNA_ERR(xdna, "Query telemetry failed, status %d", ret); 448 goto free_buf; 449 } 450 451 if (buf_sz < resp.size) { 452 ret = -EINVAL; 453 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size); 454 goto free_buf; 455 } 456 457 size = min(size, resp.size); 458 if (copy_to_user(buf, addr, size)) { 459 ret = -EFAULT; 460 XDNA_ERR(xdna, "Failed to copy telemetry to user space"); 461 goto free_buf; 462 } 463 464 header->major = resp.major; 465 header->minor = resp.minor; 466 467 free_buf: 468 amdxdna_free_msg_buffer(xdna, buf_sz, addr, dma_addr); 469 return ret; 470 } 471 472 int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size, 473 void *handle, int (*cb)(void*, void __iomem *, size_t)) 474 { 475 struct async_event_msg_req req = { 0 }; 476 struct xdna_mailbox_msg msg = { 477 .send_data = (u8 *)&req, 478 .send_size = sizeof(req), 479 .handle = handle, 480 .opcode = MSG_OP_REGISTER_ASYNC_EVENT_MSG, 481 .notify_cb = cb, 482 }; 483 484 req.buf_addr = addr; 485 req.buf_size = size; 486 487 XDNA_DBG(ndev->aie.xdna, "Register addr 0x%llx size 0x%x", addr, size); 488 return xdna_mailbox_send_msg(ndev->aie.mgmt_chann, &msg, TX_TIMEOUT); 489 } 490 491 int aie2_config_cu(struct amdxdna_hwctx *hwctx, 492 int (*notify_cb)(void *, void __iomem *, size_t)) 493 { 494 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 495 struct amdxdna_dev *xdna = hwctx->client->xdna; 496 u32 shift = xdna->dev_info->dev_mem_buf_shift; 497 struct config_cu_req req = { 0 }; 498 struct xdna_mailbox_msg msg; 499 struct drm_gem_object *gobj; 500 struct amdxdna_gem_obj *abo; 501 int i; 502 503 if (!chann) 504 return -ENODEV; 505 506 if (!hwctx->cus) 507 return 0; 508 509 if (hwctx->cus->num_cus > MAX_NUM_CUS) { 510 XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS); 511 return -EINVAL; 512 } 513 514 for (i = 0; i < hwctx->cus->num_cus; i++) { 515 struct amdxdna_cu_config *cu = &hwctx->cus->cu_configs[i]; 516 517 if (XDNA_MBZ_DBG(xdna, cu->pad, sizeof(cu->pad))) 518 return -EINVAL; 519 520 gobj = drm_gem_object_lookup(hwctx->client->filp, cu->cu_bo); 521 if (!gobj) { 522 XDNA_ERR(xdna, "Lookup GEM object failed"); 523 return -EINVAL; 524 } 525 abo = to_xdna_obj(gobj); 526 527 if (abo->type != AMDXDNA_BO_DEV) { 528 drm_gem_object_put(gobj); 529 XDNA_ERR(xdna, "Invalid BO type"); 530 return -EINVAL; 531 } 532 533 req.cfgs[i] = FIELD_PREP(AIE2_MSG_CFG_CU_PDI_ADDR, 534 amdxdna_gem_dev_addr(abo) >> shift); 535 req.cfgs[i] |= FIELD_PREP(AIE2_MSG_CFG_CU_FUNC, cu->cu_func); 536 XDNA_DBG(xdna, "CU %d full addr 0x%llx, cfg 0x%x", i, 537 amdxdna_gem_dev_addr(abo), req.cfgs[i]); 538 drm_gem_object_put(gobj); 539 } 540 req.num_cus = hwctx->cus->num_cus; 541 542 msg.send_data = (u8 *)&req; 543 msg.send_size = sizeof(req); 544 msg.handle = hwctx; 545 msg.opcode = MSG_OP_CONFIG_CU; 546 msg.notify_cb = notify_cb; 547 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 548 } 549 550 static int aie2_init_exec_cu_req(struct amdxdna_gem_obj *cmd_bo, void *req, 551 size_t *size, u32 *msg_op) 552 { 553 struct execute_buffer_req *cu_req = req; 554 u32 cmd_len; 555 void *cmd; 556 557 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 558 if (cmd_len > sizeof(cu_req->payload)) 559 return -EINVAL; 560 561 cu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 562 if (cu_req->cu_idx == INVALID_CU_IDX) 563 return -EINVAL; 564 565 memcpy(cu_req->payload, cmd, cmd_len); 566 567 *size = sizeof(*cu_req); 568 *msg_op = MSG_OP_EXECUTE_BUFFER_CF; 569 return 0; 570 } 571 572 static int aie2_init_exec_dpu_req(struct amdxdna_gem_obj *cmd_bo, void *req, 573 size_t *size, u32 *msg_op) 574 { 575 struct exec_dpu_req *dpu_req = req; 576 struct amdxdna_cmd_start_npu *sn; 577 u32 cmd_len; 578 579 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 580 if (cmd_len - sizeof(*sn) > sizeof(dpu_req->payload)) 581 return -EINVAL; 582 583 dpu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 584 if (dpu_req->cu_idx == INVALID_CU_IDX) 585 return -EINVAL; 586 587 dpu_req->inst_buf_addr = sn->buffer; 588 dpu_req->inst_size = sn->buffer_size; 589 dpu_req->inst_prop_cnt = sn->prop_count; 590 memcpy(dpu_req->payload, sn->prop_args, cmd_len - sizeof(*sn)); 591 592 *size = sizeof(*dpu_req); 593 *msg_op = MSG_OP_EXEC_DPU; 594 return 0; 595 } 596 597 static void aie2_init_exec_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) 598 { 599 struct cmd_chain_req *chain_req = req; 600 601 chain_req->buf_addr = slot_addr; 602 chain_req->buf_size = size; 603 chain_req->count = cmd_cnt; 604 } 605 606 static void aie2_init_npu_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) 607 { 608 struct cmd_chain_npu_req *npu_chain_req = req; 609 610 npu_chain_req->flags = 0; 611 npu_chain_req->reserved = 0; 612 npu_chain_req->buf_addr = slot_addr; 613 npu_chain_req->buf_size = size; 614 npu_chain_req->count = cmd_cnt; 615 } 616 617 static int 618 aie2_cmdlist_fill_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 619 { 620 struct cmd_chain_slot_execbuf_cf *cf_slot = slot; 621 u32 cmd_len; 622 void *cmd; 623 624 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 625 if (*size < sizeof(*cf_slot) + cmd_len) 626 return -EINVAL; 627 628 cf_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 629 if (cf_slot->cu_idx == INVALID_CU_IDX) 630 return -EINVAL; 631 632 cf_slot->arg_cnt = cmd_len / sizeof(u32); 633 memcpy(cf_slot->args, cmd, cmd_len); 634 /* Accurate slot size to hint firmware to do necessary copy */ 635 *size = sizeof(*cf_slot) + cmd_len; 636 return 0; 637 } 638 639 static int 640 aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 641 { 642 struct cmd_chain_slot_dpu *dpu_slot = slot; 643 struct amdxdna_cmd_start_npu *sn; 644 u32 cmd_len; 645 u32 arg_sz; 646 647 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 648 arg_sz = cmd_len - sizeof(*sn); 649 if (cmd_len < sizeof(*sn) || arg_sz > MAX_DPU_ARGS_SIZE) 650 return -EINVAL; 651 652 if (*size < sizeof(*dpu_slot) + arg_sz) 653 return -EINVAL; 654 655 dpu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 656 if (dpu_slot->cu_idx == INVALID_CU_IDX) 657 return -EINVAL; 658 659 dpu_slot->inst_buf_addr = sn->buffer; 660 dpu_slot->inst_size = sn->buffer_size; 661 dpu_slot->inst_prop_cnt = sn->prop_count; 662 dpu_slot->arg_cnt = arg_sz / sizeof(u32); 663 memcpy(dpu_slot->args, sn->prop_args, arg_sz); 664 665 /* Accurate slot size to hint firmware to do necessary copy */ 666 *size = sizeof(*dpu_slot) + arg_sz; 667 return 0; 668 } 669 670 static int aie2_cmdlist_unsupp(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 671 { 672 return -EOPNOTSUPP; 673 } 674 675 static u32 aie2_get_chain_msg_op(u32 cmd_op) 676 { 677 switch (cmd_op) { 678 case ERT_START_CU: 679 return MSG_OP_CHAIN_EXEC_BUFFER_CF; 680 case ERT_START_NPU: 681 return MSG_OP_CHAIN_EXEC_DPU; 682 default: 683 break; 684 } 685 686 return MSG_OP_MAX_OPCODE; 687 } 688 689 static struct aie2_exec_msg_ops legacy_exec_message_ops = { 690 .init_cu_req = aie2_init_exec_cu_req, 691 .init_dpu_req = aie2_init_exec_dpu_req, 692 .init_chain_req = aie2_init_exec_chain_req, 693 .fill_cf_slot = aie2_cmdlist_fill_cf, 694 .fill_dpu_slot = aie2_cmdlist_fill_dpu, 695 .fill_preempt_slot = aie2_cmdlist_unsupp, 696 .fill_elf_slot = aie2_cmdlist_unsupp, 697 .get_chain_msg_op = aie2_get_chain_msg_op, 698 }; 699 700 static int 701 aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 702 { 703 struct cmd_chain_slot_npu *npu_slot = slot; 704 u32 cmd_len; 705 void *cmd; 706 707 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 708 if (*size < sizeof(*npu_slot) + cmd_len) 709 return -EINVAL; 710 711 memset(npu_slot, 0, sizeof(*npu_slot)); 712 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 713 if (npu_slot->cu_idx == INVALID_CU_IDX) 714 return -EINVAL; 715 716 npu_slot->type = EXEC_NPU_TYPE_NON_ELF; 717 npu_slot->arg_cnt = cmd_len / sizeof(u32); 718 memcpy(npu_slot->args, cmd, cmd_len); 719 720 *size = sizeof(*npu_slot) + cmd_len; 721 return 0; 722 } 723 724 static int 725 aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 726 { 727 struct cmd_chain_slot_npu *npu_slot = slot; 728 struct amdxdna_cmd_start_npu *sn; 729 u32 cmd_len; 730 u32 arg_sz; 731 732 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 733 arg_sz = cmd_len - sizeof(*sn); 734 if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE) 735 return -EINVAL; 736 737 if (*size < sizeof(*npu_slot) + arg_sz) 738 return -EINVAL; 739 740 memset(npu_slot, 0, sizeof(*npu_slot)); 741 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 742 if (npu_slot->cu_idx == INVALID_CU_IDX) 743 return -EINVAL; 744 745 npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF; 746 npu_slot->inst_buf_addr = sn->buffer; 747 npu_slot->inst_size = sn->buffer_size; 748 npu_slot->inst_prop_cnt = sn->prop_count; 749 npu_slot->arg_cnt = arg_sz / sizeof(u32); 750 memcpy(npu_slot->args, sn->prop_args, arg_sz); 751 752 *size = sizeof(*npu_slot) + arg_sz; 753 return 0; 754 } 755 756 static int 757 aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 758 { 759 struct cmd_chain_slot_npu *npu_slot = slot; 760 struct amdxdna_cmd_preempt_data *pd; 761 u32 cmd_len; 762 u32 arg_sz; 763 764 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 765 arg_sz = cmd_len - sizeof(*pd); 766 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) 767 return -EINVAL; 768 769 if (*size < sizeof(*npu_slot) + arg_sz) 770 return -EINVAL; 771 772 memset(npu_slot, 0, sizeof(*npu_slot)); 773 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 774 if (npu_slot->cu_idx == INVALID_CU_IDX) 775 return -EINVAL; 776 777 npu_slot->type = EXEC_NPU_TYPE_PREEMPT; 778 npu_slot->inst_buf_addr = pd->inst_buf; 779 npu_slot->save_buf_addr = pd->save_buf; 780 npu_slot->restore_buf_addr = pd->restore_buf; 781 npu_slot->inst_size = pd->inst_size; 782 npu_slot->save_size = pd->save_size; 783 npu_slot->restore_size = pd->restore_size; 784 npu_slot->inst_prop_cnt = pd->inst_prop_cnt; 785 npu_slot->arg_cnt = arg_sz / sizeof(u32); 786 memcpy(npu_slot->args, pd->prop_args, arg_sz); 787 788 *size = sizeof(*npu_slot) + arg_sz; 789 return 0; 790 } 791 792 static int 793 aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 794 { 795 struct cmd_chain_slot_npu *npu_slot = slot; 796 struct amdxdna_cmd_preempt_data *pd; 797 u32 cmd_len; 798 u32 arg_sz; 799 800 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 801 arg_sz = cmd_len - sizeof(*pd); 802 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) 803 return -EINVAL; 804 805 if (*size < sizeof(*npu_slot) + arg_sz) 806 return -EINVAL; 807 808 memset(npu_slot, 0, sizeof(*npu_slot)); 809 npu_slot->type = EXEC_NPU_TYPE_ELF; 810 npu_slot->inst_buf_addr = pd->inst_buf; 811 npu_slot->save_buf_addr = pd->save_buf; 812 npu_slot->restore_buf_addr = pd->restore_buf; 813 npu_slot->inst_size = pd->inst_size; 814 npu_slot->save_size = pd->save_size; 815 npu_slot->restore_size = pd->restore_size; 816 npu_slot->inst_prop_cnt = pd->inst_prop_cnt; 817 npu_slot->arg_cnt = 1; 818 npu_slot->args[0] = AIE2_EXEC_BUFFER_KERNEL_OP_TXN; 819 820 *size = struct_size(npu_slot, args, npu_slot->arg_cnt); 821 return 0; 822 } 823 824 static u32 aie2_get_npu_chain_msg_op(u32 cmd_op) 825 { 826 return MSG_OP_CHAIN_EXEC_NPU; 827 } 828 829 static struct aie2_exec_msg_ops npu_exec_message_ops = { 830 .init_cu_req = aie2_init_exec_cu_req, 831 .init_dpu_req = aie2_init_exec_dpu_req, 832 .init_chain_req = aie2_init_npu_chain_req, 833 .fill_cf_slot = aie2_cmdlist_fill_npu_cf, 834 .fill_dpu_slot = aie2_cmdlist_fill_npu_dpu, 835 .fill_preempt_slot = aie2_cmdlist_fill_npu_preempt, 836 .fill_elf_slot = aie2_cmdlist_fill_npu_elf, 837 .get_chain_msg_op = aie2_get_npu_chain_msg_op, 838 }; 839 840 static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo, 841 size_t *size, u32 *msg_op) 842 { 843 struct amdxdna_dev *xdna = cmd_abo->client->xdna; 844 int ret; 845 u32 op; 846 847 op = amdxdna_cmd_get_op(cmd_abo); 848 switch (op) { 849 case ERT_START_CU: 850 ret = EXEC_MSG_OPS(xdna)->init_cu_req(cmd_abo, req, size, msg_op); 851 if (ret) { 852 XDNA_DBG(xdna, "Init CU req failed ret %d", ret); 853 return ret; 854 } 855 break; 856 case ERT_START_NPU: 857 ret = EXEC_MSG_OPS(xdna)->init_dpu_req(cmd_abo, req, size, msg_op); 858 if (ret) { 859 XDNA_DBG(xdna, "Init DPU req failed ret %d", ret); 860 return ret; 861 } 862 863 break; 864 default: 865 XDNA_ERR(xdna, "Unsupported op %d", op); 866 ret = -EOPNOTSUPP; 867 break; 868 } 869 870 return ret; 871 } 872 873 static int 874 aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo, 875 size_t *size, u32 *cmd_op) 876 { 877 struct amdxdna_dev *xdna = cmd_abo->client->xdna; 878 int ret; 879 u32 op; 880 881 op = amdxdna_cmd_get_op(cmd_abo); 882 if (*cmd_op == ERT_INVALID_CMD) 883 *cmd_op = op; 884 else if (op != *cmd_op) 885 return -EINVAL; 886 887 switch (op) { 888 case ERT_START_CU: 889 ret = EXEC_MSG_OPS(xdna)->fill_cf_slot(cmd_abo, slot, size); 890 break; 891 case ERT_START_NPU: 892 ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size); 893 break; 894 case ERT_START_NPU_PREEMPT: 895 if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT)) 896 return -EOPNOTSUPP; 897 ret = EXEC_MSG_OPS(xdna)->fill_preempt_slot(cmd_abo, slot, size); 898 break; 899 case ERT_START_NPU_PREEMPT_ELF: 900 if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT)) 901 return -EOPNOTSUPP; 902 ret = EXEC_MSG_OPS(xdna)->fill_elf_slot(cmd_abo, slot, size); 903 break; 904 default: 905 XDNA_INFO(xdna, "Unsupported op %d", op); 906 ret = -EOPNOTSUPP; 907 break; 908 } 909 910 return ret; 911 } 912 913 void aie2_msg_init(struct amdxdna_dev_hdl *ndev) 914 { 915 if (AIE_FEATURE_ON(&ndev->aie, AIE2_NPU_COMMAND)) 916 ndev->exec_msg_ops = &npu_exec_message_ops; 917 else 918 ndev->exec_msg_ops = &legacy_exec_message_ops; 919 } 920 921 static inline struct amdxdna_gem_obj * 922 aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job) 923 { 924 int idx = get_job_idx(job->seq); 925 926 return job->hwctx->priv->cmd_buf[idx]; 927 } 928 929 int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 930 int (*notify_cb)(void *, void __iomem *, size_t)) 931 { 932 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 933 struct amdxdna_dev *xdna = hwctx->client->xdna; 934 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 935 struct xdna_mailbox_msg msg; 936 union exec_req req; 937 int ret; 938 939 if (!chann) 940 return -ENODEV; 941 942 ret = aie2_init_exec_req(&req, cmd_abo, &msg.send_size, &msg.opcode); 943 if (ret) 944 return ret; 945 946 msg.handle = job; 947 msg.notify_cb = notify_cb; 948 msg.send_data = (u8 *)&req; 949 print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req, 950 0x40, false); 951 952 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 953 if (ret) { 954 XDNA_ERR(xdna, "Send message failed"); 955 return ret; 956 } 957 958 return 0; 959 } 960 961 int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, 962 struct amdxdna_sched_job *job, 963 int (*notify_cb)(void *, void __iomem *, size_t)) 964 { 965 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 966 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 967 struct amdxdna_client *client = hwctx->client; 968 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 969 void *cmd_buf = amdxdna_gem_vmap(cmdbuf_abo); 970 struct amdxdna_dev *xdna = client->xdna; 971 struct amdxdna_cmd_chain *payload; 972 struct xdna_mailbox_msg msg; 973 union exec_chain_req req; 974 u32 payload_len, ccnt; 975 u32 offset = 0; 976 size_t size; 977 int ret; 978 u32 op; 979 u32 i; 980 981 if (!cmd_buf) 982 return -ENOMEM; 983 984 op = amdxdna_cmd_get_op(cmd_abo); 985 payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len); 986 if (op != ERT_CMD_CHAIN) { 987 XDNA_DBG(xdna, "Invalid op code %d", op); 988 return -EINVAL; 989 } 990 991 if (!payload) { 992 XDNA_DBG(xdna, "Failed to get command payload"); 993 return -EINVAL; 994 } 995 996 ccnt = payload->command_count; 997 if (payload_len < struct_size(payload, data, ccnt)) { 998 XDNA_DBG(xdna, "Invalid command count %d", ccnt); 999 return -EINVAL; 1000 } 1001 1002 op = ERT_INVALID_CMD; 1003 for (i = 0; i < ccnt; i++) { 1004 u32 boh = (u32)(payload->data[i]); 1005 struct amdxdna_gem_obj *abo; 1006 1007 abo = amdxdna_gem_get_obj(client, boh, AMDXDNA_BO_SHARE); 1008 if (!abo) { 1009 XDNA_ERR(xdna, "Failed to find cmd BO %d", boh); 1010 return -ENOENT; 1011 } 1012 1013 size = cmdbuf_abo->mem.size - offset; 1014 ret = aie2_cmdlist_fill_slot(cmd_buf + offset, abo, &size, &op); 1015 amdxdna_gem_put_obj(abo); 1016 if (ret) 1017 return ret; 1018 1019 offset += size; 1020 } 1021 1022 XDNA_DBG(xdna, "Total %d commands:", ccnt); 1023 print_hex_dump_debug("cmdbufs: ", DUMP_PREFIX_OFFSET, 16, 4, 1024 cmd_buf, offset, false); 1025 1026 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); 1027 if (msg.opcode == MSG_OP_MAX_OPCODE) 1028 return -EOPNOTSUPP; 1029 1030 /* The offset is the accumulated total size of the cmd buffer */ 1031 EXEC_MSG_OPS(xdna)->init_chain_req(&req, amdxdna_gem_dev_addr(cmdbuf_abo), 1032 offset, ccnt); 1033 drm_clflush_virt_range(cmd_buf, offset); 1034 1035 msg.handle = job; 1036 msg.notify_cb = notify_cb; 1037 msg.send_data = (u8 *)&req; 1038 msg.send_size = sizeof(req); 1039 print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4, 1040 &req, msg.send_size, false); 1041 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1042 if (ret) { 1043 XDNA_ERR(xdna, "Send message failed"); 1044 return ret; 1045 } 1046 1047 return 0; 1048 } 1049 1050 int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx, 1051 struct amdxdna_sched_job *job, 1052 int (*notify_cb)(void *, void __iomem *, size_t)) 1053 { 1054 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 1055 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1056 struct amdxdna_dev *xdna = hwctx->client->xdna; 1057 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 1058 void *cmd_buf = amdxdna_gem_vmap(cmdbuf_abo); 1059 struct xdna_mailbox_msg msg; 1060 union exec_chain_req req; 1061 u32 op = ERT_INVALID_CMD; 1062 size_t size; 1063 int ret; 1064 1065 if (!cmd_buf) 1066 return -ENOMEM; 1067 1068 size = cmdbuf_abo->mem.size; 1069 ret = aie2_cmdlist_fill_slot(cmd_buf, cmd_abo, &size, &op); 1070 if (ret) 1071 return ret; 1072 1073 print_hex_dump_debug("cmdbuf: ", DUMP_PREFIX_OFFSET, 16, 4, cmd_buf, size, false); 1074 1075 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); 1076 if (msg.opcode == MSG_OP_MAX_OPCODE) 1077 return -EOPNOTSUPP; 1078 1079 EXEC_MSG_OPS(xdna)->init_chain_req(&req, amdxdna_gem_dev_addr(cmdbuf_abo), size, 1); 1080 drm_clflush_virt_range(cmd_buf, size); 1081 1082 msg.handle = job; 1083 msg.notify_cb = notify_cb; 1084 msg.send_data = (u8 *)&req; 1085 msg.send_size = sizeof(req); 1086 print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4, 1087 &req, msg.send_size, false); 1088 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1089 if (ret) { 1090 XDNA_ERR(hwctx->client->xdna, "Send message failed"); 1091 return ret; 1092 } 1093 1094 return 0; 1095 } 1096 1097 int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 1098 int (*notify_cb)(void *, void __iomem *, size_t)) 1099 { 1100 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1101 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); 1102 struct amdxdna_dev *xdna = hwctx->client->xdna; 1103 struct xdna_mailbox_msg msg; 1104 struct sync_bo_req req; 1105 int ret = 0; 1106 1107 req.src_addr = 0; 1108 req.dst_addr = amdxdna_dev_bo_offset(abo); 1109 req.size = abo->mem.size; 1110 1111 /* Device to Host */ 1112 req.type = FIELD_PREP(AIE2_MSG_SYNC_BO_SRC_TYPE, SYNC_BO_DEV_MEM) | 1113 FIELD_PREP(AIE2_MSG_SYNC_BO_DST_TYPE, SYNC_BO_HOST_MEM); 1114 1115 XDNA_DBG(xdna, "sync %d bytes src(0x%llx) to dst(0x%llx) completed", 1116 req.size, req.src_addr, req.dst_addr); 1117 1118 msg.handle = job; 1119 msg.notify_cb = notify_cb; 1120 msg.send_data = (u8 *)&req; 1121 msg.send_size = sizeof(req); 1122 msg.opcode = MSG_OP_SYNC_BO; 1123 1124 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1125 if (ret) { 1126 XDNA_ERR(xdna, "Send message failed"); 1127 return ret; 1128 } 1129 1130 return 0; 1131 } 1132 1133 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 1134 int (*notify_cb)(void *, void __iomem *, size_t)) 1135 { 1136 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1137 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); 1138 struct amdxdna_dev *xdna = hwctx->client->xdna; 1139 struct config_debug_bo_req req; 1140 struct xdna_mailbox_msg msg; 1141 1142 if (job->drv_cmd->opcode == ATTACH_DEBUG_BO) 1143 req.config = DEBUG_BO_REGISTER; 1144 else 1145 req.config = DEBUG_BO_UNREGISTER; 1146 1147 req.offset = amdxdna_dev_bo_offset(abo); 1148 req.size = abo->mem.size; 1149 1150 XDNA_DBG(xdna, "offset 0x%llx size 0x%llx config %d", 1151 req.offset, req.size, req.config); 1152 1153 msg.handle = job; 1154 msg.notify_cb = notify_cb; 1155 msg.send_data = (u8 *)&req; 1156 msg.send_size = sizeof(req); 1157 msg.opcode = MSG_OP_CONFIG_DEBUG_BO; 1158 1159 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1160 } 1161 1162 int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id, 1163 struct app_health_report *report) 1164 { 1165 DECLARE_AIE_MSG(get_app_health, MSG_OP_GET_APP_HEALTH); 1166 struct amdxdna_dev *xdna = ndev->aie.xdna; 1167 struct app_health_report *buf; 1168 dma_addr_t dma_addr; 1169 u32 buf_size; 1170 int ret; 1171 1172 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_APP_HEALTH)) { 1173 XDNA_DBG(xdna, "App health feature not supported"); 1174 return -EOPNOTSUPP; 1175 } 1176 1177 buf_size = sizeof(*report); 1178 buf = amdxdna_alloc_msg_buffer(xdna, &buf_size, &dma_addr); 1179 if (IS_ERR(buf)) { 1180 XDNA_ERR(xdna, "Failed to allocate buffer for app health"); 1181 return PTR_ERR(buf); 1182 } 1183 1184 req.buf_addr = dma_addr; 1185 req.context_id = context_id; 1186 req.buf_size = buf_size; 1187 1188 drm_clflush_virt_range(buf, req.buf_size); 1189 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 1190 if (ret) { 1191 XDNA_ERR(xdna, "Get app health failed, ret %d status 0x%x", ret, resp.status); 1192 goto free_buf; 1193 } 1194 1195 /* Copy the report to caller's buffer */ 1196 memcpy(report, buf, sizeof(*report)); 1197 1198 free_buf: 1199 amdxdna_free_msg_buffer(xdna, buf_size, buf, dma_addr); 1200 return ret; 1201 } 1202 1203 static int 1204 aie2_runtime_update_ctx_prop(struct amdxdna_dev_hdl *ndev, 1205 struct amdxdna_hwctx *ctx, u32 type, u32 value) 1206 { 1207 DECLARE_AIE_MSG(update_property, MSG_OP_UPDATE_PROPERTY); 1208 struct amdxdna_dev *xdna = ndev->aie.xdna; 1209 int ret; 1210 1211 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_UPDATE_PROPERTY)) 1212 return -EOPNOTSUPP; 1213 1214 if (ctx) 1215 req.context_id = ctx->fw_ctx_id; 1216 else 1217 req.context_id = AIE2_UPDATE_PROPERTY_ALL_CTX; 1218 1219 req.time_quota_us = value; 1220 req.type = type; 1221 1222 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 1223 if (ret) { 1224 XDNA_ERR(xdna, "%s update property failed, type %d ret %d", 1225 ctx ? ctx->name : "ctx.all", type, ret); 1226 return ret; 1227 } 1228 1229 return 0; 1230 } 1231 1232 int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us) 1233 { 1234 struct amdxdna_dev *xdna = ndev->aie.xdna; 1235 int ret; 1236 1237 ret = aie2_runtime_update_ctx_prop(ndev, NULL, UPDATE_PROPERTY_TIME_QUOTA, us); 1238 if (ret == -EOPNOTSUPP) { 1239 XDNA_DBG(xdna, "update time quota not support, skipped"); 1240 ret = 0; 1241 } else if (!ret) { 1242 XDNA_DBG(xdna, "Ctx exec time quantum updated to %u us", us); 1243 } 1244 return ret; 1245 } 1246 1247 int aie2_get_dev_revision(struct amdxdna_dev_hdl *ndev, enum aie2_dev_revision *rev) 1248 { 1249 DECLARE_AIE_MSG(get_dev_revision, MSG_OP_GET_DEV_REVISION); 1250 struct amdxdna_dev *xdna = ndev->aie.xdna; 1251 int ret; 1252 1253 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_GET_DEV_REVISION)) 1254 return -EOPNOTSUPP; 1255 1256 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 1257 if (ret) 1258 return ret; 1259 1260 *rev = resp.rev; 1261 1262 if (*rev < AIE2_DEV_REVISION_STXA || *rev >= AIE2_DEV_REVISION_UNKN) { 1263 XDNA_ERR(xdna, "Unknown device revision: %d (raw fuse: 0x%x)", 1264 *rev, resp.raw_fuse_data); 1265 return -EINVAL; 1266 } 1267 1268 XDNA_DBG(xdna, "Device revision: %d (raw fuse: 0x%x)", *rev, resp.raw_fuse_data); 1269 return 0; 1270 } 1271