1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_cache.h> 8 #include <drm/drm_device.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/drm_print.h> 12 #include <drm/gpu_scheduler.h> 13 #include <linux/bitfield.h> 14 #include <linux/errno.h> 15 #include <linux/pci.h> 16 #include <linux/types.h> 17 #include <linux/xarray.h> 18 19 #include "aie.h" 20 #include "aie2_msg_priv.h" 21 #include "aie2_pci.h" 22 #include "amdxdna_ctx.h" 23 #include "amdxdna_gem.h" 24 #include "amdxdna_mailbox.h" 25 #include "amdxdna_mailbox_helper.h" 26 #include "amdxdna_pci_drv.h" 27 28 #define EXEC_MSG_OPS(xdna) ((xdna)->dev_handle->exec_msg_ops) 29 30 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev) 31 { 32 DECLARE_AIE_MSG(suspend, MSG_OP_SUSPEND); 33 int ret; 34 35 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 36 if (ret) { 37 XDNA_ERR(ndev->aie.xdna, "Failed to suspend fw, ret %d", ret); 38 return ret; 39 } 40 41 return aie_psp_waitmode_poll(ndev->aie.psp_hdl); 42 } 43 44 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev) 45 { 46 DECLARE_AIE_MSG(suspend, MSG_OP_RESUME); 47 48 return aie_send_mgmt_msg_wait(&ndev->aie, &msg); 49 } 50 51 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value) 52 { 53 DECLARE_AIE_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG); 54 int ret; 55 56 req.type = type; 57 req.value = value; 58 59 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 60 if (ret) { 61 XDNA_ERR(ndev->aie.xdna, "Failed to set runtime config, ret %d", ret); 62 return ret; 63 } 64 65 return 0; 66 } 67 68 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value) 69 { 70 DECLARE_AIE_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG); 71 int ret; 72 73 req.type = type; 74 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 75 if (ret) { 76 XDNA_ERR(ndev->aie.xdna, "Failed to get runtime config, ret %d", ret); 77 return ret; 78 } 79 80 *value = resp.value; 81 return 0; 82 } 83 84 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid) 85 { 86 DECLARE_AIE_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID); 87 88 req.pasid = pasid; 89 90 return aie_send_mgmt_msg_wait(&ndev->aie, &msg); 91 } 92 93 int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, 94 struct amdxdna_drm_query_aie_version *version) 95 { 96 DECLARE_AIE_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION); 97 struct amdxdna_dev *xdna = ndev->aie.xdna; 98 int ret; 99 100 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 101 if (ret) 102 return ret; 103 104 XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed", 105 resp.major, resp.minor); 106 107 version->major = resp.major; 108 version->minor = resp.minor; 109 110 return 0; 111 } 112 113 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, 114 struct amdxdna_drm_query_aie_metadata *metadata) 115 { 116 DECLARE_AIE_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO); 117 int ret; 118 119 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 120 if (ret) 121 return ret; 122 123 metadata->col_size = resp.info.size; 124 metadata->cols = resp.info.cols; 125 metadata->rows = resp.info.rows; 126 127 metadata->version.major = resp.info.major; 128 metadata->version.minor = resp.info.minor; 129 130 metadata->core.row_count = resp.info.core_rows; 131 metadata->core.row_start = resp.info.core_row_start; 132 metadata->core.dma_channel_count = resp.info.core_dma_channels; 133 metadata->core.lock_count = resp.info.core_locks; 134 metadata->core.event_reg_count = resp.info.core_events; 135 136 metadata->mem.row_count = resp.info.mem_rows; 137 metadata->mem.row_start = resp.info.mem_row_start; 138 metadata->mem.dma_channel_count = resp.info.mem_dma_channels; 139 metadata->mem.lock_count = resp.info.mem_locks; 140 metadata->mem.event_reg_count = resp.info.mem_events; 141 142 metadata->shim.row_count = resp.info.shim_rows; 143 metadata->shim.row_start = resp.info.shim_row_start; 144 metadata->shim.dma_channel_count = resp.info.shim_dma_channels; 145 metadata->shim.lock_count = resp.info.shim_locks; 146 metadata->shim.event_reg_count = resp.info.shim_events; 147 148 return 0; 149 } 150 151 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, 152 struct amdxdna_fw_ver *fw_ver) 153 { 154 DECLARE_AIE_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION); 155 int ret; 156 157 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 158 if (ret) 159 return ret; 160 161 fw_ver->major = resp.major; 162 fw_ver->minor = resp.minor; 163 fw_ver->sub = resp.sub; 164 fw_ver->build = resp.build; 165 166 return 0; 167 } 168 169 static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id) 170 { 171 DECLARE_AIE_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); 172 struct amdxdna_dev *xdna = ndev->aie.xdna; 173 int ret; 174 175 req.context_id = id; 176 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 177 if (ret && ret != -ENODEV) 178 XDNA_WARN(xdna, "Destroy context failed, ret %d", ret); 179 else if (ret == -ENODEV) 180 XDNA_DBG(xdna, "Destroy context: device already stopped"); 181 182 return ret; 183 } 184 185 static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev, 186 struct amdxdna_hwctx *hwctx) 187 { 188 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_PREEMPT)) 189 return PRIORITY_HIGH; 190 191 switch (hwctx->qos.priority) { 192 case AMDXDNA_QOS_REALTIME_PRIORITY: 193 return PRIORITY_REALTIME; 194 case AMDXDNA_QOS_HIGH_PRIORITY: 195 return PRIORITY_HIGH; 196 case AMDXDNA_QOS_NORMAL_PRIORITY: 197 return PRIORITY_NORMAL; 198 case AMDXDNA_QOS_LOW_PRIORITY: 199 return PRIORITY_LOW; 200 default: 201 return PRIORITY_HIGH; 202 } 203 } 204 205 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 206 { 207 DECLARE_AIE_MSG(create_ctx, MSG_OP_CREATE_CONTEXT); 208 struct amdxdna_dev *xdna = ndev->aie.xdna; 209 struct xdna_mailbox_chann_res x2i; 210 struct xdna_mailbox_chann_res i2x; 211 struct cq_pair *cq_pair; 212 u32 intr_reg; 213 int ret; 214 215 req.aie_type = 1; 216 req.start_col = hwctx->start_col; 217 req.num_col = hwctx->num_col; 218 req.num_unused_col = hwctx->num_unused_col; 219 req.num_cq_pairs_requested = 1; 220 req.pasid = amdxdna_pasid_on(hwctx->client) ? hwctx->client->pasid : 0; 221 req.context_priority = aie2_get_context_priority(ndev, hwctx); 222 223 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 224 if (ret) 225 return ret; 226 227 hwctx->fw_ctx_id = resp.context_id; 228 if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1)) 229 return -EINVAL; 230 231 if (ndev->force_preempt_enabled) { 232 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id); 233 if (ret) { 234 XDNA_ERR(xdna, "failed to enable force preempt %d", ret); 235 goto del_ctx_req; 236 } 237 } 238 239 cq_pair = &resp.cq_pair[0]; 240 x2i.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.head_addr); 241 x2i.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.tail_addr); 242 x2i.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->x2i_q.buf_addr); 243 x2i.rb_size = cq_pair->x2i_q.buf_size; 244 245 i2x.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.head_addr); 246 i2x.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.tail_addr); 247 i2x.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->i2x_q.buf_addr); 248 i2x.rb_size = cq_pair->i2x_q.buf_size; 249 250 ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id); 251 if (ret == -EINVAL) { 252 XDNA_ERR(xdna, "Alloc IRQ failed %d", ret); 253 goto del_ctx_req; 254 } 255 256 intr_reg = i2x.mb_head_ptr_reg + 4; 257 hwctx->priv->mbox_chann = xdna_mailbox_alloc_channel(ndev->mbox); 258 if (!hwctx->priv->mbox_chann) { 259 XDNA_ERR(xdna, "Not able to create channel"); 260 ret = -EINVAL; 261 goto del_ctx_req; 262 } 263 264 /* +1 for the config_cu message that can be in-flight concurrently */ 265 ret = xdna_mailbox_start_channel(hwctx->priv->mbox_chann, &x2i, &i2x, 266 intr_reg, ret, HWCTX_MAX_CMDS + 1); 267 if (ret) { 268 XDNA_ERR(xdna, "Not able to create channel"); 269 ret = -EINVAL; 270 goto free_channel; 271 } 272 ndev->hwctx_num++; 273 274 XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id); 275 XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid); 276 277 return 0; 278 279 free_channel: 280 xdna_mailbox_free_channel(hwctx->priv->mbox_chann); 281 hwctx->priv->mbox_chann = NULL; 282 del_ctx_req: 283 aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); 284 return ret; 285 } 286 287 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 288 { 289 struct amdxdna_dev *xdna = ndev->aie.xdna; 290 int ret; 291 292 if (!hwctx->priv->mbox_chann) 293 return 0; 294 295 xdna_mailbox_stop_channel(hwctx->priv->mbox_chann); 296 ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); 297 xdna_mailbox_free_channel(hwctx->priv->mbox_chann); 298 XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id); 299 hwctx->priv->mbox_chann = NULL; 300 hwctx->fw_ctx_id = -1; 301 ndev->hwctx_num--; 302 303 return ret; 304 } 305 306 static int aie2_send_host_buf_msgs(struct amdxdna_dev_hdl *ndev, u32 context_id, 307 u64 addr, u64 size, u32 initial_opcode) 308 { 309 DECLARE_AIE_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER); 310 struct amdxdna_dev *xdna = ndev->aie.xdna; 311 size_t chunk_size; 312 int ret; 313 314 chunk_size = xdna->dev_info->dev_mem_size; 315 if (!size || !IS_ALIGNED(size, chunk_size)) { 316 XDNA_ERR(xdna, "Invalid size 0x%llx for chunk 0x%lx", 317 size, chunk_size); 318 return -EINVAL; 319 } 320 321 msg.opcode = initial_opcode; 322 do { 323 req.context_id = context_id; 324 req.buf_addr = addr; 325 req.buf_size = chunk_size; 326 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 327 if (ret) { 328 XDNA_ERR(xdna, "fw ctx %d addr 0x%llx size 0x%lx", 329 context_id, addr, chunk_size); 330 return ret; 331 } 332 333 XDNA_DBG(xdna, "fw ctx %d host buf op 0x%x addr 0x%llx size 0x%lx", 334 context_id, msg.opcode, addr, chunk_size); 335 336 addr += chunk_size; 337 size -= chunk_size; 338 msg.opcode = MSG_OP_ADD_HOST_BUFFER; 339 } while (size); 340 341 return 0; 342 } 343 344 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size) 345 { 346 return aie2_send_host_buf_msgs(ndev, context_id, addr, size, 347 MSG_OP_MAP_HOST_BUFFER); 348 } 349 350 int aie2_add_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size) 351 { 352 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_ADD_HOST_BUFFER)) 353 return -EOPNOTSUPP; 354 355 return aie2_send_host_buf_msgs(ndev, context_id, addr, size, 356 MSG_OP_ADD_HOST_BUFFER); 357 } 358 359 static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg) 360 { 361 u32 *bitmap = arg; 362 363 *bitmap |= GENMASK(hwctx->start_col + hwctx->num_col - 1, hwctx->start_col); 364 365 return 0; 366 } 367 368 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, 369 u32 size, u32 *cols_filled) 370 { 371 DECLARE_AIE_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS); 372 struct amdxdna_dev *xdna = ndev->aie.xdna; 373 u32 buf_sz, aie_bitmap = 0; 374 struct amdxdna_client *client; 375 dma_addr_t dma_addr; 376 u8 *buff_addr; 377 int ret; 378 379 buf_sz = ndev->aie.metadata.cols * ndev->aie.metadata.col_size; 380 buff_addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr); 381 if (IS_ERR(buff_addr)) 382 return PTR_ERR(buff_addr); 383 384 /* Go through each hardware context and mark the AIE columns that are active */ 385 list_for_each_entry(client, &xdna->client_list, node) 386 amdxdna_hwctx_walk(client, &aie_bitmap, amdxdna_hwctx_col_map); 387 388 *cols_filled = 0; 389 req.dump_buff_addr = dma_addr; 390 req.dump_buff_size = buf_sz; 391 req.num_cols = hweight32(aie_bitmap); 392 req.aie_bitmap = aie_bitmap; 393 394 drm_clflush_virt_range(buff_addr, req.dump_buff_size); /* device can access */ 395 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 396 if (ret) { 397 XDNA_ERR(xdna, "Error during NPU query, status %d", ret); 398 goto fail; 399 } 400 401 XDNA_DBG(xdna, "Query NPU status completed"); 402 403 if (buf_sz < resp.size) { 404 ret = -EINVAL; 405 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size); 406 goto fail; 407 } 408 409 size = min(size, resp.size); 410 if (copy_to_user(buf, buff_addr, size)) { 411 ret = -EFAULT; 412 XDNA_ERR(xdna, "Failed to copy NPU status to user space"); 413 goto fail; 414 } 415 416 *cols_filled = aie_bitmap; 417 418 fail: 419 amdxdna_free_msg_buffer(xdna, buf_sz, buff_addr, dma_addr); 420 return ret; 421 } 422 423 int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, 424 char __user *buf, u32 size, 425 struct amdxdna_drm_query_telemetry_header *header) 426 { 427 DECLARE_AIE_MSG(get_telemetry, MSG_OP_GET_TELEMETRY); 428 struct amdxdna_dev *xdna = ndev->aie.xdna; 429 dma_addr_t dma_addr; 430 u32 buf_sz; 431 u8 *addr; 432 int ret; 433 434 if (header->type >= MAX_TELEMETRY_TYPE) 435 return -EINVAL; 436 437 buf_sz = min(size, SZ_4M); 438 addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr); 439 if (IS_ERR(addr)) 440 return PTR_ERR(addr); 441 442 req.buf_addr = dma_addr; 443 req.buf_size = buf_sz; 444 req.type = header->type; 445 446 drm_clflush_virt_range(addr, req.buf_size); /* device can access */ 447 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 448 if (ret) { 449 XDNA_ERR(xdna, "Query telemetry failed, status %d", ret); 450 goto free_buf; 451 } 452 453 if (buf_sz < resp.size) { 454 ret = -EINVAL; 455 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size); 456 goto free_buf; 457 } 458 459 size = min(size, resp.size); 460 if (copy_to_user(buf, addr, size)) { 461 ret = -EFAULT; 462 XDNA_ERR(xdna, "Failed to copy telemetry to user space"); 463 goto free_buf; 464 } 465 466 header->major = resp.major; 467 header->minor = resp.minor; 468 469 free_buf: 470 amdxdna_free_msg_buffer(xdna, buf_sz, addr, dma_addr); 471 return ret; 472 } 473 474 int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size, 475 void *handle, int (*cb)(void*, void __iomem *, size_t)) 476 { 477 struct async_event_msg_req req = { 0 }; 478 struct xdna_mailbox_msg msg = { 479 .send_data = (u8 *)&req, 480 .send_size = sizeof(req), 481 .handle = handle, 482 .opcode = MSG_OP_REGISTER_ASYNC_EVENT_MSG, 483 .notify_cb = cb, 484 }; 485 486 req.buf_addr = addr; 487 req.buf_size = size; 488 489 XDNA_DBG(ndev->aie.xdna, "Register addr 0x%llx size 0x%x", addr, size); 490 return xdna_mailbox_send_msg(ndev->aie.mgmt_chann, &msg); 491 } 492 493 int aie2_config_cu(struct amdxdna_hwctx *hwctx, 494 int (*notify_cb)(void *, void __iomem *, size_t)) 495 { 496 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 497 struct amdxdna_dev *xdna = hwctx->client->xdna; 498 u32 shift = xdna->dev_info->dev_mem_buf_shift; 499 struct config_cu_req req = { 0 }; 500 struct xdna_mailbox_msg msg; 501 struct drm_gem_object *gobj; 502 struct amdxdna_gem_obj *abo; 503 int i; 504 505 if (!chann) 506 return -ENODEV; 507 508 if (!hwctx->cus) 509 return 0; 510 511 if (hwctx->cus->num_cus > MAX_NUM_CUS) { 512 XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS); 513 return -EINVAL; 514 } 515 516 for (i = 0; i < hwctx->cus->num_cus; i++) { 517 struct amdxdna_cu_config *cu = &hwctx->cus->cu_configs[i]; 518 519 if (XDNA_MBZ_DBG(xdna, cu->pad, sizeof(cu->pad))) 520 return -EINVAL; 521 522 gobj = drm_gem_object_lookup(hwctx->client->filp, cu->cu_bo); 523 if (!gobj) { 524 XDNA_ERR(xdna, "Lookup GEM object failed"); 525 return -EINVAL; 526 } 527 abo = to_xdna_obj(gobj); 528 529 if (abo->type != AMDXDNA_BO_DEV) { 530 drm_gem_object_put(gobj); 531 XDNA_ERR(xdna, "Invalid BO type"); 532 return -EINVAL; 533 } 534 535 req.cfgs[i] = FIELD_PREP(AIE2_MSG_CFG_CU_PDI_ADDR, 536 amdxdna_gem_dev_addr(abo) >> shift); 537 req.cfgs[i] |= FIELD_PREP(AIE2_MSG_CFG_CU_FUNC, cu->cu_func); 538 XDNA_DBG(xdna, "CU %d full addr 0x%llx, cfg 0x%x", i, 539 amdxdna_gem_dev_addr(abo), req.cfgs[i]); 540 drm_gem_object_put(gobj); 541 } 542 req.num_cus = hwctx->cus->num_cus; 543 544 msg.send_data = (u8 *)&req; 545 msg.send_size = sizeof(req); 546 msg.handle = hwctx; 547 msg.opcode = MSG_OP_CONFIG_CU; 548 msg.notify_cb = notify_cb; 549 550 return xdna_mailbox_send_msg(chann, &msg); 551 } 552 553 static int aie2_init_exec_cu_req(struct amdxdna_gem_obj *cmd_bo, void *req, 554 size_t *size, u32 *msg_op) 555 { 556 struct execute_buffer_req *cu_req = req; 557 u32 cmd_len; 558 void *cmd; 559 560 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 561 if (cmd_len > sizeof(cu_req->payload)) 562 return -EINVAL; 563 564 cu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 565 if (cu_req->cu_idx == INVALID_CU_IDX) 566 return -EINVAL; 567 568 memcpy(cu_req->payload, cmd, cmd_len); 569 570 *size = sizeof(*cu_req); 571 *msg_op = MSG_OP_EXECUTE_BUFFER_CF; 572 return 0; 573 } 574 575 static int aie2_init_exec_dpu_req(struct amdxdna_gem_obj *cmd_bo, void *req, 576 size_t *size, u32 *msg_op) 577 { 578 struct exec_dpu_req *dpu_req = req; 579 struct amdxdna_cmd_start_npu *sn; 580 u32 cmd_len; 581 582 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 583 if (cmd_len - sizeof(*sn) > sizeof(dpu_req->payload)) 584 return -EINVAL; 585 586 dpu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 587 if (dpu_req->cu_idx == INVALID_CU_IDX) 588 return -EINVAL; 589 590 dpu_req->inst_buf_addr = sn->buffer; 591 dpu_req->inst_size = sn->buffer_size; 592 dpu_req->inst_prop_cnt = sn->prop_count; 593 memcpy(dpu_req->payload, sn->prop_args, cmd_len - sizeof(*sn)); 594 595 *size = sizeof(*dpu_req); 596 *msg_op = MSG_OP_EXEC_DPU; 597 return 0; 598 } 599 600 static void aie2_init_exec_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) 601 { 602 struct cmd_chain_req *chain_req = req; 603 604 chain_req->buf_addr = slot_addr; 605 chain_req->buf_size = size; 606 chain_req->count = cmd_cnt; 607 } 608 609 static void aie2_init_npu_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) 610 { 611 struct cmd_chain_npu_req *npu_chain_req = req; 612 613 npu_chain_req->flags = 0; 614 npu_chain_req->reserved = 0; 615 npu_chain_req->buf_addr = slot_addr; 616 npu_chain_req->buf_size = size; 617 npu_chain_req->count = cmd_cnt; 618 } 619 620 static int 621 aie2_cmdlist_fill_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 622 { 623 struct cmd_chain_slot_execbuf_cf *cf_slot = slot; 624 u32 cmd_len; 625 void *cmd; 626 627 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 628 if (*size < sizeof(*cf_slot) + cmd_len) 629 return -EINVAL; 630 631 cf_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 632 if (cf_slot->cu_idx == INVALID_CU_IDX) 633 return -EINVAL; 634 635 cf_slot->arg_cnt = cmd_len / sizeof(u32); 636 memcpy(cf_slot->args, cmd, cmd_len); 637 /* Accurate slot size to hint firmware to do necessary copy */ 638 *size = sizeof(*cf_slot) + cmd_len; 639 return 0; 640 } 641 642 static int 643 aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 644 { 645 struct cmd_chain_slot_dpu *dpu_slot = slot; 646 struct amdxdna_cmd_start_npu *sn; 647 u32 cmd_len; 648 u32 arg_sz; 649 650 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 651 arg_sz = cmd_len - sizeof(*sn); 652 if (cmd_len < sizeof(*sn) || arg_sz > MAX_DPU_ARGS_SIZE) 653 return -EINVAL; 654 655 if (*size < sizeof(*dpu_slot) + arg_sz) 656 return -EINVAL; 657 658 dpu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 659 if (dpu_slot->cu_idx == INVALID_CU_IDX) 660 return -EINVAL; 661 662 dpu_slot->inst_buf_addr = sn->buffer; 663 dpu_slot->inst_size = sn->buffer_size; 664 dpu_slot->inst_prop_cnt = sn->prop_count; 665 dpu_slot->arg_cnt = arg_sz / sizeof(u32); 666 memcpy(dpu_slot->args, sn->prop_args, arg_sz); 667 668 /* Accurate slot size to hint firmware to do necessary copy */ 669 *size = sizeof(*dpu_slot) + arg_sz; 670 return 0; 671 } 672 673 static int aie2_cmdlist_unsupp(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 674 { 675 return -EOPNOTSUPP; 676 } 677 678 static u32 aie2_get_chain_msg_op(u32 cmd_op) 679 { 680 switch (cmd_op) { 681 case ERT_START_CU: 682 return MSG_OP_CHAIN_EXEC_BUFFER_CF; 683 case ERT_START_NPU: 684 return MSG_OP_CHAIN_EXEC_DPU; 685 default: 686 break; 687 } 688 689 return MSG_OP_MAX_OPCODE; 690 } 691 692 static struct aie2_exec_msg_ops legacy_exec_message_ops = { 693 .init_cu_req = aie2_init_exec_cu_req, 694 .init_dpu_req = aie2_init_exec_dpu_req, 695 .init_chain_req = aie2_init_exec_chain_req, 696 .fill_cf_slot = aie2_cmdlist_fill_cf, 697 .fill_dpu_slot = aie2_cmdlist_fill_dpu, 698 .fill_preempt_slot = aie2_cmdlist_unsupp, 699 .fill_elf_slot = aie2_cmdlist_unsupp, 700 .get_chain_msg_op = aie2_get_chain_msg_op, 701 }; 702 703 static int 704 aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 705 { 706 struct cmd_chain_slot_npu *npu_slot = slot; 707 u32 cmd_len; 708 void *cmd; 709 710 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 711 if (*size < sizeof(*npu_slot) + cmd_len) 712 return -EINVAL; 713 714 memset(npu_slot, 0, sizeof(*npu_slot)); 715 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 716 if (npu_slot->cu_idx == INVALID_CU_IDX) 717 return -EINVAL; 718 719 npu_slot->type = EXEC_NPU_TYPE_NON_ELF; 720 npu_slot->arg_cnt = cmd_len / sizeof(u32); 721 memcpy(npu_slot->args, cmd, cmd_len); 722 723 *size = sizeof(*npu_slot) + cmd_len; 724 return 0; 725 } 726 727 static int 728 aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 729 { 730 struct cmd_chain_slot_npu *npu_slot = slot; 731 struct amdxdna_cmd_start_npu *sn; 732 u32 cmd_len; 733 u32 arg_sz; 734 735 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 736 arg_sz = cmd_len - sizeof(*sn); 737 if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE) 738 return -EINVAL; 739 740 if (*size < sizeof(*npu_slot) + arg_sz) 741 return -EINVAL; 742 743 memset(npu_slot, 0, sizeof(*npu_slot)); 744 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 745 if (npu_slot->cu_idx == INVALID_CU_IDX) 746 return -EINVAL; 747 748 npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF; 749 npu_slot->inst_buf_addr = sn->buffer; 750 npu_slot->inst_size = sn->buffer_size; 751 npu_slot->inst_prop_cnt = sn->prop_count; 752 npu_slot->arg_cnt = arg_sz / sizeof(u32); 753 memcpy(npu_slot->args, sn->prop_args, arg_sz); 754 755 *size = sizeof(*npu_slot) + arg_sz; 756 return 0; 757 } 758 759 static int 760 aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 761 { 762 struct cmd_chain_slot_npu *npu_slot = slot; 763 struct amdxdna_cmd_preempt_data *pd; 764 u32 cmd_len; 765 u32 arg_sz; 766 767 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 768 arg_sz = cmd_len - sizeof(*pd); 769 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) 770 return -EINVAL; 771 772 if (*size < sizeof(*npu_slot) + arg_sz) 773 return -EINVAL; 774 775 memset(npu_slot, 0, sizeof(*npu_slot)); 776 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 777 if (npu_slot->cu_idx == INVALID_CU_IDX) 778 return -EINVAL; 779 780 npu_slot->type = EXEC_NPU_TYPE_PREEMPT; 781 npu_slot->inst_buf_addr = pd->inst_buf; 782 npu_slot->save_buf_addr = pd->save_buf; 783 npu_slot->restore_buf_addr = pd->restore_buf; 784 npu_slot->inst_size = pd->inst_size; 785 npu_slot->save_size = pd->save_size; 786 npu_slot->restore_size = pd->restore_size; 787 npu_slot->inst_prop_cnt = pd->inst_prop_cnt; 788 npu_slot->arg_cnt = arg_sz / sizeof(u32); 789 memcpy(npu_slot->args, pd->prop_args, arg_sz); 790 791 *size = sizeof(*npu_slot) + arg_sz; 792 return 0; 793 } 794 795 static int 796 aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 797 { 798 struct cmd_chain_slot_npu *npu_slot = slot; 799 struct amdxdna_cmd_preempt_data *pd; 800 u32 cmd_len; 801 u32 arg_sz; 802 803 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 804 arg_sz = cmd_len - sizeof(*pd); 805 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) 806 return -EINVAL; 807 808 if (*size < sizeof(*npu_slot) + arg_sz) 809 return -EINVAL; 810 811 memset(npu_slot, 0, sizeof(*npu_slot)); 812 npu_slot->type = EXEC_NPU_TYPE_ELF; 813 npu_slot->inst_buf_addr = pd->inst_buf; 814 npu_slot->save_buf_addr = pd->save_buf; 815 npu_slot->restore_buf_addr = pd->restore_buf; 816 npu_slot->inst_size = pd->inst_size; 817 npu_slot->save_size = pd->save_size; 818 npu_slot->restore_size = pd->restore_size; 819 npu_slot->inst_prop_cnt = pd->inst_prop_cnt; 820 npu_slot->arg_cnt = 1; 821 npu_slot->args[0] = AIE2_EXEC_BUFFER_KERNEL_OP_TXN; 822 823 *size = struct_size(npu_slot, args, npu_slot->arg_cnt); 824 return 0; 825 } 826 827 static u32 aie2_get_npu_chain_msg_op(u32 cmd_op) 828 { 829 return MSG_OP_CHAIN_EXEC_NPU; 830 } 831 832 static struct aie2_exec_msg_ops npu_exec_message_ops = { 833 .init_cu_req = aie2_init_exec_cu_req, 834 .init_dpu_req = aie2_init_exec_dpu_req, 835 .init_chain_req = aie2_init_npu_chain_req, 836 .fill_cf_slot = aie2_cmdlist_fill_npu_cf, 837 .fill_dpu_slot = aie2_cmdlist_fill_npu_dpu, 838 .fill_preempt_slot = aie2_cmdlist_fill_npu_preempt, 839 .fill_elf_slot = aie2_cmdlist_fill_npu_elf, 840 .get_chain_msg_op = aie2_get_npu_chain_msg_op, 841 }; 842 843 static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo, 844 size_t *size, u32 *msg_op) 845 { 846 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(cmd_abo)->dev); 847 int ret; 848 u32 op; 849 850 op = amdxdna_cmd_get_op(cmd_abo); 851 switch (op) { 852 case ERT_START_CU: 853 ret = EXEC_MSG_OPS(xdna)->init_cu_req(cmd_abo, req, size, msg_op); 854 if (ret) { 855 XDNA_DBG(xdna, "Init CU req failed ret %d", ret); 856 return ret; 857 } 858 break; 859 case ERT_START_NPU: 860 ret = EXEC_MSG_OPS(xdna)->init_dpu_req(cmd_abo, req, size, msg_op); 861 if (ret) { 862 XDNA_DBG(xdna, "Init DPU req failed ret %d", ret); 863 return ret; 864 } 865 866 break; 867 default: 868 XDNA_ERR(xdna, "Unsupported op %d", op); 869 ret = -EOPNOTSUPP; 870 break; 871 } 872 873 return ret; 874 } 875 876 static int 877 aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo, 878 size_t *size, u32 *cmd_op) 879 { 880 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(cmd_abo)->dev); 881 int ret; 882 u32 op; 883 884 op = amdxdna_cmd_get_op(cmd_abo); 885 if (*cmd_op == ERT_INVALID_CMD) 886 *cmd_op = op; 887 else if (op != *cmd_op) 888 return -EINVAL; 889 890 switch (op) { 891 case ERT_START_CU: 892 ret = EXEC_MSG_OPS(xdna)->fill_cf_slot(cmd_abo, slot, size); 893 break; 894 case ERT_START_NPU: 895 ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size); 896 break; 897 case ERT_START_NPU_PREEMPT: 898 if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT)) 899 return -EOPNOTSUPP; 900 ret = EXEC_MSG_OPS(xdna)->fill_preempt_slot(cmd_abo, slot, size); 901 break; 902 case ERT_START_NPU_PREEMPT_ELF: 903 if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT)) 904 return -EOPNOTSUPP; 905 ret = EXEC_MSG_OPS(xdna)->fill_elf_slot(cmd_abo, slot, size); 906 break; 907 default: 908 XDNA_INFO(xdna, "Unsupported op %d", op); 909 ret = -EOPNOTSUPP; 910 break; 911 } 912 913 return ret; 914 } 915 916 void aie2_msg_init(struct amdxdna_dev_hdl *ndev) 917 { 918 if (AIE_FEATURE_ON(&ndev->aie, AIE2_NPU_COMMAND)) 919 ndev->exec_msg_ops = &npu_exec_message_ops; 920 else 921 ndev->exec_msg_ops = &legacy_exec_message_ops; 922 } 923 924 static inline struct amdxdna_gem_obj * 925 aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job) 926 { 927 int idx = get_job_idx(job->seq); 928 929 return job->hwctx->priv->cmd_buf[idx]; 930 } 931 932 int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 933 int (*notify_cb)(void *, void __iomem *, size_t)) 934 { 935 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 936 struct amdxdna_dev *xdna = hwctx->client->xdna; 937 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 938 struct xdna_mailbox_msg msg; 939 union exec_req req; 940 int ret; 941 942 if (!chann) 943 return -ENODEV; 944 945 ret = aie2_init_exec_req(&req, cmd_abo, &msg.send_size, &msg.opcode); 946 if (ret) 947 return ret; 948 949 msg.handle = job; 950 msg.notify_cb = notify_cb; 951 msg.send_data = (u8 *)&req; 952 print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req, 953 0x40, false); 954 955 ret = xdna_mailbox_send_msg(chann, &msg); 956 if (ret) { 957 XDNA_ERR(xdna, "Send message failed"); 958 return ret; 959 } 960 961 return 0; 962 } 963 964 int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, 965 struct amdxdna_sched_job *job, 966 int (*notify_cb)(void *, void __iomem *, size_t)) 967 { 968 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 969 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 970 struct amdxdna_client *client = hwctx->client; 971 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 972 void *cmd_buf = amdxdna_gem_vmap(cmdbuf_abo); 973 struct amdxdna_dev *xdna = client->xdna; 974 struct amdxdna_cmd_chain *payload; 975 struct xdna_mailbox_msg msg; 976 union exec_chain_req req; 977 u32 payload_len, ccnt; 978 u32 offset = 0; 979 size_t size; 980 int ret; 981 u32 op; 982 u32 i; 983 984 if (!cmd_buf) 985 return -ENOMEM; 986 987 op = amdxdna_cmd_get_op(cmd_abo); 988 payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len); 989 if (op != ERT_CMD_CHAIN) { 990 XDNA_DBG(xdna, "Invalid op code %d", op); 991 return -EINVAL; 992 } 993 994 if (!payload) { 995 XDNA_DBG(xdna, "Failed to get command payload"); 996 return -EINVAL; 997 } 998 999 ccnt = payload->command_count; 1000 if (!ccnt || payload_len < struct_size(payload, data, ccnt)) { 1001 XDNA_DBG(xdna, "Invalid command count %d", ccnt); 1002 return -EINVAL; 1003 } 1004 1005 op = ERT_INVALID_CMD; 1006 for (i = 0; i < ccnt; i++) { 1007 u32 boh = (u32)(payload->data[i]); 1008 struct amdxdna_gem_obj *abo; 1009 1010 abo = amdxdna_gem_get_obj(client, boh, AMDXDNA_BO_SHARE); 1011 if (!abo) { 1012 XDNA_ERR(xdna, "Failed to find cmd BO %d", boh); 1013 return -ENOENT; 1014 } 1015 1016 size = cmdbuf_abo->mem.size - offset; 1017 ret = aie2_cmdlist_fill_slot(cmd_buf + offset, abo, &size, &op); 1018 amdxdna_gem_put_obj(abo); 1019 if (ret) 1020 return ret; 1021 1022 offset += size; 1023 } 1024 1025 XDNA_DBG(xdna, "Total %d commands:", ccnt); 1026 print_hex_dump_debug("cmdbufs: ", DUMP_PREFIX_OFFSET, 16, 4, 1027 cmd_buf, offset, false); 1028 1029 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); 1030 if (msg.opcode == MSG_OP_MAX_OPCODE) 1031 return -EOPNOTSUPP; 1032 1033 /* The offset is the accumulated total size of the cmd buffer */ 1034 EXEC_MSG_OPS(xdna)->init_chain_req(&req, amdxdna_gem_dev_addr(cmdbuf_abo), 1035 offset, ccnt); 1036 drm_clflush_virt_range(cmd_buf, offset); 1037 1038 msg.handle = job; 1039 msg.notify_cb = notify_cb; 1040 msg.send_data = (u8 *)&req; 1041 msg.send_size = sizeof(req); 1042 print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4, 1043 &req, msg.send_size, false); 1044 ret = xdna_mailbox_send_msg(chann, &msg); 1045 if (ret) { 1046 XDNA_ERR(xdna, "Send message failed"); 1047 return ret; 1048 } 1049 1050 return 0; 1051 } 1052 1053 int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx, 1054 struct amdxdna_sched_job *job, 1055 int (*notify_cb)(void *, void __iomem *, size_t)) 1056 { 1057 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 1058 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1059 struct amdxdna_dev *xdna = hwctx->client->xdna; 1060 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 1061 void *cmd_buf = amdxdna_gem_vmap(cmdbuf_abo); 1062 struct xdna_mailbox_msg msg; 1063 union exec_chain_req req; 1064 u32 op = ERT_INVALID_CMD; 1065 size_t size; 1066 int ret; 1067 1068 if (!cmd_buf) 1069 return -ENOMEM; 1070 1071 size = cmdbuf_abo->mem.size; 1072 ret = aie2_cmdlist_fill_slot(cmd_buf, cmd_abo, &size, &op); 1073 if (ret) 1074 return ret; 1075 1076 print_hex_dump_debug("cmdbuf: ", DUMP_PREFIX_OFFSET, 16, 4, cmd_buf, size, false); 1077 1078 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); 1079 if (msg.opcode == MSG_OP_MAX_OPCODE) 1080 return -EOPNOTSUPP; 1081 1082 EXEC_MSG_OPS(xdna)->init_chain_req(&req, amdxdna_gem_dev_addr(cmdbuf_abo), size, 1); 1083 drm_clflush_virt_range(cmd_buf, size); 1084 1085 msg.handle = job; 1086 msg.notify_cb = notify_cb; 1087 msg.send_data = (u8 *)&req; 1088 msg.send_size = sizeof(req); 1089 print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4, 1090 &req, msg.send_size, false); 1091 ret = xdna_mailbox_send_msg(chann, &msg); 1092 if (ret) { 1093 XDNA_ERR(hwctx->client->xdna, "Send message failed"); 1094 return ret; 1095 } 1096 1097 return 0; 1098 } 1099 1100 int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 1101 int (*notify_cb)(void *, void __iomem *, size_t)) 1102 { 1103 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1104 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); 1105 struct amdxdna_dev *xdna = hwctx->client->xdna; 1106 struct xdna_mailbox_msg msg; 1107 struct sync_bo_req req; 1108 int ret = 0; 1109 1110 req.src_addr = 0; 1111 req.dst_addr = amdxdna_dev_bo_offset(abo); 1112 req.size = abo->mem.size; 1113 1114 /* Device to Host */ 1115 req.type = FIELD_PREP(AIE2_MSG_SYNC_BO_SRC_TYPE, SYNC_BO_DEV_MEM) | 1116 FIELD_PREP(AIE2_MSG_SYNC_BO_DST_TYPE, SYNC_BO_HOST_MEM); 1117 1118 XDNA_DBG(xdna, "sync %d bytes src(0x%llx) to dst(0x%llx) completed", 1119 req.size, req.src_addr, req.dst_addr); 1120 1121 msg.handle = job; 1122 msg.notify_cb = notify_cb; 1123 msg.send_data = (u8 *)&req; 1124 msg.send_size = sizeof(req); 1125 msg.opcode = MSG_OP_SYNC_BO; 1126 1127 ret = xdna_mailbox_send_msg(chann, &msg); 1128 if (ret) { 1129 XDNA_ERR(xdna, "Send message failed"); 1130 return ret; 1131 } 1132 1133 return 0; 1134 } 1135 1136 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 1137 int (*notify_cb)(void *, void __iomem *, size_t)) 1138 { 1139 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1140 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); 1141 struct amdxdna_dev *xdna = hwctx->client->xdna; 1142 struct config_debug_bo_req req; 1143 struct xdna_mailbox_msg msg; 1144 1145 if (job->drv_cmd->opcode == ATTACH_DEBUG_BO) 1146 req.config = DEBUG_BO_REGISTER; 1147 else 1148 req.config = DEBUG_BO_UNREGISTER; 1149 1150 req.offset = amdxdna_dev_bo_offset(abo); 1151 req.size = abo->mem.size; 1152 1153 XDNA_DBG(xdna, "offset 0x%llx size 0x%llx config %d", 1154 req.offset, req.size, req.config); 1155 1156 msg.handle = job; 1157 msg.notify_cb = notify_cb; 1158 msg.send_data = (u8 *)&req; 1159 msg.send_size = sizeof(req); 1160 msg.opcode = MSG_OP_CONFIG_DEBUG_BO; 1161 1162 return xdna_mailbox_send_msg(chann, &msg); 1163 } 1164 1165 int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id, 1166 struct app_health_report *report) 1167 { 1168 DECLARE_AIE_MSG(get_app_health, MSG_OP_GET_APP_HEALTH); 1169 struct amdxdna_dev *xdna = ndev->aie.xdna; 1170 struct app_health_report *buf; 1171 dma_addr_t dma_addr; 1172 u32 buf_size; 1173 int ret; 1174 1175 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_APP_HEALTH)) { 1176 XDNA_DBG(xdna, "App health feature not supported"); 1177 return -EOPNOTSUPP; 1178 } 1179 1180 buf_size = sizeof(*report); 1181 buf = amdxdna_alloc_msg_buffer(xdna, &buf_size, &dma_addr); 1182 if (IS_ERR(buf)) { 1183 XDNA_ERR(xdna, "Failed to allocate buffer for app health"); 1184 return PTR_ERR(buf); 1185 } 1186 1187 req.buf_addr = dma_addr; 1188 req.context_id = context_id; 1189 req.buf_size = buf_size; 1190 1191 drm_clflush_virt_range(buf, req.buf_size); 1192 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 1193 if (ret) { 1194 XDNA_ERR(xdna, "Get app health failed, ret %d status 0x%x", ret, resp.status); 1195 goto free_buf; 1196 } 1197 1198 /* Copy the report to caller's buffer */ 1199 memcpy(report, buf, sizeof(*report)); 1200 1201 free_buf: 1202 amdxdna_free_msg_buffer(xdna, buf_size, buf, dma_addr); 1203 return ret; 1204 } 1205 1206 static int 1207 aie2_runtime_update_ctx_prop(struct amdxdna_dev_hdl *ndev, 1208 struct amdxdna_hwctx *ctx, u32 type, u32 value) 1209 { 1210 DECLARE_AIE_MSG(update_property, MSG_OP_UPDATE_PROPERTY); 1211 struct amdxdna_dev *xdna = ndev->aie.xdna; 1212 int ret; 1213 1214 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_UPDATE_PROPERTY)) 1215 return -EOPNOTSUPP; 1216 1217 if (ctx) 1218 req.context_id = ctx->fw_ctx_id; 1219 else 1220 req.context_id = AIE2_UPDATE_PROPERTY_ALL_CTX; 1221 1222 req.time_quota_us = value; 1223 req.type = type; 1224 1225 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 1226 if (ret) { 1227 XDNA_ERR(xdna, "%s update property failed, type %d ret %d", 1228 ctx ? ctx->name : "ctx.all", type, ret); 1229 return ret; 1230 } 1231 1232 return 0; 1233 } 1234 1235 int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us) 1236 { 1237 struct amdxdna_dev *xdna = ndev->aie.xdna; 1238 int ret; 1239 1240 ret = aie2_runtime_update_ctx_prop(ndev, NULL, UPDATE_PROPERTY_TIME_QUOTA, us); 1241 if (ret == -EOPNOTSUPP) { 1242 XDNA_DBG(xdna, "update time quota not support, skipped"); 1243 ret = 0; 1244 } else if (!ret) { 1245 XDNA_DBG(xdna, "Ctx exec time quantum updated to %u us", us); 1246 } 1247 return ret; 1248 } 1249 1250 int aie2_get_dev_revision(struct amdxdna_dev_hdl *ndev, enum aie2_dev_revision *rev) 1251 { 1252 DECLARE_AIE_MSG(get_dev_revision, MSG_OP_GET_DEV_REVISION); 1253 struct amdxdna_dev *xdna = ndev->aie.xdna; 1254 int ret; 1255 1256 if (!AIE_FEATURE_ON(&ndev->aie, AIE2_GET_DEV_REVISION)) 1257 return -EOPNOTSUPP; 1258 1259 ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg); 1260 if (ret) 1261 return ret; 1262 1263 *rev = resp.rev; 1264 1265 if (*rev < AIE2_DEV_REVISION_STXA || *rev >= AIE2_DEV_REVISION_UNKN) { 1266 XDNA_ERR(xdna, "Unknown device revision: %d (raw fuse: 0x%x)", 1267 *rev, resp.raw_fuse_data); 1268 return -EINVAL; 1269 } 1270 1271 XDNA_DBG(xdna, "Device revision: %d (raw fuse: 0x%x)", *rev, resp.raw_fuse_data); 1272 return 0; 1273 } 1274