1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_cache.h> 8 #include <drm/drm_device.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/drm_print.h> 12 #include <drm/gpu_scheduler.h> 13 #include <linux/bitfield.h> 14 #include <linux/errno.h> 15 #include <linux/pci.h> 16 #include <linux/types.h> 17 #include <linux/xarray.h> 18 19 #include "aie2_msg_priv.h" 20 #include "aie2_pci.h" 21 #include "amdxdna_ctx.h" 22 #include "amdxdna_gem.h" 23 #include "amdxdna_mailbox.h" 24 #include "amdxdna_mailbox_helper.h" 25 #include "amdxdna_pci_drv.h" 26 27 #define DECLARE_AIE2_MSG(name, op) \ 28 DECLARE_XDNA_MSG_COMMON(name, op, MAX_AIE2_STATUS_CODE) 29 30 #define EXEC_MSG_OPS(xdna) ((xdna)->dev_handle->exec_msg_ops) 31 32 static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev, 33 struct xdna_mailbox_msg *msg) 34 { 35 struct amdxdna_dev *xdna = ndev->xdna; 36 struct xdna_notify *hdl = msg->handle; 37 int ret; 38 39 if (!ndev->mgmt_chann) 40 return -ENODEV; 41 42 ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg); 43 if (ret == -ETIME) 44 aie2_destroy_mgmt_chann(ndev); 45 46 if (!ret && *hdl->status != AIE2_STATUS_SUCCESS) { 47 XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x", 48 msg->opcode, *hdl->data); 49 ret = -EINVAL; 50 } 51 52 return ret; 53 } 54 55 void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size, 56 dma_addr_t *dma_addr) 57 { 58 struct amdxdna_dev *xdna = ndev->xdna; 59 int order; 60 61 *size = max(*size, SZ_8K); 62 order = get_order(*size); 63 if (order > MAX_PAGE_ORDER) 64 return ERR_PTR(-EINVAL); 65 *size = PAGE_SIZE << order; 66 67 if (amdxdna_iova_on(xdna)) 68 return amdxdna_iommu_alloc(xdna, *size, dma_addr); 69 70 return dma_alloc_noncoherent(xdna->ddev.dev, *size, dma_addr, 71 DMA_FROM_DEVICE, GFP_KERNEL); 72 } 73 74 void aie2_free_msg_buffer(struct amdxdna_dev_hdl *ndev, size_t size, 75 void *cpu_addr, dma_addr_t dma_addr) 76 { 77 struct amdxdna_dev *xdna = ndev->xdna; 78 79 if (amdxdna_iova_on(xdna)) { 80 amdxdna_iommu_free(xdna, size, cpu_addr, dma_addr); 81 return; 82 } 83 84 dma_free_noncoherent(xdna->ddev.dev, size, cpu_addr, dma_addr, DMA_FROM_DEVICE); 85 } 86 87 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev) 88 { 89 DECLARE_AIE2_MSG(suspend, MSG_OP_SUSPEND); 90 int ret; 91 92 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 93 if (ret) { 94 XDNA_ERR(ndev->xdna, "Failed to suspend fw, ret %d", ret); 95 return ret; 96 } 97 98 return aie2_psp_waitmode_poll(ndev->psp_hdl); 99 } 100 101 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev) 102 { 103 DECLARE_AIE2_MSG(suspend, MSG_OP_RESUME); 104 105 return aie2_send_mgmt_msg_wait(ndev, &msg); 106 } 107 108 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value) 109 { 110 DECLARE_AIE2_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG); 111 int ret; 112 113 req.type = type; 114 req.value = value; 115 116 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 117 if (ret) { 118 XDNA_ERR(ndev->xdna, "Failed to set runtime config, ret %d", ret); 119 return ret; 120 } 121 122 return 0; 123 } 124 125 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value) 126 { 127 DECLARE_AIE2_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG); 128 int ret; 129 130 req.type = type; 131 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 132 if (ret) { 133 XDNA_ERR(ndev->xdna, "Failed to get runtime config, ret %d", ret); 134 return ret; 135 } 136 137 *value = resp.value; 138 return 0; 139 } 140 141 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid) 142 { 143 DECLARE_AIE2_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID); 144 145 req.pasid = pasid; 146 147 return aie2_send_mgmt_msg_wait(ndev, &msg); 148 } 149 150 int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version) 151 { 152 DECLARE_AIE2_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION); 153 struct amdxdna_dev *xdna = ndev->xdna; 154 int ret; 155 156 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 157 if (ret) 158 return ret; 159 160 XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed", 161 resp.major, resp.minor); 162 163 version->major = resp.major; 164 version->minor = resp.minor; 165 166 return 0; 167 } 168 169 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata) 170 { 171 DECLARE_AIE2_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO); 172 int ret; 173 174 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 175 if (ret) 176 return ret; 177 178 metadata->size = resp.info.size; 179 metadata->cols = resp.info.cols; 180 metadata->rows = resp.info.rows; 181 182 metadata->version.major = resp.info.major; 183 metadata->version.minor = resp.info.minor; 184 185 metadata->core.row_count = resp.info.core_rows; 186 metadata->core.row_start = resp.info.core_row_start; 187 metadata->core.dma_channel_count = resp.info.core_dma_channels; 188 metadata->core.lock_count = resp.info.core_locks; 189 metadata->core.event_reg_count = resp.info.core_events; 190 191 metadata->mem.row_count = resp.info.mem_rows; 192 metadata->mem.row_start = resp.info.mem_row_start; 193 metadata->mem.dma_channel_count = resp.info.mem_dma_channels; 194 metadata->mem.lock_count = resp.info.mem_locks; 195 metadata->mem.event_reg_count = resp.info.mem_events; 196 197 metadata->shim.row_count = resp.info.shim_rows; 198 metadata->shim.row_start = resp.info.shim_row_start; 199 metadata->shim.dma_channel_count = resp.info.shim_dma_channels; 200 metadata->shim.lock_count = resp.info.shim_locks; 201 metadata->shim.event_reg_count = resp.info.shim_events; 202 203 return 0; 204 } 205 206 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, 207 struct amdxdna_fw_ver *fw_ver) 208 { 209 DECLARE_AIE2_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION); 210 int ret; 211 212 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 213 if (ret) 214 return ret; 215 216 fw_ver->major = resp.major; 217 fw_ver->minor = resp.minor; 218 fw_ver->sub = resp.sub; 219 fw_ver->build = resp.build; 220 221 return 0; 222 } 223 224 static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id) 225 { 226 DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); 227 struct amdxdna_dev *xdna = ndev->xdna; 228 int ret; 229 230 req.context_id = id; 231 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 232 if (ret && ret != -ENODEV) 233 XDNA_WARN(xdna, "Destroy context failed, ret %d", ret); 234 else if (ret == -ENODEV) 235 XDNA_DBG(xdna, "Destroy context: device already stopped"); 236 237 return ret; 238 } 239 240 static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev, 241 struct amdxdna_hwctx *hwctx) 242 { 243 if (!AIE2_FEATURE_ON(ndev, AIE2_PREEMPT)) 244 return PRIORITY_HIGH; 245 246 switch (hwctx->qos.priority) { 247 case AMDXDNA_QOS_REALTIME_PRIORITY: 248 return PRIORITY_REALTIME; 249 case AMDXDNA_QOS_HIGH_PRIORITY: 250 return PRIORITY_HIGH; 251 case AMDXDNA_QOS_NORMAL_PRIORITY: 252 return PRIORITY_NORMAL; 253 case AMDXDNA_QOS_LOW_PRIORITY: 254 return PRIORITY_LOW; 255 default: 256 return PRIORITY_HIGH; 257 } 258 } 259 260 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 261 { 262 DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT); 263 struct amdxdna_dev *xdna = ndev->xdna; 264 struct xdna_mailbox_chann_res x2i; 265 struct xdna_mailbox_chann_res i2x; 266 struct cq_pair *cq_pair; 267 u32 intr_reg; 268 int ret; 269 270 req.aie_type = 1; 271 req.start_col = hwctx->start_col; 272 req.num_col = hwctx->num_col; 273 req.num_unused_col = hwctx->num_unused_col; 274 req.num_cq_pairs_requested = 1; 275 req.pasid = amdxdna_pasid_on(hwctx->client) ? hwctx->client->pasid : 0; 276 req.context_priority = aie2_get_context_priority(ndev, hwctx); 277 278 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 279 if (ret) 280 return ret; 281 282 hwctx->fw_ctx_id = resp.context_id; 283 if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1)) 284 return -EINVAL; 285 286 if (ndev->force_preempt_enabled) { 287 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id); 288 if (ret) { 289 XDNA_ERR(xdna, "failed to enable force preempt %d", ret); 290 goto del_ctx_req; 291 } 292 } 293 294 cq_pair = &resp.cq_pair[0]; 295 x2i.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.head_addr); 296 x2i.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.tail_addr); 297 x2i.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->x2i_q.buf_addr); 298 x2i.rb_size = cq_pair->x2i_q.buf_size; 299 300 i2x.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.head_addr); 301 i2x.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.tail_addr); 302 i2x.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->i2x_q.buf_addr); 303 i2x.rb_size = cq_pair->i2x_q.buf_size; 304 305 ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id); 306 if (ret == -EINVAL) { 307 XDNA_ERR(xdna, "Alloc IRQ failed %d", ret); 308 goto del_ctx_req; 309 } 310 311 intr_reg = i2x.mb_head_ptr_reg + 4; 312 hwctx->priv->mbox_chann = xdna_mailbox_alloc_channel(ndev->mbox); 313 if (!hwctx->priv->mbox_chann) { 314 XDNA_ERR(xdna, "Not able to create channel"); 315 ret = -EINVAL; 316 goto del_ctx_req; 317 } 318 319 ret = xdna_mailbox_start_channel(hwctx->priv->mbox_chann, &x2i, &i2x, 320 intr_reg, ret); 321 if (ret) { 322 XDNA_ERR(xdna, "Not able to create channel"); 323 ret = -EINVAL; 324 goto free_channel; 325 } 326 ndev->hwctx_num++; 327 328 XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id); 329 XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid); 330 331 return 0; 332 333 free_channel: 334 xdna_mailbox_free_channel(hwctx->priv->mbox_chann); 335 del_ctx_req: 336 aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); 337 return ret; 338 } 339 340 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 341 { 342 struct amdxdna_dev *xdna = ndev->xdna; 343 int ret; 344 345 if (!hwctx->priv->mbox_chann) 346 return 0; 347 348 xdna_mailbox_stop_channel(hwctx->priv->mbox_chann); 349 ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); 350 xdna_mailbox_free_channel(hwctx->priv->mbox_chann); 351 XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id); 352 hwctx->priv->mbox_chann = NULL; 353 hwctx->fw_ctx_id = -1; 354 ndev->hwctx_num--; 355 356 return ret; 357 } 358 359 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size) 360 { 361 DECLARE_AIE2_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER); 362 struct amdxdna_dev *xdna = ndev->xdna; 363 int ret; 364 365 req.context_id = context_id; 366 req.buf_addr = addr; 367 req.buf_size = size; 368 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 369 if (ret) 370 return ret; 371 372 XDNA_DBG(xdna, "fw ctx %d map host buf addr 0x%llx size 0x%llx", 373 context_id, addr, size); 374 375 return 0; 376 } 377 378 static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg) 379 { 380 u32 *bitmap = arg; 381 382 *bitmap |= GENMASK(hwctx->start_col + hwctx->num_col - 1, hwctx->start_col); 383 384 return 0; 385 } 386 387 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, 388 u32 size, u32 *cols_filled) 389 { 390 DECLARE_AIE2_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS); 391 struct amdxdna_dev *xdna = ndev->xdna; 392 u32 buf_sz = size, aie_bitmap = 0; 393 struct amdxdna_client *client; 394 dma_addr_t dma_addr; 395 u8 *buff_addr; 396 int ret; 397 398 buff_addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr); 399 if (IS_ERR(buff_addr)) 400 return PTR_ERR(buff_addr); 401 402 /* Go through each hardware context and mark the AIE columns that are active */ 403 list_for_each_entry(client, &xdna->client_list, node) 404 amdxdna_hwctx_walk(client, &aie_bitmap, amdxdna_hwctx_col_map); 405 406 *cols_filled = 0; 407 req.dump_buff_addr = dma_addr; 408 req.dump_buff_size = buf_sz; 409 req.num_cols = hweight32(aie_bitmap); 410 req.aie_bitmap = aie_bitmap; 411 412 drm_clflush_virt_range(buff_addr, size); /* device can access */ 413 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 414 if (ret) { 415 XDNA_ERR(xdna, "Error during NPU query, status %d", ret); 416 goto fail; 417 } 418 419 XDNA_DBG(xdna, "Query NPU status completed"); 420 421 if (size < resp.size) { 422 ret = -EINVAL; 423 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size); 424 goto fail; 425 } 426 427 if (copy_to_user(buf, buff_addr, resp.size)) { 428 ret = -EFAULT; 429 XDNA_ERR(xdna, "Failed to copy NPU status to user space"); 430 goto fail; 431 } 432 433 *cols_filled = aie_bitmap; 434 435 fail: 436 aie2_free_msg_buffer(ndev, buf_sz, buff_addr, dma_addr); 437 return ret; 438 } 439 440 int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, 441 char __user *buf, u32 size, 442 struct amdxdna_drm_query_telemetry_header *header) 443 { 444 DECLARE_AIE2_MSG(get_telemetry, MSG_OP_GET_TELEMETRY); 445 struct amdxdna_dev *xdna = ndev->xdna; 446 dma_addr_t dma_addr; 447 u32 buf_sz = size; 448 u8 *addr; 449 int ret; 450 451 if (header->type >= MAX_TELEMETRY_TYPE) 452 return -EINVAL; 453 454 addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr); 455 if (IS_ERR(addr)) 456 return PTR_ERR(addr); 457 458 req.buf_addr = dma_addr; 459 req.buf_size = buf_sz; 460 req.type = header->type; 461 462 drm_clflush_virt_range(addr, size); /* device can access */ 463 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 464 if (ret) { 465 XDNA_ERR(xdna, "Query telemetry failed, status %d", ret); 466 goto free_buf; 467 } 468 469 if (size < resp.size) { 470 ret = -EINVAL; 471 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size); 472 goto free_buf; 473 } 474 475 if (copy_to_user(buf, addr, resp.size)) { 476 ret = -EFAULT; 477 XDNA_ERR(xdna, "Failed to copy telemetry to user space"); 478 goto free_buf; 479 } 480 481 header->major = resp.major; 482 header->minor = resp.minor; 483 484 free_buf: 485 aie2_free_msg_buffer(ndev, buf_sz, addr, dma_addr); 486 return ret; 487 } 488 489 int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size, 490 void *handle, int (*cb)(void*, void __iomem *, size_t)) 491 { 492 struct async_event_msg_req req = { 0 }; 493 struct xdna_mailbox_msg msg = { 494 .send_data = (u8 *)&req, 495 .send_size = sizeof(req), 496 .handle = handle, 497 .opcode = MSG_OP_REGISTER_ASYNC_EVENT_MSG, 498 .notify_cb = cb, 499 }; 500 501 req.buf_addr = addr; 502 req.buf_size = size; 503 504 XDNA_DBG(ndev->xdna, "Register addr 0x%llx size 0x%x", addr, size); 505 return xdna_mailbox_send_msg(ndev->mgmt_chann, &msg, TX_TIMEOUT); 506 } 507 508 int aie2_config_cu(struct amdxdna_hwctx *hwctx, 509 int (*notify_cb)(void *, void __iomem *, size_t)) 510 { 511 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 512 struct amdxdna_dev *xdna = hwctx->client->xdna; 513 u32 shift = xdna->dev_info->dev_mem_buf_shift; 514 struct config_cu_req req = { 0 }; 515 struct xdna_mailbox_msg msg; 516 struct drm_gem_object *gobj; 517 struct amdxdna_gem_obj *abo; 518 int i; 519 520 if (!chann) 521 return -ENODEV; 522 523 if (!hwctx->cus) 524 return 0; 525 526 if (hwctx->cus->num_cus > MAX_NUM_CUS) { 527 XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS); 528 return -EINVAL; 529 } 530 531 for (i = 0; i < hwctx->cus->num_cus; i++) { 532 struct amdxdna_cu_config *cu = &hwctx->cus->cu_configs[i]; 533 534 if (XDNA_MBZ_DBG(xdna, cu->pad, sizeof(cu->pad))) 535 return -EINVAL; 536 537 gobj = drm_gem_object_lookup(hwctx->client->filp, cu->cu_bo); 538 if (!gobj) { 539 XDNA_ERR(xdna, "Lookup GEM object failed"); 540 return -EINVAL; 541 } 542 abo = to_xdna_obj(gobj); 543 544 if (abo->type != AMDXDNA_BO_DEV) { 545 drm_gem_object_put(gobj); 546 XDNA_ERR(xdna, "Invalid BO type"); 547 return -EINVAL; 548 } 549 550 req.cfgs[i] = FIELD_PREP(AIE2_MSG_CFG_CU_PDI_ADDR, 551 abo->mem.dev_addr >> shift); 552 req.cfgs[i] |= FIELD_PREP(AIE2_MSG_CFG_CU_FUNC, cu->cu_func); 553 XDNA_DBG(xdna, "CU %d full addr 0x%llx, cfg 0x%x", i, 554 abo->mem.dev_addr, req.cfgs[i]); 555 drm_gem_object_put(gobj); 556 } 557 req.num_cus = hwctx->cus->num_cus; 558 559 msg.send_data = (u8 *)&req; 560 msg.send_size = sizeof(req); 561 msg.handle = hwctx; 562 msg.opcode = MSG_OP_CONFIG_CU; 563 msg.notify_cb = notify_cb; 564 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 565 } 566 567 static int aie2_init_exec_cu_req(struct amdxdna_gem_obj *cmd_bo, void *req, 568 size_t *size, u32 *msg_op) 569 { 570 struct execute_buffer_req *cu_req = req; 571 u32 cmd_len; 572 void *cmd; 573 574 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 575 if (cmd_len > sizeof(cu_req->payload)) 576 return -EINVAL; 577 578 cu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 579 if (cu_req->cu_idx == INVALID_CU_IDX) 580 return -EINVAL; 581 582 memcpy(cu_req->payload, cmd, cmd_len); 583 584 *size = sizeof(*cu_req); 585 *msg_op = MSG_OP_EXECUTE_BUFFER_CF; 586 return 0; 587 } 588 589 static int aie2_init_exec_dpu_req(struct amdxdna_gem_obj *cmd_bo, void *req, 590 size_t *size, u32 *msg_op) 591 { 592 struct exec_dpu_req *dpu_req = req; 593 struct amdxdna_cmd_start_npu *sn; 594 u32 cmd_len; 595 596 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 597 if (cmd_len - sizeof(*sn) > sizeof(dpu_req->payload)) 598 return -EINVAL; 599 600 dpu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 601 if (dpu_req->cu_idx == INVALID_CU_IDX) 602 return -EINVAL; 603 604 dpu_req->inst_buf_addr = sn->buffer; 605 dpu_req->inst_size = sn->buffer_size; 606 dpu_req->inst_prop_cnt = sn->prop_count; 607 memcpy(dpu_req->payload, sn->prop_args, cmd_len - sizeof(*sn)); 608 609 *size = sizeof(*dpu_req); 610 *msg_op = MSG_OP_EXEC_DPU; 611 return 0; 612 } 613 614 static void aie2_init_exec_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) 615 { 616 struct cmd_chain_req *chain_req = req; 617 618 chain_req->buf_addr = slot_addr; 619 chain_req->buf_size = size; 620 chain_req->count = cmd_cnt; 621 } 622 623 static void aie2_init_npu_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) 624 { 625 struct cmd_chain_npu_req *npu_chain_req = req; 626 627 npu_chain_req->flags = 0; 628 npu_chain_req->reserved = 0; 629 npu_chain_req->buf_addr = slot_addr; 630 npu_chain_req->buf_size = size; 631 npu_chain_req->count = cmd_cnt; 632 } 633 634 static int 635 aie2_cmdlist_fill_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 636 { 637 struct cmd_chain_slot_execbuf_cf *cf_slot = slot; 638 u32 cmd_len; 639 void *cmd; 640 641 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 642 if (*size < sizeof(*cf_slot) + cmd_len) 643 return -EINVAL; 644 645 cf_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 646 if (cf_slot->cu_idx == INVALID_CU_IDX) 647 return -EINVAL; 648 649 cf_slot->arg_cnt = cmd_len / sizeof(u32); 650 memcpy(cf_slot->args, cmd, cmd_len); 651 /* Accurate slot size to hint firmware to do necessary copy */ 652 *size = sizeof(*cf_slot) + cmd_len; 653 return 0; 654 } 655 656 static int 657 aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 658 { 659 struct cmd_chain_slot_dpu *dpu_slot = slot; 660 struct amdxdna_cmd_start_npu *sn; 661 u32 cmd_len; 662 u32 arg_sz; 663 664 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 665 arg_sz = cmd_len - sizeof(*sn); 666 if (cmd_len < sizeof(*sn) || arg_sz > MAX_DPU_ARGS_SIZE) 667 return -EINVAL; 668 669 if (*size < sizeof(*dpu_slot) + arg_sz) 670 return -EINVAL; 671 672 dpu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 673 if (dpu_slot->cu_idx == INVALID_CU_IDX) 674 return -EINVAL; 675 676 dpu_slot->inst_buf_addr = sn->buffer; 677 dpu_slot->inst_size = sn->buffer_size; 678 dpu_slot->inst_prop_cnt = sn->prop_count; 679 dpu_slot->arg_cnt = arg_sz / sizeof(u32); 680 memcpy(dpu_slot->args, sn->prop_args, arg_sz); 681 682 /* Accurate slot size to hint firmware to do necessary copy */ 683 *size = sizeof(*dpu_slot) + arg_sz; 684 return 0; 685 } 686 687 static int aie2_cmdlist_unsupp(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 688 { 689 return -EOPNOTSUPP; 690 } 691 692 static u32 aie2_get_chain_msg_op(u32 cmd_op) 693 { 694 switch (cmd_op) { 695 case ERT_START_CU: 696 return MSG_OP_CHAIN_EXEC_BUFFER_CF; 697 case ERT_START_NPU: 698 return MSG_OP_CHAIN_EXEC_DPU; 699 default: 700 break; 701 } 702 703 return MSG_OP_MAX_OPCODE; 704 } 705 706 static struct aie2_exec_msg_ops legacy_exec_message_ops = { 707 .init_cu_req = aie2_init_exec_cu_req, 708 .init_dpu_req = aie2_init_exec_dpu_req, 709 .init_chain_req = aie2_init_exec_chain_req, 710 .fill_cf_slot = aie2_cmdlist_fill_cf, 711 .fill_dpu_slot = aie2_cmdlist_fill_dpu, 712 .fill_preempt_slot = aie2_cmdlist_unsupp, 713 .fill_elf_slot = aie2_cmdlist_unsupp, 714 .get_chain_msg_op = aie2_get_chain_msg_op, 715 }; 716 717 static int 718 aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 719 { 720 struct cmd_chain_slot_npu *npu_slot = slot; 721 u32 cmd_len; 722 void *cmd; 723 724 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 725 if (*size < sizeof(*npu_slot) + cmd_len) 726 return -EINVAL; 727 728 memset(npu_slot, 0, sizeof(*npu_slot)); 729 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 730 if (npu_slot->cu_idx == INVALID_CU_IDX) 731 return -EINVAL; 732 733 npu_slot->type = EXEC_NPU_TYPE_NON_ELF; 734 npu_slot->arg_cnt = cmd_len / sizeof(u32); 735 memcpy(npu_slot->args, cmd, cmd_len); 736 737 *size = sizeof(*npu_slot) + cmd_len; 738 return 0; 739 } 740 741 static int 742 aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 743 { 744 struct cmd_chain_slot_npu *npu_slot = slot; 745 struct amdxdna_cmd_start_npu *sn; 746 u32 cmd_len; 747 u32 arg_sz; 748 749 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 750 arg_sz = cmd_len - sizeof(*sn); 751 if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE) 752 return -EINVAL; 753 754 if (*size < sizeof(*npu_slot) + arg_sz) 755 return -EINVAL; 756 757 memset(npu_slot, 0, sizeof(*npu_slot)); 758 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 759 if (npu_slot->cu_idx == INVALID_CU_IDX) 760 return -EINVAL; 761 762 npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF; 763 npu_slot->inst_buf_addr = sn->buffer; 764 npu_slot->inst_size = sn->buffer_size; 765 npu_slot->inst_prop_cnt = sn->prop_count; 766 npu_slot->arg_cnt = arg_sz / sizeof(u32); 767 memcpy(npu_slot->args, sn->prop_args, arg_sz); 768 769 *size = sizeof(*npu_slot) + arg_sz; 770 return 0; 771 } 772 773 static int 774 aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 775 { 776 struct cmd_chain_slot_npu *npu_slot = slot; 777 struct amdxdna_cmd_preempt_data *pd; 778 u32 cmd_len; 779 u32 arg_sz; 780 781 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 782 arg_sz = cmd_len - sizeof(*pd); 783 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) 784 return -EINVAL; 785 786 if (*size < sizeof(*npu_slot) + arg_sz) 787 return -EINVAL; 788 789 memset(npu_slot, 0, sizeof(*npu_slot)); 790 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 791 if (npu_slot->cu_idx == INVALID_CU_IDX) 792 return -EINVAL; 793 794 npu_slot->type = EXEC_NPU_TYPE_PREEMPT; 795 npu_slot->inst_buf_addr = pd->inst_buf; 796 npu_slot->save_buf_addr = pd->save_buf; 797 npu_slot->restore_buf_addr = pd->restore_buf; 798 npu_slot->inst_size = pd->inst_size; 799 npu_slot->save_size = pd->save_size; 800 npu_slot->restore_size = pd->restore_size; 801 npu_slot->inst_prop_cnt = pd->inst_prop_cnt; 802 npu_slot->arg_cnt = arg_sz / sizeof(u32); 803 memcpy(npu_slot->args, pd->prop_args, arg_sz); 804 805 *size = sizeof(*npu_slot) + arg_sz; 806 return 0; 807 } 808 809 static int 810 aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 811 { 812 struct cmd_chain_slot_npu *npu_slot = slot; 813 struct amdxdna_cmd_preempt_data *pd; 814 u32 cmd_len; 815 u32 arg_sz; 816 817 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 818 arg_sz = cmd_len - sizeof(*pd); 819 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) 820 return -EINVAL; 821 822 if (*size < sizeof(*npu_slot) + arg_sz) 823 return -EINVAL; 824 825 memset(npu_slot, 0, sizeof(*npu_slot)); 826 npu_slot->type = EXEC_NPU_TYPE_ELF; 827 npu_slot->inst_buf_addr = pd->inst_buf; 828 npu_slot->save_buf_addr = pd->save_buf; 829 npu_slot->restore_buf_addr = pd->restore_buf; 830 npu_slot->inst_size = pd->inst_size; 831 npu_slot->save_size = pd->save_size; 832 npu_slot->restore_size = pd->restore_size; 833 npu_slot->inst_prop_cnt = pd->inst_prop_cnt; 834 npu_slot->arg_cnt = 1; 835 npu_slot->args[0] = AIE2_EXEC_BUFFER_KERNEL_OP_TXN; 836 837 *size = struct_size(npu_slot, args, npu_slot->arg_cnt); 838 return 0; 839 } 840 841 static u32 aie2_get_npu_chain_msg_op(u32 cmd_op) 842 { 843 return MSG_OP_CHAIN_EXEC_NPU; 844 } 845 846 static struct aie2_exec_msg_ops npu_exec_message_ops = { 847 .init_cu_req = aie2_init_exec_cu_req, 848 .init_dpu_req = aie2_init_exec_dpu_req, 849 .init_chain_req = aie2_init_npu_chain_req, 850 .fill_cf_slot = aie2_cmdlist_fill_npu_cf, 851 .fill_dpu_slot = aie2_cmdlist_fill_npu_dpu, 852 .fill_preempt_slot = aie2_cmdlist_fill_npu_preempt, 853 .fill_elf_slot = aie2_cmdlist_fill_npu_elf, 854 .get_chain_msg_op = aie2_get_npu_chain_msg_op, 855 }; 856 857 static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo, 858 size_t *size, u32 *msg_op) 859 { 860 struct amdxdna_dev *xdna = cmd_abo->client->xdna; 861 int ret; 862 u32 op; 863 864 865 op = amdxdna_cmd_get_op(cmd_abo); 866 switch (op) { 867 case ERT_START_CU: 868 ret = EXEC_MSG_OPS(xdna)->init_cu_req(cmd_abo, req, size, msg_op); 869 if (ret) { 870 XDNA_DBG(xdna, "Init CU req failed ret %d", ret); 871 return ret; 872 } 873 break; 874 case ERT_START_NPU: 875 ret = EXEC_MSG_OPS(xdna)->init_dpu_req(cmd_abo, req, size, msg_op); 876 if (ret) { 877 XDNA_DBG(xdna, "Init DPU req failed ret %d", ret); 878 return ret; 879 } 880 881 break; 882 default: 883 XDNA_ERR(xdna, "Unsupported op %d", op); 884 ret = -EOPNOTSUPP; 885 break; 886 } 887 888 return ret; 889 } 890 891 static int 892 aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo, 893 size_t *size, u32 *cmd_op) 894 { 895 struct amdxdna_dev *xdna = cmd_abo->client->xdna; 896 int ret; 897 u32 op; 898 899 op = amdxdna_cmd_get_op(cmd_abo); 900 if (*cmd_op == ERT_INVALID_CMD) 901 *cmd_op = op; 902 else if (op != *cmd_op) 903 return -EINVAL; 904 905 switch (op) { 906 case ERT_START_CU: 907 ret = EXEC_MSG_OPS(xdna)->fill_cf_slot(cmd_abo, slot, size); 908 break; 909 case ERT_START_NPU: 910 ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size); 911 break; 912 case ERT_START_NPU_PREEMPT: 913 if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT)) 914 return -EOPNOTSUPP; 915 ret = EXEC_MSG_OPS(xdna)->fill_preempt_slot(cmd_abo, slot, size); 916 break; 917 case ERT_START_NPU_PREEMPT_ELF: 918 if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT)) 919 return -EOPNOTSUPP; 920 ret = EXEC_MSG_OPS(xdna)->fill_elf_slot(cmd_abo, slot, size); 921 break; 922 default: 923 XDNA_INFO(xdna, "Unsupported op %d", op); 924 ret = -EOPNOTSUPP; 925 break; 926 } 927 928 return ret; 929 } 930 931 void aie2_msg_init(struct amdxdna_dev_hdl *ndev) 932 { 933 if (AIE2_FEATURE_ON(ndev, AIE2_NPU_COMMAND)) 934 ndev->exec_msg_ops = &npu_exec_message_ops; 935 else 936 ndev->exec_msg_ops = &legacy_exec_message_ops; 937 } 938 939 void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev) 940 { 941 struct amdxdna_dev *xdna = ndev->xdna; 942 943 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 944 945 if (!ndev->mgmt_chann) 946 return; 947 948 xdna_mailbox_stop_channel(ndev->mgmt_chann); 949 xdna_mailbox_free_channel(ndev->mgmt_chann); 950 ndev->mgmt_chann = NULL; 951 } 952 953 static inline struct amdxdna_gem_obj * 954 aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job) 955 { 956 int idx = get_job_idx(job->seq); 957 958 return job->hwctx->priv->cmd_buf[idx]; 959 } 960 961 int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 962 int (*notify_cb)(void *, void __iomem *, size_t)) 963 { 964 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 965 struct amdxdna_dev *xdna = hwctx->client->xdna; 966 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 967 struct xdna_mailbox_msg msg; 968 union exec_req req; 969 int ret; 970 971 if (!chann) 972 return -ENODEV; 973 974 ret = aie2_init_exec_req(&req, cmd_abo, &msg.send_size, &msg.opcode); 975 if (ret) 976 return ret; 977 978 msg.handle = job; 979 msg.notify_cb = notify_cb; 980 msg.send_data = (u8 *)&req; 981 print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req, 982 0x40, false); 983 984 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 985 if (ret) { 986 XDNA_ERR(xdna, "Send message failed"); 987 return ret; 988 } 989 990 return 0; 991 } 992 993 int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, 994 struct amdxdna_sched_job *job, 995 int (*notify_cb)(void *, void __iomem *, size_t)) 996 { 997 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 998 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 999 struct amdxdna_client *client = hwctx->client; 1000 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 1001 struct amdxdna_dev *xdna = client->xdna; 1002 struct amdxdna_cmd_chain *payload; 1003 struct xdna_mailbox_msg msg; 1004 union exec_chain_req req; 1005 u32 payload_len, ccnt; 1006 u32 offset = 0; 1007 size_t size; 1008 int ret; 1009 u32 op; 1010 u32 i; 1011 1012 op = amdxdna_cmd_get_op(cmd_abo); 1013 payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len); 1014 if (op != ERT_CMD_CHAIN) { 1015 XDNA_DBG(xdna, "Invalid op code %d", op); 1016 return -EINVAL; 1017 } 1018 1019 if (!payload) { 1020 XDNA_DBG(xdna, "Failed to get command payload"); 1021 return -EINVAL; 1022 } 1023 1024 ccnt = payload->command_count; 1025 if (payload_len < struct_size(payload, data, ccnt)) { 1026 XDNA_DBG(xdna, "Invalid command count %d", ccnt); 1027 return -EINVAL; 1028 } 1029 1030 op = ERT_INVALID_CMD; 1031 for (i = 0; i < ccnt; i++) { 1032 u32 boh = (u32)(payload->data[i]); 1033 struct amdxdna_gem_obj *abo; 1034 1035 abo = amdxdna_gem_get_obj(client, boh, AMDXDNA_BO_CMD); 1036 if (!abo) { 1037 XDNA_ERR(xdna, "Failed to find cmd BO %d", boh); 1038 return -ENOENT; 1039 } 1040 1041 size = cmdbuf_abo->mem.size - offset; 1042 ret = aie2_cmdlist_fill_slot(cmdbuf_abo->mem.kva + offset, 1043 abo, &size, &op); 1044 amdxdna_gem_put_obj(abo); 1045 if (ret) 1046 return ret; 1047 1048 offset += size; 1049 } 1050 1051 XDNA_DBG(xdna, "Total %d commands:", ccnt); 1052 print_hex_dump_debug("cmdbufs: ", DUMP_PREFIX_OFFSET, 16, 4, 1053 cmdbuf_abo->mem.kva, offset, false); 1054 1055 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); 1056 if (msg.opcode == MSG_OP_MAX_OPCODE) 1057 return -EOPNOTSUPP; 1058 1059 /* The offset is the accumulated total size of the cmd buffer */ 1060 EXEC_MSG_OPS(xdna)->init_chain_req(&req, cmdbuf_abo->mem.dev_addr, 1061 offset, ccnt); 1062 drm_clflush_virt_range(cmdbuf_abo->mem.kva, offset); 1063 1064 msg.handle = job; 1065 msg.notify_cb = notify_cb; 1066 msg.send_data = (u8 *)&req; 1067 msg.send_size = sizeof(req); 1068 print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4, 1069 &req, msg.send_size, false); 1070 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1071 if (ret) { 1072 XDNA_ERR(xdna, "Send message failed"); 1073 return ret; 1074 } 1075 1076 return 0; 1077 } 1078 1079 int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx, 1080 struct amdxdna_sched_job *job, 1081 int (*notify_cb)(void *, void __iomem *, size_t)) 1082 { 1083 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 1084 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1085 struct amdxdna_dev *xdna = hwctx->client->xdna; 1086 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 1087 struct xdna_mailbox_msg msg; 1088 union exec_chain_req req; 1089 u32 op = ERT_INVALID_CMD; 1090 size_t size; 1091 int ret; 1092 1093 size = cmdbuf_abo->mem.size; 1094 ret = aie2_cmdlist_fill_slot(cmdbuf_abo->mem.kva, cmd_abo, &size, &op); 1095 if (ret) 1096 return ret; 1097 1098 print_hex_dump_debug("cmdbuf: ", DUMP_PREFIX_OFFSET, 16, 4, 1099 cmdbuf_abo->mem.kva, size, false); 1100 1101 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); 1102 if (msg.opcode == MSG_OP_MAX_OPCODE) 1103 return -EOPNOTSUPP; 1104 1105 EXEC_MSG_OPS(xdna)->init_chain_req(&req, cmdbuf_abo->mem.dev_addr, 1106 size, 1); 1107 drm_clflush_virt_range(cmdbuf_abo->mem.kva, size); 1108 1109 msg.handle = job; 1110 msg.notify_cb = notify_cb; 1111 msg.send_data = (u8 *)&req; 1112 msg.send_size = sizeof(req); 1113 print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4, 1114 &req, msg.send_size, false); 1115 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1116 if (ret) { 1117 XDNA_ERR(hwctx->client->xdna, "Send message failed"); 1118 return ret; 1119 } 1120 1121 return 0; 1122 } 1123 1124 int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 1125 int (*notify_cb)(void *, void __iomem *, size_t)) 1126 { 1127 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1128 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); 1129 struct amdxdna_dev *xdna = hwctx->client->xdna; 1130 struct xdna_mailbox_msg msg; 1131 struct sync_bo_req req; 1132 int ret = 0; 1133 1134 req.src_addr = 0; 1135 req.dst_addr = amdxdna_dev_bo_offset(abo); 1136 req.size = abo->mem.size; 1137 1138 /* Device to Host */ 1139 req.type = FIELD_PREP(AIE2_MSG_SYNC_BO_SRC_TYPE, SYNC_BO_DEV_MEM) | 1140 FIELD_PREP(AIE2_MSG_SYNC_BO_DST_TYPE, SYNC_BO_HOST_MEM); 1141 1142 XDNA_DBG(xdna, "sync %d bytes src(0x%llx) to dst(0x%llx) completed", 1143 req.size, req.src_addr, req.dst_addr); 1144 1145 msg.handle = job; 1146 msg.notify_cb = notify_cb; 1147 msg.send_data = (u8 *)&req; 1148 msg.send_size = sizeof(req); 1149 msg.opcode = MSG_OP_SYNC_BO; 1150 1151 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1152 if (ret) { 1153 XDNA_ERR(xdna, "Send message failed"); 1154 return ret; 1155 } 1156 1157 return 0; 1158 } 1159 1160 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 1161 int (*notify_cb)(void *, void __iomem *, size_t)) 1162 { 1163 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1164 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); 1165 struct amdxdna_dev *xdna = hwctx->client->xdna; 1166 struct config_debug_bo_req req; 1167 struct xdna_mailbox_msg msg; 1168 1169 if (job->drv_cmd->opcode == ATTACH_DEBUG_BO) 1170 req.config = DEBUG_BO_REGISTER; 1171 else 1172 req.config = DEBUG_BO_UNREGISTER; 1173 1174 req.offset = amdxdna_dev_bo_offset(abo); 1175 req.size = abo->mem.size; 1176 1177 XDNA_DBG(xdna, "offset 0x%llx size 0x%llx config %d", 1178 req.offset, req.size, req.config); 1179 1180 msg.handle = job; 1181 msg.notify_cb = notify_cb; 1182 msg.send_data = (u8 *)&req; 1183 msg.send_size = sizeof(req); 1184 msg.opcode = MSG_OP_CONFIG_DEBUG_BO; 1185 1186 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1187 } 1188 1189 int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id, 1190 struct app_health_report *report) 1191 { 1192 DECLARE_AIE2_MSG(get_app_health, MSG_OP_GET_APP_HEALTH); 1193 struct amdxdna_dev *xdna = ndev->xdna; 1194 struct app_health_report *buf; 1195 dma_addr_t dma_addr; 1196 u32 buf_size; 1197 int ret; 1198 1199 if (!AIE2_FEATURE_ON(ndev, AIE2_APP_HEALTH)) { 1200 XDNA_DBG(xdna, "App health feature not supported"); 1201 return -EOPNOTSUPP; 1202 } 1203 1204 buf_size = sizeof(*report); 1205 buf = aie2_alloc_msg_buffer(ndev, &buf_size, &dma_addr); 1206 if (IS_ERR(buf)) { 1207 XDNA_ERR(xdna, "Failed to allocate buffer for app health"); 1208 return PTR_ERR(buf); 1209 } 1210 1211 req.buf_addr = dma_addr; 1212 req.context_id = context_id; 1213 req.buf_size = buf_size; 1214 1215 drm_clflush_virt_range(buf, sizeof(*report)); 1216 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 1217 if (ret) { 1218 XDNA_ERR(xdna, "Get app health failed, ret %d status 0x%x", ret, resp.status); 1219 goto free_buf; 1220 } 1221 1222 /* Copy the report to caller's buffer */ 1223 memcpy(report, buf, sizeof(*report)); 1224 1225 free_buf: 1226 aie2_free_msg_buffer(ndev, buf_size, buf, dma_addr); 1227 return ret; 1228 } 1229