1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_cache.h> 8 #include <drm/drm_device.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/drm_print.h> 12 #include <drm/gpu_scheduler.h> 13 #include <linux/bitfield.h> 14 #include <linux/errno.h> 15 #include <linux/pci.h> 16 #include <linux/types.h> 17 #include <linux/xarray.h> 18 19 #include "aie2_msg_priv.h" 20 #include "aie2_pci.h" 21 #include "amdxdna_ctx.h" 22 #include "amdxdna_gem.h" 23 #include "amdxdna_mailbox.h" 24 #include "amdxdna_mailbox_helper.h" 25 #include "amdxdna_pci_drv.h" 26 27 #define DECLARE_AIE2_MSG(name, op) \ 28 DECLARE_XDNA_MSG_COMMON(name, op, MAX_AIE2_STATUS_CODE) 29 30 #define EXEC_MSG_OPS(xdna) ((xdna)->dev_handle->exec_msg_ops) 31 32 static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev, 33 struct xdna_mailbox_msg *msg) 34 { 35 struct amdxdna_dev *xdna = ndev->xdna; 36 struct xdna_notify *hdl = msg->handle; 37 int ret; 38 39 if (!ndev->mgmt_chann) 40 return -ENODEV; 41 42 drm_WARN_ON(&xdna->ddev, xdna->rpm_on && !mutex_is_locked(&xdna->dev_lock)); 43 ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg); 44 if (ret == -ETIME) { 45 xdna_mailbox_stop_channel(ndev->mgmt_chann); 46 xdna_mailbox_destroy_channel(ndev->mgmt_chann); 47 ndev->mgmt_chann = NULL; 48 } 49 50 if (!ret && *hdl->status != AIE2_STATUS_SUCCESS) { 51 XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x", 52 msg->opcode, *hdl->data); 53 ret = -EINVAL; 54 } 55 56 return ret; 57 } 58 59 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev) 60 { 61 DECLARE_AIE2_MSG(suspend, MSG_OP_SUSPEND); 62 63 return aie2_send_mgmt_msg_wait(ndev, &msg); 64 } 65 66 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev) 67 { 68 DECLARE_AIE2_MSG(suspend, MSG_OP_RESUME); 69 70 return aie2_send_mgmt_msg_wait(ndev, &msg); 71 } 72 73 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value) 74 { 75 DECLARE_AIE2_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG); 76 int ret; 77 78 req.type = type; 79 req.value = value; 80 81 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 82 if (ret) { 83 XDNA_ERR(ndev->xdna, "Failed to set runtime config, ret %d", ret); 84 return ret; 85 } 86 87 return 0; 88 } 89 90 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value) 91 { 92 DECLARE_AIE2_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG); 93 int ret; 94 95 req.type = type; 96 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 97 if (ret) { 98 XDNA_ERR(ndev->xdna, "Failed to get runtime config, ret %d", ret); 99 return ret; 100 } 101 102 *value = resp.value; 103 return 0; 104 } 105 106 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid) 107 { 108 DECLARE_AIE2_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID); 109 110 req.pasid = pasid; 111 112 return aie2_send_mgmt_msg_wait(ndev, &msg); 113 } 114 115 int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version) 116 { 117 DECLARE_AIE2_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION); 118 struct amdxdna_dev *xdna = ndev->xdna; 119 int ret; 120 121 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 122 if (ret) 123 return ret; 124 125 XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed", 126 resp.major, resp.minor); 127 128 version->major = resp.major; 129 version->minor = resp.minor; 130 131 return 0; 132 } 133 134 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata) 135 { 136 DECLARE_AIE2_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO); 137 int ret; 138 139 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 140 if (ret) 141 return ret; 142 143 metadata->size = resp.info.size; 144 metadata->cols = resp.info.cols; 145 metadata->rows = resp.info.rows; 146 147 metadata->version.major = resp.info.major; 148 metadata->version.minor = resp.info.minor; 149 150 metadata->core.row_count = resp.info.core_rows; 151 metadata->core.row_start = resp.info.core_row_start; 152 metadata->core.dma_channel_count = resp.info.core_dma_channels; 153 metadata->core.lock_count = resp.info.core_locks; 154 metadata->core.event_reg_count = resp.info.core_events; 155 156 metadata->mem.row_count = resp.info.mem_rows; 157 metadata->mem.row_start = resp.info.mem_row_start; 158 metadata->mem.dma_channel_count = resp.info.mem_dma_channels; 159 metadata->mem.lock_count = resp.info.mem_locks; 160 metadata->mem.event_reg_count = resp.info.mem_events; 161 162 metadata->shim.row_count = resp.info.shim_rows; 163 metadata->shim.row_start = resp.info.shim_row_start; 164 metadata->shim.dma_channel_count = resp.info.shim_dma_channels; 165 metadata->shim.lock_count = resp.info.shim_locks; 166 metadata->shim.event_reg_count = resp.info.shim_events; 167 168 return 0; 169 } 170 171 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, 172 struct amdxdna_fw_ver *fw_ver) 173 { 174 DECLARE_AIE2_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION); 175 int ret; 176 177 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 178 if (ret) 179 return ret; 180 181 fw_ver->major = resp.major; 182 fw_ver->minor = resp.minor; 183 fw_ver->sub = resp.sub; 184 fw_ver->build = resp.build; 185 186 return 0; 187 } 188 189 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 190 { 191 DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT); 192 struct amdxdna_dev *xdna = ndev->xdna; 193 struct xdna_mailbox_chann_res x2i; 194 struct xdna_mailbox_chann_res i2x; 195 struct cq_pair *cq_pair; 196 u32 intr_reg; 197 int ret; 198 199 req.aie_type = 1; 200 req.start_col = hwctx->start_col; 201 req.num_col = hwctx->num_col; 202 req.num_cq_pairs_requested = 1; 203 req.pasid = hwctx->client->pasid; 204 req.context_priority = 2; 205 206 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 207 if (ret) 208 return ret; 209 210 hwctx->fw_ctx_id = resp.context_id; 211 WARN_ONCE(hwctx->fw_ctx_id == -1, "Unexpected context id"); 212 213 cq_pair = &resp.cq_pair[0]; 214 x2i.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.head_addr); 215 x2i.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.tail_addr); 216 x2i.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->x2i_q.buf_addr); 217 x2i.rb_size = cq_pair->x2i_q.buf_size; 218 219 i2x.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.head_addr); 220 i2x.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.tail_addr); 221 i2x.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->i2x_q.buf_addr); 222 i2x.rb_size = cq_pair->i2x_q.buf_size; 223 224 ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id); 225 if (ret == -EINVAL) { 226 XDNA_ERR(xdna, "not able to create channel"); 227 goto out_destroy_context; 228 } 229 230 intr_reg = i2x.mb_head_ptr_reg + 4; 231 hwctx->priv->mbox_chann = xdna_mailbox_create_channel(ndev->mbox, &x2i, &i2x, 232 intr_reg, ret); 233 if (!hwctx->priv->mbox_chann) { 234 XDNA_ERR(xdna, "not able to create channel"); 235 ret = -EINVAL; 236 goto out_destroy_context; 237 } 238 ndev->hwctx_num++; 239 240 XDNA_DBG(xdna, "%s mailbox channel irq: %d, msix_id: %d", 241 hwctx->name, ret, resp.msix_id); 242 XDNA_DBG(xdna, "%s created fw ctx %d pasid %d", hwctx->name, 243 hwctx->fw_ctx_id, hwctx->client->pasid); 244 245 return 0; 246 247 out_destroy_context: 248 aie2_destroy_context(ndev, hwctx); 249 return ret; 250 } 251 252 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 253 { 254 DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); 255 struct amdxdna_dev *xdna = ndev->xdna; 256 int ret; 257 258 if (hwctx->fw_ctx_id == -1) 259 return 0; 260 261 xdna_mailbox_stop_channel(hwctx->priv->mbox_chann); 262 263 req.context_id = hwctx->fw_ctx_id; 264 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 265 if (ret) 266 XDNA_WARN(xdna, "%s destroy context failed, ret %d", hwctx->name, ret); 267 268 xdna_mailbox_destroy_channel(hwctx->priv->mbox_chann); 269 XDNA_DBG(xdna, "%s destroyed fw ctx %d", hwctx->name, 270 hwctx->fw_ctx_id); 271 hwctx->priv->mbox_chann = NULL; 272 hwctx->fw_ctx_id = -1; 273 ndev->hwctx_num--; 274 275 return ret; 276 } 277 278 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size) 279 { 280 DECLARE_AIE2_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER); 281 struct amdxdna_dev *xdna = ndev->xdna; 282 int ret; 283 284 req.context_id = context_id; 285 req.buf_addr = addr; 286 req.buf_size = size; 287 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 288 if (ret) 289 return ret; 290 291 XDNA_DBG(xdna, "fw ctx %d map host buf addr 0x%llx size 0x%llx", 292 context_id, addr, size); 293 294 return 0; 295 } 296 297 static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg) 298 { 299 u32 *bitmap = arg; 300 301 *bitmap |= GENMASK(hwctx->start_col + hwctx->num_col - 1, hwctx->start_col); 302 303 return 0; 304 } 305 306 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, 307 u32 size, u32 *cols_filled) 308 { 309 DECLARE_AIE2_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS); 310 struct amdxdna_dev *xdna = ndev->xdna; 311 struct amdxdna_client *client; 312 dma_addr_t dma_addr; 313 u32 aie_bitmap = 0; 314 u8 *buff_addr; 315 int ret; 316 317 buff_addr = dma_alloc_noncoherent(xdna->ddev.dev, size, &dma_addr, 318 DMA_FROM_DEVICE, GFP_KERNEL); 319 if (!buff_addr) 320 return -ENOMEM; 321 322 /* Go through each hardware context and mark the AIE columns that are active */ 323 list_for_each_entry(client, &xdna->client_list, node) 324 amdxdna_hwctx_walk(client, &aie_bitmap, amdxdna_hwctx_col_map); 325 326 *cols_filled = 0; 327 req.dump_buff_addr = dma_addr; 328 req.dump_buff_size = size; 329 req.num_cols = hweight32(aie_bitmap); 330 req.aie_bitmap = aie_bitmap; 331 332 drm_clflush_virt_range(buff_addr, size); /* device can access */ 333 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 334 if (ret) { 335 XDNA_ERR(xdna, "Error during NPU query, status %d", ret); 336 goto fail; 337 } 338 339 XDNA_DBG(xdna, "Query NPU status completed"); 340 341 if (size < resp.size) { 342 ret = -EINVAL; 343 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size); 344 goto fail; 345 } 346 347 if (copy_to_user(buf, buff_addr, resp.size)) { 348 ret = -EFAULT; 349 XDNA_ERR(xdna, "Failed to copy NPU status to user space"); 350 goto fail; 351 } 352 353 *cols_filled = aie_bitmap; 354 355 fail: 356 dma_free_noncoherent(xdna->ddev.dev, size, buff_addr, dma_addr, DMA_FROM_DEVICE); 357 return ret; 358 } 359 360 int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, 361 char __user *buf, u32 size, 362 struct amdxdna_drm_query_telemetry_header *header) 363 { 364 DECLARE_AIE2_MSG(get_telemetry, MSG_OP_GET_TELEMETRY); 365 struct amdxdna_dev *xdna = ndev->xdna; 366 dma_addr_t dma_addr; 367 u8 *addr; 368 int ret; 369 370 if (header->type >= MAX_TELEMETRY_TYPE) 371 return -EINVAL; 372 373 addr = dma_alloc_noncoherent(xdna->ddev.dev, size, &dma_addr, 374 DMA_FROM_DEVICE, GFP_KERNEL); 375 if (!addr) 376 return -ENOMEM; 377 378 req.buf_addr = dma_addr; 379 req.buf_size = size; 380 req.type = header->type; 381 382 drm_clflush_virt_range(addr, size); /* device can access */ 383 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 384 if (ret) { 385 XDNA_ERR(xdna, "Query telemetry failed, status %d", ret); 386 goto free_buf; 387 } 388 389 if (size < resp.size) { 390 ret = -EINVAL; 391 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size); 392 goto free_buf; 393 } 394 395 if (copy_to_user(buf, addr, resp.size)) { 396 ret = -EFAULT; 397 XDNA_ERR(xdna, "Failed to copy telemetry to user space"); 398 goto free_buf; 399 } 400 401 header->major = resp.major; 402 header->minor = resp.minor; 403 404 free_buf: 405 dma_free_noncoherent(xdna->ddev.dev, size, addr, dma_addr, DMA_FROM_DEVICE); 406 return ret; 407 } 408 409 int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size, 410 void *handle, int (*cb)(void*, void __iomem *, size_t)) 411 { 412 struct async_event_msg_req req = { 0 }; 413 struct xdna_mailbox_msg msg = { 414 .send_data = (u8 *)&req, 415 .send_size = sizeof(req), 416 .handle = handle, 417 .opcode = MSG_OP_REGISTER_ASYNC_EVENT_MSG, 418 .notify_cb = cb, 419 }; 420 421 req.buf_addr = addr; 422 req.buf_size = size; 423 424 XDNA_DBG(ndev->xdna, "Register addr 0x%llx size 0x%x", addr, size); 425 return xdna_mailbox_send_msg(ndev->mgmt_chann, &msg, TX_TIMEOUT); 426 } 427 428 int aie2_config_cu(struct amdxdna_hwctx *hwctx, 429 int (*notify_cb)(void *, void __iomem *, size_t)) 430 { 431 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 432 struct amdxdna_dev *xdna = hwctx->client->xdna; 433 u32 shift = xdna->dev_info->dev_mem_buf_shift; 434 struct config_cu_req req = { 0 }; 435 struct xdna_mailbox_msg msg; 436 struct drm_gem_object *gobj; 437 struct amdxdna_gem_obj *abo; 438 int i; 439 440 if (!chann) 441 return -ENODEV; 442 443 if (hwctx->cus->num_cus > MAX_NUM_CUS) { 444 XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS); 445 return -EINVAL; 446 } 447 448 for (i = 0; i < hwctx->cus->num_cus; i++) { 449 struct amdxdna_cu_config *cu = &hwctx->cus->cu_configs[i]; 450 451 if (XDNA_MBZ_DBG(xdna, cu->pad, sizeof(cu->pad))) 452 return -EINVAL; 453 454 gobj = drm_gem_object_lookup(hwctx->client->filp, cu->cu_bo); 455 if (!gobj) { 456 XDNA_ERR(xdna, "Lookup GEM object failed"); 457 return -EINVAL; 458 } 459 abo = to_xdna_obj(gobj); 460 461 if (abo->type != AMDXDNA_BO_DEV) { 462 drm_gem_object_put(gobj); 463 XDNA_ERR(xdna, "Invalid BO type"); 464 return -EINVAL; 465 } 466 467 req.cfgs[i] = FIELD_PREP(AIE2_MSG_CFG_CU_PDI_ADDR, 468 abo->mem.dev_addr >> shift); 469 req.cfgs[i] |= FIELD_PREP(AIE2_MSG_CFG_CU_FUNC, cu->cu_func); 470 XDNA_DBG(xdna, "CU %d full addr 0x%llx, cfg 0x%x", i, 471 abo->mem.dev_addr, req.cfgs[i]); 472 drm_gem_object_put(gobj); 473 } 474 req.num_cus = hwctx->cus->num_cus; 475 476 msg.send_data = (u8 *)&req; 477 msg.send_size = sizeof(req); 478 msg.handle = hwctx; 479 msg.opcode = MSG_OP_CONFIG_CU; 480 msg.notify_cb = notify_cb; 481 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 482 } 483 484 static int aie2_init_exec_cu_req(struct amdxdna_gem_obj *cmd_bo, void *req, 485 size_t *size, u32 *msg_op) 486 { 487 struct execute_buffer_req *cu_req = req; 488 u32 cmd_len; 489 void *cmd; 490 491 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 492 if (cmd_len > sizeof(cu_req->payload)) 493 return -EINVAL; 494 495 cu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 496 if (cu_req->cu_idx == INVALID_CU_IDX) 497 return -EINVAL; 498 499 memcpy(cu_req->payload, cmd, cmd_len); 500 501 *size = sizeof(*cu_req); 502 *msg_op = MSG_OP_EXECUTE_BUFFER_CF; 503 return 0; 504 } 505 506 static int aie2_init_exec_dpu_req(struct amdxdna_gem_obj *cmd_bo, void *req, 507 size_t *size, u32 *msg_op) 508 { 509 struct exec_dpu_req *dpu_req = req; 510 struct amdxdna_cmd_start_npu *sn; 511 u32 cmd_len; 512 513 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 514 if (cmd_len - sizeof(*sn) > sizeof(dpu_req->payload)) 515 return -EINVAL; 516 517 dpu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 518 if (dpu_req->cu_idx == INVALID_CU_IDX) 519 return -EINVAL; 520 521 dpu_req->inst_buf_addr = sn->buffer; 522 dpu_req->inst_size = sn->buffer_size; 523 dpu_req->inst_prop_cnt = sn->prop_count; 524 memcpy(dpu_req->payload, sn->prop_args, cmd_len - sizeof(*sn)); 525 526 *size = sizeof(*dpu_req); 527 *msg_op = MSG_OP_EXEC_DPU; 528 return 0; 529 } 530 531 static void aie2_init_exec_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) 532 { 533 struct cmd_chain_req *chain_req = req; 534 535 chain_req->buf_addr = slot_addr; 536 chain_req->buf_size = size; 537 chain_req->count = cmd_cnt; 538 } 539 540 static void aie2_init_npu_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) 541 { 542 struct cmd_chain_npu_req *npu_chain_req = req; 543 544 npu_chain_req->flags = 0; 545 npu_chain_req->reserved = 0; 546 npu_chain_req->buf_addr = slot_addr; 547 npu_chain_req->buf_size = size; 548 npu_chain_req->count = cmd_cnt; 549 } 550 551 static int 552 aie2_cmdlist_fill_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 553 { 554 struct cmd_chain_slot_execbuf_cf *cf_slot = slot; 555 u32 cmd_len; 556 void *cmd; 557 558 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 559 if (*size < sizeof(*cf_slot) + cmd_len) 560 return -EINVAL; 561 562 cf_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 563 if (cf_slot->cu_idx == INVALID_CU_IDX) 564 return -EINVAL; 565 566 cf_slot->arg_cnt = cmd_len / sizeof(u32); 567 memcpy(cf_slot->args, cmd, cmd_len); 568 /* Accurate slot size to hint firmware to do necessary copy */ 569 *size = sizeof(*cf_slot) + cmd_len; 570 return 0; 571 } 572 573 static int 574 aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 575 { 576 struct cmd_chain_slot_dpu *dpu_slot = slot; 577 struct amdxdna_cmd_start_npu *sn; 578 u32 cmd_len; 579 u32 arg_sz; 580 581 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 582 arg_sz = cmd_len - sizeof(*sn); 583 if (cmd_len < sizeof(*sn) || arg_sz > MAX_DPU_ARGS_SIZE) 584 return -EINVAL; 585 586 if (*size < sizeof(*dpu_slot) + arg_sz) 587 return -EINVAL; 588 589 dpu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 590 if (dpu_slot->cu_idx == INVALID_CU_IDX) 591 return -EINVAL; 592 593 dpu_slot->inst_buf_addr = sn->buffer; 594 dpu_slot->inst_size = sn->buffer_size; 595 dpu_slot->inst_prop_cnt = sn->prop_count; 596 dpu_slot->arg_cnt = arg_sz / sizeof(u32); 597 memcpy(dpu_slot->args, sn->prop_args, arg_sz); 598 599 /* Accurate slot size to hint firmware to do necessary copy */ 600 *size = sizeof(*dpu_slot) + arg_sz; 601 return 0; 602 } 603 604 static u32 aie2_get_chain_msg_op(u32 cmd_op) 605 { 606 switch (cmd_op) { 607 case ERT_START_CU: 608 return MSG_OP_CHAIN_EXEC_BUFFER_CF; 609 case ERT_START_NPU: 610 return MSG_OP_CHAIN_EXEC_DPU; 611 default: 612 break; 613 } 614 615 return MSG_OP_MAX_OPCODE; 616 } 617 618 static struct aie2_exec_msg_ops legacy_exec_message_ops = { 619 .init_cu_req = aie2_init_exec_cu_req, 620 .init_dpu_req = aie2_init_exec_dpu_req, 621 .init_chain_req = aie2_init_exec_chain_req, 622 .fill_cf_slot = aie2_cmdlist_fill_cf, 623 .fill_dpu_slot = aie2_cmdlist_fill_dpu, 624 .get_chain_msg_op = aie2_get_chain_msg_op, 625 }; 626 627 static int 628 aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 629 { 630 struct cmd_chain_slot_npu *npu_slot = slot; 631 u32 cmd_len; 632 void *cmd; 633 634 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 635 if (*size < sizeof(*npu_slot) + cmd_len) 636 return -EINVAL; 637 638 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 639 if (npu_slot->cu_idx == INVALID_CU_IDX) 640 return -EINVAL; 641 642 memset(npu_slot, 0, sizeof(*npu_slot)); 643 npu_slot->type = EXEC_NPU_TYPE_NON_ELF; 644 npu_slot->arg_cnt = cmd_len / sizeof(u32); 645 memcpy(npu_slot->args, cmd, cmd_len); 646 647 *size = sizeof(*npu_slot) + cmd_len; 648 return 0; 649 } 650 651 static int 652 aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 653 { 654 struct cmd_chain_slot_npu *npu_slot = slot; 655 struct amdxdna_cmd_start_npu *sn; 656 u32 cmd_len; 657 u32 arg_sz; 658 659 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 660 arg_sz = cmd_len - sizeof(*sn); 661 if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE) 662 return -EINVAL; 663 664 if (*size < sizeof(*npu_slot) + arg_sz) 665 return -EINVAL; 666 667 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 668 if (npu_slot->cu_idx == INVALID_CU_IDX) 669 return -EINVAL; 670 671 memset(npu_slot, 0, sizeof(*npu_slot)); 672 npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF; 673 npu_slot->inst_buf_addr = sn->buffer; 674 npu_slot->inst_size = sn->buffer_size; 675 npu_slot->inst_prop_cnt = sn->prop_count; 676 npu_slot->arg_cnt = arg_sz / sizeof(u32); 677 memcpy(npu_slot->args, sn->prop_args, arg_sz); 678 679 *size = sizeof(*npu_slot) + arg_sz; 680 return 0; 681 } 682 683 static u32 aie2_get_npu_chain_msg_op(u32 cmd_op) 684 { 685 return MSG_OP_CHAIN_EXEC_NPU; 686 } 687 688 static struct aie2_exec_msg_ops npu_exec_message_ops = { 689 .init_cu_req = aie2_init_exec_cu_req, 690 .init_dpu_req = aie2_init_exec_dpu_req, 691 .init_chain_req = aie2_init_npu_chain_req, 692 .fill_cf_slot = aie2_cmdlist_fill_npu_cf, 693 .fill_dpu_slot = aie2_cmdlist_fill_npu_dpu, 694 .get_chain_msg_op = aie2_get_npu_chain_msg_op, 695 }; 696 697 static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo, 698 size_t *size, u32 *msg_op) 699 { 700 struct amdxdna_dev *xdna = cmd_abo->client->xdna; 701 int ret; 702 u32 op; 703 704 705 op = amdxdna_cmd_get_op(cmd_abo); 706 switch (op) { 707 case ERT_START_CU: 708 ret = EXEC_MSG_OPS(xdna)->init_cu_req(cmd_abo, req, size, msg_op); 709 if (ret) { 710 XDNA_DBG(xdna, "Init CU req failed ret %d", ret); 711 return ret; 712 } 713 break; 714 case ERT_START_NPU: 715 ret = EXEC_MSG_OPS(xdna)->init_dpu_req(cmd_abo, req, size, msg_op); 716 if (ret) { 717 XDNA_DBG(xdna, "Init DPU req failed ret %d", ret); 718 return ret; 719 } 720 721 break; 722 default: 723 XDNA_ERR(xdna, "Unsupported op %d", op); 724 ret = -EOPNOTSUPP; 725 break; 726 } 727 728 return ret; 729 } 730 731 static int 732 aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo, 733 size_t *size, u32 *cmd_op) 734 { 735 struct amdxdna_dev *xdna = cmd_abo->client->xdna; 736 int ret; 737 u32 op; 738 739 op = amdxdna_cmd_get_op(cmd_abo); 740 if (*cmd_op == ERT_INVALID_CMD) 741 *cmd_op = op; 742 else if (op != *cmd_op) 743 return -EINVAL; 744 745 switch (op) { 746 case ERT_START_CU: 747 ret = EXEC_MSG_OPS(xdna)->fill_cf_slot(cmd_abo, slot, size); 748 break; 749 case ERT_START_NPU: 750 ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size); 751 break; 752 default: 753 XDNA_INFO(xdna, "Unsupported op %d", op); 754 ret = -EOPNOTSUPP; 755 break; 756 } 757 758 return ret; 759 } 760 761 void aie2_msg_init(struct amdxdna_dev_hdl *ndev) 762 { 763 if (AIE2_FEATURE_ON(ndev, AIE2_NPU_COMMAND)) 764 ndev->exec_msg_ops = &npu_exec_message_ops; 765 else 766 ndev->exec_msg_ops = &legacy_exec_message_ops; 767 } 768 769 static inline struct amdxdna_gem_obj * 770 aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job) 771 { 772 int idx = get_job_idx(job->seq); 773 774 return job->hwctx->priv->cmd_buf[idx]; 775 } 776 777 int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 778 int (*notify_cb)(void *, void __iomem *, size_t)) 779 { 780 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 781 struct amdxdna_dev *xdna = hwctx->client->xdna; 782 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 783 struct xdna_mailbox_msg msg; 784 union exec_req req; 785 int ret; 786 787 if (!chann) 788 return -ENODEV; 789 790 ret = aie2_init_exec_req(&req, cmd_abo, &msg.send_size, &msg.opcode); 791 if (ret) 792 return ret; 793 794 msg.handle = job; 795 msg.notify_cb = notify_cb; 796 msg.send_data = (u8 *)&req; 797 print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req, 798 0x40, false); 799 800 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 801 if (ret) { 802 XDNA_ERR(xdna, "Send message failed"); 803 return ret; 804 } 805 806 return 0; 807 } 808 809 int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, 810 struct amdxdna_sched_job *job, 811 int (*notify_cb)(void *, void __iomem *, size_t)) 812 { 813 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 814 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 815 struct amdxdna_client *client = hwctx->client; 816 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 817 struct amdxdna_dev *xdna = client->xdna; 818 struct amdxdna_cmd_chain *payload; 819 struct xdna_mailbox_msg msg; 820 union exec_chain_req req; 821 u32 payload_len; 822 u32 offset = 0; 823 size_t size; 824 int ret; 825 u32 op; 826 u32 i; 827 828 op = amdxdna_cmd_get_op(cmd_abo); 829 payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len); 830 if (op != ERT_CMD_CHAIN || !payload || 831 payload_len < struct_size(payload, data, payload->command_count)) 832 return -EINVAL; 833 834 op = ERT_INVALID_CMD; 835 for (i = 0; i < payload->command_count; i++) { 836 u32 boh = (u32)(payload->data[i]); 837 struct amdxdna_gem_obj *abo; 838 839 abo = amdxdna_gem_get_obj(client, boh, AMDXDNA_BO_CMD); 840 if (!abo) { 841 XDNA_ERR(xdna, "Failed to find cmd BO %d", boh); 842 return -ENOENT; 843 } 844 845 size = cmdbuf_abo->mem.size - offset; 846 ret = aie2_cmdlist_fill_slot(cmdbuf_abo->mem.kva + offset, 847 abo, &size, &op); 848 amdxdna_gem_put_obj(abo); 849 if (ret) 850 return ret; 851 852 offset += size; 853 } 854 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); 855 if (msg.opcode == MSG_OP_MAX_OPCODE) 856 return -EOPNOTSUPP; 857 858 /* The offset is the accumulated total size of the cmd buffer */ 859 EXEC_MSG_OPS(xdna)->init_chain_req(&req, cmdbuf_abo->mem.dev_addr, 860 offset, payload->command_count); 861 drm_clflush_virt_range(cmdbuf_abo->mem.kva, offset); 862 863 msg.handle = job; 864 msg.notify_cb = notify_cb; 865 msg.send_data = (u8 *)&req; 866 msg.send_size = sizeof(req); 867 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 868 if (ret) { 869 XDNA_ERR(xdna, "Send message failed"); 870 return ret; 871 } 872 873 return 0; 874 } 875 876 int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx, 877 struct amdxdna_sched_job *job, 878 int (*notify_cb)(void *, void __iomem *, size_t)) 879 { 880 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 881 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 882 struct amdxdna_dev *xdna = hwctx->client->xdna; 883 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 884 struct xdna_mailbox_msg msg; 885 union exec_chain_req req; 886 u32 op = ERT_INVALID_CMD; 887 size_t size; 888 int ret; 889 890 size = cmdbuf_abo->mem.size; 891 ret = aie2_cmdlist_fill_slot(cmdbuf_abo->mem.kva, cmd_abo, &size, &op); 892 if (ret) 893 return ret; 894 895 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); 896 if (msg.opcode == MSG_OP_MAX_OPCODE) 897 return -EOPNOTSUPP; 898 899 EXEC_MSG_OPS(xdna)->init_chain_req(&req, cmdbuf_abo->mem.dev_addr, 900 size, 1); 901 drm_clflush_virt_range(cmdbuf_abo->mem.kva, size); 902 903 msg.handle = job; 904 msg.notify_cb = notify_cb; 905 msg.send_data = (u8 *)&req; 906 msg.send_size = sizeof(req); 907 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 908 if (ret) { 909 XDNA_ERR(hwctx->client->xdna, "Send message failed"); 910 return ret; 911 } 912 913 return 0; 914 } 915 916 int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 917 int (*notify_cb)(void *, void __iomem *, size_t)) 918 { 919 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 920 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); 921 struct amdxdna_dev *xdna = hwctx->client->xdna; 922 struct xdna_mailbox_msg msg; 923 struct sync_bo_req req; 924 int ret = 0; 925 926 req.src_addr = 0; 927 req.dst_addr = amdxdna_dev_bo_offset(abo); 928 req.size = abo->mem.size; 929 930 /* Device to Host */ 931 req.type = FIELD_PREP(AIE2_MSG_SYNC_BO_SRC_TYPE, SYNC_BO_DEV_MEM) | 932 FIELD_PREP(AIE2_MSG_SYNC_BO_DST_TYPE, SYNC_BO_HOST_MEM); 933 934 XDNA_DBG(xdna, "sync %d bytes src(0x%llx) to dst(0x%llx) completed", 935 req.size, req.src_addr, req.dst_addr); 936 937 msg.handle = job; 938 msg.notify_cb = notify_cb; 939 msg.send_data = (u8 *)&req; 940 msg.send_size = sizeof(req); 941 msg.opcode = MSG_OP_SYNC_BO; 942 943 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 944 if (ret) { 945 XDNA_ERR(xdna, "Send message failed"); 946 return ret; 947 } 948 949 return 0; 950 } 951 952 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 953 int (*notify_cb)(void *, void __iomem *, size_t)) 954 { 955 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 956 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); 957 struct amdxdna_dev *xdna = hwctx->client->xdna; 958 struct config_debug_bo_req req; 959 struct xdna_mailbox_msg msg; 960 961 if (job->drv_cmd->opcode == ATTACH_DEBUG_BO) 962 req.config = DEBUG_BO_REGISTER; 963 else 964 req.config = DEBUG_BO_UNREGISTER; 965 966 req.offset = amdxdna_dev_bo_offset(abo); 967 req.size = abo->mem.size; 968 969 XDNA_DBG(xdna, "offset 0x%llx size 0x%llx config %d", 970 req.offset, req.size, req.config); 971 972 msg.handle = job; 973 msg.notify_cb = notify_cb; 974 msg.send_data = (u8 *)&req; 975 msg.send_size = sizeof(req); 976 msg.opcode = MSG_OP_CONFIG_DEBUG_BO; 977 978 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 979 } 980