1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_cache.h> 8 #include <drm/drm_device.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/drm_print.h> 12 #include <drm/gpu_scheduler.h> 13 #include <linux/bitfield.h> 14 #include <linux/errno.h> 15 #include <linux/pci.h> 16 #include <linux/types.h> 17 #include <linux/xarray.h> 18 19 #include "aie2_msg_priv.h" 20 #include "aie2_pci.h" 21 #include "amdxdna_ctx.h" 22 #include "amdxdna_gem.h" 23 #include "amdxdna_mailbox.h" 24 #include "amdxdna_mailbox_helper.h" 25 #include "amdxdna_pci_drv.h" 26 27 #define DECLARE_AIE2_MSG(name, op) \ 28 DECLARE_XDNA_MSG_COMMON(name, op, MAX_AIE2_STATUS_CODE) 29 30 static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev, 31 struct xdna_mailbox_msg *msg) 32 { 33 struct amdxdna_dev *xdna = ndev->xdna; 34 struct xdna_notify *hdl = msg->handle; 35 int ret; 36 37 if (!ndev->mgmt_chann) 38 return -ENODEV; 39 40 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 41 ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg); 42 if (ret == -ETIME) { 43 xdna_mailbox_stop_channel(ndev->mgmt_chann); 44 xdna_mailbox_destroy_channel(ndev->mgmt_chann); 45 ndev->mgmt_chann = NULL; 46 } 47 48 if (!ret && *hdl->data != AIE2_STATUS_SUCCESS) { 49 XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x", 50 msg->opcode, *hdl->data); 51 ret = -EINVAL; 52 } 53 54 return ret; 55 } 56 57 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev) 58 { 59 DECLARE_AIE2_MSG(suspend, MSG_OP_SUSPEND); 60 61 return aie2_send_mgmt_msg_wait(ndev, &msg); 62 } 63 64 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev) 65 { 66 DECLARE_AIE2_MSG(suspend, MSG_OP_RESUME); 67 68 return aie2_send_mgmt_msg_wait(ndev, &msg); 69 } 70 71 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value) 72 { 73 DECLARE_AIE2_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG); 74 int ret; 75 76 req.type = type; 77 req.value = value; 78 79 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 80 if (ret) { 81 XDNA_ERR(ndev->xdna, "Failed to set runtime config, ret %d", ret); 82 return ret; 83 } 84 85 return 0; 86 } 87 88 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value) 89 { 90 DECLARE_AIE2_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG); 91 int ret; 92 93 req.type = type; 94 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 95 if (ret) { 96 XDNA_ERR(ndev->xdna, "Failed to get runtime config, ret %d", ret); 97 return ret; 98 } 99 100 *value = resp.value; 101 return 0; 102 } 103 104 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid) 105 { 106 DECLARE_AIE2_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID); 107 108 req.pasid = pasid; 109 110 return aie2_send_mgmt_msg_wait(ndev, &msg); 111 } 112 113 int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version) 114 { 115 DECLARE_AIE2_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION); 116 struct amdxdna_dev *xdna = ndev->xdna; 117 int ret; 118 119 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 120 if (ret) 121 return ret; 122 123 XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed", 124 resp.major, resp.minor); 125 126 version->major = resp.major; 127 version->minor = resp.minor; 128 129 return 0; 130 } 131 132 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata) 133 { 134 DECLARE_AIE2_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO); 135 int ret; 136 137 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 138 if (ret) 139 return ret; 140 141 metadata->size = resp.info.size; 142 metadata->cols = resp.info.cols; 143 metadata->rows = resp.info.rows; 144 145 metadata->version.major = resp.info.major; 146 metadata->version.minor = resp.info.minor; 147 148 metadata->core.row_count = resp.info.core_rows; 149 metadata->core.row_start = resp.info.core_row_start; 150 metadata->core.dma_channel_count = resp.info.core_dma_channels; 151 metadata->core.lock_count = resp.info.core_locks; 152 metadata->core.event_reg_count = resp.info.core_events; 153 154 metadata->mem.row_count = resp.info.mem_rows; 155 metadata->mem.row_start = resp.info.mem_row_start; 156 metadata->mem.dma_channel_count = resp.info.mem_dma_channels; 157 metadata->mem.lock_count = resp.info.mem_locks; 158 metadata->mem.event_reg_count = resp.info.mem_events; 159 160 metadata->shim.row_count = resp.info.shim_rows; 161 metadata->shim.row_start = resp.info.shim_row_start; 162 metadata->shim.dma_channel_count = resp.info.shim_dma_channels; 163 metadata->shim.lock_count = resp.info.shim_locks; 164 metadata->shim.event_reg_count = resp.info.shim_events; 165 166 return 0; 167 } 168 169 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, 170 struct amdxdna_fw_ver *fw_ver) 171 { 172 DECLARE_AIE2_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION); 173 int ret; 174 175 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 176 if (ret) 177 return ret; 178 179 fw_ver->major = resp.major; 180 fw_ver->minor = resp.minor; 181 fw_ver->sub = resp.sub; 182 fw_ver->build = resp.build; 183 184 return 0; 185 } 186 187 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 188 { 189 DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT); 190 struct amdxdna_dev *xdna = ndev->xdna; 191 struct xdna_mailbox_chann_res x2i; 192 struct xdna_mailbox_chann_res i2x; 193 struct cq_pair *cq_pair; 194 u32 intr_reg; 195 int ret; 196 197 req.aie_type = 1; 198 req.start_col = hwctx->start_col; 199 req.num_col = hwctx->num_col; 200 req.num_cq_pairs_requested = 1; 201 req.pasid = hwctx->client->pasid; 202 req.context_priority = 2; 203 204 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 205 if (ret) 206 return ret; 207 208 hwctx->fw_ctx_id = resp.context_id; 209 WARN_ONCE(hwctx->fw_ctx_id == -1, "Unexpected context id"); 210 211 cq_pair = &resp.cq_pair[0]; 212 x2i.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.head_addr); 213 x2i.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.tail_addr); 214 x2i.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->x2i_q.buf_addr); 215 x2i.rb_size = cq_pair->x2i_q.buf_size; 216 217 i2x.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.head_addr); 218 i2x.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.tail_addr); 219 i2x.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->i2x_q.buf_addr); 220 i2x.rb_size = cq_pair->i2x_q.buf_size; 221 222 ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id); 223 if (ret == -EINVAL) { 224 XDNA_ERR(xdna, "not able to create channel"); 225 goto out_destroy_context; 226 } 227 228 intr_reg = i2x.mb_head_ptr_reg + 4; 229 hwctx->priv->mbox_chann = xdna_mailbox_create_channel(ndev->mbox, &x2i, &i2x, 230 intr_reg, ret); 231 if (!hwctx->priv->mbox_chann) { 232 XDNA_ERR(xdna, "not able to create channel"); 233 ret = -EINVAL; 234 goto out_destroy_context; 235 } 236 237 XDNA_DBG(xdna, "%s mailbox channel irq: %d, msix_id: %d", 238 hwctx->name, ret, resp.msix_id); 239 XDNA_DBG(xdna, "%s created fw ctx %d pasid %d", hwctx->name, 240 hwctx->fw_ctx_id, hwctx->client->pasid); 241 242 return 0; 243 244 out_destroy_context: 245 aie2_destroy_context(ndev, hwctx); 246 return ret; 247 } 248 249 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 250 { 251 DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); 252 struct amdxdna_dev *xdna = ndev->xdna; 253 int ret; 254 255 if (hwctx->fw_ctx_id == -1) 256 return 0; 257 258 xdna_mailbox_stop_channel(hwctx->priv->mbox_chann); 259 260 req.context_id = hwctx->fw_ctx_id; 261 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 262 if (ret) 263 XDNA_WARN(xdna, "%s destroy context failed, ret %d", hwctx->name, ret); 264 265 xdna_mailbox_destroy_channel(hwctx->priv->mbox_chann); 266 XDNA_DBG(xdna, "%s destroyed fw ctx %d", hwctx->name, 267 hwctx->fw_ctx_id); 268 hwctx->priv->mbox_chann = NULL; 269 hwctx->fw_ctx_id = -1; 270 271 return ret; 272 } 273 274 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size) 275 { 276 DECLARE_AIE2_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER); 277 struct amdxdna_dev *xdna = ndev->xdna; 278 int ret; 279 280 req.context_id = context_id; 281 req.buf_addr = addr; 282 req.buf_size = size; 283 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 284 if (ret) 285 return ret; 286 287 XDNA_DBG(xdna, "fw ctx %d map host buf addr 0x%llx size 0x%llx", 288 context_id, addr, size); 289 290 return 0; 291 } 292 293 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, 294 u32 size, u32 *cols_filled) 295 { 296 DECLARE_AIE2_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS); 297 struct amdxdna_dev *xdna = ndev->xdna; 298 struct amdxdna_client *client; 299 struct amdxdna_hwctx *hwctx; 300 unsigned long hwctx_id; 301 dma_addr_t dma_addr; 302 u32 aie_bitmap = 0; 303 u8 *buff_addr; 304 int ret, idx; 305 306 buff_addr = dma_alloc_noncoherent(xdna->ddev.dev, size, &dma_addr, 307 DMA_FROM_DEVICE, GFP_KERNEL); 308 if (!buff_addr) 309 return -ENOMEM; 310 311 /* Go through each hardware context and mark the AIE columns that are active */ 312 list_for_each_entry(client, &xdna->client_list, node) { 313 idx = srcu_read_lock(&client->hwctx_srcu); 314 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) 315 aie_bitmap |= amdxdna_hwctx_col_map(hwctx); 316 srcu_read_unlock(&client->hwctx_srcu, idx); 317 } 318 319 *cols_filled = 0; 320 req.dump_buff_addr = dma_addr; 321 req.dump_buff_size = size; 322 req.num_cols = hweight32(aie_bitmap); 323 req.aie_bitmap = aie_bitmap; 324 325 drm_clflush_virt_range(buff_addr, size); /* device can access */ 326 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 327 if (ret) { 328 XDNA_ERR(xdna, "Error during NPU query, status %d", ret); 329 goto fail; 330 } 331 332 if (resp.status != AIE2_STATUS_SUCCESS) { 333 XDNA_ERR(xdna, "Query NPU status failed, status 0x%x", resp.status); 334 ret = -EINVAL; 335 goto fail; 336 } 337 XDNA_DBG(xdna, "Query NPU status completed"); 338 339 if (size < resp.size) { 340 ret = -EINVAL; 341 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size); 342 goto fail; 343 } 344 345 if (copy_to_user(buf, buff_addr, resp.size)) { 346 ret = -EFAULT; 347 XDNA_ERR(xdna, "Failed to copy NPU status to user space"); 348 goto fail; 349 } 350 351 *cols_filled = aie_bitmap; 352 353 fail: 354 dma_free_noncoherent(xdna->ddev.dev, size, buff_addr, dma_addr, DMA_FROM_DEVICE); 355 return ret; 356 } 357 358 int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size, 359 void *handle, int (*cb)(void*, const u32 *, size_t)) 360 { 361 struct async_event_msg_req req = { 0 }; 362 struct xdna_mailbox_msg msg = { 363 .send_data = (u8 *)&req, 364 .send_size = sizeof(req), 365 .handle = handle, 366 .opcode = MSG_OP_REGISTER_ASYNC_EVENT_MSG, 367 .notify_cb = cb, 368 }; 369 370 req.buf_addr = addr; 371 req.buf_size = size; 372 373 XDNA_DBG(ndev->xdna, "Register addr 0x%llx size 0x%x", addr, size); 374 return xdna_mailbox_send_msg(ndev->mgmt_chann, &msg, TX_TIMEOUT); 375 } 376 377 int aie2_config_cu(struct amdxdna_hwctx *hwctx) 378 { 379 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 380 struct amdxdna_dev *xdna = hwctx->client->xdna; 381 u32 shift = xdna->dev_info->dev_mem_buf_shift; 382 DECLARE_AIE2_MSG(config_cu, MSG_OP_CONFIG_CU); 383 struct drm_gem_object *gobj; 384 struct amdxdna_gem_obj *abo; 385 int ret, i; 386 387 if (!chann) 388 return -ENODEV; 389 390 if (hwctx->cus->num_cus > MAX_NUM_CUS) { 391 XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS); 392 return -EINVAL; 393 } 394 395 for (i = 0; i < hwctx->cus->num_cus; i++) { 396 struct amdxdna_cu_config *cu = &hwctx->cus->cu_configs[i]; 397 398 if (XDNA_MBZ_DBG(xdna, cu->pad, sizeof(cu->pad))) 399 return -EINVAL; 400 401 gobj = drm_gem_object_lookup(hwctx->client->filp, cu->cu_bo); 402 if (!gobj) { 403 XDNA_ERR(xdna, "Lookup GEM object failed"); 404 return -EINVAL; 405 } 406 abo = to_xdna_obj(gobj); 407 408 if (abo->type != AMDXDNA_BO_DEV) { 409 drm_gem_object_put(gobj); 410 XDNA_ERR(xdna, "Invalid BO type"); 411 return -EINVAL; 412 } 413 414 req.cfgs[i] = FIELD_PREP(AIE2_MSG_CFG_CU_PDI_ADDR, 415 abo->mem.dev_addr >> shift); 416 req.cfgs[i] |= FIELD_PREP(AIE2_MSG_CFG_CU_FUNC, cu->cu_func); 417 XDNA_DBG(xdna, "CU %d full addr 0x%llx, cfg 0x%x", i, 418 abo->mem.dev_addr, req.cfgs[i]); 419 drm_gem_object_put(gobj); 420 } 421 req.num_cus = hwctx->cus->num_cus; 422 423 ret = xdna_send_msg_wait(xdna, chann, &msg); 424 if (ret == -ETIME) 425 aie2_destroy_context(xdna->dev_handle, hwctx); 426 427 if (resp.status == AIE2_STATUS_SUCCESS) { 428 XDNA_DBG(xdna, "Configure %d CUs, ret %d", req.num_cus, ret); 429 return 0; 430 } 431 432 XDNA_ERR(xdna, "Command opcode 0x%x failed, status 0x%x ret %d", 433 msg.opcode, resp.status, ret); 434 return ret; 435 } 436 437 int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 438 int (*notify_cb)(void *, const u32 *, size_t)) 439 { 440 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 441 struct amdxdna_dev *xdna = hwctx->client->xdna; 442 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 443 union { 444 struct execute_buffer_req ebuf; 445 struct exec_dpu_req dpu; 446 } req; 447 struct xdna_mailbox_msg msg; 448 u32 payload_len; 449 void *payload; 450 int cu_idx; 451 int ret; 452 u32 op; 453 454 if (!chann) 455 return -ENODEV; 456 457 payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len); 458 if (!payload) { 459 XDNA_ERR(xdna, "Invalid command, cannot get payload"); 460 return -EINVAL; 461 } 462 463 cu_idx = amdxdna_cmd_get_cu_idx(cmd_abo); 464 if (cu_idx < 0) { 465 XDNA_DBG(xdna, "Invalid cu idx"); 466 return -EINVAL; 467 } 468 469 op = amdxdna_cmd_get_op(cmd_abo); 470 switch (op) { 471 case ERT_START_CU: 472 if (unlikely(payload_len > sizeof(req.ebuf.payload))) 473 XDNA_DBG(xdna, "Invalid ebuf payload len: %d", payload_len); 474 req.ebuf.cu_idx = cu_idx; 475 memcpy(req.ebuf.payload, payload, sizeof(req.ebuf.payload)); 476 msg.send_size = sizeof(req.ebuf); 477 msg.opcode = MSG_OP_EXECUTE_BUFFER_CF; 478 break; 479 case ERT_START_NPU: { 480 struct amdxdna_cmd_start_npu *sn = payload; 481 482 if (unlikely(payload_len - sizeof(*sn) > sizeof(req.dpu.payload))) 483 XDNA_DBG(xdna, "Invalid dpu payload len: %d", payload_len); 484 req.dpu.inst_buf_addr = sn->buffer; 485 req.dpu.inst_size = sn->buffer_size; 486 req.dpu.inst_prop_cnt = sn->prop_count; 487 req.dpu.cu_idx = cu_idx; 488 memcpy(req.dpu.payload, sn->prop_args, sizeof(req.dpu.payload)); 489 msg.send_size = sizeof(req.dpu); 490 msg.opcode = MSG_OP_EXEC_DPU; 491 break; 492 } 493 default: 494 XDNA_DBG(xdna, "Invalid ERT cmd op code: %d", op); 495 return -EINVAL; 496 } 497 msg.handle = job; 498 msg.notify_cb = notify_cb; 499 msg.send_data = (u8 *)&req; 500 print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req, 501 0x40, false); 502 503 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 504 if (ret) { 505 XDNA_ERR(xdna, "Send message failed"); 506 return ret; 507 } 508 509 return 0; 510 } 511 512 static int 513 aie2_cmdlist_fill_one_slot_cf(void *cmd_buf, u32 offset, 514 struct amdxdna_gem_obj *abo, u32 *size) 515 { 516 struct cmd_chain_slot_execbuf_cf *buf = cmd_buf + offset; 517 int cu_idx = amdxdna_cmd_get_cu_idx(abo); 518 u32 payload_len; 519 void *payload; 520 521 if (cu_idx < 0) 522 return -EINVAL; 523 524 payload = amdxdna_cmd_get_payload(abo, &payload_len); 525 if (!payload) 526 return -EINVAL; 527 528 if (!slot_cf_has_space(offset, payload_len)) 529 return -ENOSPC; 530 531 buf->cu_idx = cu_idx; 532 buf->arg_cnt = payload_len / sizeof(u32); 533 memcpy(buf->args, payload, payload_len); 534 /* Accurate buf size to hint firmware to do necessary copy */ 535 *size = sizeof(*buf) + payload_len; 536 return 0; 537 } 538 539 static int 540 aie2_cmdlist_fill_one_slot_dpu(void *cmd_buf, u32 offset, 541 struct amdxdna_gem_obj *abo, u32 *size) 542 { 543 struct cmd_chain_slot_dpu *buf = cmd_buf + offset; 544 int cu_idx = amdxdna_cmd_get_cu_idx(abo); 545 struct amdxdna_cmd_start_npu *sn; 546 u32 payload_len; 547 void *payload; 548 u32 arg_sz; 549 550 if (cu_idx < 0) 551 return -EINVAL; 552 553 payload = amdxdna_cmd_get_payload(abo, &payload_len); 554 if (!payload) 555 return -EINVAL; 556 sn = payload; 557 arg_sz = payload_len - sizeof(*sn); 558 if (payload_len < sizeof(*sn) || arg_sz > MAX_DPU_ARGS_SIZE) 559 return -EINVAL; 560 561 if (!slot_dpu_has_space(offset, arg_sz)) 562 return -ENOSPC; 563 564 buf->inst_buf_addr = sn->buffer; 565 buf->inst_size = sn->buffer_size; 566 buf->inst_prop_cnt = sn->prop_count; 567 buf->cu_idx = cu_idx; 568 buf->arg_cnt = arg_sz / sizeof(u32); 569 memcpy(buf->args, sn->prop_args, arg_sz); 570 571 /* Accurate buf size to hint firmware to do necessary copy */ 572 *size += sizeof(*buf) + arg_sz; 573 return 0; 574 } 575 576 static int 577 aie2_cmdlist_fill_one_slot(u32 op, struct amdxdna_gem_obj *cmdbuf_abo, u32 offset, 578 struct amdxdna_gem_obj *abo, u32 *size) 579 { 580 u32 this_op = amdxdna_cmd_get_op(abo); 581 void *cmd_buf = cmdbuf_abo->mem.kva; 582 int ret; 583 584 if (this_op != op) { 585 ret = -EINVAL; 586 goto done; 587 } 588 589 switch (op) { 590 case ERT_START_CU: 591 ret = aie2_cmdlist_fill_one_slot_cf(cmd_buf, offset, abo, size); 592 break; 593 case ERT_START_NPU: 594 ret = aie2_cmdlist_fill_one_slot_dpu(cmd_buf, offset, abo, size); 595 break; 596 default: 597 ret = -EOPNOTSUPP; 598 } 599 600 done: 601 if (ret) { 602 XDNA_ERR(abo->client->xdna, "Can't fill slot for cmd op %d ret %d", 603 op, ret); 604 } 605 return ret; 606 } 607 608 static inline struct amdxdna_gem_obj * 609 aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job) 610 { 611 int idx = get_job_idx(job->seq); 612 613 return job->hwctx->priv->cmd_buf[idx]; 614 } 615 616 static void 617 aie2_cmdlist_prepare_request(struct cmd_chain_req *req, 618 struct amdxdna_gem_obj *cmdbuf_abo, u32 size, u32 cnt) 619 { 620 req->buf_addr = cmdbuf_abo->mem.dev_addr; 621 req->buf_size = size; 622 req->count = cnt; 623 drm_clflush_virt_range(cmdbuf_abo->mem.kva, size); 624 XDNA_DBG(cmdbuf_abo->client->xdna, "Command buf addr 0x%llx size 0x%x count %d", 625 req->buf_addr, size, cnt); 626 } 627 628 static inline u32 629 aie2_cmd_op_to_msg_op(u32 op) 630 { 631 switch (op) { 632 case ERT_START_CU: 633 return MSG_OP_CHAIN_EXEC_BUFFER_CF; 634 case ERT_START_NPU: 635 return MSG_OP_CHAIN_EXEC_DPU; 636 default: 637 return MSG_OP_MAX_OPCODE; 638 } 639 } 640 641 int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, 642 struct amdxdna_sched_job *job, 643 int (*notify_cb)(void *, const u32 *, size_t)) 644 { 645 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 646 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 647 struct amdxdna_client *client = hwctx->client; 648 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 649 struct amdxdna_cmd_chain *payload; 650 struct xdna_mailbox_msg msg; 651 struct cmd_chain_req req; 652 u32 payload_len; 653 u32 offset = 0; 654 u32 size; 655 int ret; 656 u32 op; 657 u32 i; 658 659 op = amdxdna_cmd_get_op(cmd_abo); 660 payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len); 661 if (op != ERT_CMD_CHAIN || !payload || 662 payload_len < struct_size(payload, data, payload->command_count)) 663 return -EINVAL; 664 665 for (i = 0; i < payload->command_count; i++) { 666 u32 boh = (u32)(payload->data[i]); 667 struct amdxdna_gem_obj *abo; 668 669 abo = amdxdna_gem_get_obj(client, boh, AMDXDNA_BO_CMD); 670 if (!abo) { 671 XDNA_ERR(client->xdna, "Failed to find cmd BO %d", boh); 672 return -ENOENT; 673 } 674 675 /* All sub-cmd should have same op, use the first one. */ 676 if (i == 0) 677 op = amdxdna_cmd_get_op(abo); 678 679 ret = aie2_cmdlist_fill_one_slot(op, cmdbuf_abo, offset, abo, &size); 680 amdxdna_gem_put_obj(abo); 681 if (ret) 682 return -EINVAL; 683 684 offset += size; 685 } 686 687 /* The offset is the accumulated total size of the cmd buffer */ 688 aie2_cmdlist_prepare_request(&req, cmdbuf_abo, offset, payload->command_count); 689 690 msg.opcode = aie2_cmd_op_to_msg_op(op); 691 if (msg.opcode == MSG_OP_MAX_OPCODE) 692 return -EOPNOTSUPP; 693 msg.handle = job; 694 msg.notify_cb = notify_cb; 695 msg.send_data = (u8 *)&req; 696 msg.send_size = sizeof(req); 697 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 698 if (ret) { 699 XDNA_ERR(hwctx->client->xdna, "Send message failed"); 700 return ret; 701 } 702 703 return 0; 704 } 705 706 int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx, 707 struct amdxdna_sched_job *job, 708 int (*notify_cb)(void *, const u32 *, size_t)) 709 { 710 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 711 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 712 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 713 struct xdna_mailbox_msg msg; 714 struct cmd_chain_req req; 715 u32 size; 716 int ret; 717 u32 op; 718 719 op = amdxdna_cmd_get_op(cmd_abo); 720 ret = aie2_cmdlist_fill_one_slot(op, cmdbuf_abo, 0, cmd_abo, &size); 721 if (ret) 722 return ret; 723 724 aie2_cmdlist_prepare_request(&req, cmdbuf_abo, size, 1); 725 726 msg.opcode = aie2_cmd_op_to_msg_op(op); 727 if (msg.opcode == MSG_OP_MAX_OPCODE) 728 return -EOPNOTSUPP; 729 msg.handle = job; 730 msg.notify_cb = notify_cb; 731 msg.send_data = (u8 *)&req; 732 msg.send_size = sizeof(req); 733 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 734 if (ret) { 735 XDNA_ERR(hwctx->client->xdna, "Send message failed"); 736 return ret; 737 } 738 739 return 0; 740 } 741 742 int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 743 int (*notify_cb)(void *, const u32 *, size_t)) 744 { 745 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 746 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); 747 struct amdxdna_dev *xdna = hwctx->client->xdna; 748 struct xdna_mailbox_msg msg; 749 struct sync_bo_req req; 750 int ret = 0; 751 752 req.src_addr = 0; 753 req.dst_addr = abo->mem.dev_addr - hwctx->client->dev_heap->mem.dev_addr; 754 req.size = abo->mem.size; 755 756 /* Device to Host */ 757 req.type = FIELD_PREP(AIE2_MSG_SYNC_BO_SRC_TYPE, SYNC_BO_DEV_MEM) | 758 FIELD_PREP(AIE2_MSG_SYNC_BO_DST_TYPE, SYNC_BO_HOST_MEM); 759 760 XDNA_DBG(xdna, "sync %d bytes src(0x%llx) to dst(0x%llx) completed", 761 req.size, req.src_addr, req.dst_addr); 762 763 msg.handle = job; 764 msg.notify_cb = notify_cb; 765 msg.send_data = (u8 *)&req; 766 msg.send_size = sizeof(req); 767 msg.opcode = MSG_OP_SYNC_BO; 768 769 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 770 if (ret) { 771 XDNA_ERR(xdna, "Send message failed"); 772 return ret; 773 } 774 775 return 0; 776 } 777