1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_cache.h> 8 #include <drm/drm_device.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/drm_print.h> 12 #include <drm/gpu_scheduler.h> 13 #include <linux/bitfield.h> 14 #include <linux/errno.h> 15 #include <linux/pci.h> 16 #include <linux/types.h> 17 #include <linux/xarray.h> 18 19 #include "aie2_msg_priv.h" 20 #include "aie2_pci.h" 21 #include "amdxdna_ctx.h" 22 #include "amdxdna_gem.h" 23 #include "amdxdna_mailbox.h" 24 #include "amdxdna_mailbox_helper.h" 25 #include "amdxdna_pci_drv.h" 26 27 #define DECLARE_AIE2_MSG(name, op) \ 28 DECLARE_XDNA_MSG_COMMON(name, op, MAX_AIE2_STATUS_CODE) 29 30 #define EXEC_MSG_OPS(xdna) ((xdna)->dev_handle->exec_msg_ops) 31 32 static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev, 33 struct xdna_mailbox_msg *msg) 34 { 35 struct amdxdna_dev *xdna = ndev->xdna; 36 struct xdna_notify *hdl = msg->handle; 37 int ret; 38 39 if (!ndev->mgmt_chann) 40 return -ENODEV; 41 42 ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg); 43 if (ret == -ETIME) 44 aie2_destroy_mgmt_chann(ndev); 45 46 if (!ret && *hdl->status != AIE2_STATUS_SUCCESS) { 47 XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x", 48 msg->opcode, *hdl->data); 49 ret = -EINVAL; 50 } 51 52 return ret; 53 } 54 55 void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size, 56 dma_addr_t *dma_addr) 57 { 58 struct amdxdna_dev *xdna = ndev->xdna; 59 int order; 60 61 *size = max(*size, SZ_8K); 62 order = get_order(*size); 63 if (order > MAX_PAGE_ORDER) 64 return NULL; 65 *size = PAGE_SIZE << order; 66 67 return dma_alloc_noncoherent(xdna->ddev.dev, *size, dma_addr, 68 DMA_FROM_DEVICE, GFP_KERNEL); 69 } 70 71 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev) 72 { 73 DECLARE_AIE2_MSG(suspend, MSG_OP_SUSPEND); 74 int ret; 75 76 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 77 if (ret) { 78 XDNA_ERR(ndev->xdna, "Failed to suspend fw, ret %d", ret); 79 return ret; 80 } 81 82 return aie2_psp_waitmode_poll(ndev->psp_hdl); 83 } 84 85 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev) 86 { 87 DECLARE_AIE2_MSG(suspend, MSG_OP_RESUME); 88 89 return aie2_send_mgmt_msg_wait(ndev, &msg); 90 } 91 92 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value) 93 { 94 DECLARE_AIE2_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG); 95 int ret; 96 97 req.type = type; 98 req.value = value; 99 100 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 101 if (ret) { 102 XDNA_ERR(ndev->xdna, "Failed to set runtime config, ret %d", ret); 103 return ret; 104 } 105 106 return 0; 107 } 108 109 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value) 110 { 111 DECLARE_AIE2_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG); 112 int ret; 113 114 req.type = type; 115 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 116 if (ret) { 117 XDNA_ERR(ndev->xdna, "Failed to get runtime config, ret %d", ret); 118 return ret; 119 } 120 121 *value = resp.value; 122 return 0; 123 } 124 125 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid) 126 { 127 DECLARE_AIE2_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID); 128 129 req.pasid = pasid; 130 131 return aie2_send_mgmt_msg_wait(ndev, &msg); 132 } 133 134 int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version) 135 { 136 DECLARE_AIE2_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION); 137 struct amdxdna_dev *xdna = ndev->xdna; 138 int ret; 139 140 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 141 if (ret) 142 return ret; 143 144 XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed", 145 resp.major, resp.minor); 146 147 version->major = resp.major; 148 version->minor = resp.minor; 149 150 return 0; 151 } 152 153 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata) 154 { 155 DECLARE_AIE2_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO); 156 int ret; 157 158 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 159 if (ret) 160 return ret; 161 162 metadata->size = resp.info.size; 163 metadata->cols = resp.info.cols; 164 metadata->rows = resp.info.rows; 165 166 metadata->version.major = resp.info.major; 167 metadata->version.minor = resp.info.minor; 168 169 metadata->core.row_count = resp.info.core_rows; 170 metadata->core.row_start = resp.info.core_row_start; 171 metadata->core.dma_channel_count = resp.info.core_dma_channels; 172 metadata->core.lock_count = resp.info.core_locks; 173 metadata->core.event_reg_count = resp.info.core_events; 174 175 metadata->mem.row_count = resp.info.mem_rows; 176 metadata->mem.row_start = resp.info.mem_row_start; 177 metadata->mem.dma_channel_count = resp.info.mem_dma_channels; 178 metadata->mem.lock_count = resp.info.mem_locks; 179 metadata->mem.event_reg_count = resp.info.mem_events; 180 181 metadata->shim.row_count = resp.info.shim_rows; 182 metadata->shim.row_start = resp.info.shim_row_start; 183 metadata->shim.dma_channel_count = resp.info.shim_dma_channels; 184 metadata->shim.lock_count = resp.info.shim_locks; 185 metadata->shim.event_reg_count = resp.info.shim_events; 186 187 return 0; 188 } 189 190 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, 191 struct amdxdna_fw_ver *fw_ver) 192 { 193 DECLARE_AIE2_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION); 194 int ret; 195 196 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 197 if (ret) 198 return ret; 199 200 fw_ver->major = resp.major; 201 fw_ver->minor = resp.minor; 202 fw_ver->sub = resp.sub; 203 fw_ver->build = resp.build; 204 205 return 0; 206 } 207 208 static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id) 209 { 210 DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT); 211 struct amdxdna_dev *xdna = ndev->xdna; 212 int ret; 213 214 req.context_id = id; 215 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 216 if (ret && ret != -ENODEV) 217 XDNA_WARN(xdna, "Destroy context failed, ret %d", ret); 218 else if (ret == -ENODEV) 219 XDNA_DBG(xdna, "Destroy context: device already stopped"); 220 221 return ret; 222 } 223 224 static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev, 225 struct amdxdna_hwctx *hwctx) 226 { 227 if (!AIE2_FEATURE_ON(ndev, AIE2_PREEMPT)) 228 return PRIORITY_HIGH; 229 230 switch (hwctx->qos.priority) { 231 case AMDXDNA_QOS_REALTIME_PRIORITY: 232 return PRIORITY_REALTIME; 233 case AMDXDNA_QOS_HIGH_PRIORITY: 234 return PRIORITY_HIGH; 235 case AMDXDNA_QOS_NORMAL_PRIORITY: 236 return PRIORITY_NORMAL; 237 case AMDXDNA_QOS_LOW_PRIORITY: 238 return PRIORITY_LOW; 239 default: 240 return PRIORITY_HIGH; 241 } 242 } 243 244 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 245 { 246 DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT); 247 struct amdxdna_dev *xdna = ndev->xdna; 248 struct xdna_mailbox_chann_res x2i; 249 struct xdna_mailbox_chann_res i2x; 250 struct cq_pair *cq_pair; 251 u32 intr_reg; 252 int ret; 253 254 req.aie_type = 1; 255 req.start_col = hwctx->start_col; 256 req.num_col = hwctx->num_col; 257 req.num_unused_col = hwctx->num_unused_col; 258 req.num_cq_pairs_requested = 1; 259 req.pasid = hwctx->client->pasid; 260 req.context_priority = aie2_get_context_priority(ndev, hwctx); 261 262 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 263 if (ret) 264 return ret; 265 266 hwctx->fw_ctx_id = resp.context_id; 267 if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1)) 268 return -EINVAL; 269 270 if (ndev->force_preempt_enabled) { 271 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id); 272 if (ret) { 273 XDNA_ERR(xdna, "failed to enable force preempt %d", ret); 274 goto del_ctx_req; 275 } 276 } 277 278 cq_pair = &resp.cq_pair[0]; 279 x2i.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.head_addr); 280 x2i.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.tail_addr); 281 x2i.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->x2i_q.buf_addr); 282 x2i.rb_size = cq_pair->x2i_q.buf_size; 283 284 i2x.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.head_addr); 285 i2x.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.tail_addr); 286 i2x.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->i2x_q.buf_addr); 287 i2x.rb_size = cq_pair->i2x_q.buf_size; 288 289 ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id); 290 if (ret == -EINVAL) { 291 XDNA_ERR(xdna, "Alloc IRQ failed %d", ret); 292 goto del_ctx_req; 293 } 294 295 intr_reg = i2x.mb_head_ptr_reg + 4; 296 hwctx->priv->mbox_chann = xdna_mailbox_alloc_channel(ndev->mbox); 297 if (!hwctx->priv->mbox_chann) { 298 XDNA_ERR(xdna, "Not able to create channel"); 299 ret = -EINVAL; 300 goto del_ctx_req; 301 } 302 303 ret = xdna_mailbox_start_channel(hwctx->priv->mbox_chann, &x2i, &i2x, 304 intr_reg, ret); 305 if (ret) { 306 XDNA_ERR(xdna, "Not able to create channel"); 307 ret = -EINVAL; 308 goto free_channel; 309 } 310 ndev->hwctx_num++; 311 312 XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id); 313 XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid); 314 315 return 0; 316 317 free_channel: 318 xdna_mailbox_free_channel(hwctx->priv->mbox_chann); 319 del_ctx_req: 320 aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); 321 return ret; 322 } 323 324 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx) 325 { 326 struct amdxdna_dev *xdna = ndev->xdna; 327 int ret; 328 329 if (!hwctx->priv->mbox_chann) 330 return 0; 331 332 xdna_mailbox_stop_channel(hwctx->priv->mbox_chann); 333 ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id); 334 xdna_mailbox_free_channel(hwctx->priv->mbox_chann); 335 XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id); 336 hwctx->priv->mbox_chann = NULL; 337 hwctx->fw_ctx_id = -1; 338 ndev->hwctx_num--; 339 340 return ret; 341 } 342 343 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size) 344 { 345 DECLARE_AIE2_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER); 346 struct amdxdna_dev *xdna = ndev->xdna; 347 int ret; 348 349 req.context_id = context_id; 350 req.buf_addr = addr; 351 req.buf_size = size; 352 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 353 if (ret) 354 return ret; 355 356 XDNA_DBG(xdna, "fw ctx %d map host buf addr 0x%llx size 0x%llx", 357 context_id, addr, size); 358 359 return 0; 360 } 361 362 static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg) 363 { 364 u32 *bitmap = arg; 365 366 *bitmap |= GENMASK(hwctx->start_col + hwctx->num_col - 1, hwctx->start_col); 367 368 return 0; 369 } 370 371 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, 372 u32 size, u32 *cols_filled) 373 { 374 DECLARE_AIE2_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS); 375 struct amdxdna_dev *xdna = ndev->xdna; 376 u32 buf_sz = size, aie_bitmap = 0; 377 struct amdxdna_client *client; 378 dma_addr_t dma_addr; 379 u8 *buff_addr; 380 int ret; 381 382 buff_addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr); 383 if (!buff_addr) 384 return -ENOMEM; 385 386 /* Go through each hardware context and mark the AIE columns that are active */ 387 list_for_each_entry(client, &xdna->client_list, node) 388 amdxdna_hwctx_walk(client, &aie_bitmap, amdxdna_hwctx_col_map); 389 390 *cols_filled = 0; 391 req.dump_buff_addr = dma_addr; 392 req.dump_buff_size = buf_sz; 393 req.num_cols = hweight32(aie_bitmap); 394 req.aie_bitmap = aie_bitmap; 395 396 drm_clflush_virt_range(buff_addr, size); /* device can access */ 397 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 398 if (ret) { 399 XDNA_ERR(xdna, "Error during NPU query, status %d", ret); 400 goto fail; 401 } 402 403 XDNA_DBG(xdna, "Query NPU status completed"); 404 405 if (size < resp.size) { 406 ret = -EINVAL; 407 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size); 408 goto fail; 409 } 410 411 if (copy_to_user(buf, buff_addr, resp.size)) { 412 ret = -EFAULT; 413 XDNA_ERR(xdna, "Failed to copy NPU status to user space"); 414 goto fail; 415 } 416 417 *cols_filled = aie_bitmap; 418 419 fail: 420 aie2_free_msg_buffer(ndev, buf_sz, buff_addr, dma_addr); 421 return ret; 422 } 423 424 int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, 425 char __user *buf, u32 size, 426 struct amdxdna_drm_query_telemetry_header *header) 427 { 428 DECLARE_AIE2_MSG(get_telemetry, MSG_OP_GET_TELEMETRY); 429 struct amdxdna_dev *xdna = ndev->xdna; 430 dma_addr_t dma_addr; 431 u32 buf_sz = size; 432 u8 *addr; 433 int ret; 434 435 if (header->type >= MAX_TELEMETRY_TYPE) 436 return -EINVAL; 437 438 addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr); 439 if (!addr) 440 return -ENOMEM; 441 442 req.buf_addr = dma_addr; 443 req.buf_size = buf_sz; 444 req.type = header->type; 445 446 drm_clflush_virt_range(addr, size); /* device can access */ 447 ret = aie2_send_mgmt_msg_wait(ndev, &msg); 448 if (ret) { 449 XDNA_ERR(xdna, "Query telemetry failed, status %d", ret); 450 goto free_buf; 451 } 452 453 if (size < resp.size) { 454 ret = -EINVAL; 455 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size); 456 goto free_buf; 457 } 458 459 if (copy_to_user(buf, addr, resp.size)) { 460 ret = -EFAULT; 461 XDNA_ERR(xdna, "Failed to copy telemetry to user space"); 462 goto free_buf; 463 } 464 465 header->major = resp.major; 466 header->minor = resp.minor; 467 468 free_buf: 469 aie2_free_msg_buffer(ndev, buf_sz, addr, dma_addr); 470 return ret; 471 } 472 473 int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size, 474 void *handle, int (*cb)(void*, void __iomem *, size_t)) 475 { 476 struct async_event_msg_req req = { 0 }; 477 struct xdna_mailbox_msg msg = { 478 .send_data = (u8 *)&req, 479 .send_size = sizeof(req), 480 .handle = handle, 481 .opcode = MSG_OP_REGISTER_ASYNC_EVENT_MSG, 482 .notify_cb = cb, 483 }; 484 485 req.buf_addr = addr; 486 req.buf_size = size; 487 488 XDNA_DBG(ndev->xdna, "Register addr 0x%llx size 0x%x", addr, size); 489 return xdna_mailbox_send_msg(ndev->mgmt_chann, &msg, TX_TIMEOUT); 490 } 491 492 int aie2_config_cu(struct amdxdna_hwctx *hwctx, 493 int (*notify_cb)(void *, void __iomem *, size_t)) 494 { 495 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 496 struct amdxdna_dev *xdna = hwctx->client->xdna; 497 u32 shift = xdna->dev_info->dev_mem_buf_shift; 498 struct config_cu_req req = { 0 }; 499 struct xdna_mailbox_msg msg; 500 struct drm_gem_object *gobj; 501 struct amdxdna_gem_obj *abo; 502 int i; 503 504 if (!chann) 505 return -ENODEV; 506 507 if (!hwctx->cus) 508 return 0; 509 510 if (hwctx->cus->num_cus > MAX_NUM_CUS) { 511 XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS); 512 return -EINVAL; 513 } 514 515 for (i = 0; i < hwctx->cus->num_cus; i++) { 516 struct amdxdna_cu_config *cu = &hwctx->cus->cu_configs[i]; 517 518 if (XDNA_MBZ_DBG(xdna, cu->pad, sizeof(cu->pad))) 519 return -EINVAL; 520 521 gobj = drm_gem_object_lookup(hwctx->client->filp, cu->cu_bo); 522 if (!gobj) { 523 XDNA_ERR(xdna, "Lookup GEM object failed"); 524 return -EINVAL; 525 } 526 abo = to_xdna_obj(gobj); 527 528 if (abo->type != AMDXDNA_BO_DEV) { 529 drm_gem_object_put(gobj); 530 XDNA_ERR(xdna, "Invalid BO type"); 531 return -EINVAL; 532 } 533 534 req.cfgs[i] = FIELD_PREP(AIE2_MSG_CFG_CU_PDI_ADDR, 535 abo->mem.dev_addr >> shift); 536 req.cfgs[i] |= FIELD_PREP(AIE2_MSG_CFG_CU_FUNC, cu->cu_func); 537 XDNA_DBG(xdna, "CU %d full addr 0x%llx, cfg 0x%x", i, 538 abo->mem.dev_addr, req.cfgs[i]); 539 drm_gem_object_put(gobj); 540 } 541 req.num_cus = hwctx->cus->num_cus; 542 543 msg.send_data = (u8 *)&req; 544 msg.send_size = sizeof(req); 545 msg.handle = hwctx; 546 msg.opcode = MSG_OP_CONFIG_CU; 547 msg.notify_cb = notify_cb; 548 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 549 } 550 551 static int aie2_init_exec_cu_req(struct amdxdna_gem_obj *cmd_bo, void *req, 552 size_t *size, u32 *msg_op) 553 { 554 struct execute_buffer_req *cu_req = req; 555 u32 cmd_len; 556 void *cmd; 557 558 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 559 if (cmd_len > sizeof(cu_req->payload)) 560 return -EINVAL; 561 562 cu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 563 if (cu_req->cu_idx == INVALID_CU_IDX) 564 return -EINVAL; 565 566 memcpy(cu_req->payload, cmd, cmd_len); 567 568 *size = sizeof(*cu_req); 569 *msg_op = MSG_OP_EXECUTE_BUFFER_CF; 570 return 0; 571 } 572 573 static int aie2_init_exec_dpu_req(struct amdxdna_gem_obj *cmd_bo, void *req, 574 size_t *size, u32 *msg_op) 575 { 576 struct exec_dpu_req *dpu_req = req; 577 struct amdxdna_cmd_start_npu *sn; 578 u32 cmd_len; 579 580 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 581 if (cmd_len - sizeof(*sn) > sizeof(dpu_req->payload)) 582 return -EINVAL; 583 584 dpu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 585 if (dpu_req->cu_idx == INVALID_CU_IDX) 586 return -EINVAL; 587 588 dpu_req->inst_buf_addr = sn->buffer; 589 dpu_req->inst_size = sn->buffer_size; 590 dpu_req->inst_prop_cnt = sn->prop_count; 591 memcpy(dpu_req->payload, sn->prop_args, cmd_len - sizeof(*sn)); 592 593 *size = sizeof(*dpu_req); 594 *msg_op = MSG_OP_EXEC_DPU; 595 return 0; 596 } 597 598 static void aie2_init_exec_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) 599 { 600 struct cmd_chain_req *chain_req = req; 601 602 chain_req->buf_addr = slot_addr; 603 chain_req->buf_size = size; 604 chain_req->count = cmd_cnt; 605 } 606 607 static void aie2_init_npu_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt) 608 { 609 struct cmd_chain_npu_req *npu_chain_req = req; 610 611 npu_chain_req->flags = 0; 612 npu_chain_req->reserved = 0; 613 npu_chain_req->buf_addr = slot_addr; 614 npu_chain_req->buf_size = size; 615 npu_chain_req->count = cmd_cnt; 616 } 617 618 static int 619 aie2_cmdlist_fill_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 620 { 621 struct cmd_chain_slot_execbuf_cf *cf_slot = slot; 622 u32 cmd_len; 623 void *cmd; 624 625 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 626 if (*size < sizeof(*cf_slot) + cmd_len) 627 return -EINVAL; 628 629 cf_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 630 if (cf_slot->cu_idx == INVALID_CU_IDX) 631 return -EINVAL; 632 633 cf_slot->arg_cnt = cmd_len / sizeof(u32); 634 memcpy(cf_slot->args, cmd, cmd_len); 635 /* Accurate slot size to hint firmware to do necessary copy */ 636 *size = sizeof(*cf_slot) + cmd_len; 637 return 0; 638 } 639 640 static int 641 aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 642 { 643 struct cmd_chain_slot_dpu *dpu_slot = slot; 644 struct amdxdna_cmd_start_npu *sn; 645 u32 cmd_len; 646 u32 arg_sz; 647 648 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 649 arg_sz = cmd_len - sizeof(*sn); 650 if (cmd_len < sizeof(*sn) || arg_sz > MAX_DPU_ARGS_SIZE) 651 return -EINVAL; 652 653 if (*size < sizeof(*dpu_slot) + arg_sz) 654 return -EINVAL; 655 656 dpu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 657 if (dpu_slot->cu_idx == INVALID_CU_IDX) 658 return -EINVAL; 659 660 dpu_slot->inst_buf_addr = sn->buffer; 661 dpu_slot->inst_size = sn->buffer_size; 662 dpu_slot->inst_prop_cnt = sn->prop_count; 663 dpu_slot->arg_cnt = arg_sz / sizeof(u32); 664 memcpy(dpu_slot->args, sn->prop_args, arg_sz); 665 666 /* Accurate slot size to hint firmware to do necessary copy */ 667 *size = sizeof(*dpu_slot) + arg_sz; 668 return 0; 669 } 670 671 static int aie2_cmdlist_unsupp(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 672 { 673 return -EOPNOTSUPP; 674 } 675 676 static u32 aie2_get_chain_msg_op(u32 cmd_op) 677 { 678 switch (cmd_op) { 679 case ERT_START_CU: 680 return MSG_OP_CHAIN_EXEC_BUFFER_CF; 681 case ERT_START_NPU: 682 return MSG_OP_CHAIN_EXEC_DPU; 683 default: 684 break; 685 } 686 687 return MSG_OP_MAX_OPCODE; 688 } 689 690 static struct aie2_exec_msg_ops legacy_exec_message_ops = { 691 .init_cu_req = aie2_init_exec_cu_req, 692 .init_dpu_req = aie2_init_exec_dpu_req, 693 .init_chain_req = aie2_init_exec_chain_req, 694 .fill_cf_slot = aie2_cmdlist_fill_cf, 695 .fill_dpu_slot = aie2_cmdlist_fill_dpu, 696 .fill_preempt_slot = aie2_cmdlist_unsupp, 697 .fill_elf_slot = aie2_cmdlist_unsupp, 698 .get_chain_msg_op = aie2_get_chain_msg_op, 699 }; 700 701 static int 702 aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 703 { 704 struct cmd_chain_slot_npu *npu_slot = slot; 705 u32 cmd_len; 706 void *cmd; 707 708 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 709 if (*size < sizeof(*npu_slot) + cmd_len) 710 return -EINVAL; 711 712 memset(npu_slot, 0, sizeof(*npu_slot)); 713 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 714 if (npu_slot->cu_idx == INVALID_CU_IDX) 715 return -EINVAL; 716 717 npu_slot->type = EXEC_NPU_TYPE_NON_ELF; 718 npu_slot->arg_cnt = cmd_len / sizeof(u32); 719 memcpy(npu_slot->args, cmd, cmd_len); 720 721 *size = sizeof(*npu_slot) + cmd_len; 722 return 0; 723 } 724 725 static int 726 aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 727 { 728 struct cmd_chain_slot_npu *npu_slot = slot; 729 struct amdxdna_cmd_start_npu *sn; 730 u32 cmd_len; 731 u32 arg_sz; 732 733 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 734 arg_sz = cmd_len - sizeof(*sn); 735 if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE) 736 return -EINVAL; 737 738 if (*size < sizeof(*npu_slot) + arg_sz) 739 return -EINVAL; 740 741 memset(npu_slot, 0, sizeof(*npu_slot)); 742 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 743 if (npu_slot->cu_idx == INVALID_CU_IDX) 744 return -EINVAL; 745 746 npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF; 747 npu_slot->inst_buf_addr = sn->buffer; 748 npu_slot->inst_size = sn->buffer_size; 749 npu_slot->inst_prop_cnt = sn->prop_count; 750 npu_slot->arg_cnt = arg_sz / sizeof(u32); 751 memcpy(npu_slot->args, sn->prop_args, arg_sz); 752 753 *size = sizeof(*npu_slot) + arg_sz; 754 return 0; 755 } 756 757 static int 758 aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 759 { 760 struct cmd_chain_slot_npu *npu_slot = slot; 761 struct amdxdna_cmd_preempt_data *pd; 762 u32 cmd_len; 763 u32 arg_sz; 764 765 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 766 arg_sz = cmd_len - sizeof(*pd); 767 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) 768 return -EINVAL; 769 770 if (*size < sizeof(*npu_slot) + arg_sz) 771 return -EINVAL; 772 773 memset(npu_slot, 0, sizeof(*npu_slot)); 774 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo); 775 if (npu_slot->cu_idx == INVALID_CU_IDX) 776 return -EINVAL; 777 778 npu_slot->type = EXEC_NPU_TYPE_PREEMPT; 779 npu_slot->inst_buf_addr = pd->inst_buf; 780 npu_slot->save_buf_addr = pd->save_buf; 781 npu_slot->restore_buf_addr = pd->restore_buf; 782 npu_slot->inst_size = pd->inst_size; 783 npu_slot->save_size = pd->save_size; 784 npu_slot->restore_size = pd->restore_size; 785 npu_slot->inst_prop_cnt = pd->inst_prop_cnt; 786 npu_slot->arg_cnt = arg_sz / sizeof(u32); 787 memcpy(npu_slot->args, pd->prop_args, arg_sz); 788 789 *size = sizeof(*npu_slot) + arg_sz; 790 return 0; 791 } 792 793 static int 794 aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size) 795 { 796 struct cmd_chain_slot_npu *npu_slot = slot; 797 struct amdxdna_cmd_preempt_data *pd; 798 u32 cmd_len; 799 u32 arg_sz; 800 801 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len); 802 arg_sz = cmd_len - sizeof(*pd); 803 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE) 804 return -EINVAL; 805 806 if (*size < sizeof(*npu_slot) + arg_sz) 807 return -EINVAL; 808 809 memset(npu_slot, 0, sizeof(*npu_slot)); 810 npu_slot->type = EXEC_NPU_TYPE_ELF; 811 npu_slot->inst_buf_addr = pd->inst_buf; 812 npu_slot->save_buf_addr = pd->save_buf; 813 npu_slot->restore_buf_addr = pd->restore_buf; 814 npu_slot->inst_size = pd->inst_size; 815 npu_slot->save_size = pd->save_size; 816 npu_slot->restore_size = pd->restore_size; 817 npu_slot->inst_prop_cnt = pd->inst_prop_cnt; 818 npu_slot->arg_cnt = 1; 819 npu_slot->args[0] = AIE2_EXEC_BUFFER_KERNEL_OP_TXN; 820 821 *size = struct_size(npu_slot, args, npu_slot->arg_cnt); 822 return 0; 823 } 824 825 static u32 aie2_get_npu_chain_msg_op(u32 cmd_op) 826 { 827 return MSG_OP_CHAIN_EXEC_NPU; 828 } 829 830 static struct aie2_exec_msg_ops npu_exec_message_ops = { 831 .init_cu_req = aie2_init_exec_cu_req, 832 .init_dpu_req = aie2_init_exec_dpu_req, 833 .init_chain_req = aie2_init_npu_chain_req, 834 .fill_cf_slot = aie2_cmdlist_fill_npu_cf, 835 .fill_dpu_slot = aie2_cmdlist_fill_npu_dpu, 836 .fill_preempt_slot = aie2_cmdlist_fill_npu_preempt, 837 .fill_elf_slot = aie2_cmdlist_fill_npu_elf, 838 .get_chain_msg_op = aie2_get_npu_chain_msg_op, 839 }; 840 841 static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo, 842 size_t *size, u32 *msg_op) 843 { 844 struct amdxdna_dev *xdna = cmd_abo->client->xdna; 845 int ret; 846 u32 op; 847 848 849 op = amdxdna_cmd_get_op(cmd_abo); 850 switch (op) { 851 case ERT_START_CU: 852 ret = EXEC_MSG_OPS(xdna)->init_cu_req(cmd_abo, req, size, msg_op); 853 if (ret) { 854 XDNA_DBG(xdna, "Init CU req failed ret %d", ret); 855 return ret; 856 } 857 break; 858 case ERT_START_NPU: 859 ret = EXEC_MSG_OPS(xdna)->init_dpu_req(cmd_abo, req, size, msg_op); 860 if (ret) { 861 XDNA_DBG(xdna, "Init DPU req failed ret %d", ret); 862 return ret; 863 } 864 865 break; 866 default: 867 XDNA_ERR(xdna, "Unsupported op %d", op); 868 ret = -EOPNOTSUPP; 869 break; 870 } 871 872 return ret; 873 } 874 875 static int 876 aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo, 877 size_t *size, u32 *cmd_op) 878 { 879 struct amdxdna_dev *xdna = cmd_abo->client->xdna; 880 int ret; 881 u32 op; 882 883 op = amdxdna_cmd_get_op(cmd_abo); 884 if (*cmd_op == ERT_INVALID_CMD) 885 *cmd_op = op; 886 else if (op != *cmd_op) 887 return -EINVAL; 888 889 switch (op) { 890 case ERT_START_CU: 891 ret = EXEC_MSG_OPS(xdna)->fill_cf_slot(cmd_abo, slot, size); 892 break; 893 case ERT_START_NPU: 894 ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size); 895 break; 896 case ERT_START_NPU_PREEMPT: 897 if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT)) 898 return -EOPNOTSUPP; 899 ret = EXEC_MSG_OPS(xdna)->fill_preempt_slot(cmd_abo, slot, size); 900 break; 901 case ERT_START_NPU_PREEMPT_ELF: 902 if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT)) 903 return -EOPNOTSUPP; 904 ret = EXEC_MSG_OPS(xdna)->fill_elf_slot(cmd_abo, slot, size); 905 break; 906 default: 907 XDNA_INFO(xdna, "Unsupported op %d", op); 908 ret = -EOPNOTSUPP; 909 break; 910 } 911 912 return ret; 913 } 914 915 void aie2_msg_init(struct amdxdna_dev_hdl *ndev) 916 { 917 if (AIE2_FEATURE_ON(ndev, AIE2_NPU_COMMAND)) 918 ndev->exec_msg_ops = &npu_exec_message_ops; 919 else 920 ndev->exec_msg_ops = &legacy_exec_message_ops; 921 } 922 923 void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev) 924 { 925 struct amdxdna_dev *xdna = ndev->xdna; 926 927 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 928 929 if (!ndev->mgmt_chann) 930 return; 931 932 xdna_mailbox_stop_channel(ndev->mgmt_chann); 933 xdna_mailbox_free_channel(ndev->mgmt_chann); 934 ndev->mgmt_chann = NULL; 935 } 936 937 static inline struct amdxdna_gem_obj * 938 aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job) 939 { 940 int idx = get_job_idx(job->seq); 941 942 return job->hwctx->priv->cmd_buf[idx]; 943 } 944 945 int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 946 int (*notify_cb)(void *, void __iomem *, size_t)) 947 { 948 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 949 struct amdxdna_dev *xdna = hwctx->client->xdna; 950 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 951 struct xdna_mailbox_msg msg; 952 union exec_req req; 953 int ret; 954 955 if (!chann) 956 return -ENODEV; 957 958 ret = aie2_init_exec_req(&req, cmd_abo, &msg.send_size, &msg.opcode); 959 if (ret) 960 return ret; 961 962 msg.handle = job; 963 msg.notify_cb = notify_cb; 964 msg.send_data = (u8 *)&req; 965 print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req, 966 0x40, false); 967 968 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 969 if (ret) { 970 XDNA_ERR(xdna, "Send message failed"); 971 return ret; 972 } 973 974 return 0; 975 } 976 977 int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, 978 struct amdxdna_sched_job *job, 979 int (*notify_cb)(void *, void __iomem *, size_t)) 980 { 981 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 982 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 983 struct amdxdna_client *client = hwctx->client; 984 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 985 struct amdxdna_dev *xdna = client->xdna; 986 struct amdxdna_cmd_chain *payload; 987 struct xdna_mailbox_msg msg; 988 union exec_chain_req req; 989 u32 payload_len; 990 u32 offset = 0; 991 size_t size; 992 int ret; 993 u32 op; 994 u32 i; 995 996 op = amdxdna_cmd_get_op(cmd_abo); 997 payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len); 998 if (op != ERT_CMD_CHAIN || !payload || 999 payload_len < struct_size(payload, data, payload->command_count)) 1000 return -EINVAL; 1001 1002 op = ERT_INVALID_CMD; 1003 for (i = 0; i < payload->command_count; i++) { 1004 u32 boh = (u32)(payload->data[i]); 1005 struct amdxdna_gem_obj *abo; 1006 1007 abo = amdxdna_gem_get_obj(client, boh, AMDXDNA_BO_CMD); 1008 if (!abo) { 1009 XDNA_ERR(xdna, "Failed to find cmd BO %d", boh); 1010 return -ENOENT; 1011 } 1012 1013 size = cmdbuf_abo->mem.size - offset; 1014 ret = aie2_cmdlist_fill_slot(cmdbuf_abo->mem.kva + offset, 1015 abo, &size, &op); 1016 amdxdna_gem_put_obj(abo); 1017 if (ret) 1018 return ret; 1019 1020 offset += size; 1021 } 1022 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); 1023 if (msg.opcode == MSG_OP_MAX_OPCODE) 1024 return -EOPNOTSUPP; 1025 1026 /* The offset is the accumulated total size of the cmd buffer */ 1027 EXEC_MSG_OPS(xdna)->init_chain_req(&req, cmdbuf_abo->mem.dev_addr, 1028 offset, payload->command_count); 1029 drm_clflush_virt_range(cmdbuf_abo->mem.kva, offset); 1030 1031 msg.handle = job; 1032 msg.notify_cb = notify_cb; 1033 msg.send_data = (u8 *)&req; 1034 msg.send_size = sizeof(req); 1035 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1036 if (ret) { 1037 XDNA_ERR(xdna, "Send message failed"); 1038 return ret; 1039 } 1040 1041 return 0; 1042 } 1043 1044 int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx, 1045 struct amdxdna_sched_job *job, 1046 int (*notify_cb)(void *, void __iomem *, size_t)) 1047 { 1048 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job); 1049 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1050 struct amdxdna_dev *xdna = hwctx->client->xdna; 1051 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 1052 struct xdna_mailbox_msg msg; 1053 union exec_chain_req req; 1054 u32 op = ERT_INVALID_CMD; 1055 size_t size; 1056 int ret; 1057 1058 size = cmdbuf_abo->mem.size; 1059 ret = aie2_cmdlist_fill_slot(cmdbuf_abo->mem.kva, cmd_abo, &size, &op); 1060 if (ret) 1061 return ret; 1062 1063 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op); 1064 if (msg.opcode == MSG_OP_MAX_OPCODE) 1065 return -EOPNOTSUPP; 1066 1067 EXEC_MSG_OPS(xdna)->init_chain_req(&req, cmdbuf_abo->mem.dev_addr, 1068 size, 1); 1069 drm_clflush_virt_range(cmdbuf_abo->mem.kva, size); 1070 1071 msg.handle = job; 1072 msg.notify_cb = notify_cb; 1073 msg.send_data = (u8 *)&req; 1074 msg.send_size = sizeof(req); 1075 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1076 if (ret) { 1077 XDNA_ERR(hwctx->client->xdna, "Send message failed"); 1078 return ret; 1079 } 1080 1081 return 0; 1082 } 1083 1084 int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 1085 int (*notify_cb)(void *, void __iomem *, size_t)) 1086 { 1087 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1088 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); 1089 struct amdxdna_dev *xdna = hwctx->client->xdna; 1090 struct xdna_mailbox_msg msg; 1091 struct sync_bo_req req; 1092 int ret = 0; 1093 1094 req.src_addr = 0; 1095 req.dst_addr = amdxdna_dev_bo_offset(abo); 1096 req.size = abo->mem.size; 1097 1098 /* Device to Host */ 1099 req.type = FIELD_PREP(AIE2_MSG_SYNC_BO_SRC_TYPE, SYNC_BO_DEV_MEM) | 1100 FIELD_PREP(AIE2_MSG_SYNC_BO_DST_TYPE, SYNC_BO_HOST_MEM); 1101 1102 XDNA_DBG(xdna, "sync %d bytes src(0x%llx) to dst(0x%llx) completed", 1103 req.size, req.src_addr, req.dst_addr); 1104 1105 msg.handle = job; 1106 msg.notify_cb = notify_cb; 1107 msg.send_data = (u8 *)&req; 1108 msg.send_size = sizeof(req); 1109 msg.opcode = MSG_OP_SYNC_BO; 1110 1111 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1112 if (ret) { 1113 XDNA_ERR(xdna, "Send message failed"); 1114 return ret; 1115 } 1116 1117 return 0; 1118 } 1119 1120 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 1121 int (*notify_cb)(void *, void __iomem *, size_t)) 1122 { 1123 struct mailbox_channel *chann = hwctx->priv->mbox_chann; 1124 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]); 1125 struct amdxdna_dev *xdna = hwctx->client->xdna; 1126 struct config_debug_bo_req req; 1127 struct xdna_mailbox_msg msg; 1128 1129 if (job->drv_cmd->opcode == ATTACH_DEBUG_BO) 1130 req.config = DEBUG_BO_REGISTER; 1131 else 1132 req.config = DEBUG_BO_UNREGISTER; 1133 1134 req.offset = amdxdna_dev_bo_offset(abo); 1135 req.size = abo->mem.size; 1136 1137 XDNA_DBG(xdna, "offset 0x%llx size 0x%llx config %d", 1138 req.offset, req.size, req.config); 1139 1140 msg.handle = job; 1141 msg.notify_cb = notify_cb; 1142 msg.send_data = (u8 *)&req; 1143 msg.send_size = sizeof(req); 1144 msg.opcode = MSG_OP_CONFIG_DEBUG_BO; 1145 1146 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT); 1147 } 1148