1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_gem.h> 9 #include <drm/drm_gem_shmem_helper.h> 10 #include <drm/drm_print.h> 11 #include <drm/drm_syncobj.h> 12 #include <linux/hmm.h> 13 #include <linux/types.h> 14 #include <linux/xarray.h> 15 #include <trace/events/amdxdna.h> 16 17 #include "aie2_msg_priv.h" 18 #include "aie2_pci.h" 19 #include "aie2_solver.h" 20 #include "amdxdna_ctx.h" 21 #include "amdxdna_gem.h" 22 #include "amdxdna_mailbox.h" 23 #include "amdxdna_pci_drv.h" 24 #include "amdxdna_pm.h" 25 26 static bool force_cmdlist; 27 module_param(force_cmdlist, bool, 0600); 28 MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default false)"); 29 30 #define HWCTX_MAX_TIMEOUT 60000 /* milliseconds */ 31 32 static void aie2_job_release(struct kref *ref) 33 { 34 struct amdxdna_sched_job *job; 35 36 job = container_of(ref, struct amdxdna_sched_job, refcnt); 37 amdxdna_sched_job_cleanup(job); 38 atomic64_inc(&job->hwctx->job_free_cnt); 39 wake_up(&job->hwctx->priv->job_free_wq); 40 if (job->out_fence) 41 dma_fence_put(job->out_fence); 42 kfree(job); 43 } 44 45 static void aie2_job_put(struct amdxdna_sched_job *job) 46 { 47 kref_put(&job->refcnt, aie2_job_release); 48 } 49 50 static void aie2_hwctx_status_shift_stop(struct amdxdna_hwctx *hwctx) 51 { 52 hwctx->old_status = hwctx->status; 53 hwctx->status = HWCTX_STAT_STOP; 54 } 55 56 static void aie2_hwctx_status_restore(struct amdxdna_hwctx *hwctx) 57 { 58 hwctx->status = hwctx->old_status; 59 } 60 61 /* The bad_job is used in aie2_sched_job_timedout, otherwise, set it to NULL */ 62 static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx, 63 struct drm_sched_job *bad_job) 64 { 65 drm_sched_stop(&hwctx->priv->sched, bad_job); 66 aie2_destroy_context(xdna->dev_handle, hwctx); 67 } 68 69 static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) 70 { 71 struct amdxdna_gem_obj *heap = hwctx->priv->heap; 72 int ret; 73 74 ret = aie2_create_context(xdna->dev_handle, hwctx); 75 if (ret) { 76 XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret); 77 goto out; 78 } 79 80 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 81 heap->mem.userptr, heap->mem.size); 82 if (ret) { 83 XDNA_ERR(xdna, "Map host buf failed, ret %d", ret); 84 goto out; 85 } 86 87 if (hwctx->status != HWCTX_STAT_READY) { 88 XDNA_DBG(xdna, "hwctx is not ready, status %d", hwctx->status); 89 goto out; 90 } 91 92 ret = aie2_config_cu(hwctx, NULL); 93 if (ret) { 94 XDNA_ERR(xdna, "Config cu failed, ret %d", ret); 95 goto out; 96 } 97 98 out: 99 drm_sched_start(&hwctx->priv->sched, 0); 100 XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret); 101 return ret; 102 } 103 104 static struct dma_fence *aie2_cmd_get_out_fence(struct amdxdna_hwctx *hwctx, u64 seq) 105 { 106 struct dma_fence *fence, *out_fence = NULL; 107 int ret; 108 109 fence = drm_syncobj_fence_get(hwctx->priv->syncobj); 110 if (!fence) 111 return NULL; 112 113 ret = dma_fence_chain_find_seqno(&fence, seq); 114 if (ret) 115 goto out; 116 117 out_fence = dma_fence_get(dma_fence_chain_contained(fence)); 118 119 out: 120 dma_fence_put(fence); 121 return out_fence; 122 } 123 124 static void aie2_hwctx_wait_for_idle(struct amdxdna_hwctx *hwctx) 125 { 126 struct dma_fence *fence; 127 128 fence = aie2_cmd_get_out_fence(hwctx, hwctx->priv->seq - 1); 129 if (!fence) 130 return; 131 132 /* Wait up to 2 seconds for fw to finish all pending requests */ 133 dma_fence_wait_timeout(fence, false, msecs_to_jiffies(2000)); 134 dma_fence_put(fence); 135 } 136 137 static int aie2_hwctx_suspend_cb(struct amdxdna_hwctx *hwctx, void *arg) 138 { 139 struct amdxdna_dev *xdna = hwctx->client->xdna; 140 141 aie2_hwctx_wait_for_idle(hwctx); 142 aie2_hwctx_stop(xdna, hwctx, NULL); 143 aie2_hwctx_status_shift_stop(hwctx); 144 145 return 0; 146 } 147 148 void aie2_hwctx_suspend(struct amdxdna_client *client) 149 { 150 struct amdxdna_dev *xdna = client->xdna; 151 152 /* 153 * Command timeout is unlikely. But if it happens, it doesn't 154 * break the system. aie2_hwctx_stop() will destroy mailbox 155 * and abort all commands. 156 */ 157 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 158 amdxdna_hwctx_walk(client, NULL, aie2_hwctx_suspend_cb); 159 } 160 161 static int aie2_hwctx_resume_cb(struct amdxdna_hwctx *hwctx, void *arg) 162 { 163 struct amdxdna_dev *xdna = hwctx->client->xdna; 164 165 aie2_hwctx_status_restore(hwctx); 166 return aie2_hwctx_restart(xdna, hwctx); 167 } 168 169 int aie2_hwctx_resume(struct amdxdna_client *client) 170 { 171 /* 172 * The resume path cannot guarantee that mailbox channel can be 173 * regenerated. If this happen, when submit message to this 174 * mailbox channel, error will return. 175 */ 176 return amdxdna_hwctx_walk(client, NULL, aie2_hwctx_resume_cb); 177 } 178 179 static void 180 aie2_sched_notify(struct amdxdna_sched_job *job) 181 { 182 struct dma_fence *fence = job->fence; 183 184 trace_xdna_job(&job->base, job->hwctx->name, "signaled fence", job->seq); 185 186 amdxdna_pm_suspend_put(job->hwctx->client->xdna); 187 job->hwctx->priv->completed++; 188 dma_fence_signal(fence); 189 190 up(&job->hwctx->priv->job_sem); 191 job->job_done = true; 192 dma_fence_put(fence); 193 mmput_async(job->mm); 194 aie2_job_put(job); 195 } 196 197 static int 198 aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size) 199 { 200 struct amdxdna_sched_job *job = handle; 201 struct amdxdna_gem_obj *cmd_abo; 202 int ret = 0; 203 u32 status; 204 205 cmd_abo = job->cmd_bo; 206 207 if (unlikely(job->job_timeout)) { 208 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_TIMEOUT); 209 ret = -EINVAL; 210 goto out; 211 } 212 213 if (unlikely(!data) || unlikely(size != sizeof(u32))) { 214 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT); 215 ret = -EINVAL; 216 goto out; 217 } 218 219 status = readl(data); 220 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); 221 if (status == AIE2_STATUS_SUCCESS) 222 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); 223 else 224 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ERROR); 225 226 out: 227 aie2_sched_notify(job); 228 return ret; 229 } 230 231 static int 232 aie2_sched_drvcmd_resp_handler(void *handle, void __iomem *data, size_t size) 233 { 234 struct amdxdna_sched_job *job = handle; 235 int ret = 0; 236 237 if (unlikely(!data)) 238 goto out; 239 240 if (unlikely(size != sizeof(u32))) { 241 ret = -EINVAL; 242 goto out; 243 } 244 245 job->drv_cmd->result = readl(data); 246 247 out: 248 aie2_sched_notify(job); 249 return ret; 250 } 251 252 static int 253 aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) 254 { 255 struct amdxdna_sched_job *job = handle; 256 struct amdxdna_gem_obj *cmd_abo; 257 struct amdxdna_dev *xdna; 258 u32 fail_cmd_status; 259 u32 fail_cmd_idx; 260 u32 cmd_status; 261 int ret = 0; 262 263 cmd_abo = job->cmd_bo; 264 265 if (unlikely(job->job_timeout)) { 266 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_TIMEOUT); 267 ret = -EINVAL; 268 goto out; 269 } 270 271 if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) { 272 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT); 273 ret = -EINVAL; 274 goto out; 275 } 276 277 cmd_status = readl(data + offsetof(struct cmd_chain_resp, status)); 278 xdna = job->hwctx->client->xdna; 279 XDNA_DBG(xdna, "Status 0x%x", cmd_status); 280 if (cmd_status == AIE2_STATUS_SUCCESS) { 281 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); 282 goto out; 283 } 284 285 /* Slow path to handle error, read from ringbuf on BAR */ 286 fail_cmd_idx = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_idx)); 287 fail_cmd_status = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_status)); 288 XDNA_DBG(xdna, "Failed cmd idx %d, status 0x%x", 289 fail_cmd_idx, fail_cmd_status); 290 291 if (fail_cmd_status == AIE2_STATUS_SUCCESS) { 292 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT); 293 ret = -EINVAL; 294 goto out; 295 } 296 amdxdna_cmd_set_state(cmd_abo, fail_cmd_status); 297 298 if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) { 299 struct amdxdna_cmd_chain *cc = amdxdna_cmd_get_payload(cmd_abo, NULL); 300 301 cc->error_index = fail_cmd_idx; 302 if (cc->error_index >= cc->command_count) 303 cc->error_index = 0; 304 } 305 out: 306 aie2_sched_notify(job); 307 return ret; 308 } 309 310 static struct dma_fence * 311 aie2_sched_job_run(struct drm_sched_job *sched_job) 312 { 313 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 314 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 315 struct amdxdna_hwctx *hwctx = job->hwctx; 316 struct dma_fence *fence; 317 int ret; 318 319 if (!mmget_not_zero(job->mm)) 320 return ERR_PTR(-ESRCH); 321 322 kref_get(&job->refcnt); 323 fence = dma_fence_get(job->fence); 324 325 if (job->drv_cmd) { 326 switch (job->drv_cmd->opcode) { 327 case SYNC_DEBUG_BO: 328 ret = aie2_sync_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); 329 break; 330 case ATTACH_DEBUG_BO: 331 ret = aie2_config_debug_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); 332 break; 333 default: 334 ret = -EINVAL; 335 break; 336 } 337 goto out; 338 } 339 340 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_NEW); 341 342 if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) 343 ret = aie2_cmdlist_multi_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); 344 else if (force_cmdlist) 345 ret = aie2_cmdlist_single_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); 346 else 347 ret = aie2_execbuf(hwctx, job, aie2_sched_resp_handler); 348 349 out: 350 if (ret) { 351 dma_fence_put(job->fence); 352 aie2_job_put(job); 353 mmput(job->mm); 354 fence = ERR_PTR(ret); 355 } 356 trace_xdna_job(sched_job, hwctx->name, "sent to device", job->seq); 357 358 return fence; 359 } 360 361 static void aie2_sched_job_free(struct drm_sched_job *sched_job) 362 { 363 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 364 struct amdxdna_hwctx *hwctx = job->hwctx; 365 366 trace_xdna_job(sched_job, hwctx->name, "job free", job->seq); 367 if (!job->job_done) 368 up(&hwctx->priv->job_sem); 369 370 drm_sched_job_cleanup(sched_job); 371 aie2_job_put(job); 372 } 373 374 static enum drm_gpu_sched_stat 375 aie2_sched_job_timedout(struct drm_sched_job *sched_job) 376 { 377 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 378 struct amdxdna_hwctx *hwctx = job->hwctx; 379 struct amdxdna_dev *xdna; 380 381 xdna = hwctx->client->xdna; 382 trace_xdna_job(sched_job, hwctx->name, "job timedout", job->seq); 383 job->job_timeout = true; 384 mutex_lock(&xdna->dev_lock); 385 aie2_hwctx_stop(xdna, hwctx, sched_job); 386 387 aie2_hwctx_restart(xdna, hwctx); 388 mutex_unlock(&xdna->dev_lock); 389 390 return DRM_GPU_SCHED_STAT_RESET; 391 } 392 393 static const struct drm_sched_backend_ops sched_ops = { 394 .run_job = aie2_sched_job_run, 395 .free_job = aie2_sched_job_free, 396 .timedout_job = aie2_sched_job_timedout, 397 }; 398 399 static int aie2_hwctx_col_list(struct amdxdna_hwctx *hwctx) 400 { 401 struct amdxdna_dev *xdna = hwctx->client->xdna; 402 struct amdxdna_dev_hdl *ndev; 403 int start, end, first, last; 404 u32 width = 1, entries = 0; 405 int i; 406 407 if (!hwctx->num_tiles) { 408 XDNA_ERR(xdna, "Number of tiles is zero"); 409 return -EINVAL; 410 } 411 412 ndev = xdna->dev_handle; 413 if (unlikely(!ndev->metadata.core.row_count)) { 414 XDNA_WARN(xdna, "Core tile row count is zero"); 415 return -EINVAL; 416 } 417 418 hwctx->num_col = hwctx->num_tiles / ndev->metadata.core.row_count; 419 if (!hwctx->num_col || hwctx->num_col > ndev->total_col) { 420 XDNA_ERR(xdna, "Invalid num_col %d", hwctx->num_col); 421 return -EINVAL; 422 } 423 424 if (ndev->priv->col_align == COL_ALIGN_NATURE) 425 width = hwctx->num_col; 426 427 /* 428 * In range [start, end], find out columns that is multiple of width. 429 * 'first' is the first column, 430 * 'last' is the last column, 431 * 'entries' is the total number of columns. 432 */ 433 start = xdna->dev_info->first_col; 434 end = ndev->total_col - hwctx->num_col; 435 if (start > 0 && end == 0) { 436 XDNA_DBG(xdna, "Force start from col 0"); 437 start = 0; 438 } 439 first = start + (width - start % width) % width; 440 last = end - end % width; 441 if (last >= first) 442 entries = (last - first) / width + 1; 443 XDNA_DBG(xdna, "start %d end %d first %d last %d", 444 start, end, first, last); 445 446 if (unlikely(!entries)) { 447 XDNA_ERR(xdna, "Start %d end %d width %d", 448 start, end, width); 449 return -EINVAL; 450 } 451 452 hwctx->col_list = kmalloc_array(entries, sizeof(*hwctx->col_list), GFP_KERNEL); 453 if (!hwctx->col_list) 454 return -ENOMEM; 455 456 hwctx->col_list_len = entries; 457 hwctx->col_list[0] = first; 458 for (i = 1; i < entries; i++) 459 hwctx->col_list[i] = hwctx->col_list[i - 1] + width; 460 461 print_hex_dump_debug("col_list: ", DUMP_PREFIX_OFFSET, 16, 4, hwctx->col_list, 462 entries * sizeof(*hwctx->col_list), false); 463 return 0; 464 } 465 466 static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx) 467 { 468 struct amdxdna_dev *xdna = hwctx->client->xdna; 469 struct alloc_requests *xrs_req; 470 int ret; 471 472 xrs_req = kzalloc(sizeof(*xrs_req), GFP_KERNEL); 473 if (!xrs_req) 474 return -ENOMEM; 475 476 xrs_req->cdo.start_cols = hwctx->col_list; 477 xrs_req->cdo.cols_len = hwctx->col_list_len; 478 xrs_req->cdo.ncols = hwctx->num_col; 479 xrs_req->cdo.qos_cap.opc = hwctx->max_opc; 480 481 xrs_req->rqos.gops = hwctx->qos.gops; 482 xrs_req->rqos.fps = hwctx->qos.fps; 483 xrs_req->rqos.dma_bw = hwctx->qos.dma_bandwidth; 484 xrs_req->rqos.latency = hwctx->qos.latency; 485 xrs_req->rqos.exec_time = hwctx->qos.frame_exec_time; 486 xrs_req->rqos.priority = hwctx->qos.priority; 487 488 xrs_req->rid = (uintptr_t)hwctx; 489 490 ret = xrs_allocate_resource(xdna->xrs_hdl, xrs_req, hwctx); 491 if (ret) 492 XDNA_ERR(xdna, "Allocate AIE resource failed, ret %d", ret); 493 494 kfree(xrs_req); 495 return ret; 496 } 497 498 static void aie2_release_resource(struct amdxdna_hwctx *hwctx) 499 { 500 struct amdxdna_dev *xdna = hwctx->client->xdna; 501 int ret; 502 503 ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); 504 if (ret) 505 XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret); 506 } 507 508 static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx) 509 { 510 struct amdxdna_dev *xdna = hwctx->client->xdna; 511 struct drm_file *filp = hwctx->client->filp; 512 struct drm_syncobj *syncobj; 513 u32 hdl; 514 int ret; 515 516 hwctx->syncobj_hdl = AMDXDNA_INVALID_FENCE_HANDLE; 517 518 ret = drm_syncobj_create(&syncobj, 0, NULL); 519 if (ret) { 520 XDNA_ERR(xdna, "Create ctx syncobj failed, ret %d", ret); 521 return ret; 522 } 523 ret = drm_syncobj_get_handle(filp, syncobj, &hdl); 524 if (ret) { 525 drm_syncobj_put(syncobj); 526 XDNA_ERR(xdna, "Create ctx syncobj handle failed, ret %d", ret); 527 return ret; 528 } 529 hwctx->priv->syncobj = syncobj; 530 hwctx->syncobj_hdl = hdl; 531 532 return 0; 533 } 534 535 static void aie2_ctx_syncobj_destroy(struct amdxdna_hwctx *hwctx) 536 { 537 /* 538 * The syncobj_hdl is owned by user space and will be cleaned up 539 * separately. 540 */ 541 drm_syncobj_put(hwctx->priv->syncobj); 542 } 543 544 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) 545 { 546 struct amdxdna_client *client = hwctx->client; 547 struct amdxdna_dev *xdna = client->xdna; 548 const struct drm_sched_init_args args = { 549 .ops = &sched_ops, 550 .num_rqs = DRM_SCHED_PRIORITY_COUNT, 551 .credit_limit = HWCTX_MAX_CMDS, 552 .timeout = msecs_to_jiffies(HWCTX_MAX_TIMEOUT), 553 .name = "amdxdna_js", 554 .dev = xdna->ddev.dev, 555 }; 556 struct drm_gpu_scheduler *sched; 557 struct amdxdna_hwctx_priv *priv; 558 struct amdxdna_gem_obj *heap; 559 struct amdxdna_dev_hdl *ndev; 560 int i, ret; 561 562 priv = kzalloc(sizeof(*hwctx->priv), GFP_KERNEL); 563 if (!priv) 564 return -ENOMEM; 565 hwctx->priv = priv; 566 567 mutex_lock(&client->mm_lock); 568 heap = client->dev_heap; 569 if (!heap) { 570 XDNA_ERR(xdna, "The client dev heap object not exist"); 571 mutex_unlock(&client->mm_lock); 572 ret = -ENOENT; 573 goto free_priv; 574 } 575 drm_gem_object_get(to_gobj(heap)); 576 mutex_unlock(&client->mm_lock); 577 priv->heap = heap; 578 sema_init(&priv->job_sem, HWCTX_MAX_CMDS); 579 580 ret = amdxdna_gem_pin(heap); 581 if (ret) { 582 XDNA_ERR(xdna, "Dev heap pin failed, ret %d", ret); 583 goto put_heap; 584 } 585 586 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { 587 struct amdxdna_gem_obj *abo; 588 struct amdxdna_drm_create_bo args = { 589 .flags = 0, 590 .type = AMDXDNA_BO_DEV, 591 .vaddr = 0, 592 .size = MAX_CHAIN_CMDBUF_SIZE, 593 }; 594 595 abo = amdxdna_drm_alloc_dev_bo(&xdna->ddev, &args, client->filp); 596 if (IS_ERR(abo)) { 597 ret = PTR_ERR(abo); 598 goto free_cmd_bufs; 599 } 600 601 XDNA_DBG(xdna, "Command buf %d addr 0x%llx size 0x%lx", 602 i, abo->mem.dev_addr, abo->mem.size); 603 priv->cmd_buf[i] = abo; 604 } 605 606 sched = &priv->sched; 607 mutex_init(&priv->io_lock); 608 609 fs_reclaim_acquire(GFP_KERNEL); 610 might_lock(&priv->io_lock); 611 fs_reclaim_release(GFP_KERNEL); 612 613 ret = drm_sched_init(sched, &args); 614 if (ret) { 615 XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret); 616 goto free_cmd_bufs; 617 } 618 619 ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL, 620 &sched, 1, NULL); 621 if (ret) { 622 XDNA_ERR(xdna, "Failed to initial sched entiry. ret %d", ret); 623 goto free_sched; 624 } 625 626 ret = aie2_hwctx_col_list(hwctx); 627 if (ret) { 628 XDNA_ERR(xdna, "Create col list failed, ret %d", ret); 629 goto free_entity; 630 } 631 632 ret = amdxdna_pm_resume_get(xdna); 633 if (ret) 634 goto free_col_list; 635 636 ret = aie2_alloc_resource(hwctx); 637 if (ret) { 638 XDNA_ERR(xdna, "Alloc hw resource failed, ret %d", ret); 639 goto suspend_put; 640 } 641 642 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 643 heap->mem.userptr, heap->mem.size); 644 if (ret) { 645 XDNA_ERR(xdna, "Map host buffer failed, ret %d", ret); 646 goto release_resource; 647 } 648 649 ret = aie2_ctx_syncobj_create(hwctx); 650 if (ret) { 651 XDNA_ERR(xdna, "Create syncobj failed, ret %d", ret); 652 goto release_resource; 653 } 654 amdxdna_pm_suspend_put(xdna); 655 656 hwctx->status = HWCTX_STAT_INIT; 657 ndev = xdna->dev_handle; 658 ndev->hwctx_num++; 659 init_waitqueue_head(&priv->job_free_wq); 660 661 XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); 662 663 return 0; 664 665 release_resource: 666 aie2_release_resource(hwctx); 667 suspend_put: 668 amdxdna_pm_suspend_put(xdna); 669 free_col_list: 670 kfree(hwctx->col_list); 671 free_entity: 672 drm_sched_entity_destroy(&priv->entity); 673 free_sched: 674 drm_sched_fini(&priv->sched); 675 free_cmd_bufs: 676 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { 677 if (!priv->cmd_buf[i]) 678 continue; 679 drm_gem_object_put(to_gobj(priv->cmd_buf[i])); 680 } 681 amdxdna_gem_unpin(heap); 682 put_heap: 683 drm_gem_object_put(to_gobj(heap)); 684 free_priv: 685 kfree(priv); 686 return ret; 687 } 688 689 void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx) 690 { 691 struct amdxdna_dev_hdl *ndev; 692 struct amdxdna_dev *xdna; 693 int idx; 694 695 xdna = hwctx->client->xdna; 696 ndev = xdna->dev_handle; 697 ndev->hwctx_num--; 698 699 XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq); 700 drm_sched_entity_destroy(&hwctx->priv->entity); 701 702 aie2_hwctx_wait_for_idle(hwctx); 703 704 /* Request fw to destroy hwctx and cancel the rest pending requests */ 705 aie2_release_resource(hwctx); 706 707 /* Wait for all submitted jobs to be completed or canceled */ 708 wait_event(hwctx->priv->job_free_wq, 709 atomic64_read(&hwctx->job_submit_cnt) == 710 atomic64_read(&hwctx->job_free_cnt)); 711 712 drm_sched_fini(&hwctx->priv->sched); 713 aie2_ctx_syncobj_destroy(hwctx); 714 715 for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++) 716 drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx])); 717 amdxdna_gem_unpin(hwctx->priv->heap); 718 drm_gem_object_put(to_gobj(hwctx->priv->heap)); 719 720 mutex_destroy(&hwctx->priv->io_lock); 721 kfree(hwctx->col_list); 722 kfree(hwctx->priv); 723 kfree(hwctx->cus); 724 } 725 726 static int aie2_config_cu_resp_handler(void *handle, void __iomem *data, size_t size) 727 { 728 struct amdxdna_hwctx *hwctx = handle; 729 730 amdxdna_pm_suspend_put(hwctx->client->xdna); 731 return 0; 732 } 733 734 static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size) 735 { 736 struct amdxdna_hwctx_param_config_cu *config = buf; 737 struct amdxdna_dev *xdna = hwctx->client->xdna; 738 u32 total_size; 739 int ret; 740 741 XDNA_DBG(xdna, "Config %d CU to %s", config->num_cus, hwctx->name); 742 if (XDNA_MBZ_DBG(xdna, config->pad, sizeof(config->pad))) 743 return -EINVAL; 744 745 if (hwctx->status != HWCTX_STAT_INIT) { 746 XDNA_ERR(xdna, "Not support re-config CU"); 747 return -EINVAL; 748 } 749 750 if (!config->num_cus) { 751 XDNA_ERR(xdna, "Number of CU is zero"); 752 return -EINVAL; 753 } 754 755 total_size = struct_size(config, cu_configs, config->num_cus); 756 if (total_size > size) { 757 XDNA_ERR(xdna, "CU config larger than size"); 758 return -EINVAL; 759 } 760 761 hwctx->cus = kmemdup(config, total_size, GFP_KERNEL); 762 if (!hwctx->cus) 763 return -ENOMEM; 764 765 ret = amdxdna_pm_resume_get(xdna); 766 if (ret) 767 goto free_cus; 768 769 ret = aie2_config_cu(hwctx, aie2_config_cu_resp_handler); 770 if (ret) { 771 XDNA_ERR(xdna, "Config CU to firmware failed, ret %d", ret); 772 goto pm_suspend_put; 773 } 774 775 wmb(); /* To avoid locking in command submit when check status */ 776 hwctx->status = HWCTX_STAT_READY; 777 778 return 0; 779 780 pm_suspend_put: 781 amdxdna_pm_suspend_put(xdna); 782 free_cus: 783 kfree(hwctx->cus); 784 hwctx->cus = NULL; 785 return ret; 786 } 787 788 static void aie2_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq) 789 { 790 struct dma_fence *out_fence = aie2_cmd_get_out_fence(hwctx, seq); 791 792 if (!out_fence) { 793 XDNA_ERR(hwctx->client->xdna, "Failed to get fence"); 794 return; 795 } 796 797 dma_fence_wait_timeout(out_fence, false, MAX_SCHEDULE_TIMEOUT); 798 dma_fence_put(out_fence); 799 } 800 801 static int aie2_hwctx_cfg_debug_bo(struct amdxdna_hwctx *hwctx, u32 bo_hdl, 802 bool attach) 803 { 804 struct amdxdna_client *client = hwctx->client; 805 struct amdxdna_dev *xdna = client->xdna; 806 struct amdxdna_drv_cmd cmd = { 0 }; 807 struct amdxdna_gem_obj *abo; 808 u64 seq; 809 int ret; 810 811 abo = amdxdna_gem_get_obj(client, bo_hdl, AMDXDNA_BO_DEV); 812 if (!abo) { 813 XDNA_ERR(xdna, "Get bo %d failed", bo_hdl); 814 return -EINVAL; 815 } 816 817 if (attach) { 818 if (abo->assigned_hwctx != AMDXDNA_INVALID_CTX_HANDLE) { 819 ret = -EBUSY; 820 goto put_obj; 821 } 822 cmd.opcode = ATTACH_DEBUG_BO; 823 } else { 824 if (abo->assigned_hwctx != hwctx->id) { 825 ret = -EINVAL; 826 goto put_obj; 827 } 828 cmd.opcode = DETACH_DEBUG_BO; 829 } 830 831 ret = amdxdna_cmd_submit(client, &cmd, AMDXDNA_INVALID_BO_HANDLE, 832 &bo_hdl, 1, hwctx->id, &seq); 833 if (ret) { 834 XDNA_ERR(xdna, "Submit command failed"); 835 goto put_obj; 836 } 837 838 aie2_cmd_wait(hwctx, seq); 839 if (cmd.result) { 840 XDNA_ERR(xdna, "Response failure 0x%x", cmd.result); 841 goto put_obj; 842 } 843 844 if (attach) 845 abo->assigned_hwctx = hwctx->id; 846 else 847 abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE; 848 849 XDNA_DBG(xdna, "Config debug BO %d to %s", bo_hdl, hwctx->name); 850 851 put_obj: 852 amdxdna_gem_put_obj(abo); 853 return ret; 854 } 855 856 int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size) 857 { 858 struct amdxdna_dev *xdna = hwctx->client->xdna; 859 860 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 861 switch (type) { 862 case DRM_AMDXDNA_HWCTX_CONFIG_CU: 863 return aie2_hwctx_cu_config(hwctx, buf, size); 864 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF: 865 return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, true); 866 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF: 867 return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, false); 868 default: 869 XDNA_DBG(xdna, "Not supported type %d", type); 870 return -EOPNOTSUPP; 871 } 872 } 873 874 int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl) 875 { 876 struct amdxdna_client *client = hwctx->client; 877 struct amdxdna_dev *xdna = client->xdna; 878 struct amdxdna_drv_cmd cmd = { 0 }; 879 u64 seq; 880 int ret; 881 882 cmd.opcode = SYNC_DEBUG_BO; 883 ret = amdxdna_cmd_submit(client, &cmd, AMDXDNA_INVALID_BO_HANDLE, 884 &debug_bo_hdl, 1, hwctx->id, &seq); 885 if (ret) { 886 XDNA_ERR(xdna, "Submit command failed"); 887 return ret; 888 } 889 890 aie2_cmd_wait(hwctx, seq); 891 if (cmd.result) { 892 XDNA_ERR(xdna, "Response failure 0x%x", cmd.result); 893 return -EINVAL; 894 } 895 896 return 0; 897 } 898 899 static int aie2_populate_range(struct amdxdna_gem_obj *abo) 900 { 901 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 902 struct amdxdna_umap *mapp; 903 unsigned long timeout; 904 struct mm_struct *mm; 905 bool found; 906 int ret; 907 908 timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 909 again: 910 found = false; 911 down_write(&xdna->notifier_lock); 912 list_for_each_entry(mapp, &abo->mem.umap_list, node) { 913 if (mapp->invalid) { 914 found = true; 915 break; 916 } 917 } 918 919 if (!found) { 920 abo->mem.map_invalid = false; 921 up_write(&xdna->notifier_lock); 922 return 0; 923 } 924 kref_get(&mapp->refcnt); 925 up_write(&xdna->notifier_lock); 926 927 XDNA_DBG(xdna, "populate memory range %lx %lx", 928 mapp->vma->vm_start, mapp->vma->vm_end); 929 mm = mapp->notifier.mm; 930 if (!mmget_not_zero(mm)) { 931 amdxdna_umap_put(mapp); 932 return -EFAULT; 933 } 934 935 mapp->range.notifier_seq = mmu_interval_read_begin(&mapp->notifier); 936 mmap_read_lock(mm); 937 ret = hmm_range_fault(&mapp->range); 938 mmap_read_unlock(mm); 939 if (ret) { 940 if (time_after(jiffies, timeout)) { 941 ret = -ETIME; 942 goto put_mm; 943 } 944 945 if (ret == -EBUSY) { 946 amdxdna_umap_put(mapp); 947 goto again; 948 } 949 950 goto put_mm; 951 } 952 953 down_write(&xdna->notifier_lock); 954 if (mmu_interval_read_retry(&mapp->notifier, mapp->range.notifier_seq)) { 955 up_write(&xdna->notifier_lock); 956 amdxdna_umap_put(mapp); 957 goto again; 958 } 959 mapp->invalid = false; 960 up_write(&xdna->notifier_lock); 961 amdxdna_umap_put(mapp); 962 goto again; 963 964 put_mm: 965 amdxdna_umap_put(mapp); 966 mmput(mm); 967 return ret; 968 } 969 970 int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq) 971 { 972 struct amdxdna_dev *xdna = hwctx->client->xdna; 973 struct ww_acquire_ctx acquire_ctx; 974 struct dma_fence_chain *chain; 975 struct amdxdna_gem_obj *abo; 976 unsigned long timeout = 0; 977 int ret, i; 978 979 ret = down_interruptible(&hwctx->priv->job_sem); 980 if (ret) { 981 XDNA_ERR(xdna, "Grab job sem failed, ret %d", ret); 982 return ret; 983 } 984 985 chain = dma_fence_chain_alloc(); 986 if (!chain) { 987 XDNA_ERR(xdna, "Alloc fence chain failed"); 988 ret = -ENOMEM; 989 goto up_sem; 990 } 991 992 ret = drm_sched_job_init(&job->base, &hwctx->priv->entity, 1, hwctx, 993 hwctx->client->filp->client_id); 994 if (ret) { 995 XDNA_ERR(xdna, "DRM job init failed, ret %d", ret); 996 goto free_chain; 997 } 998 999 ret = amdxdna_pm_resume_get(xdna); 1000 if (ret) 1001 goto cleanup_job; 1002 1003 retry: 1004 ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1005 if (ret) { 1006 XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret); 1007 goto suspend_put; 1008 } 1009 1010 for (i = 0; i < job->bo_cnt; i++) { 1011 ret = dma_resv_reserve_fences(job->bos[i]->resv, 1); 1012 if (ret) { 1013 XDNA_WARN(xdna, "Failed to reserve fences %d", ret); 1014 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1015 goto suspend_put; 1016 } 1017 } 1018 1019 down_read(&xdna->notifier_lock); 1020 for (i = 0; i < job->bo_cnt; i++) { 1021 abo = to_xdna_obj(job->bos[i]); 1022 if (abo->mem.map_invalid) { 1023 up_read(&xdna->notifier_lock); 1024 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1025 if (!timeout) { 1026 timeout = jiffies + 1027 msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1028 } else if (time_after(jiffies, timeout)) { 1029 ret = -ETIME; 1030 goto suspend_put; 1031 } 1032 1033 ret = aie2_populate_range(abo); 1034 if (ret) 1035 goto suspend_put; 1036 goto retry; 1037 } 1038 } 1039 1040 mutex_lock(&hwctx->priv->io_lock); 1041 drm_sched_job_arm(&job->base); 1042 job->out_fence = dma_fence_get(&job->base.s_fence->finished); 1043 for (i = 0; i < job->bo_cnt; i++) 1044 dma_resv_add_fence(job->bos[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE); 1045 job->seq = hwctx->priv->seq++; 1046 kref_get(&job->refcnt); 1047 drm_sched_entity_push_job(&job->base); 1048 1049 *seq = job->seq; 1050 drm_syncobj_add_point(hwctx->priv->syncobj, chain, job->out_fence, *seq); 1051 mutex_unlock(&hwctx->priv->io_lock); 1052 1053 up_read(&xdna->notifier_lock); 1054 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1055 1056 aie2_job_put(job); 1057 atomic64_inc(&hwctx->job_submit_cnt); 1058 1059 return 0; 1060 1061 suspend_put: 1062 amdxdna_pm_suspend_put(xdna); 1063 cleanup_job: 1064 drm_sched_job_cleanup(&job->base); 1065 free_chain: 1066 dma_fence_chain_free(chain); 1067 up_sem: 1068 up(&hwctx->priv->job_sem); 1069 job->job_done = true; 1070 return ret; 1071 } 1072 1073 void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, 1074 unsigned long cur_seq) 1075 { 1076 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 1077 struct drm_gem_object *gobj = to_gobj(abo); 1078 long ret; 1079 1080 ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP, 1081 true, MAX_SCHEDULE_TIMEOUT); 1082 if (!ret || ret == -ERESTARTSYS) 1083 XDNA_ERR(xdna, "Failed to wait for bo, ret %ld", ret); 1084 } 1085