1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_gem.h> 9 #include <drm/drm_gem_shmem_helper.h> 10 #include <drm/drm_print.h> 11 #include <drm/drm_syncobj.h> 12 #include <linux/hmm.h> 13 #include <linux/types.h> 14 #include <linux/xarray.h> 15 #include <trace/events/amdxdna.h> 16 17 #include "aie2_msg_priv.h" 18 #include "aie2_pci.h" 19 #include "aie2_solver.h" 20 #include "amdxdna_ctx.h" 21 #include "amdxdna_gem.h" 22 #include "amdxdna_mailbox.h" 23 #include "amdxdna_pci_drv.h" 24 #include "amdxdna_pm.h" 25 26 static bool force_cmdlist; 27 module_param(force_cmdlist, bool, 0600); 28 MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default false)"); 29 30 #define HWCTX_MAX_TIMEOUT 60000 /* milliseconds */ 31 32 static void aie2_job_release(struct kref *ref) 33 { 34 struct amdxdna_sched_job *job; 35 36 job = container_of(ref, struct amdxdna_sched_job, refcnt); 37 amdxdna_sched_job_cleanup(job); 38 atomic64_inc(&job->hwctx->job_free_cnt); 39 wake_up(&job->hwctx->priv->job_free_wq); 40 if (job->out_fence) 41 dma_fence_put(job->out_fence); 42 kfree(job); 43 } 44 45 static void aie2_job_put(struct amdxdna_sched_job *job) 46 { 47 kref_put(&job->refcnt, aie2_job_release); 48 } 49 50 static void aie2_hwctx_status_shift_stop(struct amdxdna_hwctx *hwctx) 51 { 52 hwctx->old_status = hwctx->status; 53 hwctx->status = HWCTX_STAT_STOP; 54 } 55 56 static void aie2_hwctx_status_restore(struct amdxdna_hwctx *hwctx) 57 { 58 hwctx->status = hwctx->old_status; 59 } 60 61 /* The bad_job is used in aie2_sched_job_timedout, otherwise, set it to NULL */ 62 static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx, 63 struct drm_sched_job *bad_job) 64 { 65 drm_sched_stop(&hwctx->priv->sched, bad_job); 66 aie2_destroy_context(xdna->dev_handle, hwctx); 67 } 68 69 static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) 70 { 71 struct amdxdna_gem_obj *heap = hwctx->priv->heap; 72 int ret; 73 74 ret = aie2_create_context(xdna->dev_handle, hwctx); 75 if (ret) { 76 XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret); 77 goto out; 78 } 79 80 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 81 heap->mem.userptr, heap->mem.size); 82 if (ret) { 83 XDNA_ERR(xdna, "Map host buf failed, ret %d", ret); 84 goto out; 85 } 86 87 if (hwctx->status != HWCTX_STAT_READY) { 88 XDNA_DBG(xdna, "hwctx is not ready, status %d", hwctx->status); 89 goto out; 90 } 91 92 ret = aie2_config_cu(hwctx, NULL); 93 if (ret) { 94 XDNA_ERR(xdna, "Config cu failed, ret %d", ret); 95 goto out; 96 } 97 98 out: 99 drm_sched_start(&hwctx->priv->sched, 0); 100 XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret); 101 return ret; 102 } 103 104 static struct dma_fence *aie2_cmd_get_out_fence(struct amdxdna_hwctx *hwctx, u64 seq) 105 { 106 struct dma_fence *fence, *out_fence = NULL; 107 int ret; 108 109 fence = drm_syncobj_fence_get(hwctx->priv->syncobj); 110 if (!fence) 111 return NULL; 112 113 ret = dma_fence_chain_find_seqno(&fence, seq); 114 if (ret) 115 goto out; 116 117 out_fence = dma_fence_get(dma_fence_chain_contained(fence)); 118 119 out: 120 dma_fence_put(fence); 121 return out_fence; 122 } 123 124 static void aie2_hwctx_wait_for_idle(struct amdxdna_hwctx *hwctx) 125 { 126 struct dma_fence *fence; 127 128 fence = aie2_cmd_get_out_fence(hwctx, hwctx->priv->seq - 1); 129 if (!fence) 130 return; 131 132 /* Wait up to 2 seconds for fw to finish all pending requests */ 133 dma_fence_wait_timeout(fence, false, msecs_to_jiffies(2000)); 134 dma_fence_put(fence); 135 } 136 137 static int aie2_hwctx_suspend_cb(struct amdxdna_hwctx *hwctx, void *arg) 138 { 139 struct amdxdna_dev *xdna = hwctx->client->xdna; 140 141 aie2_hwctx_wait_for_idle(hwctx); 142 aie2_hwctx_stop(xdna, hwctx, NULL); 143 aie2_hwctx_status_shift_stop(hwctx); 144 145 return 0; 146 } 147 148 void aie2_hwctx_suspend(struct amdxdna_client *client) 149 { 150 struct amdxdna_dev *xdna = client->xdna; 151 152 /* 153 * Command timeout is unlikely. But if it happens, it doesn't 154 * break the system. aie2_hwctx_stop() will destroy mailbox 155 * and abort all commands. 156 */ 157 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 158 amdxdna_hwctx_walk(client, NULL, aie2_hwctx_suspend_cb); 159 } 160 161 static int aie2_hwctx_resume_cb(struct amdxdna_hwctx *hwctx, void *arg) 162 { 163 struct amdxdna_dev *xdna = hwctx->client->xdna; 164 165 aie2_hwctx_status_restore(hwctx); 166 return aie2_hwctx_restart(xdna, hwctx); 167 } 168 169 int aie2_hwctx_resume(struct amdxdna_client *client) 170 { 171 /* 172 * The resume path cannot guarantee that mailbox channel can be 173 * regenerated. If this happen, when submit message to this 174 * mailbox channel, error will return. 175 */ 176 return amdxdna_hwctx_walk(client, NULL, aie2_hwctx_resume_cb); 177 } 178 179 static void 180 aie2_sched_notify(struct amdxdna_sched_job *job) 181 { 182 struct dma_fence *fence = job->fence; 183 184 trace_xdna_job(&job->base, job->hwctx->name, "signaled fence", job->seq); 185 186 amdxdna_pm_suspend_put(job->hwctx->client->xdna); 187 job->hwctx->priv->completed++; 188 dma_fence_signal(fence); 189 190 up(&job->hwctx->priv->job_sem); 191 job->job_done = true; 192 mmput_async(job->mm); 193 aie2_job_put(job); 194 } 195 196 static int 197 aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size) 198 { 199 struct amdxdna_sched_job *job = handle; 200 struct amdxdna_gem_obj *cmd_abo; 201 int ret = 0; 202 u32 status; 203 204 cmd_abo = job->cmd_bo; 205 206 if (unlikely(job->job_timeout)) { 207 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_TIMEOUT); 208 ret = -EINVAL; 209 goto out; 210 } 211 212 if (unlikely(!data) || unlikely(size != sizeof(u32))) { 213 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT); 214 ret = -EINVAL; 215 goto out; 216 } 217 218 status = readl(data); 219 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); 220 if (status == AIE2_STATUS_SUCCESS) 221 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); 222 else 223 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ERROR); 224 225 out: 226 aie2_sched_notify(job); 227 return ret; 228 } 229 230 static int 231 aie2_sched_drvcmd_resp_handler(void *handle, void __iomem *data, size_t size) 232 { 233 struct amdxdna_sched_job *job = handle; 234 int ret = 0; 235 236 if (unlikely(!data)) 237 goto out; 238 239 if (unlikely(size != sizeof(u32))) { 240 ret = -EINVAL; 241 goto out; 242 } 243 244 job->drv_cmd->result = readl(data); 245 246 out: 247 aie2_sched_notify(job); 248 return ret; 249 } 250 251 static int 252 aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) 253 { 254 struct amdxdna_sched_job *job = handle; 255 struct amdxdna_gem_obj *cmd_abo; 256 struct amdxdna_dev *xdna; 257 u32 fail_cmd_status; 258 u32 fail_cmd_idx; 259 u32 cmd_status; 260 int ret = 0; 261 262 cmd_abo = job->cmd_bo; 263 264 if (unlikely(job->job_timeout)) { 265 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_TIMEOUT); 266 ret = -EINVAL; 267 goto out; 268 } 269 270 if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) { 271 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT); 272 ret = -EINVAL; 273 goto out; 274 } 275 276 cmd_status = readl(data + offsetof(struct cmd_chain_resp, status)); 277 xdna = job->hwctx->client->xdna; 278 XDNA_DBG(xdna, "Status 0x%x", cmd_status); 279 if (cmd_status == AIE2_STATUS_SUCCESS) { 280 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); 281 goto out; 282 } 283 284 /* Slow path to handle error, read from ringbuf on BAR */ 285 fail_cmd_idx = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_idx)); 286 fail_cmd_status = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_status)); 287 XDNA_DBG(xdna, "Failed cmd idx %d, status 0x%x", 288 fail_cmd_idx, fail_cmd_status); 289 290 if (fail_cmd_status == AIE2_STATUS_SUCCESS) { 291 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ABORT); 292 ret = -EINVAL; 293 goto out; 294 } 295 amdxdna_cmd_set_state(cmd_abo, fail_cmd_status); 296 297 if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) { 298 struct amdxdna_cmd_chain *cc = amdxdna_cmd_get_payload(cmd_abo, NULL); 299 300 cc->error_index = fail_cmd_idx; 301 if (cc->error_index >= cc->command_count) 302 cc->error_index = 0; 303 } 304 out: 305 aie2_sched_notify(job); 306 return ret; 307 } 308 309 static struct dma_fence * 310 aie2_sched_job_run(struct drm_sched_job *sched_job) 311 { 312 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 313 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 314 struct amdxdna_hwctx *hwctx = job->hwctx; 315 struct dma_fence *fence; 316 int ret; 317 318 if (!mmget_not_zero(job->mm)) 319 return ERR_PTR(-ESRCH); 320 321 kref_get(&job->refcnt); 322 fence = dma_fence_get(job->fence); 323 324 if (job->drv_cmd) { 325 switch (job->drv_cmd->opcode) { 326 case SYNC_DEBUG_BO: 327 ret = aie2_sync_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); 328 break; 329 case ATTACH_DEBUG_BO: 330 ret = aie2_config_debug_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); 331 break; 332 default: 333 ret = -EINVAL; 334 break; 335 } 336 goto out; 337 } 338 339 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_NEW); 340 341 if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) 342 ret = aie2_cmdlist_multi_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); 343 else if (force_cmdlist) 344 ret = aie2_cmdlist_single_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); 345 else 346 ret = aie2_execbuf(hwctx, job, aie2_sched_resp_handler); 347 348 out: 349 if (ret) { 350 dma_fence_put(job->fence); 351 aie2_job_put(job); 352 mmput(job->mm); 353 fence = ERR_PTR(ret); 354 } 355 trace_xdna_job(sched_job, hwctx->name, "sent to device", job->seq); 356 357 return fence; 358 } 359 360 static void aie2_sched_job_free(struct drm_sched_job *sched_job) 361 { 362 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 363 struct amdxdna_hwctx *hwctx = job->hwctx; 364 365 trace_xdna_job(sched_job, hwctx->name, "job free", job->seq); 366 if (!job->job_done) 367 up(&hwctx->priv->job_sem); 368 369 drm_sched_job_cleanup(sched_job); 370 aie2_job_put(job); 371 } 372 373 static enum drm_gpu_sched_stat 374 aie2_sched_job_timedout(struct drm_sched_job *sched_job) 375 { 376 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 377 struct amdxdna_hwctx *hwctx = job->hwctx; 378 struct amdxdna_dev *xdna; 379 380 xdna = hwctx->client->xdna; 381 trace_xdna_job(sched_job, hwctx->name, "job timedout", job->seq); 382 job->job_timeout = true; 383 mutex_lock(&xdna->dev_lock); 384 aie2_hwctx_stop(xdna, hwctx, sched_job); 385 386 aie2_hwctx_restart(xdna, hwctx); 387 mutex_unlock(&xdna->dev_lock); 388 389 return DRM_GPU_SCHED_STAT_RESET; 390 } 391 392 static const struct drm_sched_backend_ops sched_ops = { 393 .run_job = aie2_sched_job_run, 394 .free_job = aie2_sched_job_free, 395 .timedout_job = aie2_sched_job_timedout, 396 }; 397 398 static int aie2_hwctx_col_list(struct amdxdna_hwctx *hwctx) 399 { 400 struct amdxdna_dev *xdna = hwctx->client->xdna; 401 struct amdxdna_dev_hdl *ndev; 402 int start, end, first, last; 403 u32 width = 1, entries = 0; 404 int i; 405 406 if (!hwctx->num_tiles) { 407 XDNA_ERR(xdna, "Number of tiles is zero"); 408 return -EINVAL; 409 } 410 411 ndev = xdna->dev_handle; 412 if (unlikely(!ndev->metadata.core.row_count)) { 413 XDNA_WARN(xdna, "Core tile row count is zero"); 414 return -EINVAL; 415 } 416 417 hwctx->num_col = hwctx->num_tiles / ndev->metadata.core.row_count; 418 if (!hwctx->num_col || hwctx->num_col > ndev->total_col) { 419 XDNA_ERR(xdna, "Invalid num_col %d", hwctx->num_col); 420 return -EINVAL; 421 } 422 423 if (ndev->priv->col_align == COL_ALIGN_NATURE) 424 width = hwctx->num_col; 425 426 /* 427 * In range [start, end], find out columns that is multiple of width. 428 * 'first' is the first column, 429 * 'last' is the last column, 430 * 'entries' is the total number of columns. 431 */ 432 start = xdna->dev_info->first_col; 433 end = ndev->total_col - hwctx->num_col; 434 if (start > 0 && end == 0) { 435 XDNA_DBG(xdna, "Force start from col 0"); 436 start = 0; 437 } 438 first = start + (width - start % width) % width; 439 last = end - end % width; 440 if (last >= first) 441 entries = (last - first) / width + 1; 442 XDNA_DBG(xdna, "start %d end %d first %d last %d", 443 start, end, first, last); 444 445 if (unlikely(!entries)) { 446 XDNA_ERR(xdna, "Start %d end %d width %d", 447 start, end, width); 448 return -EINVAL; 449 } 450 451 hwctx->col_list = kmalloc_array(entries, sizeof(*hwctx->col_list), GFP_KERNEL); 452 if (!hwctx->col_list) 453 return -ENOMEM; 454 455 hwctx->col_list_len = entries; 456 hwctx->col_list[0] = first; 457 for (i = 1; i < entries; i++) 458 hwctx->col_list[i] = hwctx->col_list[i - 1] + width; 459 460 print_hex_dump_debug("col_list: ", DUMP_PREFIX_OFFSET, 16, 4, hwctx->col_list, 461 entries * sizeof(*hwctx->col_list), false); 462 return 0; 463 } 464 465 static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx) 466 { 467 struct amdxdna_dev *xdna = hwctx->client->xdna; 468 struct alloc_requests *xrs_req; 469 int ret; 470 471 xrs_req = kzalloc(sizeof(*xrs_req), GFP_KERNEL); 472 if (!xrs_req) 473 return -ENOMEM; 474 475 xrs_req->cdo.start_cols = hwctx->col_list; 476 xrs_req->cdo.cols_len = hwctx->col_list_len; 477 xrs_req->cdo.ncols = hwctx->num_col; 478 xrs_req->cdo.qos_cap.opc = hwctx->max_opc; 479 480 xrs_req->rqos.gops = hwctx->qos.gops; 481 xrs_req->rqos.fps = hwctx->qos.fps; 482 xrs_req->rqos.dma_bw = hwctx->qos.dma_bandwidth; 483 xrs_req->rqos.latency = hwctx->qos.latency; 484 xrs_req->rqos.exec_time = hwctx->qos.frame_exec_time; 485 xrs_req->rqos.priority = hwctx->qos.priority; 486 487 xrs_req->rid = (uintptr_t)hwctx; 488 489 ret = xrs_allocate_resource(xdna->xrs_hdl, xrs_req, hwctx); 490 if (ret) 491 XDNA_ERR(xdna, "Allocate AIE resource failed, ret %d", ret); 492 493 kfree(xrs_req); 494 return ret; 495 } 496 497 static void aie2_release_resource(struct amdxdna_hwctx *hwctx) 498 { 499 struct amdxdna_dev *xdna = hwctx->client->xdna; 500 int ret; 501 502 ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); 503 if (ret) 504 XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret); 505 } 506 507 static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx) 508 { 509 struct amdxdna_dev *xdna = hwctx->client->xdna; 510 struct drm_file *filp = hwctx->client->filp; 511 struct drm_syncobj *syncobj; 512 u32 hdl; 513 int ret; 514 515 hwctx->syncobj_hdl = AMDXDNA_INVALID_FENCE_HANDLE; 516 517 ret = drm_syncobj_create(&syncobj, 0, NULL); 518 if (ret) { 519 XDNA_ERR(xdna, "Create ctx syncobj failed, ret %d", ret); 520 return ret; 521 } 522 ret = drm_syncobj_get_handle(filp, syncobj, &hdl); 523 if (ret) { 524 drm_syncobj_put(syncobj); 525 XDNA_ERR(xdna, "Create ctx syncobj handle failed, ret %d", ret); 526 return ret; 527 } 528 hwctx->priv->syncobj = syncobj; 529 hwctx->syncobj_hdl = hdl; 530 531 return 0; 532 } 533 534 static void aie2_ctx_syncobj_destroy(struct amdxdna_hwctx *hwctx) 535 { 536 /* 537 * The syncobj_hdl is owned by user space and will be cleaned up 538 * separately. 539 */ 540 drm_syncobj_put(hwctx->priv->syncobj); 541 } 542 543 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) 544 { 545 struct amdxdna_client *client = hwctx->client; 546 struct amdxdna_dev *xdna = client->xdna; 547 const struct drm_sched_init_args args = { 548 .ops = &sched_ops, 549 .num_rqs = DRM_SCHED_PRIORITY_COUNT, 550 .credit_limit = HWCTX_MAX_CMDS, 551 .timeout = msecs_to_jiffies(HWCTX_MAX_TIMEOUT), 552 .name = "amdxdna_js", 553 .dev = xdna->ddev.dev, 554 }; 555 struct drm_gpu_scheduler *sched; 556 struct amdxdna_hwctx_priv *priv; 557 struct amdxdna_gem_obj *heap; 558 int i, ret; 559 560 priv = kzalloc(sizeof(*hwctx->priv), GFP_KERNEL); 561 if (!priv) 562 return -ENOMEM; 563 hwctx->priv = priv; 564 565 mutex_lock(&client->mm_lock); 566 heap = client->dev_heap; 567 if (!heap) { 568 XDNA_ERR(xdna, "The client dev heap object not exist"); 569 mutex_unlock(&client->mm_lock); 570 ret = -ENOENT; 571 goto free_priv; 572 } 573 drm_gem_object_get(to_gobj(heap)); 574 mutex_unlock(&client->mm_lock); 575 priv->heap = heap; 576 sema_init(&priv->job_sem, HWCTX_MAX_CMDS); 577 578 ret = amdxdna_gem_pin(heap); 579 if (ret) { 580 XDNA_ERR(xdna, "Dev heap pin failed, ret %d", ret); 581 goto put_heap; 582 } 583 584 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { 585 struct amdxdna_gem_obj *abo; 586 struct amdxdna_drm_create_bo args = { 587 .flags = 0, 588 .type = AMDXDNA_BO_DEV, 589 .vaddr = 0, 590 .size = MAX_CHAIN_CMDBUF_SIZE, 591 }; 592 593 abo = amdxdna_drm_alloc_dev_bo(&xdna->ddev, &args, client->filp); 594 if (IS_ERR(abo)) { 595 ret = PTR_ERR(abo); 596 goto free_cmd_bufs; 597 } 598 599 XDNA_DBG(xdna, "Command buf %d addr 0x%llx size 0x%lx", 600 i, abo->mem.dev_addr, abo->mem.size); 601 priv->cmd_buf[i] = abo; 602 } 603 604 sched = &priv->sched; 605 mutex_init(&priv->io_lock); 606 607 fs_reclaim_acquire(GFP_KERNEL); 608 might_lock(&priv->io_lock); 609 fs_reclaim_release(GFP_KERNEL); 610 611 ret = drm_sched_init(sched, &args); 612 if (ret) { 613 XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret); 614 goto free_cmd_bufs; 615 } 616 617 ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL, 618 &sched, 1, NULL); 619 if (ret) { 620 XDNA_ERR(xdna, "Failed to initial sched entiry. ret %d", ret); 621 goto free_sched; 622 } 623 624 ret = aie2_hwctx_col_list(hwctx); 625 if (ret) { 626 XDNA_ERR(xdna, "Create col list failed, ret %d", ret); 627 goto free_entity; 628 } 629 630 ret = amdxdna_pm_resume_get(xdna); 631 if (ret) 632 goto free_col_list; 633 634 ret = aie2_alloc_resource(hwctx); 635 if (ret) { 636 XDNA_ERR(xdna, "Alloc hw resource failed, ret %d", ret); 637 goto suspend_put; 638 } 639 640 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 641 heap->mem.userptr, heap->mem.size); 642 if (ret) { 643 XDNA_ERR(xdna, "Map host buffer failed, ret %d", ret); 644 goto release_resource; 645 } 646 647 ret = aie2_ctx_syncobj_create(hwctx); 648 if (ret) { 649 XDNA_ERR(xdna, "Create syncobj failed, ret %d", ret); 650 goto release_resource; 651 } 652 amdxdna_pm_suspend_put(xdna); 653 654 hwctx->status = HWCTX_STAT_INIT; 655 init_waitqueue_head(&priv->job_free_wq); 656 657 XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); 658 659 return 0; 660 661 release_resource: 662 aie2_release_resource(hwctx); 663 suspend_put: 664 amdxdna_pm_suspend_put(xdna); 665 free_col_list: 666 kfree(hwctx->col_list); 667 free_entity: 668 drm_sched_entity_destroy(&priv->entity); 669 free_sched: 670 drm_sched_fini(&priv->sched); 671 free_cmd_bufs: 672 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { 673 if (!priv->cmd_buf[i]) 674 continue; 675 drm_gem_object_put(to_gobj(priv->cmd_buf[i])); 676 } 677 amdxdna_gem_unpin(heap); 678 put_heap: 679 drm_gem_object_put(to_gobj(heap)); 680 free_priv: 681 kfree(priv); 682 return ret; 683 } 684 685 void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx) 686 { 687 struct amdxdna_dev *xdna; 688 int idx; 689 690 xdna = hwctx->client->xdna; 691 692 XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq); 693 aie2_hwctx_wait_for_idle(hwctx); 694 695 /* Request fw to destroy hwctx and cancel the rest pending requests */ 696 aie2_release_resource(hwctx); 697 698 mutex_unlock(&xdna->dev_lock); 699 drm_sched_entity_destroy(&hwctx->priv->entity); 700 701 /* Wait for all submitted jobs to be completed or canceled */ 702 wait_event(hwctx->priv->job_free_wq, 703 atomic64_read(&hwctx->job_submit_cnt) == 704 atomic64_read(&hwctx->job_free_cnt)); 705 mutex_lock(&xdna->dev_lock); 706 707 drm_sched_fini(&hwctx->priv->sched); 708 aie2_ctx_syncobj_destroy(hwctx); 709 710 for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++) 711 drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx])); 712 amdxdna_gem_unpin(hwctx->priv->heap); 713 drm_gem_object_put(to_gobj(hwctx->priv->heap)); 714 715 mutex_destroy(&hwctx->priv->io_lock); 716 kfree(hwctx->col_list); 717 kfree(hwctx->priv); 718 kfree(hwctx->cus); 719 } 720 721 static int aie2_config_cu_resp_handler(void *handle, void __iomem *data, size_t size) 722 { 723 struct amdxdna_hwctx *hwctx = handle; 724 725 amdxdna_pm_suspend_put(hwctx->client->xdna); 726 return 0; 727 } 728 729 static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size) 730 { 731 struct amdxdna_hwctx_param_config_cu *config = buf; 732 struct amdxdna_dev *xdna = hwctx->client->xdna; 733 u32 total_size; 734 int ret; 735 736 XDNA_DBG(xdna, "Config %d CU to %s", config->num_cus, hwctx->name); 737 if (XDNA_MBZ_DBG(xdna, config->pad, sizeof(config->pad))) 738 return -EINVAL; 739 740 if (hwctx->status != HWCTX_STAT_INIT) { 741 XDNA_ERR(xdna, "Not support re-config CU"); 742 return -EINVAL; 743 } 744 745 if (!config->num_cus) { 746 XDNA_ERR(xdna, "Number of CU is zero"); 747 return -EINVAL; 748 } 749 750 total_size = struct_size(config, cu_configs, config->num_cus); 751 if (total_size > size) { 752 XDNA_ERR(xdna, "CU config larger than size"); 753 return -EINVAL; 754 } 755 756 hwctx->cus = kmemdup(config, total_size, GFP_KERNEL); 757 if (!hwctx->cus) 758 return -ENOMEM; 759 760 ret = amdxdna_pm_resume_get(xdna); 761 if (ret) 762 goto free_cus; 763 764 ret = aie2_config_cu(hwctx, aie2_config_cu_resp_handler); 765 if (ret) { 766 XDNA_ERR(xdna, "Config CU to firmware failed, ret %d", ret); 767 goto pm_suspend_put; 768 } 769 770 wmb(); /* To avoid locking in command submit when check status */ 771 hwctx->status = HWCTX_STAT_READY; 772 773 return 0; 774 775 pm_suspend_put: 776 amdxdna_pm_suspend_put(xdna); 777 free_cus: 778 kfree(hwctx->cus); 779 hwctx->cus = NULL; 780 return ret; 781 } 782 783 static void aie2_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq) 784 { 785 struct dma_fence *out_fence = aie2_cmd_get_out_fence(hwctx, seq); 786 787 if (!out_fence) { 788 XDNA_ERR(hwctx->client->xdna, "Failed to get fence"); 789 return; 790 } 791 792 dma_fence_wait_timeout(out_fence, false, MAX_SCHEDULE_TIMEOUT); 793 dma_fence_put(out_fence); 794 } 795 796 static int aie2_hwctx_cfg_debug_bo(struct amdxdna_hwctx *hwctx, u32 bo_hdl, 797 bool attach) 798 { 799 struct amdxdna_client *client = hwctx->client; 800 struct amdxdna_dev *xdna = client->xdna; 801 struct amdxdna_drv_cmd cmd = { 0 }; 802 struct amdxdna_gem_obj *abo; 803 u64 seq; 804 int ret; 805 806 abo = amdxdna_gem_get_obj(client, bo_hdl, AMDXDNA_BO_DEV); 807 if (!abo) { 808 XDNA_ERR(xdna, "Get bo %d failed", bo_hdl); 809 return -EINVAL; 810 } 811 812 if (attach) { 813 if (abo->assigned_hwctx != AMDXDNA_INVALID_CTX_HANDLE) { 814 ret = -EBUSY; 815 goto put_obj; 816 } 817 cmd.opcode = ATTACH_DEBUG_BO; 818 } else { 819 if (abo->assigned_hwctx != hwctx->id) { 820 ret = -EINVAL; 821 goto put_obj; 822 } 823 cmd.opcode = DETACH_DEBUG_BO; 824 } 825 826 ret = amdxdna_cmd_submit(client, &cmd, AMDXDNA_INVALID_BO_HANDLE, 827 &bo_hdl, 1, hwctx->id, &seq); 828 if (ret) { 829 XDNA_ERR(xdna, "Submit command failed"); 830 goto put_obj; 831 } 832 833 aie2_cmd_wait(hwctx, seq); 834 if (cmd.result) { 835 XDNA_ERR(xdna, "Response failure 0x%x", cmd.result); 836 goto put_obj; 837 } 838 839 if (attach) 840 abo->assigned_hwctx = hwctx->id; 841 else 842 abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE; 843 844 XDNA_DBG(xdna, "Config debug BO %d to %s", bo_hdl, hwctx->name); 845 846 put_obj: 847 amdxdna_gem_put_obj(abo); 848 return ret; 849 } 850 851 int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size) 852 { 853 struct amdxdna_dev *xdna = hwctx->client->xdna; 854 855 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 856 switch (type) { 857 case DRM_AMDXDNA_HWCTX_CONFIG_CU: 858 return aie2_hwctx_cu_config(hwctx, buf, size); 859 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF: 860 return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, true); 861 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF: 862 return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, false); 863 default: 864 XDNA_DBG(xdna, "Not supported type %d", type); 865 return -EOPNOTSUPP; 866 } 867 } 868 869 int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl) 870 { 871 struct amdxdna_client *client = hwctx->client; 872 struct amdxdna_dev *xdna = client->xdna; 873 struct amdxdna_drv_cmd cmd = { 0 }; 874 u64 seq; 875 int ret; 876 877 cmd.opcode = SYNC_DEBUG_BO; 878 ret = amdxdna_cmd_submit(client, &cmd, AMDXDNA_INVALID_BO_HANDLE, 879 &debug_bo_hdl, 1, hwctx->id, &seq); 880 if (ret) { 881 XDNA_ERR(xdna, "Submit command failed"); 882 return ret; 883 } 884 885 aie2_cmd_wait(hwctx, seq); 886 if (cmd.result) { 887 XDNA_ERR(xdna, "Response failure 0x%x", cmd.result); 888 return -EINVAL; 889 } 890 891 return 0; 892 } 893 894 static int aie2_populate_range(struct amdxdna_gem_obj *abo) 895 { 896 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 897 struct amdxdna_umap *mapp; 898 unsigned long timeout; 899 struct mm_struct *mm; 900 bool found; 901 int ret; 902 903 timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 904 again: 905 found = false; 906 down_write(&xdna->notifier_lock); 907 list_for_each_entry(mapp, &abo->mem.umap_list, node) { 908 if (mapp->invalid) { 909 found = true; 910 break; 911 } 912 } 913 914 if (!found) { 915 abo->mem.map_invalid = false; 916 up_write(&xdna->notifier_lock); 917 return 0; 918 } 919 kref_get(&mapp->refcnt); 920 up_write(&xdna->notifier_lock); 921 922 XDNA_DBG(xdna, "populate memory range %lx %lx", 923 mapp->vma->vm_start, mapp->vma->vm_end); 924 mm = mapp->notifier.mm; 925 if (!mmget_not_zero(mm)) { 926 amdxdna_umap_put(mapp); 927 return -EFAULT; 928 } 929 930 mapp->range.notifier_seq = mmu_interval_read_begin(&mapp->notifier); 931 mmap_read_lock(mm); 932 ret = hmm_range_fault(&mapp->range); 933 mmap_read_unlock(mm); 934 if (ret) { 935 if (time_after(jiffies, timeout)) { 936 ret = -ETIME; 937 goto put_mm; 938 } 939 940 if (ret == -EBUSY) { 941 amdxdna_umap_put(mapp); 942 goto again; 943 } 944 945 goto put_mm; 946 } 947 948 down_write(&xdna->notifier_lock); 949 if (mmu_interval_read_retry(&mapp->notifier, mapp->range.notifier_seq)) { 950 up_write(&xdna->notifier_lock); 951 amdxdna_umap_put(mapp); 952 goto again; 953 } 954 mapp->invalid = false; 955 up_write(&xdna->notifier_lock); 956 amdxdna_umap_put(mapp); 957 goto again; 958 959 put_mm: 960 amdxdna_umap_put(mapp); 961 mmput(mm); 962 return ret; 963 } 964 965 int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq) 966 { 967 struct amdxdna_dev *xdna = hwctx->client->xdna; 968 struct ww_acquire_ctx acquire_ctx; 969 struct dma_fence_chain *chain; 970 struct amdxdna_gem_obj *abo; 971 unsigned long timeout = 0; 972 int ret, i; 973 974 ret = down_interruptible(&hwctx->priv->job_sem); 975 if (ret) { 976 XDNA_ERR(xdna, "Grab job sem failed, ret %d", ret); 977 return ret; 978 } 979 980 chain = dma_fence_chain_alloc(); 981 if (!chain) { 982 XDNA_ERR(xdna, "Alloc fence chain failed"); 983 ret = -ENOMEM; 984 goto up_sem; 985 } 986 987 ret = drm_sched_job_init(&job->base, &hwctx->priv->entity, 1, hwctx, 988 hwctx->client->filp->client_id); 989 if (ret) { 990 XDNA_ERR(xdna, "DRM job init failed, ret %d", ret); 991 goto free_chain; 992 } 993 994 ret = amdxdna_pm_resume_get(xdna); 995 if (ret) 996 goto cleanup_job; 997 998 retry: 999 ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1000 if (ret) { 1001 XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret); 1002 goto suspend_put; 1003 } 1004 1005 for (i = 0; i < job->bo_cnt; i++) { 1006 ret = dma_resv_reserve_fences(job->bos[i]->resv, 1); 1007 if (ret) { 1008 XDNA_WARN(xdna, "Failed to reserve fences %d", ret); 1009 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1010 goto suspend_put; 1011 } 1012 } 1013 1014 down_read(&xdna->notifier_lock); 1015 for (i = 0; i < job->bo_cnt; i++) { 1016 abo = to_xdna_obj(job->bos[i]); 1017 if (abo->mem.map_invalid) { 1018 up_read(&xdna->notifier_lock); 1019 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1020 if (!timeout) { 1021 timeout = jiffies + 1022 msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1023 } else if (time_after(jiffies, timeout)) { 1024 ret = -ETIME; 1025 goto suspend_put; 1026 } 1027 1028 ret = aie2_populate_range(abo); 1029 if (ret) 1030 goto suspend_put; 1031 goto retry; 1032 } 1033 } 1034 1035 mutex_lock(&hwctx->priv->io_lock); 1036 drm_sched_job_arm(&job->base); 1037 job->out_fence = dma_fence_get(&job->base.s_fence->finished); 1038 for (i = 0; i < job->bo_cnt; i++) 1039 dma_resv_add_fence(job->bos[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE); 1040 job->seq = hwctx->priv->seq++; 1041 kref_get(&job->refcnt); 1042 drm_sched_entity_push_job(&job->base); 1043 1044 *seq = job->seq; 1045 drm_syncobj_add_point(hwctx->priv->syncobj, chain, job->out_fence, *seq); 1046 mutex_unlock(&hwctx->priv->io_lock); 1047 1048 up_read(&xdna->notifier_lock); 1049 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1050 1051 aie2_job_put(job); 1052 atomic64_inc(&hwctx->job_submit_cnt); 1053 1054 return 0; 1055 1056 suspend_put: 1057 amdxdna_pm_suspend_put(xdna); 1058 cleanup_job: 1059 drm_sched_job_cleanup(&job->base); 1060 free_chain: 1061 dma_fence_chain_free(chain); 1062 up_sem: 1063 up(&hwctx->priv->job_sem); 1064 job->job_done = true; 1065 return ret; 1066 } 1067 1068 void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, 1069 unsigned long cur_seq) 1070 { 1071 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 1072 struct drm_gem_object *gobj = to_gobj(abo); 1073 long ret; 1074 1075 ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP, 1076 true, MAX_SCHEDULE_TIMEOUT); 1077 if (!ret || ret == -ERESTARTSYS) 1078 XDNA_ERR(xdna, "Failed to wait for bo, ret %ld", ret); 1079 } 1080