1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_gem.h> 9 #include <drm/drm_gem_shmem_helper.h> 10 #include <drm/drm_print.h> 11 #include <drm/drm_syncobj.h> 12 #include <linux/hmm.h> 13 #include <linux/types.h> 14 #include <linux/xarray.h> 15 #include <trace/events/amdxdna.h> 16 17 #include "aie2_msg_priv.h" 18 #include "aie2_pci.h" 19 #include "aie2_solver.h" 20 #include "amdxdna_ctx.h" 21 #include "amdxdna_gem.h" 22 #include "amdxdna_mailbox.h" 23 #include "amdxdna_pci_drv.h" 24 #include "amdxdna_pm.h" 25 26 static bool force_cmdlist = true; 27 module_param(force_cmdlist, bool, 0600); 28 MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default true)"); 29 30 uint tdr_timeout_ms = 2000; 31 module_param(tdr_timeout_ms, int, 0400); 32 MODULE_PARM_DESC(tdr_timeout_ms, "TDR (Timeout Detection and Recovery) timeout in milliseconds (0 = disable)"); 33 34 struct aie2_ctx_health { 35 struct amdxdna_ctx_health header; 36 u32 txn_op_idx; 37 u32 ctx_pc; 38 u32 fatal_error_type; 39 u32 fatal_error_exception_type; 40 u32 fatal_error_exception_pc; 41 u32 fatal_error_app_module; 42 }; 43 44 static inline void aie2_tdr_signal(struct amdxdna_dev *xdna) 45 { 46 WRITE_ONCE(xdna->dev_handle->last_signal_ts, jiffies); 47 } 48 49 static bool aie2_tdr_detect(struct amdxdna_dev *xdna) 50 { 51 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 52 unsigned long last = READ_ONCE(ndev->last_signal_ts); 53 54 if (!tdr_timeout_ms) 55 return false; 56 57 if (!time_after(jiffies, last + msecs_to_jiffies(tdr_timeout_ms))) 58 return false; 59 60 XDNA_ERR(xdna, "TDR timeout detected"); 61 return true; 62 } 63 64 static void aie2_cmd_release(struct kref *ref) 65 { 66 struct amdxdna_drv_cmd *drv_cmd = container_of(ref, struct amdxdna_drv_cmd, refcnt); 67 68 kfree(drv_cmd); 69 } 70 71 static void aie2_cmd_put(struct amdxdna_drv_cmd *drv_cmd) 72 { 73 kref_put(&drv_cmd->refcnt, aie2_cmd_release); 74 } 75 76 static void aie2_job_release(struct kref *ref) 77 { 78 struct amdxdna_sched_job *job; 79 80 job = container_of(ref, struct amdxdna_sched_job, refcnt); 81 82 amdxdna_sched_job_cleanup(job); 83 atomic64_inc(&job->hwctx->job_free_cnt); 84 wake_up(&job->hwctx->priv->job_free_wq); 85 if (job->out_fence) 86 dma_fence_put(job->out_fence); 87 if (job->drv_cmd) 88 aie2_cmd_put(job->drv_cmd); 89 kfree(job->aie2_job_health); 90 kfree(job); 91 } 92 93 static void aie2_job_put(struct amdxdna_sched_job *job) 94 { 95 kref_put(&job->refcnt, aie2_job_release); 96 } 97 98 /* The bad_job is used in aie2_sched_job_timedout, otherwise, set it to NULL */ 99 static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx, 100 struct drm_sched_job *bad_job) 101 { 102 drm_sched_stop(&hwctx->priv->sched, bad_job); 103 aie2_destroy_context(xdna->dev_handle, hwctx); 104 drm_sched_start(&hwctx->priv->sched, 0); 105 } 106 107 static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) 108 { 109 struct amdxdna_gem_obj *heap = hwctx->priv->heap; 110 unsigned long heap_id; 111 int ret; 112 113 ret = aie2_create_context(xdna->dev_handle, hwctx); 114 if (ret) { 115 XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret); 116 goto out; 117 } 118 119 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 120 amdxdna_obj_dma_addr(heap), 121 heap->mem.size); 122 if (ret) { 123 XDNA_ERR(xdna, "Map host buf failed, ret %d", ret); 124 goto out; 125 } 126 127 xa_for_each_range(&hwctx->client->dev_heap_xa, heap_id, heap, 1, 128 hwctx->last_attached_heap) { 129 ret = aie2_add_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 130 amdxdna_obj_dma_addr(heap), 131 heap->mem.size); 132 if (ret) { 133 XDNA_ERR(xdna, "Add heap %ld failed ret %d", heap_id, ret); 134 goto out; 135 } 136 } 137 138 ret = aie2_config_cu(hwctx, NULL); 139 if (ret) { 140 XDNA_ERR(xdna, "Config cu failed, ret %d", ret); 141 goto out; 142 } 143 144 out: 145 XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret); 146 return ret; 147 } 148 149 static struct dma_fence *aie2_cmd_get_out_fence(struct amdxdna_hwctx *hwctx, u64 seq) 150 { 151 struct dma_fence *fence, *out_fence = NULL; 152 int ret; 153 154 fence = drm_syncobj_fence_get(hwctx->priv->syncobj); 155 if (!fence) 156 return NULL; 157 158 ret = dma_fence_chain_find_seqno(&fence, seq); 159 if (ret) 160 goto out; 161 162 out_fence = dma_fence_get(dma_fence_chain_contained(fence)); 163 164 out: 165 dma_fence_put(fence); 166 return out_fence; 167 } 168 169 static void aie2_hwctx_wait_for_idle(struct amdxdna_hwctx *hwctx) 170 { 171 struct dma_fence *fence; 172 173 fence = aie2_cmd_get_out_fence(hwctx, hwctx->priv->seq - 1); 174 if (!fence) 175 return; 176 177 /* Wait up to 2 seconds for fw to finish all pending requests */ 178 dma_fence_wait_timeout(fence, false, msecs_to_jiffies(2000)); 179 dma_fence_put(fence); 180 } 181 182 static int aie2_hwctx_suspend_cb(struct amdxdna_hwctx *hwctx, void *arg) 183 { 184 struct amdxdna_dev *xdna = hwctx->client->xdna; 185 186 aie2_hwctx_wait_for_idle(hwctx); 187 aie2_hwctx_stop(xdna, hwctx, NULL); 188 189 return 0; 190 } 191 192 void aie2_hwctx_suspend(struct amdxdna_client *client) 193 { 194 struct amdxdna_dev *xdna = client->xdna; 195 196 /* 197 * Command timeout is unlikely. But if it happens, it doesn't 198 * break the system. aie2_hwctx_stop() will destroy mailbox 199 * and abort all commands. 200 */ 201 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 202 amdxdna_hwctx_walk(client, NULL, aie2_hwctx_suspend_cb); 203 } 204 205 static int aie2_hwctx_resume_cb(struct amdxdna_hwctx *hwctx, void *arg) 206 { 207 struct amdxdna_dev *xdna = hwctx->client->xdna; 208 209 return aie2_hwctx_restart(xdna, hwctx); 210 } 211 212 int aie2_hwctx_resume(struct amdxdna_client *client) 213 { 214 /* 215 * The resume path cannot guarantee that mailbox channel can be 216 * regenerated. If this happen, when submit message to this 217 * mailbox channel, error will return. 218 */ 219 return amdxdna_hwctx_walk(client, NULL, aie2_hwctx_resume_cb); 220 } 221 222 static void 223 aie2_sched_notify(struct amdxdna_sched_job *job) 224 { 225 struct dma_fence *fence = job->fence; 226 227 trace_xdna_job(&job->base, job->hwctx->name, "signaling fence", 228 job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); 229 230 aie2_tdr_signal(job->hwctx->client->xdna); 231 job->hwctx->priv->completed++; 232 dma_fence_signal(fence); 233 234 up(&job->hwctx->priv->job_sem); 235 job->job_done = true; 236 mmput_async(job->mm); 237 aie2_job_put(job); 238 } 239 240 static void aie2_set_cmd_timeout(struct amdxdna_sched_job *job) 241 { 242 struct aie2_ctx_health *aie2_health __free(kfree) = NULL; 243 struct amdxdna_dev *xdna = job->hwctx->client->xdna; 244 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 245 struct app_health_report *report = job->aie2_job_health; 246 u32 fail_cmd_idx = 0; 247 248 if (!report) 249 goto set_timeout; 250 251 XDNA_ERR(xdna, "Firmware timeout state capture:"); 252 XDNA_ERR(xdna, "\tVersion: %d.%d", report->major, report->minor); 253 XDNA_ERR(xdna, "\tReport size: 0x%x", report->size); 254 XDNA_ERR(xdna, "\tContext ID: %d", report->context_id); 255 XDNA_ERR(xdna, "\tDPU PC: 0x%x", report->dpu_pc); 256 XDNA_ERR(xdna, "\tTXN OP ID: 0x%x", report->txn_op_id); 257 XDNA_ERR(xdna, "\tContext PC: 0x%x", report->ctx_pc); 258 XDNA_ERR(xdna, "\tFatal error type: 0x%x", report->fatal_info.fatal_type); 259 XDNA_ERR(xdna, "\tFatal error exception type: 0x%x", report->fatal_info.exception_type); 260 XDNA_ERR(xdna, "\tFatal error exception PC: 0x%x", report->fatal_info.exception_pc); 261 XDNA_ERR(xdna, "\tFatal error app module: 0x%x", report->fatal_info.app_module); 262 XDNA_ERR(xdna, "\tFatal error task ID: %d", report->fatal_info.task_index); 263 XDNA_ERR(xdna, "\tTimed out sub command ID: %d", report->run_list_id); 264 265 fail_cmd_idx = report->run_list_id; 266 aie2_health = kzalloc_obj(*aie2_health); 267 if (!aie2_health) 268 goto set_timeout; 269 270 aie2_health->header.version = AMDXDNA_CMD_CTX_HEALTH_V1; 271 aie2_health->header.npu_gen = AMDXDNA_CMD_CTX_HEALTH_AIE2; 272 aie2_health->txn_op_idx = report->txn_op_id; 273 aie2_health->ctx_pc = report->ctx_pc; 274 aie2_health->fatal_error_type = report->fatal_info.fatal_type; 275 aie2_health->fatal_error_exception_type = report->fatal_info.exception_type; 276 aie2_health->fatal_error_exception_pc = report->fatal_info.exception_pc; 277 aie2_health->fatal_error_app_module = report->fatal_info.app_module; 278 279 set_timeout: 280 amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_TIMEOUT, 281 aie2_health, sizeof(*aie2_health)); 282 } 283 284 static int 285 aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size) 286 { 287 struct amdxdna_sched_job *job = handle; 288 struct amdxdna_gem_obj *cmd_abo; 289 int ret = 0; 290 u32 status; 291 292 cmd_abo = job->cmd_bo; 293 294 if (unlikely(job->job_timeout)) { 295 aie2_set_cmd_timeout(job); 296 ret = -EINVAL; 297 goto out; 298 } 299 300 if (unlikely(!data) || unlikely(size != sizeof(u32))) { 301 amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT, NULL, 0); 302 ret = -EINVAL; 303 goto out; 304 } 305 306 status = readl(data); 307 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); 308 if (status == AIE2_STATUS_SUCCESS) 309 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); 310 else 311 amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ERROR, NULL, 0); 312 313 out: 314 aie2_sched_notify(job); 315 return ret; 316 } 317 318 static int 319 aie2_sched_drvcmd_resp_handler(void *handle, void __iomem *data, size_t size) 320 { 321 struct amdxdna_sched_job *job = handle; 322 int ret = 0; 323 324 if (unlikely(!data || size != sizeof(u32))) { 325 job->drv_cmd->result = U32_MAX; 326 ret = -EINVAL; 327 } else { 328 job->drv_cmd->result = readl(data); 329 } 330 331 aie2_sched_notify(job); 332 return ret; 333 } 334 335 static int 336 aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) 337 { 338 struct amdxdna_sched_job *job = handle; 339 struct amdxdna_gem_obj *cmd_abo; 340 struct amdxdna_dev *xdna; 341 u32 fail_cmd_idx = 0; 342 u32 fail_cmd_status; 343 u32 cmd_status; 344 int ret = 0; 345 346 cmd_abo = job->cmd_bo; 347 348 if (unlikely(job->job_timeout)) { 349 aie2_set_cmd_timeout(job); 350 ret = -EINVAL; 351 goto out; 352 } 353 354 if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) { 355 amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT, NULL, 0); 356 ret = -EINVAL; 357 goto out; 358 } 359 360 cmd_status = readl(data + offsetof(struct cmd_chain_resp, status)); 361 xdna = job->hwctx->client->xdna; 362 XDNA_DBG(xdna, "Status 0x%x", cmd_status); 363 if (cmd_status == AIE2_STATUS_SUCCESS) { 364 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); 365 goto out; 366 } 367 368 /* Slow path to handle error, read from ringbuf on BAR */ 369 fail_cmd_idx = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_idx)); 370 fail_cmd_status = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_status)); 371 XDNA_DBG(xdna, "Failed cmd idx %d, status 0x%x", 372 fail_cmd_idx, fail_cmd_status); 373 374 if (fail_cmd_status == AIE2_STATUS_SUCCESS) { 375 amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ABORT, NULL, 0); 376 ret = -EINVAL; 377 } else { 378 amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ERROR, NULL, 0); 379 } 380 381 out: 382 aie2_sched_notify(job); 383 return ret; 384 } 385 386 static struct dma_fence * 387 aie2_sched_job_run(struct drm_sched_job *sched_job) 388 { 389 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 390 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 391 struct amdxdna_hwctx *hwctx = job->hwctx; 392 struct dma_fence *fence; 393 int ret; 394 395 trace_xdna_job(sched_job, hwctx->name, "job run", 396 job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); 397 398 if (!hwctx->priv->mbox_chann) 399 return NULL; 400 401 if (!mmget_not_zero(job->mm)) 402 return ERR_PTR(-ESRCH); 403 404 kref_get(&job->refcnt); 405 fence = dma_fence_get(job->fence); 406 407 if (job->drv_cmd) { 408 switch (job->drv_cmd->opcode) { 409 case SYNC_DEBUG_BO: 410 ret = aie2_sync_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); 411 break; 412 case ATTACH_DEBUG_BO: 413 case DETACH_DEBUG_BO: 414 ret = aie2_config_debug_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); 415 break; 416 default: 417 ret = -EINVAL; 418 break; 419 } 420 goto out; 421 } 422 423 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_NEW); 424 425 if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) 426 ret = aie2_cmdlist_multi_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); 427 else if (force_cmdlist) 428 ret = aie2_cmdlist_single_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); 429 else 430 ret = aie2_execbuf(hwctx, job, aie2_sched_resp_handler); 431 432 out: 433 if (ret) { 434 dma_fence_put(job->fence); 435 aie2_job_put(job); 436 mmput(job->mm); 437 fence = ERR_PTR(ret); 438 } else { 439 /* 440 * Command is successfully posted to hardware, update the 441 * tdr timestamp. The total pending commands are limited. 442 * So there will not be a case that driver keeps posting 443 * commands without getting any hardware respond. 444 */ 445 aie2_tdr_signal(hwctx->client->xdna); 446 } 447 trace_xdna_job(sched_job, hwctx->name, "sent to device", 448 job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); 449 450 return fence; 451 } 452 453 static void aie2_sched_job_free(struct drm_sched_job *sched_job) 454 { 455 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 456 struct amdxdna_hwctx *hwctx = job->hwctx; 457 458 /* job->drv_cmd could be freed, so use DEFAULT_IO */ 459 trace_xdna_job(sched_job, hwctx->name, "job free", 460 job->seq, DEFAULT_IO); 461 if (!job->job_done) 462 up(&hwctx->priv->job_sem); 463 464 drm_sched_job_cleanup(sched_job); 465 aie2_job_put(job); 466 } 467 468 static enum drm_gpu_sched_stat 469 aie2_sched_job_timedout(struct drm_sched_job *sched_job) 470 { 471 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 472 struct amdxdna_hwctx *hwctx = job->hwctx; 473 struct app_health_report *report; 474 struct amdxdna_dev *xdna; 475 int ret; 476 477 xdna = hwctx->client->xdna; 478 479 guard(mutex)(&xdna->dev_lock); 480 481 if (!aie2_tdr_detect(xdna)) 482 return DRM_GPU_SCHED_STAT_NO_HANG; 483 484 report = kzalloc_obj(*report); 485 if (!report) 486 goto reset_hwctx; 487 488 ret = aie2_query_app_health(xdna->dev_handle, hwctx->fw_ctx_id, report); 489 if (ret) 490 kfree(report); 491 else 492 job->aie2_job_health = report; 493 494 reset_hwctx: 495 job->job_timeout = true; 496 aie2_hwctx_stop(xdna, hwctx, sched_job); 497 498 aie2_hwctx_restart(xdna, hwctx); 499 500 return DRM_GPU_SCHED_STAT_RESET; 501 } 502 503 static const struct drm_sched_backend_ops sched_ops = { 504 .run_job = aie2_sched_job_run, 505 .free_job = aie2_sched_job_free, 506 .timedout_job = aie2_sched_job_timedout, 507 }; 508 509 static int aie2_hwctx_col_list(struct amdxdna_hwctx *hwctx) 510 { 511 struct amdxdna_dev *xdna = hwctx->client->xdna; 512 struct amdxdna_dev_hdl *ndev; 513 int start, end, first, last; 514 u32 width = 1, entries = 0; 515 int i; 516 517 if (!hwctx->num_tiles) { 518 XDNA_ERR(xdna, "Number of tiles is zero"); 519 return -EINVAL; 520 } 521 522 ndev = xdna->dev_handle; 523 if (unlikely(!ndev->aie.metadata.core.row_count)) { 524 XDNA_WARN(xdna, "Core tile row count is zero"); 525 return -EINVAL; 526 } 527 528 hwctx->num_col = hwctx->num_tiles / ndev->aie.metadata.core.row_count; 529 if (!hwctx->num_col || hwctx->num_col > ndev->total_col) { 530 XDNA_ERR(xdna, "Invalid num_col %d", hwctx->num_col); 531 return -EINVAL; 532 } 533 534 if (ndev->priv->col_align == COL_ALIGN_NATURE) 535 width = hwctx->num_col; 536 537 /* 538 * In range [start, end], find out columns that is multiple of width. 539 * 'first' is the first column, 540 * 'last' is the last column, 541 * 'entries' is the total number of columns. 542 */ 543 start = xdna->dev_info->first_col; 544 end = ndev->total_col - hwctx->num_col; 545 if (start > 0 && end == 0) { 546 XDNA_DBG(xdna, "Force start from col 0"); 547 start = 0; 548 } 549 first = start + (width - start % width) % width; 550 last = end - end % width; 551 if (last >= first) 552 entries = (last - first) / width + 1; 553 XDNA_DBG(xdna, "start %d end %d first %d last %d", 554 start, end, first, last); 555 556 if (unlikely(!entries)) { 557 XDNA_ERR(xdna, "Start %d end %d width %d", 558 start, end, width); 559 return -EINVAL; 560 } 561 562 hwctx->col_list = kmalloc_array(entries, sizeof(*hwctx->col_list), GFP_KERNEL); 563 if (!hwctx->col_list) 564 return -ENOMEM; 565 566 hwctx->col_list_len = entries; 567 hwctx->col_list[0] = first; 568 for (i = 1; i < entries; i++) 569 hwctx->col_list[i] = hwctx->col_list[i - 1] + width; 570 571 print_hex_dump_debug("col_list: ", DUMP_PREFIX_OFFSET, 16, 4, hwctx->col_list, 572 entries * sizeof(*hwctx->col_list), false); 573 return 0; 574 } 575 576 static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx) 577 { 578 struct amdxdna_dev *xdna = hwctx->client->xdna; 579 struct alloc_requests *xrs_req; 580 u32 temporal_only_col = 0; 581 int ret; 582 583 xrs_req = kzalloc_obj(*xrs_req); 584 if (!xrs_req) 585 return -ENOMEM; 586 587 if (AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_TEMPORAL_ONLY)) { 588 xrs_req->cdo.start_cols = &temporal_only_col; 589 xrs_req->cdo.cols_len = 1; 590 xrs_req->cdo.ncols = xdna->dev_handle->total_col; 591 } else { 592 xrs_req->cdo.start_cols = hwctx->col_list; 593 xrs_req->cdo.cols_len = hwctx->col_list_len; 594 xrs_req->cdo.ncols = hwctx->num_col; 595 } 596 /* Use platform opc */ 597 xrs_req->cdo.qos_cap.opc = xdna->dev_handle->priv->col_opc * hwctx->num_col; 598 599 xrs_req->rqos.gops = hwctx->qos.gops; 600 xrs_req->rqos.fps = hwctx->qos.fps; 601 xrs_req->rqos.dma_bw = hwctx->qos.dma_bandwidth; 602 xrs_req->rqos.latency = hwctx->qos.latency; 603 xrs_req->rqos.exec_time = hwctx->qos.frame_exec_time; 604 xrs_req->rqos.priority = hwctx->qos.priority; 605 606 xrs_req->rid = (uintptr_t)hwctx; 607 608 ret = xrs_allocate_resource(xdna->xrs_hdl, xrs_req, hwctx); 609 if (ret) 610 XDNA_ERR(xdna, "Allocate AIE resource failed, ret %d", ret); 611 612 kfree(xrs_req); 613 return ret; 614 } 615 616 static void aie2_release_resource(struct amdxdna_hwctx *hwctx) 617 { 618 struct amdxdna_dev *xdna = hwctx->client->xdna; 619 int ret; 620 621 ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); 622 if (ret) 623 XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret); 624 } 625 626 static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx) 627 { 628 struct amdxdna_dev *xdna = hwctx->client->xdna; 629 struct drm_file *filp = hwctx->client->filp; 630 struct drm_syncobj *syncobj; 631 u32 hdl; 632 int ret; 633 634 hwctx->syncobj_hdl = AMDXDNA_INVALID_FENCE_HANDLE; 635 636 ret = drm_syncobj_create(&syncobj, 0, NULL); 637 if (ret) { 638 XDNA_ERR(xdna, "Create ctx syncobj failed, ret %d", ret); 639 return ret; 640 } 641 ret = drm_syncobj_get_handle(filp, syncobj, &hdl); 642 if (ret) { 643 drm_syncobj_put(syncobj); 644 XDNA_ERR(xdna, "Create ctx syncobj handle failed, ret %d", ret); 645 return ret; 646 } 647 hwctx->priv->syncobj = syncobj; 648 hwctx->syncobj_hdl = hdl; 649 650 return 0; 651 } 652 653 static void aie2_ctx_syncobj_destroy(struct amdxdna_hwctx *hwctx) 654 { 655 /* 656 * The syncobj_hdl is owned by user space and will be cleaned up 657 * separately. 658 */ 659 drm_syncobj_put(hwctx->priv->syncobj); 660 } 661 662 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) 663 { 664 struct amdxdna_client *client = hwctx->client; 665 struct amdxdna_dev *xdna = client->xdna; 666 const struct drm_sched_init_args args = { 667 .ops = &sched_ops, 668 .num_rqs = DRM_SCHED_PRIORITY_COUNT, 669 .credit_limit = HWCTX_MAX_CMDS, 670 .timeout = tdr_timeout_ms ? 671 msecs_to_jiffies(tdr_timeout_ms) : 672 MAX_SCHEDULE_TIMEOUT, 673 .name = "amdxdna_js", 674 .dev = xdna->ddev.dev, 675 }; 676 struct drm_gpu_scheduler *sched; 677 struct amdxdna_hwctx_priv *priv; 678 struct amdxdna_gem_obj *heap; 679 int i, ret; 680 681 priv = kzalloc_obj(*hwctx->priv); 682 if (!priv) 683 return -ENOMEM; 684 hwctx->priv = priv; 685 686 mutex_lock(&client->mm_lock); 687 heap = xa_load(&client->dev_heap_xa, 0); 688 if (!heap) { 689 XDNA_ERR(xdna, "The client dev heap object not exist"); 690 mutex_unlock(&client->mm_lock); 691 ret = -ENOENT; 692 goto free_priv; 693 } 694 drm_gem_object_get(to_gobj(heap)); 695 mutex_unlock(&client->mm_lock); 696 priv->heap = heap; 697 sema_init(&priv->job_sem, HWCTX_MAX_CMDS); 698 699 ret = amdxdna_gem_pin(heap); 700 if (ret) { 701 XDNA_ERR(xdna, "Dev heap pin failed, ret %d", ret); 702 goto put_heap; 703 } 704 705 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { 706 struct amdxdna_gem_obj *abo; 707 struct amdxdna_drm_create_bo args = { 708 .flags = 0, 709 .type = AMDXDNA_BO_DEV, 710 .vaddr = 0, 711 .size = MAX_CHAIN_CMDBUF_SIZE, 712 }; 713 714 abo = amdxdna_drm_create_dev_bo(&xdna->ddev, &args, client->filp); 715 if (IS_ERR(abo)) { 716 ret = PTR_ERR(abo); 717 goto free_cmd_bufs; 718 } 719 720 XDNA_DBG(xdna, "Command buf %d addr 0x%llx size 0x%lx", 721 i, amdxdna_gem_dev_addr(abo), abo->mem.size); 722 priv->cmd_buf[i] = abo; 723 } 724 725 sched = &priv->sched; 726 mutex_init(&priv->io_lock); 727 728 fs_reclaim_acquire(GFP_KERNEL); 729 might_lock(&priv->io_lock); 730 fs_reclaim_release(GFP_KERNEL); 731 732 ret = drm_sched_init(sched, &args); 733 if (ret) { 734 XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret); 735 goto free_cmd_bufs; 736 } 737 738 ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL, 739 &sched, 1, NULL); 740 if (ret) { 741 XDNA_ERR(xdna, "Failed to initial sched entiry. ret %d", ret); 742 goto free_sched; 743 } 744 745 ret = aie2_hwctx_col_list(hwctx); 746 if (ret) { 747 XDNA_ERR(xdna, "Create col list failed, ret %d", ret); 748 goto free_entity; 749 } 750 751 ret = amdxdna_pm_resume_get_locked(xdna); 752 if (ret) 753 goto free_col_list; 754 755 ret = aie2_alloc_resource(hwctx); 756 if (ret) { 757 XDNA_ERR(xdna, "Alloc hw resource failed, ret %d", ret); 758 goto suspend_put; 759 } 760 761 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 762 amdxdna_obj_dma_addr(heap), 763 heap->mem.size); 764 if (ret) { 765 XDNA_ERR(xdna, "Map host buffer failed, ret %d", ret); 766 goto release_resource; 767 } 768 769 ret = amdxdna_update_heap(client, hwctx); 770 if (ret) { 771 XDNA_ERR(xdna, "Update heap failed, ret %d", ret); 772 goto release_resource; 773 } 774 775 ret = aie2_ctx_syncobj_create(hwctx); 776 if (ret) { 777 XDNA_ERR(xdna, "Create syncobj failed, ret %d", ret); 778 goto release_resource; 779 } 780 amdxdna_pm_suspend_put(xdna); 781 782 init_waitqueue_head(&priv->job_free_wq); 783 784 XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); 785 786 return 0; 787 788 release_resource: 789 aie2_release_resource(hwctx); 790 suspend_put: 791 amdxdna_pm_suspend_put(xdna); 792 free_col_list: 793 kfree(hwctx->col_list); 794 free_entity: 795 drm_sched_entity_destroy(&priv->entity); 796 free_sched: 797 drm_sched_fini(&priv->sched); 798 free_cmd_bufs: 799 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { 800 if (!priv->cmd_buf[i]) 801 continue; 802 drm_gem_object_put(to_gobj(priv->cmd_buf[i])); 803 } 804 amdxdna_gem_unpin(heap); 805 put_heap: 806 drm_gem_object_put(to_gobj(heap)); 807 free_priv: 808 kfree(priv); 809 return ret; 810 } 811 812 void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx) 813 { 814 struct amdxdna_dev *xdna; 815 int idx; 816 817 xdna = hwctx->client->xdna; 818 819 XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq); 820 aie2_hwctx_wait_for_idle(hwctx); 821 822 /* Request fw to destroy hwctx and cancel the rest pending requests */ 823 drm_sched_stop(&hwctx->priv->sched, NULL); 824 aie2_release_resource(hwctx); 825 drm_sched_start(&hwctx->priv->sched, 0); 826 827 mutex_unlock(&xdna->dev_lock); 828 drm_sched_entity_destroy(&hwctx->priv->entity); 829 830 /* Wait for all submitted jobs to be completed or canceled */ 831 wait_event(hwctx->priv->job_free_wq, 832 atomic64_read(&hwctx->job_submit_cnt) == 833 atomic64_read(&hwctx->job_free_cnt)); 834 mutex_lock(&xdna->dev_lock); 835 836 drm_sched_fini(&hwctx->priv->sched); 837 aie2_ctx_syncobj_destroy(hwctx); 838 839 for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++) 840 drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx])); 841 amdxdna_gem_unpin(hwctx->priv->heap); 842 drm_gem_object_put(to_gobj(hwctx->priv->heap)); 843 844 mutex_destroy(&hwctx->priv->io_lock); 845 kfree(hwctx->col_list); 846 kfree(hwctx->priv); 847 kfree(hwctx->cus); 848 } 849 850 static int aie2_config_cu_resp_handler(void *handle, void __iomem *data, size_t size) 851 { 852 struct amdxdna_hwctx *hwctx = handle; 853 854 amdxdna_pm_suspend_put(hwctx->client->xdna); 855 return 0; 856 } 857 858 static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size) 859 { 860 struct amdxdna_hwctx_param_config_cu *config = buf; 861 struct amdxdna_dev *xdna = hwctx->client->xdna; 862 u32 total_size; 863 int ret; 864 865 XDNA_DBG(xdna, "Config %d CU to %s", config->num_cus, hwctx->name); 866 if (XDNA_MBZ_DBG(xdna, config->pad, sizeof(config->pad))) 867 return -EINVAL; 868 869 if (hwctx->cus) { 870 XDNA_ERR(xdna, "Not support re-config CU"); 871 return -EINVAL; 872 } 873 874 if (!config->num_cus) { 875 XDNA_ERR(xdna, "Number of CU is zero"); 876 return -EINVAL; 877 } 878 879 total_size = struct_size(config, cu_configs, config->num_cus); 880 if (total_size > size) { 881 XDNA_ERR(xdna, "CU config larger than size"); 882 return -EINVAL; 883 } 884 885 hwctx->cus = kmemdup(config, total_size, GFP_KERNEL); 886 if (!hwctx->cus) 887 return -ENOMEM; 888 889 ret = amdxdna_pm_resume_get(xdna); 890 if (ret) 891 goto free_cus; 892 893 ret = aie2_config_cu(hwctx, aie2_config_cu_resp_handler); 894 if (ret) { 895 XDNA_ERR(xdna, "Config CU to firmware failed, ret %d", ret); 896 goto pm_suspend_put; 897 } 898 899 wmb(); /* To avoid locking in command submit when check status */ 900 901 return 0; 902 903 pm_suspend_put: 904 amdxdna_pm_suspend_put(xdna); 905 free_cus: 906 kfree(hwctx->cus); 907 hwctx->cus = NULL; 908 return ret; 909 } 910 911 static void aie2_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq) 912 { 913 struct dma_fence *out_fence = aie2_cmd_get_out_fence(hwctx, seq); 914 struct amdxdna_dev *xdna = hwctx->client->xdna; 915 916 if (!out_fence) { 917 XDNA_ERR(xdna, "Failed to get fence"); 918 return; 919 } 920 921 mutex_unlock(&xdna->dev_lock); 922 dma_fence_wait_timeout(out_fence, false, MAX_SCHEDULE_TIMEOUT); 923 mutex_lock(&xdna->dev_lock); 924 dma_fence_put(out_fence); 925 } 926 927 static int aie2_hwctx_cfg_debug_bo(struct amdxdna_hwctx *hwctx, u32 bo_hdl, 928 bool attach) 929 { 930 struct amdxdna_client *client = hwctx->client; 931 struct amdxdna_dev *xdna = client->xdna; 932 struct amdxdna_drv_cmd *cmd; 933 struct amdxdna_gem_obj *abo; 934 u64 seq; 935 int ret; 936 937 abo = amdxdna_gem_get_obj(client, bo_hdl, AMDXDNA_BO_DEV); 938 if (!abo) { 939 XDNA_ERR(xdna, "Get bo %d failed", bo_hdl); 940 return -EINVAL; 941 } 942 943 cmd = kzalloc_obj(*cmd); 944 if (!cmd) { 945 ret = -ENOMEM; 946 goto put_obj; 947 } 948 kref_init(&cmd->refcnt); 949 950 if (attach) { 951 if (abo->assigned_hwctx != AMDXDNA_INVALID_CTX_HANDLE) { 952 ret = -EBUSY; 953 goto put_cmd; 954 } 955 cmd->opcode = ATTACH_DEBUG_BO; 956 } else { 957 if (abo->assigned_hwctx != hwctx->id) { 958 ret = -EINVAL; 959 goto put_cmd; 960 } 961 cmd->opcode = DETACH_DEBUG_BO; 962 } 963 964 ret = amdxdna_cmd_submit(client, cmd, AMDXDNA_INVALID_BO_HANDLE, 965 &bo_hdl, 1, hwctx->id, &seq); 966 if (ret) { 967 XDNA_ERR(xdna, "Submit command failed"); 968 goto put_cmd; 969 } 970 971 aie2_cmd_wait(hwctx, seq); 972 if (cmd->result) { 973 XDNA_ERR(xdna, "Response failure 0x%x", cmd->result); 974 ret = -EINVAL; 975 goto put_cmd; 976 } 977 978 if (attach) 979 abo->assigned_hwctx = hwctx->id; 980 else 981 abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE; 982 983 XDNA_DBG(xdna, "Config debug BO %d to %s", bo_hdl, hwctx->name); 984 985 put_cmd: 986 aie2_cmd_put(cmd); 987 put_obj: 988 amdxdna_gem_put_obj(abo); 989 return ret; 990 } 991 992 int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size) 993 { 994 struct amdxdna_dev *xdna = hwctx->client->xdna; 995 996 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 997 switch (type) { 998 case DRM_AMDXDNA_HWCTX_CONFIG_CU: 999 return aie2_hwctx_cu_config(hwctx, buf, size); 1000 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF: 1001 return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, true); 1002 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF: 1003 return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, false); 1004 default: 1005 XDNA_DBG(xdna, "Not supported type %d", type); 1006 return -EOPNOTSUPP; 1007 } 1008 } 1009 1010 int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl) 1011 { 1012 struct amdxdna_client *client = hwctx->client; 1013 struct amdxdna_dev *xdna = client->xdna; 1014 struct amdxdna_drv_cmd *cmd; 1015 u64 seq; 1016 int ret; 1017 1018 cmd = kzalloc_obj(*cmd); 1019 if (!cmd) 1020 return -ENOMEM; 1021 kref_init(&cmd->refcnt); 1022 1023 cmd->opcode = SYNC_DEBUG_BO; 1024 ret = amdxdna_cmd_submit(client, cmd, AMDXDNA_INVALID_BO_HANDLE, 1025 &debug_bo_hdl, 1, hwctx->id, &seq); 1026 if (ret) { 1027 XDNA_ERR(xdna, "Submit command failed"); 1028 goto put_cmd; 1029 } 1030 1031 aie2_cmd_wait(hwctx, seq); 1032 if (cmd->result) { 1033 XDNA_ERR(xdna, "Response failure 0x%x", cmd->result); 1034 ret = -EINVAL; 1035 } 1036 1037 put_cmd: 1038 aie2_cmd_put(cmd); 1039 return ret; 1040 } 1041 1042 static int aie2_populate_range(struct amdxdna_gem_obj *abo) 1043 { 1044 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 1045 struct amdxdna_umap *mapp; 1046 unsigned long timeout; 1047 struct mm_struct *mm; 1048 bool found; 1049 int ret; 1050 1051 timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1052 again: 1053 found = false; 1054 down_write(&xdna->notifier_lock); 1055 list_for_each_entry(mapp, &abo->mem.umap_list, node) { 1056 /* 1057 * Skip entries that have already been unmapped. 1058 * 1059 * If userspace unmaps the address and later submits I/O using 1060 * it, the IOMMU will reject the access and report a fault. 1061 * Ignore such entries here. 1062 */ 1063 if (mapp->unmapped) 1064 continue; 1065 1066 if (mapp->invalid && kref_get_unless_zero(&mapp->refcnt)) { 1067 found = true; 1068 break; 1069 } 1070 } 1071 1072 if (!found) { 1073 /* 1074 * This also covers the case where all mappings have been 1075 * removed. There are no invalid mappings left to process. 1076 * Any subsequent I/O using the unmapped address will be 1077 * rejected by the IOMMU. 1078 */ 1079 abo->mem.map_invalid = false; 1080 up_write(&xdna->notifier_lock); 1081 return 0; 1082 } 1083 1084 up_write(&xdna->notifier_lock); 1085 1086 mm = mapp->notifier.mm; 1087 if (!mmget_not_zero(mm)) { 1088 amdxdna_umap_put(mapp); 1089 return -EFAULT; 1090 } 1091 1092 ret = hmm_range_fault_unlocked_timeout(&mapp->range, timeout); 1093 if (ret) 1094 goto put_mm; 1095 1096 down_write(&xdna->notifier_lock); 1097 if (mmu_interval_read_retry(&mapp->notifier, mapp->range.notifier_seq)) { 1098 up_write(&xdna->notifier_lock); 1099 amdxdna_umap_put(mapp); 1100 mmput(mm); 1101 goto again; 1102 } 1103 mapp->invalid = false; 1104 up_write(&xdna->notifier_lock); 1105 amdxdna_umap_put(mapp); 1106 mmput(mm); 1107 goto again; 1108 1109 put_mm: 1110 amdxdna_umap_put(mapp); 1111 mmput(mm); 1112 return ret == -EBUSY ? -ETIME : ret; 1113 } 1114 1115 int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq) 1116 { 1117 struct amdxdna_dev *xdna = hwctx->client->xdna; 1118 struct ww_acquire_ctx acquire_ctx; 1119 struct dma_fence_chain *chain; 1120 struct amdxdna_gem_obj *abo; 1121 unsigned long timeout = 0; 1122 int ret, i; 1123 1124 ret = down_interruptible(&hwctx->priv->job_sem); 1125 if (ret) { 1126 XDNA_ERR(xdna, "Grab job sem failed, ret %d", ret); 1127 return ret; 1128 } 1129 1130 chain = dma_fence_chain_alloc(); 1131 if (!chain) { 1132 XDNA_ERR(xdna, "Alloc fence chain failed"); 1133 ret = -ENOMEM; 1134 goto up_sem; 1135 } 1136 1137 ret = drm_sched_job_init(&job->base, &hwctx->priv->entity, 1, hwctx, 1138 hwctx->client->filp->client_id); 1139 if (ret) { 1140 XDNA_ERR(xdna, "DRM job init failed, ret %d", ret); 1141 goto free_chain; 1142 } 1143 1144 retry: 1145 ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1146 if (ret) { 1147 XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret); 1148 goto cleanup_job; 1149 } 1150 1151 for (i = 0; i < job->bo_cnt; i++) { 1152 ret = dma_resv_reserve_fences(job->bos[i]->resv, 1); 1153 if (ret) { 1154 XDNA_WARN(xdna, "Failed to reserve fences %d", ret); 1155 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1156 goto cleanup_job; 1157 } 1158 } 1159 1160 down_read(&xdna->notifier_lock); 1161 for (i = 0; i < job->bo_cnt; i++) { 1162 abo = to_xdna_obj(job->bos[i]); 1163 if (abo->mem.map_invalid) { 1164 up_read(&xdna->notifier_lock); 1165 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1166 if (!timeout) { 1167 timeout = jiffies + 1168 msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1169 } else if (time_after(jiffies, timeout)) { 1170 ret = -ETIME; 1171 goto cleanup_job; 1172 } 1173 1174 ret = aie2_populate_range(abo); 1175 if (ret) 1176 goto cleanup_job; 1177 goto retry; 1178 } 1179 } 1180 1181 mutex_lock(&hwctx->priv->io_lock); 1182 drm_sched_job_arm(&job->base); 1183 job->out_fence = dma_fence_get(&job->base.s_fence->finished); 1184 for (i = 0; i < job->bo_cnt; i++) 1185 dma_resv_add_fence(job->bos[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE); 1186 job->seq = hwctx->priv->seq++; 1187 kref_get(&job->refcnt); 1188 if (job->drv_cmd) 1189 kref_get(&job->drv_cmd->refcnt); 1190 drm_sched_entity_push_job(&job->base); 1191 1192 *seq = job->seq; 1193 drm_syncobj_add_point(hwctx->priv->syncobj, chain, job->out_fence, *seq); 1194 mutex_unlock(&hwctx->priv->io_lock); 1195 1196 up_read(&xdna->notifier_lock); 1197 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1198 1199 aie2_job_put(job); 1200 atomic64_inc(&hwctx->job_submit_cnt); 1201 1202 return 0; 1203 1204 cleanup_job: 1205 drm_sched_job_cleanup(&job->base); 1206 free_chain: 1207 dma_fence_chain_free(chain); 1208 up_sem: 1209 up(&hwctx->priv->job_sem); 1210 job->job_done = true; 1211 return ret; 1212 } 1213 1214 void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, 1215 unsigned long cur_seq) 1216 { 1217 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 1218 struct drm_gem_object *gobj = to_gobj(abo); 1219 long ret; 1220 1221 ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP, 1222 true, MAX_SCHEDULE_TIMEOUT); 1223 if (!ret) 1224 XDNA_ERR(xdna, "Failed to wait for bo, ret %ld", ret); 1225 else if (ret == -ERESTARTSYS) 1226 XDNA_DBG(xdna, "Wait for bo interrupted by signal"); 1227 } 1228 1229 int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx, 1230 struct amdxdna_gem_obj *heap) 1231 { 1232 struct amdxdna_client *client = hwctx->client; 1233 struct amdxdna_dev *xdna = client->xdna; 1234 u64 addr; 1235 int ret; 1236 1237 addr = amdxdna_obj_dma_addr(heap); 1238 ret = aie2_add_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 1239 addr, heap->mem.size); 1240 if (ret) { 1241 XDNA_ERR(xdna, "Add heap failed hwctx %s 0x%lx ret %d", 1242 hwctx->name, heap->mem.size, ret); 1243 } 1244 1245 return ret; 1246 } 1247