1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_gem.h> 9 #include <drm/drm_gem_shmem_helper.h> 10 #include <drm/drm_print.h> 11 #include <drm/drm_syncobj.h> 12 #include <linux/hmm.h> 13 #include <linux/types.h> 14 #include <linux/xarray.h> 15 #include <trace/events/amdxdna.h> 16 17 #include "aie2_msg_priv.h" 18 #include "aie2_pci.h" 19 #include "aie2_solver.h" 20 #include "amdxdna_ctx.h" 21 #include "amdxdna_gem.h" 22 #include "amdxdna_mailbox.h" 23 #include "amdxdna_pci_drv.h" 24 #include "amdxdna_pm.h" 25 26 static bool force_cmdlist = true; 27 module_param(force_cmdlist, bool, 0600); 28 MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default true)"); 29 30 uint tdr_timeout_ms = 2000; 31 module_param(tdr_timeout_ms, int, 0400); 32 MODULE_PARM_DESC(tdr_timeout_ms, "TDR (Timeout Detection and Recovery) timeout in milliseconds (0 = disable)"); 33 34 struct aie2_ctx_health { 35 struct amdxdna_ctx_health header; 36 u32 txn_op_idx; 37 u32 ctx_pc; 38 u32 fatal_error_type; 39 u32 fatal_error_exception_type; 40 u32 fatal_error_exception_pc; 41 u32 fatal_error_app_module; 42 }; 43 44 static inline void aie2_tdr_signal(struct amdxdna_dev *xdna) 45 { 46 WRITE_ONCE(xdna->dev_handle->last_signal_ts, jiffies); 47 } 48 49 static bool aie2_tdr_detect(struct amdxdna_dev *xdna) 50 { 51 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 52 unsigned long last = READ_ONCE(ndev->last_signal_ts); 53 54 if (!tdr_timeout_ms) 55 return false; 56 57 if (!time_after(jiffies, last + msecs_to_jiffies(tdr_timeout_ms))) 58 return false; 59 60 XDNA_ERR(xdna, "TDR timeout detected"); 61 return true; 62 } 63 64 static void aie2_cmd_release(struct kref *ref) 65 { 66 struct amdxdna_drv_cmd *drv_cmd = container_of(ref, struct amdxdna_drv_cmd, refcnt); 67 68 kfree(drv_cmd); 69 } 70 71 static void aie2_cmd_put(struct amdxdna_drv_cmd *drv_cmd) 72 { 73 kref_put(&drv_cmd->refcnt, aie2_cmd_release); 74 } 75 76 static void aie2_job_release(struct kref *ref) 77 { 78 struct amdxdna_sched_job *job; 79 80 job = container_of(ref, struct amdxdna_sched_job, refcnt); 81 82 amdxdna_sched_job_cleanup(job); 83 atomic64_inc(&job->hwctx->job_free_cnt); 84 wake_up(&job->hwctx->priv->job_free_wq); 85 if (job->out_fence) 86 dma_fence_put(job->out_fence); 87 if (job->drv_cmd) 88 aie2_cmd_put(job->drv_cmd); 89 kfree(job->aie2_job_health); 90 kfree(job); 91 } 92 93 static void aie2_job_put(struct amdxdna_sched_job *job) 94 { 95 kref_put(&job->refcnt, aie2_job_release); 96 } 97 98 /* The bad_job is used in aie2_sched_job_timedout, otherwise, set it to NULL */ 99 static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx, 100 struct drm_sched_job *bad_job) 101 { 102 drm_sched_stop(&hwctx->priv->sched, bad_job); 103 aie2_destroy_context(xdna->dev_handle, hwctx); 104 drm_sched_start(&hwctx->priv->sched, 0); 105 } 106 107 static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) 108 { 109 struct amdxdna_gem_obj *heap = hwctx->priv->heap; 110 unsigned long heap_id; 111 int ret; 112 113 ret = aie2_create_context(xdna->dev_handle, hwctx); 114 if (ret) { 115 XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret); 116 goto out; 117 } 118 119 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 120 amdxdna_obj_dma_addr(heap), 121 heap->mem.size); 122 if (ret) { 123 XDNA_ERR(xdna, "Map host buf failed, ret %d", ret); 124 goto out; 125 } 126 127 xa_for_each_range(&hwctx->client->dev_heap_xa, heap_id, heap, 1, 128 hwctx->last_attached_heap) { 129 ret = aie2_add_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 130 amdxdna_obj_dma_addr(heap), 131 heap->mem.size); 132 if (ret) { 133 XDNA_ERR(xdna, "Add heap %ld failed ret %d", heap_id, ret); 134 goto out; 135 } 136 } 137 138 ret = aie2_config_cu(hwctx, NULL); 139 if (ret) { 140 XDNA_ERR(xdna, "Config cu failed, ret %d", ret); 141 goto out; 142 } 143 144 out: 145 XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret); 146 return ret; 147 } 148 149 static struct dma_fence *aie2_cmd_get_out_fence(struct amdxdna_hwctx *hwctx, u64 seq) 150 { 151 struct dma_fence *fence, *out_fence = NULL; 152 int ret; 153 154 fence = drm_syncobj_fence_get(hwctx->priv->syncobj); 155 if (!fence) 156 return NULL; 157 158 ret = dma_fence_chain_find_seqno(&fence, seq); 159 if (ret) 160 goto out; 161 162 out_fence = dma_fence_get(dma_fence_chain_contained(fence)); 163 164 out: 165 dma_fence_put(fence); 166 return out_fence; 167 } 168 169 static void aie2_hwctx_wait_for_idle(struct amdxdna_hwctx *hwctx) 170 { 171 struct dma_fence *fence; 172 173 fence = aie2_cmd_get_out_fence(hwctx, hwctx->priv->seq - 1); 174 if (!fence) 175 return; 176 177 /* Wait up to 2 seconds for fw to finish all pending requests */ 178 dma_fence_wait_timeout(fence, false, msecs_to_jiffies(2000)); 179 dma_fence_put(fence); 180 } 181 182 static int aie2_hwctx_suspend_cb(struct amdxdna_hwctx *hwctx, void *arg) 183 { 184 struct amdxdna_dev *xdna = hwctx->client->xdna; 185 186 aie2_hwctx_wait_for_idle(hwctx); 187 aie2_hwctx_stop(xdna, hwctx, NULL); 188 189 return 0; 190 } 191 192 void aie2_hwctx_suspend(struct amdxdna_client *client) 193 { 194 struct amdxdna_dev *xdna = client->xdna; 195 196 /* 197 * Command timeout is unlikely. But if it happens, it doesn't 198 * break the system. aie2_hwctx_stop() will destroy mailbox 199 * and abort all commands. 200 */ 201 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 202 amdxdna_hwctx_walk(client, NULL, aie2_hwctx_suspend_cb); 203 } 204 205 static int aie2_hwctx_resume_cb(struct amdxdna_hwctx *hwctx, void *arg) 206 { 207 struct amdxdna_dev *xdna = hwctx->client->xdna; 208 209 return aie2_hwctx_restart(xdna, hwctx); 210 } 211 212 int aie2_hwctx_resume(struct amdxdna_client *client) 213 { 214 /* 215 * The resume path cannot guarantee that mailbox channel can be 216 * regenerated. If this happen, when submit message to this 217 * mailbox channel, error will return. 218 */ 219 return amdxdna_hwctx_walk(client, NULL, aie2_hwctx_resume_cb); 220 } 221 222 static void 223 aie2_sched_notify(struct amdxdna_sched_job *job) 224 { 225 struct dma_fence *fence = job->fence; 226 227 trace_xdna_job(&job->base, job->hwctx->name, "signaling fence", 228 job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); 229 230 aie2_tdr_signal(job->hwctx->client->xdna); 231 job->hwctx->priv->completed++; 232 dma_fence_signal(fence); 233 234 up(&job->hwctx->priv->job_sem); 235 job->job_done = true; 236 mmput_async(job->mm); 237 aie2_job_put(job); 238 } 239 240 static void aie2_set_cmd_timeout(struct amdxdna_sched_job *job) 241 { 242 struct aie2_ctx_health *aie2_health __free(kfree) = NULL; 243 struct amdxdna_dev *xdna = job->hwctx->client->xdna; 244 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 245 struct app_health_report *report = job->aie2_job_health; 246 u32 fail_cmd_idx = 0; 247 248 if (!report) 249 goto set_timeout; 250 251 XDNA_ERR(xdna, "Firmware timeout state capture:"); 252 XDNA_ERR(xdna, "\tVersion: %d.%d", report->major, report->minor); 253 XDNA_ERR(xdna, "\tReport size: 0x%x", report->size); 254 XDNA_ERR(xdna, "\tContext ID: %d", report->context_id); 255 XDNA_ERR(xdna, "\tDPU PC: 0x%x", report->dpu_pc); 256 XDNA_ERR(xdna, "\tTXN OP ID: 0x%x", report->txn_op_id); 257 XDNA_ERR(xdna, "\tContext PC: 0x%x", report->ctx_pc); 258 XDNA_ERR(xdna, "\tFatal error type: 0x%x", report->fatal_info.fatal_type); 259 XDNA_ERR(xdna, "\tFatal error exception type: 0x%x", report->fatal_info.exception_type); 260 XDNA_ERR(xdna, "\tFatal error exception PC: 0x%x", report->fatal_info.exception_pc); 261 XDNA_ERR(xdna, "\tFatal error app module: 0x%x", report->fatal_info.app_module); 262 XDNA_ERR(xdna, "\tFatal error task ID: %d", report->fatal_info.task_index); 263 XDNA_ERR(xdna, "\tTimed out sub command ID: %d", report->run_list_id); 264 265 fail_cmd_idx = report->run_list_id; 266 aie2_health = kzalloc_obj(*aie2_health); 267 if (!aie2_health) 268 goto set_timeout; 269 270 aie2_health->header.version = AMDXDNA_CMD_CTX_HEALTH_V1; 271 aie2_health->header.npu_gen = AMDXDNA_CMD_CTX_HEALTH_AIE2; 272 aie2_health->txn_op_idx = report->txn_op_id; 273 aie2_health->ctx_pc = report->ctx_pc; 274 aie2_health->fatal_error_type = report->fatal_info.fatal_type; 275 aie2_health->fatal_error_exception_type = report->fatal_info.exception_type; 276 aie2_health->fatal_error_exception_pc = report->fatal_info.exception_pc; 277 aie2_health->fatal_error_app_module = report->fatal_info.app_module; 278 279 set_timeout: 280 amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_TIMEOUT, 281 aie2_health, sizeof(*aie2_health)); 282 } 283 284 static int 285 aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size) 286 { 287 struct amdxdna_sched_job *job = handle; 288 struct amdxdna_gem_obj *cmd_abo; 289 int ret = 0; 290 u32 status; 291 292 cmd_abo = job->cmd_bo; 293 294 if (unlikely(job->job_timeout)) { 295 aie2_set_cmd_timeout(job); 296 ret = -EINVAL; 297 goto out; 298 } 299 300 if (unlikely(!data) || unlikely(size != sizeof(u32))) { 301 amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT, NULL, 0); 302 ret = -EINVAL; 303 goto out; 304 } 305 306 status = readl(data); 307 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); 308 if (status == AIE2_STATUS_SUCCESS) 309 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); 310 else 311 amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ERROR, NULL, 0); 312 313 out: 314 aie2_sched_notify(job); 315 return ret; 316 } 317 318 static int 319 aie2_sched_drvcmd_resp_handler(void *handle, void __iomem *data, size_t size) 320 { 321 struct amdxdna_sched_job *job = handle; 322 int ret = 0; 323 324 if (unlikely(!data || size != sizeof(u32))) { 325 job->drv_cmd->result = U32_MAX; 326 ret = -EINVAL; 327 } else { 328 job->drv_cmd->result = readl(data); 329 } 330 331 aie2_sched_notify(job); 332 return ret; 333 } 334 335 static int 336 aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) 337 { 338 struct amdxdna_sched_job *job = handle; 339 struct amdxdna_gem_obj *cmd_abo; 340 struct amdxdna_dev *xdna; 341 u32 fail_cmd_idx = 0; 342 u32 fail_cmd_status; 343 u32 cmd_status; 344 int ret = 0; 345 346 cmd_abo = job->cmd_bo; 347 348 if (unlikely(job->job_timeout)) { 349 aie2_set_cmd_timeout(job); 350 ret = -EINVAL; 351 goto out; 352 } 353 354 if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) { 355 amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT, NULL, 0); 356 ret = -EINVAL; 357 goto out; 358 } 359 360 cmd_status = readl(data + offsetof(struct cmd_chain_resp, status)); 361 xdna = job->hwctx->client->xdna; 362 XDNA_DBG(xdna, "Status 0x%x", cmd_status); 363 if (cmd_status == AIE2_STATUS_SUCCESS) { 364 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); 365 goto out; 366 } 367 368 /* Slow path to handle error, read from ringbuf on BAR */ 369 fail_cmd_idx = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_idx)); 370 fail_cmd_status = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_status)); 371 XDNA_DBG(xdna, "Failed cmd idx %d, status 0x%x", 372 fail_cmd_idx, fail_cmd_status); 373 374 if (fail_cmd_status == AIE2_STATUS_SUCCESS) { 375 amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ABORT, NULL, 0); 376 ret = -EINVAL; 377 } else { 378 amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ERROR, NULL, 0); 379 } 380 381 out: 382 aie2_sched_notify(job); 383 return ret; 384 } 385 386 static struct dma_fence * 387 aie2_sched_job_run(struct drm_sched_job *sched_job) 388 { 389 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 390 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 391 struct amdxdna_hwctx *hwctx = job->hwctx; 392 struct dma_fence *fence; 393 int ret; 394 395 trace_xdna_job(sched_job, hwctx->name, "job run", 396 job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); 397 398 if (!hwctx->priv->mbox_chann) 399 return NULL; 400 401 if (!mmget_not_zero(job->mm)) 402 return ERR_PTR(-ESRCH); 403 404 kref_get(&job->refcnt); 405 fence = dma_fence_get(job->fence); 406 407 if (job->drv_cmd) { 408 switch (job->drv_cmd->opcode) { 409 case SYNC_DEBUG_BO: 410 ret = aie2_sync_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); 411 break; 412 case ATTACH_DEBUG_BO: 413 case DETACH_DEBUG_BO: 414 ret = aie2_config_debug_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); 415 break; 416 default: 417 ret = -EINVAL; 418 break; 419 } 420 goto out; 421 } 422 423 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_NEW); 424 425 if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) 426 ret = aie2_cmdlist_multi_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); 427 else if (force_cmdlist) 428 ret = aie2_cmdlist_single_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); 429 else 430 ret = aie2_execbuf(hwctx, job, aie2_sched_resp_handler); 431 432 out: 433 if (ret) { 434 dma_fence_put(job->fence); 435 aie2_job_put(job); 436 mmput(job->mm); 437 fence = ERR_PTR(ret); 438 } else { 439 /* 440 * Command is successfully posted to hardware, update the 441 * tdr timestamp. The total pending commands are limited. 442 * So there will not be a case that driver keeps posting 443 * commands without getting any hardware respond. 444 */ 445 aie2_tdr_signal(hwctx->client->xdna); 446 } 447 trace_xdna_job(sched_job, hwctx->name, "sent to device", 448 job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); 449 450 return fence; 451 } 452 453 static void aie2_sched_job_free(struct drm_sched_job *sched_job) 454 { 455 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 456 struct amdxdna_hwctx *hwctx = job->hwctx; 457 458 /* job->drv_cmd could be freed, so use DEFAULT_IO */ 459 trace_xdna_job(sched_job, hwctx->name, "job free", 460 job->seq, DEFAULT_IO); 461 if (!job->job_done) 462 up(&hwctx->priv->job_sem); 463 464 drm_sched_job_cleanup(sched_job); 465 aie2_job_put(job); 466 } 467 468 static enum drm_gpu_sched_stat 469 aie2_sched_job_timedout(struct drm_sched_job *sched_job) 470 { 471 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 472 struct amdxdna_hwctx *hwctx = job->hwctx; 473 struct app_health_report *report; 474 struct amdxdna_dev *xdna; 475 int ret; 476 477 xdna = hwctx->client->xdna; 478 479 guard(mutex)(&xdna->dev_lock); 480 481 if (!aie2_tdr_detect(xdna)) 482 return DRM_GPU_SCHED_STAT_NO_HANG; 483 484 report = kzalloc_obj(*report); 485 if (!report) 486 goto reset_hwctx; 487 488 ret = aie2_query_app_health(xdna->dev_handle, hwctx->fw_ctx_id, report); 489 if (ret) 490 kfree(report); 491 else 492 job->aie2_job_health = report; 493 494 reset_hwctx: 495 job->job_timeout = true; 496 aie2_hwctx_stop(xdna, hwctx, sched_job); 497 498 aie2_hwctx_restart(xdna, hwctx); 499 500 return DRM_GPU_SCHED_STAT_RESET; 501 } 502 503 static const struct drm_sched_backend_ops sched_ops = { 504 .run_job = aie2_sched_job_run, 505 .free_job = aie2_sched_job_free, 506 .timedout_job = aie2_sched_job_timedout, 507 }; 508 509 static int aie2_hwctx_col_list(struct amdxdna_hwctx *hwctx) 510 { 511 struct amdxdna_dev *xdna = hwctx->client->xdna; 512 struct amdxdna_dev_hdl *ndev; 513 int start, end, first, last; 514 u32 width = 1, entries = 0; 515 int i; 516 517 if (!hwctx->num_tiles) { 518 XDNA_ERR(xdna, "Number of tiles is zero"); 519 return -EINVAL; 520 } 521 522 ndev = xdna->dev_handle; 523 if (unlikely(!ndev->aie.metadata.core.row_count)) { 524 XDNA_WARN(xdna, "Core tile row count is zero"); 525 return -EINVAL; 526 } 527 528 hwctx->num_col = hwctx->num_tiles / ndev->aie.metadata.core.row_count; 529 if (!hwctx->num_col || hwctx->num_col > ndev->total_col) { 530 XDNA_ERR(xdna, "Invalid num_col %d", hwctx->num_col); 531 return -EINVAL; 532 } 533 534 if (ndev->priv->col_align == COL_ALIGN_NATURE) 535 width = hwctx->num_col; 536 537 /* 538 * In range [start, end], find out columns that is multiple of width. 539 * 'first' is the first column, 540 * 'last' is the last column, 541 * 'entries' is the total number of columns. 542 */ 543 start = xdna->dev_info->first_col; 544 end = ndev->total_col - hwctx->num_col; 545 if (start > 0 && end == 0) { 546 XDNA_DBG(xdna, "Force start from col 0"); 547 start = 0; 548 } 549 first = start + (width - start % width) % width; 550 last = end - end % width; 551 if (last >= first) 552 entries = (last - first) / width + 1; 553 XDNA_DBG(xdna, "start %d end %d first %d last %d", 554 start, end, first, last); 555 556 if (unlikely(!entries)) { 557 XDNA_ERR(xdna, "Start %d end %d width %d", 558 start, end, width); 559 return -EINVAL; 560 } 561 562 hwctx->col_list = kmalloc_array(entries, sizeof(*hwctx->col_list), GFP_KERNEL); 563 if (!hwctx->col_list) 564 return -ENOMEM; 565 566 hwctx->col_list_len = entries; 567 hwctx->col_list[0] = first; 568 for (i = 1; i < entries; i++) 569 hwctx->col_list[i] = hwctx->col_list[i - 1] + width; 570 571 print_hex_dump_debug("col_list: ", DUMP_PREFIX_OFFSET, 16, 4, hwctx->col_list, 572 entries * sizeof(*hwctx->col_list), false); 573 return 0; 574 } 575 576 static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx) 577 { 578 struct amdxdna_dev *xdna = hwctx->client->xdna; 579 struct alloc_requests *xrs_req; 580 u32 temporal_only_col = 0; 581 int ret; 582 583 xrs_req = kzalloc_obj(*xrs_req); 584 if (!xrs_req) 585 return -ENOMEM; 586 587 if (AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_TEMPORAL_ONLY)) { 588 xrs_req->cdo.start_cols = &temporal_only_col; 589 xrs_req->cdo.cols_len = 1; 590 xrs_req->cdo.ncols = xdna->dev_handle->total_col; 591 } else { 592 xrs_req->cdo.start_cols = hwctx->col_list; 593 xrs_req->cdo.cols_len = hwctx->col_list_len; 594 xrs_req->cdo.ncols = hwctx->num_col; 595 } 596 /* Use platform opc */ 597 xrs_req->cdo.qos_cap.opc = xdna->dev_handle->priv->col_opc * hwctx->num_col; 598 599 xrs_req->rqos.gops = hwctx->qos.gops; 600 xrs_req->rqos.fps = hwctx->qos.fps; 601 xrs_req->rqos.dma_bw = hwctx->qos.dma_bandwidth; 602 xrs_req->rqos.latency = hwctx->qos.latency; 603 xrs_req->rqos.exec_time = hwctx->qos.frame_exec_time; 604 xrs_req->rqos.priority = hwctx->qos.priority; 605 606 xrs_req->rid = (uintptr_t)hwctx; 607 608 ret = xrs_allocate_resource(xdna->xrs_hdl, xrs_req, hwctx); 609 if (ret) 610 XDNA_ERR(xdna, "Allocate AIE resource failed, ret %d", ret); 611 612 kfree(xrs_req); 613 return ret; 614 } 615 616 static void aie2_release_resource(struct amdxdna_hwctx *hwctx) 617 { 618 struct amdxdna_dev *xdna = hwctx->client->xdna; 619 int ret; 620 621 ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); 622 if (ret) 623 XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret); 624 } 625 626 static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx) 627 { 628 struct amdxdna_dev *xdna = hwctx->client->xdna; 629 struct drm_file *filp = hwctx->client->filp; 630 struct drm_syncobj *syncobj; 631 u32 hdl; 632 int ret; 633 634 hwctx->syncobj_hdl = AMDXDNA_INVALID_FENCE_HANDLE; 635 636 ret = drm_syncobj_create(&syncobj, 0, NULL); 637 if (ret) { 638 XDNA_ERR(xdna, "Create ctx syncobj failed, ret %d", ret); 639 return ret; 640 } 641 ret = drm_syncobj_get_handle(filp, syncobj, &hdl); 642 if (ret) { 643 drm_syncobj_put(syncobj); 644 XDNA_ERR(xdna, "Create ctx syncobj handle failed, ret %d", ret); 645 return ret; 646 } 647 hwctx->priv->syncobj = syncobj; 648 hwctx->syncobj_hdl = hdl; 649 650 return 0; 651 } 652 653 static void aie2_ctx_syncobj_destroy(struct amdxdna_hwctx *hwctx) 654 { 655 /* 656 * The syncobj_hdl is owned by user space and will be cleaned up 657 * separately. 658 */ 659 drm_syncobj_put(hwctx->priv->syncobj); 660 } 661 662 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) 663 { 664 struct amdxdna_client *client = hwctx->client; 665 struct amdxdna_dev *xdna = client->xdna; 666 const struct drm_sched_init_args args = { 667 .ops = &sched_ops, 668 .credit_limit = HWCTX_MAX_CMDS, 669 .timeout = tdr_timeout_ms ? 670 msecs_to_jiffies(tdr_timeout_ms) : 671 MAX_SCHEDULE_TIMEOUT, 672 .name = "amdxdna_js", 673 .dev = xdna->ddev.dev, 674 }; 675 struct drm_gpu_scheduler *sched; 676 struct amdxdna_hwctx_priv *priv; 677 struct amdxdna_gem_obj *heap; 678 int i, ret; 679 680 priv = kzalloc_obj(*hwctx->priv); 681 if (!priv) 682 return -ENOMEM; 683 hwctx->priv = priv; 684 685 mutex_lock(&client->mm_lock); 686 heap = xa_load(&client->dev_heap_xa, 0); 687 if (!heap) { 688 XDNA_ERR(xdna, "The client dev heap object not exist"); 689 mutex_unlock(&client->mm_lock); 690 ret = -ENOENT; 691 goto free_priv; 692 } 693 drm_gem_object_get(to_gobj(heap)); 694 mutex_unlock(&client->mm_lock); 695 priv->heap = heap; 696 sema_init(&priv->job_sem, HWCTX_MAX_CMDS); 697 698 ret = amdxdna_gem_pin(heap); 699 if (ret) { 700 XDNA_ERR(xdna, "Dev heap pin failed, ret %d", ret); 701 goto put_heap; 702 } 703 704 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { 705 struct amdxdna_gem_obj *abo; 706 struct amdxdna_drm_create_bo args = { 707 .flags = 0, 708 .type = AMDXDNA_BO_DEV, 709 .vaddr = 0, 710 .size = MAX_CHAIN_CMDBUF_SIZE, 711 }; 712 713 abo = amdxdna_drm_create_dev_bo(&xdna->ddev, &args, client->filp); 714 if (IS_ERR(abo)) { 715 ret = PTR_ERR(abo); 716 goto free_cmd_bufs; 717 } 718 719 XDNA_DBG(xdna, "Command buf %d addr 0x%llx size 0x%lx", 720 i, amdxdna_gem_dev_addr(abo), abo->mem.size); 721 priv->cmd_buf[i] = abo; 722 } 723 724 sched = &priv->sched; 725 mutex_init(&priv->io_lock); 726 727 fs_reclaim_acquire(GFP_KERNEL); 728 might_lock(&priv->io_lock); 729 fs_reclaim_release(GFP_KERNEL); 730 731 ret = drm_sched_init(sched, &args); 732 if (ret) { 733 XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret); 734 goto free_cmd_bufs; 735 } 736 737 ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL, 738 &sched, 1, NULL); 739 if (ret) { 740 XDNA_ERR(xdna, "Failed to initial sched entiry. ret %d", ret); 741 goto free_sched; 742 } 743 744 ret = aie2_hwctx_col_list(hwctx); 745 if (ret) { 746 XDNA_ERR(xdna, "Create col list failed, ret %d", ret); 747 goto free_entity; 748 } 749 750 ret = amdxdna_pm_resume_get_locked(xdna); 751 if (ret) 752 goto free_col_list; 753 754 ret = aie2_alloc_resource(hwctx); 755 if (ret) { 756 XDNA_ERR(xdna, "Alloc hw resource failed, ret %d", ret); 757 goto suspend_put; 758 } 759 760 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 761 amdxdna_obj_dma_addr(heap), 762 heap->mem.size); 763 if (ret) { 764 XDNA_ERR(xdna, "Map host buffer failed, ret %d", ret); 765 goto release_resource; 766 } 767 768 ret = amdxdna_update_heap(client, hwctx); 769 if (ret) { 770 XDNA_ERR(xdna, "Update heap failed, ret %d", ret); 771 goto release_resource; 772 } 773 774 ret = aie2_ctx_syncobj_create(hwctx); 775 if (ret) { 776 XDNA_ERR(xdna, "Create syncobj failed, ret %d", ret); 777 goto release_resource; 778 } 779 amdxdna_pm_suspend_put(xdna); 780 781 init_waitqueue_head(&priv->job_free_wq); 782 783 XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); 784 785 return 0; 786 787 release_resource: 788 aie2_release_resource(hwctx); 789 suspend_put: 790 amdxdna_pm_suspend_put(xdna); 791 free_col_list: 792 kfree(hwctx->col_list); 793 free_entity: 794 drm_sched_entity_destroy(&priv->entity); 795 free_sched: 796 drm_sched_fini(&priv->sched); 797 free_cmd_bufs: 798 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { 799 if (!priv->cmd_buf[i]) 800 continue; 801 drm_gem_object_put(to_gobj(priv->cmd_buf[i])); 802 } 803 amdxdna_gem_unpin(heap); 804 put_heap: 805 drm_gem_object_put(to_gobj(heap)); 806 free_priv: 807 kfree(priv); 808 return ret; 809 } 810 811 void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx) 812 { 813 struct amdxdna_dev *xdna; 814 int idx; 815 816 xdna = hwctx->client->xdna; 817 818 XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq); 819 aie2_hwctx_wait_for_idle(hwctx); 820 821 /* Request fw to destroy hwctx and cancel the rest pending requests */ 822 drm_sched_stop(&hwctx->priv->sched, NULL); 823 aie2_release_resource(hwctx); 824 drm_sched_start(&hwctx->priv->sched, 0); 825 826 mutex_unlock(&xdna->dev_lock); 827 drm_sched_entity_destroy(&hwctx->priv->entity); 828 829 /* Wait for all submitted jobs to be completed or canceled */ 830 wait_event(hwctx->priv->job_free_wq, 831 atomic64_read(&hwctx->job_submit_cnt) == 832 atomic64_read(&hwctx->job_free_cnt)); 833 mutex_lock(&xdna->dev_lock); 834 835 drm_sched_fini(&hwctx->priv->sched); 836 aie2_ctx_syncobj_destroy(hwctx); 837 838 for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++) 839 drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx])); 840 amdxdna_gem_unpin(hwctx->priv->heap); 841 drm_gem_object_put(to_gobj(hwctx->priv->heap)); 842 843 mutex_destroy(&hwctx->priv->io_lock); 844 kfree(hwctx->col_list); 845 kfree(hwctx->priv); 846 kfree(hwctx->cus); 847 } 848 849 static int aie2_config_cu_resp_handler(void *handle, void __iomem *data, size_t size) 850 { 851 struct amdxdna_hwctx *hwctx = handle; 852 853 amdxdna_pm_suspend_put(hwctx->client->xdna); 854 return 0; 855 } 856 857 static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size) 858 { 859 struct amdxdna_hwctx_param_config_cu *config = buf; 860 struct amdxdna_dev *xdna = hwctx->client->xdna; 861 u32 total_size; 862 int ret; 863 864 XDNA_DBG(xdna, "Config %d CU to %s", config->num_cus, hwctx->name); 865 if (XDNA_MBZ_DBG(xdna, config->pad, sizeof(config->pad))) 866 return -EINVAL; 867 868 if (hwctx->cus) { 869 XDNA_ERR(xdna, "Not support re-config CU"); 870 return -EINVAL; 871 } 872 873 if (!config->num_cus) { 874 XDNA_ERR(xdna, "Number of CU is zero"); 875 return -EINVAL; 876 } 877 878 total_size = struct_size(config, cu_configs, config->num_cus); 879 if (total_size > size) { 880 XDNA_ERR(xdna, "CU config larger than size"); 881 return -EINVAL; 882 } 883 884 hwctx->cus = kmemdup(config, total_size, GFP_KERNEL); 885 if (!hwctx->cus) 886 return -ENOMEM; 887 888 ret = amdxdna_pm_resume_get(xdna); 889 if (ret) 890 goto free_cus; 891 892 ret = aie2_config_cu(hwctx, aie2_config_cu_resp_handler); 893 if (ret) { 894 XDNA_ERR(xdna, "Config CU to firmware failed, ret %d", ret); 895 goto pm_suspend_put; 896 } 897 898 wmb(); /* To avoid locking in command submit when check status */ 899 900 return 0; 901 902 pm_suspend_put: 903 amdxdna_pm_suspend_put(xdna); 904 free_cus: 905 kfree(hwctx->cus); 906 hwctx->cus = NULL; 907 return ret; 908 } 909 910 static void aie2_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq) 911 { 912 struct dma_fence *out_fence = aie2_cmd_get_out_fence(hwctx, seq); 913 struct amdxdna_dev *xdna = hwctx->client->xdna; 914 915 if (!out_fence) { 916 XDNA_ERR(xdna, "Failed to get fence"); 917 return; 918 } 919 920 mutex_unlock(&xdna->dev_lock); 921 dma_fence_wait_timeout(out_fence, false, MAX_SCHEDULE_TIMEOUT); 922 mutex_lock(&xdna->dev_lock); 923 dma_fence_put(out_fence); 924 } 925 926 static int aie2_hwctx_cfg_debug_bo(struct amdxdna_hwctx *hwctx, u32 bo_hdl, 927 bool attach) 928 { 929 struct amdxdna_client *client = hwctx->client; 930 struct amdxdna_dev *xdna = client->xdna; 931 struct amdxdna_drv_cmd *cmd; 932 struct amdxdna_gem_obj *abo; 933 u64 seq; 934 int ret; 935 936 abo = amdxdna_gem_get_obj(client, bo_hdl, AMDXDNA_BO_DEV); 937 if (!abo) { 938 XDNA_ERR(xdna, "Get bo %d failed", bo_hdl); 939 return -EINVAL; 940 } 941 942 cmd = kzalloc_obj(*cmd); 943 if (!cmd) { 944 ret = -ENOMEM; 945 goto put_obj; 946 } 947 kref_init(&cmd->refcnt); 948 949 if (attach) { 950 if (abo->assigned_hwctx != AMDXDNA_INVALID_CTX_HANDLE) { 951 ret = -EBUSY; 952 goto put_cmd; 953 } 954 cmd->opcode = ATTACH_DEBUG_BO; 955 } else { 956 if (abo->assigned_hwctx != hwctx->id) { 957 ret = -EINVAL; 958 goto put_cmd; 959 } 960 cmd->opcode = DETACH_DEBUG_BO; 961 } 962 963 ret = amdxdna_cmd_submit(client, cmd, AMDXDNA_INVALID_BO_HANDLE, 964 &bo_hdl, 1, hwctx->id, &seq); 965 if (ret) { 966 XDNA_ERR(xdna, "Submit command failed"); 967 goto put_cmd; 968 } 969 970 aie2_cmd_wait(hwctx, seq); 971 if (cmd->result) { 972 XDNA_ERR(xdna, "Response failure 0x%x", cmd->result); 973 ret = -EINVAL; 974 goto put_cmd; 975 } 976 977 if (attach) 978 abo->assigned_hwctx = hwctx->id; 979 else 980 abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE; 981 982 XDNA_DBG(xdna, "Config debug BO %d to %s", bo_hdl, hwctx->name); 983 984 put_cmd: 985 aie2_cmd_put(cmd); 986 put_obj: 987 amdxdna_gem_put_obj(abo); 988 return ret; 989 } 990 991 int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size) 992 { 993 struct amdxdna_dev *xdna = hwctx->client->xdna; 994 995 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 996 switch (type) { 997 case DRM_AMDXDNA_HWCTX_CONFIG_CU: 998 return aie2_hwctx_cu_config(hwctx, buf, size); 999 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF: 1000 return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, true); 1001 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF: 1002 return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, false); 1003 default: 1004 XDNA_DBG(xdna, "Not supported type %d", type); 1005 return -EOPNOTSUPP; 1006 } 1007 } 1008 1009 int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl) 1010 { 1011 struct amdxdna_client *client = hwctx->client; 1012 struct amdxdna_dev *xdna = client->xdna; 1013 struct amdxdna_drv_cmd *cmd; 1014 u64 seq; 1015 int ret; 1016 1017 cmd = kzalloc_obj(*cmd); 1018 if (!cmd) 1019 return -ENOMEM; 1020 kref_init(&cmd->refcnt); 1021 1022 cmd->opcode = SYNC_DEBUG_BO; 1023 ret = amdxdna_cmd_submit(client, cmd, AMDXDNA_INVALID_BO_HANDLE, 1024 &debug_bo_hdl, 1, hwctx->id, &seq); 1025 if (ret) { 1026 XDNA_ERR(xdna, "Submit command failed"); 1027 goto put_cmd; 1028 } 1029 1030 aie2_cmd_wait(hwctx, seq); 1031 if (cmd->result) { 1032 XDNA_ERR(xdna, "Response failure 0x%x", cmd->result); 1033 ret = -EINVAL; 1034 } 1035 1036 put_cmd: 1037 aie2_cmd_put(cmd); 1038 return ret; 1039 } 1040 1041 static int aie2_populate_range(struct amdxdna_gem_obj *abo) 1042 { 1043 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 1044 struct amdxdna_umap *mapp; 1045 unsigned long timeout; 1046 struct mm_struct *mm; 1047 bool found; 1048 int ret; 1049 1050 timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1051 again: 1052 found = false; 1053 down_write(&xdna->notifier_lock); 1054 list_for_each_entry(mapp, &abo->mem.umap_list, node) { 1055 if (mapp->invalid && kref_get_unless_zero(&mapp->refcnt)) { 1056 found = true; 1057 break; 1058 } 1059 } 1060 1061 if (!found) { 1062 abo->mem.map_invalid = false; 1063 up_write(&xdna->notifier_lock); 1064 return 0; 1065 } 1066 1067 up_write(&xdna->notifier_lock); 1068 1069 mm = mapp->notifier.mm; 1070 if (!mmget_not_zero(mm)) { 1071 amdxdna_umap_put(mapp); 1072 return -EFAULT; 1073 } 1074 1075 mapp->range.notifier_seq = mmu_interval_read_begin(&mapp->notifier); 1076 mmap_read_lock(mm); 1077 ret = hmm_range_fault(&mapp->range); 1078 mmap_read_unlock(mm); 1079 if (ret) { 1080 if (time_after(jiffies, timeout)) { 1081 ret = -ETIME; 1082 goto put_mm; 1083 } 1084 1085 if (ret == -EBUSY) { 1086 amdxdna_umap_put(mapp); 1087 mmput(mm); 1088 goto again; 1089 } 1090 1091 goto put_mm; 1092 } 1093 1094 down_write(&xdna->notifier_lock); 1095 if (mmu_interval_read_retry(&mapp->notifier, mapp->range.notifier_seq)) { 1096 up_write(&xdna->notifier_lock); 1097 amdxdna_umap_put(mapp); 1098 mmput(mm); 1099 goto again; 1100 } 1101 mapp->invalid = false; 1102 up_write(&xdna->notifier_lock); 1103 amdxdna_umap_put(mapp); 1104 mmput(mm); 1105 goto again; 1106 1107 put_mm: 1108 amdxdna_umap_put(mapp); 1109 mmput(mm); 1110 return ret; 1111 } 1112 1113 int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq) 1114 { 1115 struct amdxdna_dev *xdna = hwctx->client->xdna; 1116 struct ww_acquire_ctx acquire_ctx; 1117 struct dma_fence_chain *chain; 1118 struct amdxdna_gem_obj *abo; 1119 unsigned long timeout = 0; 1120 int ret, i; 1121 1122 ret = down_interruptible(&hwctx->priv->job_sem); 1123 if (ret) { 1124 XDNA_ERR(xdna, "Grab job sem failed, ret %d", ret); 1125 return ret; 1126 } 1127 1128 chain = dma_fence_chain_alloc(); 1129 if (!chain) { 1130 XDNA_ERR(xdna, "Alloc fence chain failed"); 1131 ret = -ENOMEM; 1132 goto up_sem; 1133 } 1134 1135 ret = drm_sched_job_init(&job->base, &hwctx->priv->entity, 1, hwctx, 1136 hwctx->client->filp->client_id); 1137 if (ret) { 1138 XDNA_ERR(xdna, "DRM job init failed, ret %d", ret); 1139 goto free_chain; 1140 } 1141 1142 retry: 1143 ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1144 if (ret) { 1145 XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret); 1146 goto cleanup_job; 1147 } 1148 1149 for (i = 0; i < job->bo_cnt; i++) { 1150 ret = dma_resv_reserve_fences(job->bos[i]->resv, 1); 1151 if (ret) { 1152 XDNA_WARN(xdna, "Failed to reserve fences %d", ret); 1153 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1154 goto cleanup_job; 1155 } 1156 } 1157 1158 down_read(&xdna->notifier_lock); 1159 for (i = 0; i < job->bo_cnt; i++) { 1160 abo = to_xdna_obj(job->bos[i]); 1161 if (abo->mem.map_invalid) { 1162 up_read(&xdna->notifier_lock); 1163 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1164 if (!timeout) { 1165 timeout = jiffies + 1166 msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1167 } else if (time_after(jiffies, timeout)) { 1168 ret = -ETIME; 1169 goto cleanup_job; 1170 } 1171 1172 ret = aie2_populate_range(abo); 1173 if (ret) 1174 goto cleanup_job; 1175 goto retry; 1176 } 1177 } 1178 1179 mutex_lock(&hwctx->priv->io_lock); 1180 drm_sched_job_arm(&job->base); 1181 job->out_fence = dma_fence_get(&job->base.s_fence->finished); 1182 for (i = 0; i < job->bo_cnt; i++) 1183 dma_resv_add_fence(job->bos[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE); 1184 job->seq = hwctx->priv->seq++; 1185 kref_get(&job->refcnt); 1186 if (job->drv_cmd) 1187 kref_get(&job->drv_cmd->refcnt); 1188 drm_sched_entity_push_job(&job->base); 1189 1190 *seq = job->seq; 1191 drm_syncobj_add_point(hwctx->priv->syncobj, chain, job->out_fence, *seq); 1192 mutex_unlock(&hwctx->priv->io_lock); 1193 1194 up_read(&xdna->notifier_lock); 1195 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1196 1197 aie2_job_put(job); 1198 atomic64_inc(&hwctx->job_submit_cnt); 1199 1200 return 0; 1201 1202 cleanup_job: 1203 drm_sched_job_cleanup(&job->base); 1204 free_chain: 1205 dma_fence_chain_free(chain); 1206 up_sem: 1207 up(&hwctx->priv->job_sem); 1208 job->job_done = true; 1209 return ret; 1210 } 1211 1212 void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, 1213 unsigned long cur_seq) 1214 { 1215 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 1216 struct drm_gem_object *gobj = to_gobj(abo); 1217 long ret; 1218 1219 ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP, 1220 true, MAX_SCHEDULE_TIMEOUT); 1221 if (!ret) 1222 XDNA_ERR(xdna, "Failed to wait for bo, ret %ld", ret); 1223 else if (ret == -ERESTARTSYS) 1224 XDNA_DBG(xdna, "Wait for bo interrupted by signal"); 1225 } 1226 1227 int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx, 1228 struct amdxdna_gem_obj *heap) 1229 { 1230 struct amdxdna_client *client = hwctx->client; 1231 struct amdxdna_dev *xdna = client->xdna; 1232 u64 addr; 1233 int ret; 1234 1235 addr = amdxdna_obj_dma_addr(heap); 1236 ret = aie2_add_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 1237 addr, heap->mem.size); 1238 if (ret) { 1239 XDNA_ERR(xdna, "Add heap failed hwctx %s 0x%lx ret %d", 1240 hwctx->name, heap->mem.size, ret); 1241 } 1242 1243 return ret; 1244 } 1245