1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_gem.h> 9 #include <drm/drm_gem_shmem_helper.h> 10 #include <drm/drm_print.h> 11 #include <drm/drm_syncobj.h> 12 #include <linux/hmm.h> 13 #include <linux/types.h> 14 #include <linux/xarray.h> 15 #include <trace/events/amdxdna.h> 16 17 #include "aie2_msg_priv.h" 18 #include "aie2_pci.h" 19 #include "aie2_solver.h" 20 #include "amdxdna_ctx.h" 21 #include "amdxdna_gem.h" 22 #include "amdxdna_mailbox.h" 23 #include "amdxdna_pci_drv.h" 24 #include "amdxdna_pm.h" 25 26 static bool force_cmdlist = true; 27 module_param(force_cmdlist, bool, 0600); 28 MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default true)"); 29 30 uint tdr_timeout_ms = 2000; 31 module_param(tdr_timeout_ms, int, 0400); 32 MODULE_PARM_DESC(tdr_timeout_ms, "TDR (Timeout Detection and Recovery) timeout in milliseconds (0 = disable)"); 33 34 struct aie2_ctx_health { 35 struct amdxdna_ctx_health header; 36 u32 txn_op_idx; 37 u32 ctx_pc; 38 u32 fatal_error_type; 39 u32 fatal_error_exception_type; 40 u32 fatal_error_exception_pc; 41 u32 fatal_error_app_module; 42 }; 43 44 static inline void aie2_tdr_signal(struct amdxdna_dev *xdna) 45 { 46 WRITE_ONCE(xdna->dev_handle->tdr_status, AIE2_TDR_SIGNALED); 47 } 48 49 static bool aie2_tdr_detect(struct amdxdna_dev *xdna) 50 { 51 struct amdxdna_dev_hdl *ndev = xdna->dev_handle; 52 53 if (READ_ONCE(ndev->tdr_status) == AIE2_TDR_WAIT) { 54 XDNA_ERR(xdna, "TDR timeout detected"); 55 return true; 56 } 57 58 WRITE_ONCE(ndev->tdr_status, AIE2_TDR_WAIT); 59 return false; 60 } 61 62 static void aie2_cmd_release(struct kref *ref) 63 { 64 struct amdxdna_drv_cmd *drv_cmd = container_of(ref, struct amdxdna_drv_cmd, refcnt); 65 66 kfree(drv_cmd); 67 } 68 69 static void aie2_cmd_put(struct amdxdna_drv_cmd *drv_cmd) 70 { 71 kref_put(&drv_cmd->refcnt, aie2_cmd_release); 72 } 73 74 static void aie2_job_release(struct kref *ref) 75 { 76 struct amdxdna_sched_job *job; 77 78 job = container_of(ref, struct amdxdna_sched_job, refcnt); 79 80 amdxdna_sched_job_cleanup(job); 81 atomic64_inc(&job->hwctx->job_free_cnt); 82 wake_up(&job->hwctx->priv->job_free_wq); 83 if (job->out_fence) 84 dma_fence_put(job->out_fence); 85 if (job->drv_cmd) 86 aie2_cmd_put(job->drv_cmd); 87 kfree(job->aie2_job_health); 88 kfree(job); 89 } 90 91 static void aie2_job_put(struct amdxdna_sched_job *job) 92 { 93 kref_put(&job->refcnt, aie2_job_release); 94 } 95 96 /* The bad_job is used in aie2_sched_job_timedout, otherwise, set it to NULL */ 97 static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx, 98 struct drm_sched_job *bad_job) 99 { 100 drm_sched_stop(&hwctx->priv->sched, bad_job); 101 aie2_destroy_context(xdna->dev_handle, hwctx); 102 drm_sched_start(&hwctx->priv->sched, 0); 103 } 104 105 static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) 106 { 107 struct amdxdna_gem_obj *heap = hwctx->priv->heap; 108 unsigned long heap_id; 109 int ret; 110 111 ret = aie2_create_context(xdna->dev_handle, hwctx); 112 if (ret) { 113 XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret); 114 goto out; 115 } 116 117 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 118 amdxdna_obj_dma_addr(heap), 119 heap->mem.size); 120 if (ret) { 121 XDNA_ERR(xdna, "Map host buf failed, ret %d", ret); 122 goto out; 123 } 124 125 xa_for_each_range(&hwctx->client->dev_heap_xa, heap_id, heap, 1, 126 hwctx->last_attached_heap) { 127 ret = aie2_add_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 128 amdxdna_obj_dma_addr(heap), 129 heap->mem.size); 130 if (ret) { 131 XDNA_ERR(xdna, "Add heap %ld failed ret %d", heap_id, ret); 132 goto out; 133 } 134 } 135 136 ret = aie2_config_cu(hwctx, NULL); 137 if (ret) { 138 XDNA_ERR(xdna, "Config cu failed, ret %d", ret); 139 goto out; 140 } 141 142 out: 143 XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret); 144 return ret; 145 } 146 147 static struct dma_fence *aie2_cmd_get_out_fence(struct amdxdna_hwctx *hwctx, u64 seq) 148 { 149 struct dma_fence *fence, *out_fence = NULL; 150 int ret; 151 152 fence = drm_syncobj_fence_get(hwctx->priv->syncobj); 153 if (!fence) 154 return NULL; 155 156 ret = dma_fence_chain_find_seqno(&fence, seq); 157 if (ret) 158 goto out; 159 160 out_fence = dma_fence_get(dma_fence_chain_contained(fence)); 161 162 out: 163 dma_fence_put(fence); 164 return out_fence; 165 } 166 167 static void aie2_hwctx_wait_for_idle(struct amdxdna_hwctx *hwctx) 168 { 169 struct dma_fence *fence; 170 171 fence = aie2_cmd_get_out_fence(hwctx, hwctx->priv->seq - 1); 172 if (!fence) 173 return; 174 175 /* Wait up to 2 seconds for fw to finish all pending requests */ 176 dma_fence_wait_timeout(fence, false, msecs_to_jiffies(2000)); 177 dma_fence_put(fence); 178 } 179 180 static int aie2_hwctx_suspend_cb(struct amdxdna_hwctx *hwctx, void *arg) 181 { 182 struct amdxdna_dev *xdna = hwctx->client->xdna; 183 184 aie2_hwctx_wait_for_idle(hwctx); 185 aie2_hwctx_stop(xdna, hwctx, NULL); 186 187 return 0; 188 } 189 190 void aie2_hwctx_suspend(struct amdxdna_client *client) 191 { 192 struct amdxdna_dev *xdna = client->xdna; 193 194 /* 195 * Command timeout is unlikely. But if it happens, it doesn't 196 * break the system. aie2_hwctx_stop() will destroy mailbox 197 * and abort all commands. 198 */ 199 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 200 amdxdna_hwctx_walk(client, NULL, aie2_hwctx_suspend_cb); 201 } 202 203 static int aie2_hwctx_resume_cb(struct amdxdna_hwctx *hwctx, void *arg) 204 { 205 struct amdxdna_dev *xdna = hwctx->client->xdna; 206 207 return aie2_hwctx_restart(xdna, hwctx); 208 } 209 210 int aie2_hwctx_resume(struct amdxdna_client *client) 211 { 212 /* 213 * The resume path cannot guarantee that mailbox channel can be 214 * regenerated. If this happen, when submit message to this 215 * mailbox channel, error will return. 216 */ 217 return amdxdna_hwctx_walk(client, NULL, aie2_hwctx_resume_cb); 218 } 219 220 static void 221 aie2_sched_notify(struct amdxdna_sched_job *job) 222 { 223 struct dma_fence *fence = job->fence; 224 225 trace_xdna_job(&job->base, job->hwctx->name, "signaling fence", 226 job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); 227 228 aie2_tdr_signal(job->hwctx->client->xdna); 229 job->hwctx->priv->completed++; 230 dma_fence_signal(fence); 231 232 up(&job->hwctx->priv->job_sem); 233 job->job_done = true; 234 mmput_async(job->mm); 235 aie2_job_put(job); 236 } 237 238 static void aie2_set_cmd_timeout(struct amdxdna_sched_job *job) 239 { 240 struct aie2_ctx_health *aie2_health __free(kfree) = NULL; 241 struct amdxdna_dev *xdna = job->hwctx->client->xdna; 242 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 243 struct app_health_report *report = job->aie2_job_health; 244 u32 fail_cmd_idx = 0; 245 246 if (!report) 247 goto set_timeout; 248 249 XDNA_ERR(xdna, "Firmware timeout state capture:"); 250 XDNA_ERR(xdna, "\tVersion: %d.%d", report->major, report->minor); 251 XDNA_ERR(xdna, "\tReport size: 0x%x", report->size); 252 XDNA_ERR(xdna, "\tContext ID: %d", report->context_id); 253 XDNA_ERR(xdna, "\tDPU PC: 0x%x", report->dpu_pc); 254 XDNA_ERR(xdna, "\tTXN OP ID: 0x%x", report->txn_op_id); 255 XDNA_ERR(xdna, "\tContext PC: 0x%x", report->ctx_pc); 256 XDNA_ERR(xdna, "\tFatal error type: 0x%x", report->fatal_info.fatal_type); 257 XDNA_ERR(xdna, "\tFatal error exception type: 0x%x", report->fatal_info.exception_type); 258 XDNA_ERR(xdna, "\tFatal error exception PC: 0x%x", report->fatal_info.exception_pc); 259 XDNA_ERR(xdna, "\tFatal error app module: 0x%x", report->fatal_info.app_module); 260 XDNA_ERR(xdna, "\tFatal error task ID: %d", report->fatal_info.task_index); 261 XDNA_ERR(xdna, "\tTimed out sub command ID: %d", report->run_list_id); 262 263 fail_cmd_idx = report->run_list_id; 264 aie2_health = kzalloc_obj(*aie2_health); 265 if (!aie2_health) 266 goto set_timeout; 267 268 aie2_health->header.version = AMDXDNA_CMD_CTX_HEALTH_V1; 269 aie2_health->header.npu_gen = AMDXDNA_CMD_CTX_HEALTH_AIE2; 270 aie2_health->txn_op_idx = report->txn_op_id; 271 aie2_health->ctx_pc = report->ctx_pc; 272 aie2_health->fatal_error_type = report->fatal_info.fatal_type; 273 aie2_health->fatal_error_exception_type = report->fatal_info.exception_type; 274 aie2_health->fatal_error_exception_pc = report->fatal_info.exception_pc; 275 aie2_health->fatal_error_app_module = report->fatal_info.app_module; 276 277 set_timeout: 278 amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_TIMEOUT, 279 aie2_health, sizeof(*aie2_health)); 280 } 281 282 static int 283 aie2_sched_resp_handler(void *handle, void __iomem *data, size_t size) 284 { 285 struct amdxdna_sched_job *job = handle; 286 struct amdxdna_gem_obj *cmd_abo; 287 int ret = 0; 288 u32 status; 289 290 cmd_abo = job->cmd_bo; 291 292 if (unlikely(job->job_timeout)) { 293 aie2_set_cmd_timeout(job); 294 ret = -EINVAL; 295 goto out; 296 } 297 298 if (unlikely(!data) || unlikely(size != sizeof(u32))) { 299 amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT, NULL, 0); 300 ret = -EINVAL; 301 goto out; 302 } 303 304 status = readl(data); 305 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); 306 if (status == AIE2_STATUS_SUCCESS) 307 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); 308 else 309 amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ERROR, NULL, 0); 310 311 out: 312 aie2_sched_notify(job); 313 return ret; 314 } 315 316 static int 317 aie2_sched_drvcmd_resp_handler(void *handle, void __iomem *data, size_t size) 318 { 319 struct amdxdna_sched_job *job = handle; 320 int ret = 0; 321 322 if (unlikely(!data || size != sizeof(u32))) { 323 job->drv_cmd->result = U32_MAX; 324 ret = -EINVAL; 325 } else { 326 job->drv_cmd->result = readl(data); 327 } 328 329 aie2_sched_notify(job); 330 return ret; 331 } 332 333 static int 334 aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) 335 { 336 struct amdxdna_sched_job *job = handle; 337 struct amdxdna_gem_obj *cmd_abo; 338 struct amdxdna_dev *xdna; 339 u32 fail_cmd_idx = 0; 340 u32 fail_cmd_status; 341 u32 cmd_status; 342 int ret = 0; 343 344 cmd_abo = job->cmd_bo; 345 346 if (unlikely(job->job_timeout)) { 347 aie2_set_cmd_timeout(job); 348 ret = -EINVAL; 349 goto out; 350 } 351 352 if (unlikely(!data) || unlikely(size != sizeof(u32) * 3)) { 353 amdxdna_cmd_set_error(cmd_abo, job, 0, ERT_CMD_STATE_ABORT, NULL, 0); 354 ret = -EINVAL; 355 goto out; 356 } 357 358 cmd_status = readl(data + offsetof(struct cmd_chain_resp, status)); 359 xdna = job->hwctx->client->xdna; 360 XDNA_DBG(xdna, "Status 0x%x", cmd_status); 361 if (cmd_status == AIE2_STATUS_SUCCESS) { 362 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_COMPLETED); 363 goto out; 364 } 365 366 /* Slow path to handle error, read from ringbuf on BAR */ 367 fail_cmd_idx = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_idx)); 368 fail_cmd_status = readl(data + offsetof(struct cmd_chain_resp, fail_cmd_status)); 369 XDNA_DBG(xdna, "Failed cmd idx %d, status 0x%x", 370 fail_cmd_idx, fail_cmd_status); 371 372 if (fail_cmd_status == AIE2_STATUS_SUCCESS) { 373 amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ABORT, NULL, 0); 374 ret = -EINVAL; 375 } else { 376 amdxdna_cmd_set_error(cmd_abo, job, fail_cmd_idx, ERT_CMD_STATE_ERROR, NULL, 0); 377 } 378 379 out: 380 aie2_sched_notify(job); 381 return ret; 382 } 383 384 static struct dma_fence * 385 aie2_sched_job_run(struct drm_sched_job *sched_job) 386 { 387 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 388 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; 389 struct amdxdna_hwctx *hwctx = job->hwctx; 390 struct dma_fence *fence; 391 int ret; 392 393 trace_xdna_job(sched_job, hwctx->name, "job run", 394 job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); 395 396 if (!hwctx->priv->mbox_chann) 397 return NULL; 398 399 if (!mmget_not_zero(job->mm)) 400 return ERR_PTR(-ESRCH); 401 402 kref_get(&job->refcnt); 403 fence = dma_fence_get(job->fence); 404 405 if (job->drv_cmd) { 406 switch (job->drv_cmd->opcode) { 407 case SYNC_DEBUG_BO: 408 ret = aie2_sync_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); 409 break; 410 case ATTACH_DEBUG_BO: 411 case DETACH_DEBUG_BO: 412 ret = aie2_config_debug_bo(hwctx, job, aie2_sched_drvcmd_resp_handler); 413 break; 414 default: 415 ret = -EINVAL; 416 break; 417 } 418 goto out; 419 } 420 421 amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_NEW); 422 423 if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) 424 ret = aie2_cmdlist_multi_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); 425 else if (force_cmdlist) 426 ret = aie2_cmdlist_single_execbuf(hwctx, job, aie2_sched_cmdlist_resp_handler); 427 else 428 ret = aie2_execbuf(hwctx, job, aie2_sched_resp_handler); 429 430 out: 431 if (ret) { 432 dma_fence_put(job->fence); 433 aie2_job_put(job); 434 mmput(job->mm); 435 fence = ERR_PTR(ret); 436 } else { 437 aie2_tdr_signal(hwctx->client->xdna); 438 } 439 trace_xdna_job(sched_job, hwctx->name, "sent to device", 440 job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO); 441 442 return fence; 443 } 444 445 static void aie2_sched_job_free(struct drm_sched_job *sched_job) 446 { 447 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 448 struct amdxdna_hwctx *hwctx = job->hwctx; 449 450 /* job->drv_cmd could be freed, so use DEFAULT_IO */ 451 trace_xdna_job(sched_job, hwctx->name, "job free", 452 job->seq, DEFAULT_IO); 453 if (!job->job_done) 454 up(&hwctx->priv->job_sem); 455 456 drm_sched_job_cleanup(sched_job); 457 aie2_job_put(job); 458 } 459 460 static enum drm_gpu_sched_stat 461 aie2_sched_job_timedout(struct drm_sched_job *sched_job) 462 { 463 struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job); 464 struct amdxdna_hwctx *hwctx = job->hwctx; 465 struct app_health_report *report; 466 struct amdxdna_dev *xdna; 467 int ret; 468 469 xdna = hwctx->client->xdna; 470 471 guard(mutex)(&xdna->dev_lock); 472 473 if (!aie2_tdr_detect(xdna)) 474 return DRM_GPU_SCHED_STAT_NO_HANG; 475 476 report = kzalloc_obj(*report); 477 if (!report) 478 goto reset_hwctx; 479 480 ret = aie2_query_app_health(xdna->dev_handle, hwctx->fw_ctx_id, report); 481 if (ret) 482 kfree(report); 483 else 484 job->aie2_job_health = report; 485 486 reset_hwctx: 487 job->job_timeout = true; 488 aie2_hwctx_stop(xdna, hwctx, sched_job); 489 490 aie2_hwctx_restart(xdna, hwctx); 491 492 return DRM_GPU_SCHED_STAT_RESET; 493 } 494 495 static const struct drm_sched_backend_ops sched_ops = { 496 .run_job = aie2_sched_job_run, 497 .free_job = aie2_sched_job_free, 498 .timedout_job = aie2_sched_job_timedout, 499 }; 500 501 static int aie2_hwctx_col_list(struct amdxdna_hwctx *hwctx) 502 { 503 struct amdxdna_dev *xdna = hwctx->client->xdna; 504 struct amdxdna_dev_hdl *ndev; 505 int start, end, first, last; 506 u32 width = 1, entries = 0; 507 int i; 508 509 if (!hwctx->num_tiles) { 510 XDNA_ERR(xdna, "Number of tiles is zero"); 511 return -EINVAL; 512 } 513 514 ndev = xdna->dev_handle; 515 if (unlikely(!ndev->aie.metadata.core.row_count)) { 516 XDNA_WARN(xdna, "Core tile row count is zero"); 517 return -EINVAL; 518 } 519 520 hwctx->num_col = hwctx->num_tiles / ndev->aie.metadata.core.row_count; 521 if (!hwctx->num_col || hwctx->num_col > ndev->total_col) { 522 XDNA_ERR(xdna, "Invalid num_col %d", hwctx->num_col); 523 return -EINVAL; 524 } 525 526 if (ndev->priv->col_align == COL_ALIGN_NATURE) 527 width = hwctx->num_col; 528 529 /* 530 * In range [start, end], find out columns that is multiple of width. 531 * 'first' is the first column, 532 * 'last' is the last column, 533 * 'entries' is the total number of columns. 534 */ 535 start = xdna->dev_info->first_col; 536 end = ndev->total_col - hwctx->num_col; 537 if (start > 0 && end == 0) { 538 XDNA_DBG(xdna, "Force start from col 0"); 539 start = 0; 540 } 541 first = start + (width - start % width) % width; 542 last = end - end % width; 543 if (last >= first) 544 entries = (last - first) / width + 1; 545 XDNA_DBG(xdna, "start %d end %d first %d last %d", 546 start, end, first, last); 547 548 if (unlikely(!entries)) { 549 XDNA_ERR(xdna, "Start %d end %d width %d", 550 start, end, width); 551 return -EINVAL; 552 } 553 554 hwctx->col_list = kmalloc_array(entries, sizeof(*hwctx->col_list), GFP_KERNEL); 555 if (!hwctx->col_list) 556 return -ENOMEM; 557 558 hwctx->col_list_len = entries; 559 hwctx->col_list[0] = first; 560 for (i = 1; i < entries; i++) 561 hwctx->col_list[i] = hwctx->col_list[i - 1] + width; 562 563 print_hex_dump_debug("col_list: ", DUMP_PREFIX_OFFSET, 16, 4, hwctx->col_list, 564 entries * sizeof(*hwctx->col_list), false); 565 return 0; 566 } 567 568 static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx) 569 { 570 struct amdxdna_dev *xdna = hwctx->client->xdna; 571 struct alloc_requests *xrs_req; 572 u32 temporal_only_col = 0; 573 int ret; 574 575 xrs_req = kzalloc_obj(*xrs_req); 576 if (!xrs_req) 577 return -ENOMEM; 578 579 if (AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_TEMPORAL_ONLY)) { 580 xrs_req->cdo.start_cols = &temporal_only_col; 581 xrs_req->cdo.cols_len = 1; 582 xrs_req->cdo.ncols = xdna->dev_handle->total_col; 583 } else { 584 xrs_req->cdo.start_cols = hwctx->col_list; 585 xrs_req->cdo.cols_len = hwctx->col_list_len; 586 xrs_req->cdo.ncols = hwctx->num_col; 587 } 588 /* Use platform opc */ 589 xrs_req->cdo.qos_cap.opc = xdna->dev_handle->priv->col_opc * hwctx->num_col; 590 591 xrs_req->rqos.gops = hwctx->qos.gops; 592 xrs_req->rqos.fps = hwctx->qos.fps; 593 xrs_req->rqos.dma_bw = hwctx->qos.dma_bandwidth; 594 xrs_req->rqos.latency = hwctx->qos.latency; 595 xrs_req->rqos.exec_time = hwctx->qos.frame_exec_time; 596 xrs_req->rqos.priority = hwctx->qos.priority; 597 598 xrs_req->rid = (uintptr_t)hwctx; 599 600 ret = xrs_allocate_resource(xdna->xrs_hdl, xrs_req, hwctx); 601 if (ret) 602 XDNA_ERR(xdna, "Allocate AIE resource failed, ret %d", ret); 603 604 kfree(xrs_req); 605 return ret; 606 } 607 608 static void aie2_release_resource(struct amdxdna_hwctx *hwctx) 609 { 610 struct amdxdna_dev *xdna = hwctx->client->xdna; 611 int ret; 612 613 ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); 614 if (ret) 615 XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret); 616 } 617 618 static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx) 619 { 620 struct amdxdna_dev *xdna = hwctx->client->xdna; 621 struct drm_file *filp = hwctx->client->filp; 622 struct drm_syncobj *syncobj; 623 u32 hdl; 624 int ret; 625 626 hwctx->syncobj_hdl = AMDXDNA_INVALID_FENCE_HANDLE; 627 628 ret = drm_syncobj_create(&syncobj, 0, NULL); 629 if (ret) { 630 XDNA_ERR(xdna, "Create ctx syncobj failed, ret %d", ret); 631 return ret; 632 } 633 ret = drm_syncobj_get_handle(filp, syncobj, &hdl); 634 if (ret) { 635 drm_syncobj_put(syncobj); 636 XDNA_ERR(xdna, "Create ctx syncobj handle failed, ret %d", ret); 637 return ret; 638 } 639 hwctx->priv->syncobj = syncobj; 640 hwctx->syncobj_hdl = hdl; 641 642 return 0; 643 } 644 645 static void aie2_ctx_syncobj_destroy(struct amdxdna_hwctx *hwctx) 646 { 647 /* 648 * The syncobj_hdl is owned by user space and will be cleaned up 649 * separately. 650 */ 651 drm_syncobj_put(hwctx->priv->syncobj); 652 } 653 654 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx) 655 { 656 struct amdxdna_client *client = hwctx->client; 657 struct amdxdna_dev *xdna = client->xdna; 658 const struct drm_sched_init_args args = { 659 .ops = &sched_ops, 660 .credit_limit = HWCTX_MAX_CMDS, 661 .timeout = msecs_to_jiffies(tdr_timeout_ms), 662 .name = "amdxdna_js", 663 .dev = xdna->ddev.dev, 664 }; 665 struct drm_gpu_scheduler *sched; 666 struct amdxdna_hwctx_priv *priv; 667 struct amdxdna_gem_obj *heap; 668 int i, ret; 669 670 priv = kzalloc_obj(*hwctx->priv); 671 if (!priv) 672 return -ENOMEM; 673 hwctx->priv = priv; 674 675 mutex_lock(&client->mm_lock); 676 heap = xa_load(&client->dev_heap_xa, 0); 677 if (!heap) { 678 XDNA_ERR(xdna, "The client dev heap object not exist"); 679 mutex_unlock(&client->mm_lock); 680 ret = -ENOENT; 681 goto free_priv; 682 } 683 drm_gem_object_get(to_gobj(heap)); 684 mutex_unlock(&client->mm_lock); 685 priv->heap = heap; 686 sema_init(&priv->job_sem, HWCTX_MAX_CMDS); 687 688 ret = amdxdna_gem_pin(heap); 689 if (ret) { 690 XDNA_ERR(xdna, "Dev heap pin failed, ret %d", ret); 691 goto put_heap; 692 } 693 694 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { 695 struct amdxdna_gem_obj *abo; 696 struct amdxdna_drm_create_bo args = { 697 .flags = 0, 698 .type = AMDXDNA_BO_DEV, 699 .vaddr = 0, 700 .size = MAX_CHAIN_CMDBUF_SIZE, 701 }; 702 703 abo = amdxdna_drm_create_dev_bo(&xdna->ddev, &args, client->filp); 704 if (IS_ERR(abo)) { 705 ret = PTR_ERR(abo); 706 goto free_cmd_bufs; 707 } 708 709 XDNA_DBG(xdna, "Command buf %d addr 0x%llx size 0x%lx", 710 i, amdxdna_gem_dev_addr(abo), abo->mem.size); 711 priv->cmd_buf[i] = abo; 712 } 713 714 sched = &priv->sched; 715 mutex_init(&priv->io_lock); 716 717 fs_reclaim_acquire(GFP_KERNEL); 718 might_lock(&priv->io_lock); 719 fs_reclaim_release(GFP_KERNEL); 720 721 ret = drm_sched_init(sched, &args); 722 if (ret) { 723 XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret); 724 goto free_cmd_bufs; 725 } 726 727 ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL, 728 &sched, 1, NULL); 729 if (ret) { 730 XDNA_ERR(xdna, "Failed to initial sched entiry. ret %d", ret); 731 goto free_sched; 732 } 733 734 ret = aie2_hwctx_col_list(hwctx); 735 if (ret) { 736 XDNA_ERR(xdna, "Create col list failed, ret %d", ret); 737 goto free_entity; 738 } 739 740 ret = amdxdna_pm_resume_get_locked(xdna); 741 if (ret) 742 goto free_col_list; 743 744 ret = aie2_alloc_resource(hwctx); 745 if (ret) { 746 XDNA_ERR(xdna, "Alloc hw resource failed, ret %d", ret); 747 goto suspend_put; 748 } 749 750 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 751 amdxdna_obj_dma_addr(heap), 752 heap->mem.size); 753 if (ret) { 754 XDNA_ERR(xdna, "Map host buffer failed, ret %d", ret); 755 goto release_resource; 756 } 757 758 ret = amdxdna_update_heap(client, hwctx); 759 if (ret) { 760 XDNA_ERR(xdna, "Update heap failed, ret %d", ret); 761 goto release_resource; 762 } 763 764 ret = aie2_ctx_syncobj_create(hwctx); 765 if (ret) { 766 XDNA_ERR(xdna, "Create syncobj failed, ret %d", ret); 767 goto release_resource; 768 } 769 amdxdna_pm_suspend_put(xdna); 770 771 init_waitqueue_head(&priv->job_free_wq); 772 773 XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); 774 775 return 0; 776 777 release_resource: 778 aie2_release_resource(hwctx); 779 suspend_put: 780 amdxdna_pm_suspend_put(xdna); 781 free_col_list: 782 kfree(hwctx->col_list); 783 free_entity: 784 drm_sched_entity_destroy(&priv->entity); 785 free_sched: 786 drm_sched_fini(&priv->sched); 787 free_cmd_bufs: 788 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { 789 if (!priv->cmd_buf[i]) 790 continue; 791 drm_gem_object_put(to_gobj(priv->cmd_buf[i])); 792 } 793 amdxdna_gem_unpin(heap); 794 put_heap: 795 drm_gem_object_put(to_gobj(heap)); 796 free_priv: 797 kfree(priv); 798 return ret; 799 } 800 801 void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx) 802 { 803 struct amdxdna_dev *xdna; 804 int idx; 805 806 xdna = hwctx->client->xdna; 807 808 XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq); 809 aie2_hwctx_wait_for_idle(hwctx); 810 811 /* Request fw to destroy hwctx and cancel the rest pending requests */ 812 drm_sched_stop(&hwctx->priv->sched, NULL); 813 aie2_release_resource(hwctx); 814 drm_sched_start(&hwctx->priv->sched, 0); 815 816 mutex_unlock(&xdna->dev_lock); 817 drm_sched_entity_destroy(&hwctx->priv->entity); 818 819 /* Wait for all submitted jobs to be completed or canceled */ 820 wait_event(hwctx->priv->job_free_wq, 821 atomic64_read(&hwctx->job_submit_cnt) == 822 atomic64_read(&hwctx->job_free_cnt)); 823 mutex_lock(&xdna->dev_lock); 824 825 drm_sched_fini(&hwctx->priv->sched); 826 aie2_ctx_syncobj_destroy(hwctx); 827 828 for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++) 829 drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx])); 830 amdxdna_gem_unpin(hwctx->priv->heap); 831 drm_gem_object_put(to_gobj(hwctx->priv->heap)); 832 833 mutex_destroy(&hwctx->priv->io_lock); 834 kfree(hwctx->col_list); 835 kfree(hwctx->priv); 836 kfree(hwctx->cus); 837 } 838 839 static int aie2_config_cu_resp_handler(void *handle, void __iomem *data, size_t size) 840 { 841 struct amdxdna_hwctx *hwctx = handle; 842 843 amdxdna_pm_suspend_put(hwctx->client->xdna); 844 return 0; 845 } 846 847 static int aie2_hwctx_cu_config(struct amdxdna_hwctx *hwctx, void *buf, u32 size) 848 { 849 struct amdxdna_hwctx_param_config_cu *config = buf; 850 struct amdxdna_dev *xdna = hwctx->client->xdna; 851 u32 total_size; 852 int ret; 853 854 XDNA_DBG(xdna, "Config %d CU to %s", config->num_cus, hwctx->name); 855 if (XDNA_MBZ_DBG(xdna, config->pad, sizeof(config->pad))) 856 return -EINVAL; 857 858 if (hwctx->cus) { 859 XDNA_ERR(xdna, "Not support re-config CU"); 860 return -EINVAL; 861 } 862 863 if (!config->num_cus) { 864 XDNA_ERR(xdna, "Number of CU is zero"); 865 return -EINVAL; 866 } 867 868 total_size = struct_size(config, cu_configs, config->num_cus); 869 if (total_size > size) { 870 XDNA_ERR(xdna, "CU config larger than size"); 871 return -EINVAL; 872 } 873 874 hwctx->cus = kmemdup(config, total_size, GFP_KERNEL); 875 if (!hwctx->cus) 876 return -ENOMEM; 877 878 ret = amdxdna_pm_resume_get_locked(xdna); 879 if (ret) 880 goto free_cus; 881 882 ret = aie2_config_cu(hwctx, aie2_config_cu_resp_handler); 883 if (ret) { 884 XDNA_ERR(xdna, "Config CU to firmware failed, ret %d", ret); 885 goto pm_suspend_put; 886 } 887 888 wmb(); /* To avoid locking in command submit when check status */ 889 890 return 0; 891 892 pm_suspend_put: 893 amdxdna_pm_suspend_put(xdna); 894 free_cus: 895 kfree(hwctx->cus); 896 hwctx->cus = NULL; 897 return ret; 898 } 899 900 static void aie2_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq) 901 { 902 struct dma_fence *out_fence = aie2_cmd_get_out_fence(hwctx, seq); 903 904 if (!out_fence) { 905 XDNA_ERR(hwctx->client->xdna, "Failed to get fence"); 906 return; 907 } 908 909 dma_fence_wait_timeout(out_fence, false, MAX_SCHEDULE_TIMEOUT); 910 dma_fence_put(out_fence); 911 } 912 913 static int aie2_hwctx_cfg_debug_bo(struct amdxdna_hwctx *hwctx, u32 bo_hdl, 914 bool attach) 915 { 916 struct amdxdna_client *client = hwctx->client; 917 struct amdxdna_dev *xdna = client->xdna; 918 struct amdxdna_drv_cmd *cmd; 919 struct amdxdna_gem_obj *abo; 920 u64 seq; 921 int ret; 922 923 abo = amdxdna_gem_get_obj(client, bo_hdl, AMDXDNA_BO_DEV); 924 if (!abo) { 925 XDNA_ERR(xdna, "Get bo %d failed", bo_hdl); 926 return -EINVAL; 927 } 928 929 cmd = kzalloc_obj(*cmd); 930 if (!cmd) { 931 ret = -ENOMEM; 932 goto put_obj; 933 } 934 kref_init(&cmd->refcnt); 935 936 if (attach) { 937 if (abo->assigned_hwctx != AMDXDNA_INVALID_CTX_HANDLE) { 938 ret = -EBUSY; 939 goto put_cmd; 940 } 941 cmd->opcode = ATTACH_DEBUG_BO; 942 } else { 943 if (abo->assigned_hwctx != hwctx->id) { 944 ret = -EINVAL; 945 goto put_cmd; 946 } 947 cmd->opcode = DETACH_DEBUG_BO; 948 } 949 950 ret = amdxdna_cmd_submit(client, cmd, AMDXDNA_INVALID_BO_HANDLE, 951 &bo_hdl, 1, hwctx->id, &seq); 952 if (ret) { 953 XDNA_ERR(xdna, "Submit command failed"); 954 goto put_cmd; 955 } 956 957 aie2_cmd_wait(hwctx, seq); 958 if (cmd->result) { 959 XDNA_ERR(xdna, "Response failure 0x%x", cmd->result); 960 ret = -EINVAL; 961 goto put_cmd; 962 } 963 964 if (attach) 965 abo->assigned_hwctx = hwctx->id; 966 else 967 abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE; 968 969 XDNA_DBG(xdna, "Config debug BO %d to %s", bo_hdl, hwctx->name); 970 971 put_cmd: 972 aie2_cmd_put(cmd); 973 put_obj: 974 amdxdna_gem_put_obj(abo); 975 return ret; 976 } 977 978 int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size) 979 { 980 struct amdxdna_dev *xdna = hwctx->client->xdna; 981 982 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); 983 switch (type) { 984 case DRM_AMDXDNA_HWCTX_CONFIG_CU: 985 return aie2_hwctx_cu_config(hwctx, buf, size); 986 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF: 987 return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, true); 988 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF: 989 return aie2_hwctx_cfg_debug_bo(hwctx, (u32)value, false); 990 default: 991 XDNA_DBG(xdna, "Not supported type %d", type); 992 return -EOPNOTSUPP; 993 } 994 } 995 996 int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl) 997 { 998 struct amdxdna_client *client = hwctx->client; 999 struct amdxdna_dev *xdna = client->xdna; 1000 struct amdxdna_drv_cmd *cmd; 1001 u64 seq; 1002 int ret; 1003 1004 cmd = kzalloc_obj(*cmd); 1005 if (!cmd) 1006 return -ENOMEM; 1007 kref_init(&cmd->refcnt); 1008 1009 cmd->opcode = SYNC_DEBUG_BO; 1010 ret = amdxdna_cmd_submit(client, cmd, AMDXDNA_INVALID_BO_HANDLE, 1011 &debug_bo_hdl, 1, hwctx->id, &seq); 1012 if (ret) { 1013 XDNA_ERR(xdna, "Submit command failed"); 1014 goto put_cmd; 1015 } 1016 1017 aie2_cmd_wait(hwctx, seq); 1018 if (cmd->result) { 1019 XDNA_ERR(xdna, "Response failure 0x%x", cmd->result); 1020 ret = -EINVAL; 1021 } 1022 1023 put_cmd: 1024 aie2_cmd_put(cmd); 1025 return ret; 1026 } 1027 1028 static int aie2_populate_range(struct amdxdna_gem_obj *abo) 1029 { 1030 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 1031 struct amdxdna_umap *mapp; 1032 unsigned long timeout; 1033 struct mm_struct *mm; 1034 bool found; 1035 int ret; 1036 1037 timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1038 again: 1039 found = false; 1040 down_write(&xdna->notifier_lock); 1041 list_for_each_entry(mapp, &abo->mem.umap_list, node) { 1042 if (mapp->invalid) { 1043 found = true; 1044 break; 1045 } 1046 } 1047 1048 if (!found) { 1049 abo->mem.map_invalid = false; 1050 up_write(&xdna->notifier_lock); 1051 return 0; 1052 } 1053 kref_get(&mapp->refcnt); 1054 up_write(&xdna->notifier_lock); 1055 1056 XDNA_DBG(xdna, "populate memory range %lx %lx", 1057 mapp->vma->vm_start, mapp->vma->vm_end); 1058 mm = mapp->notifier.mm; 1059 if (!mmget_not_zero(mm)) { 1060 amdxdna_umap_put(mapp); 1061 return -EFAULT; 1062 } 1063 1064 mapp->range.notifier_seq = mmu_interval_read_begin(&mapp->notifier); 1065 mmap_read_lock(mm); 1066 ret = hmm_range_fault(&mapp->range); 1067 mmap_read_unlock(mm); 1068 if (ret) { 1069 if (time_after(jiffies, timeout)) { 1070 ret = -ETIME; 1071 goto put_mm; 1072 } 1073 1074 if (ret == -EBUSY) { 1075 amdxdna_umap_put(mapp); 1076 mmput(mm); 1077 goto again; 1078 } 1079 1080 goto put_mm; 1081 } 1082 1083 down_write(&xdna->notifier_lock); 1084 if (mmu_interval_read_retry(&mapp->notifier, mapp->range.notifier_seq)) { 1085 up_write(&xdna->notifier_lock); 1086 amdxdna_umap_put(mapp); 1087 mmput(mm); 1088 goto again; 1089 } 1090 mapp->invalid = false; 1091 up_write(&xdna->notifier_lock); 1092 amdxdna_umap_put(mapp); 1093 mmput(mm); 1094 goto again; 1095 1096 put_mm: 1097 amdxdna_umap_put(mapp); 1098 mmput(mm); 1099 return ret; 1100 } 1101 1102 int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq) 1103 { 1104 struct amdxdna_dev *xdna = hwctx->client->xdna; 1105 struct ww_acquire_ctx acquire_ctx; 1106 struct dma_fence_chain *chain; 1107 struct amdxdna_gem_obj *abo; 1108 unsigned long timeout = 0; 1109 int ret, i; 1110 1111 ret = down_interruptible(&hwctx->priv->job_sem); 1112 if (ret) { 1113 XDNA_ERR(xdna, "Grab job sem failed, ret %d", ret); 1114 return ret; 1115 } 1116 1117 chain = dma_fence_chain_alloc(); 1118 if (!chain) { 1119 XDNA_ERR(xdna, "Alloc fence chain failed"); 1120 ret = -ENOMEM; 1121 goto up_sem; 1122 } 1123 1124 ret = drm_sched_job_init(&job->base, &hwctx->priv->entity, 1, hwctx, 1125 hwctx->client->filp->client_id); 1126 if (ret) { 1127 XDNA_ERR(xdna, "DRM job init failed, ret %d", ret); 1128 goto free_chain; 1129 } 1130 1131 retry: 1132 ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1133 if (ret) { 1134 XDNA_WARN(xdna, "Failed to lock BOs, ret %d", ret); 1135 goto cleanup_job; 1136 } 1137 1138 for (i = 0; i < job->bo_cnt; i++) { 1139 ret = dma_resv_reserve_fences(job->bos[i]->resv, 1); 1140 if (ret) { 1141 XDNA_WARN(xdna, "Failed to reserve fences %d", ret); 1142 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1143 goto cleanup_job; 1144 } 1145 } 1146 1147 down_read(&xdna->notifier_lock); 1148 for (i = 0; i < job->bo_cnt; i++) { 1149 abo = to_xdna_obj(job->bos[i]); 1150 if (abo->mem.map_invalid) { 1151 up_read(&xdna->notifier_lock); 1152 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1153 if (!timeout) { 1154 timeout = jiffies + 1155 msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1156 } else if (time_after(jiffies, timeout)) { 1157 ret = -ETIME; 1158 goto cleanup_job; 1159 } 1160 1161 ret = aie2_populate_range(abo); 1162 if (ret) 1163 goto cleanup_job; 1164 goto retry; 1165 } 1166 } 1167 1168 mutex_lock(&hwctx->priv->io_lock); 1169 drm_sched_job_arm(&job->base); 1170 job->out_fence = dma_fence_get(&job->base.s_fence->finished); 1171 for (i = 0; i < job->bo_cnt; i++) 1172 dma_resv_add_fence(job->bos[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE); 1173 job->seq = hwctx->priv->seq++; 1174 kref_get(&job->refcnt); 1175 if (job->drv_cmd) 1176 kref_get(&job->drv_cmd->refcnt); 1177 drm_sched_entity_push_job(&job->base); 1178 1179 *seq = job->seq; 1180 drm_syncobj_add_point(hwctx->priv->syncobj, chain, job->out_fence, *seq); 1181 mutex_unlock(&hwctx->priv->io_lock); 1182 1183 up_read(&xdna->notifier_lock); 1184 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); 1185 1186 aie2_job_put(job); 1187 atomic64_inc(&hwctx->job_submit_cnt); 1188 1189 return 0; 1190 1191 cleanup_job: 1192 drm_sched_job_cleanup(&job->base); 1193 free_chain: 1194 dma_fence_chain_free(chain); 1195 up_sem: 1196 up(&hwctx->priv->job_sem); 1197 job->job_done = true; 1198 return ret; 1199 } 1200 1201 void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, 1202 unsigned long cur_seq) 1203 { 1204 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 1205 struct drm_gem_object *gobj = to_gobj(abo); 1206 long ret; 1207 1208 ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP, 1209 true, MAX_SCHEDULE_TIMEOUT); 1210 if (!ret) 1211 XDNA_ERR(xdna, "Failed to wait for bo, ret %ld", ret); 1212 else if (ret == -ERESTARTSYS) 1213 XDNA_DBG(xdna, "Wait for bo interrupted by signal"); 1214 } 1215 1216 int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx, 1217 struct amdxdna_gem_obj *heap) 1218 { 1219 struct amdxdna_client *client = hwctx->client; 1220 struct amdxdna_dev *xdna = client->xdna; 1221 u64 addr; 1222 int ret; 1223 1224 ret = amdxdna_pm_resume_get_locked(xdna); 1225 if (ret) 1226 return ret; 1227 1228 addr = amdxdna_obj_dma_addr(heap); 1229 ret = aie2_add_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, 1230 addr, heap->mem.size); 1231 if (ret) { 1232 XDNA_ERR(xdna, "Add heap failed hwctx %s 0x%lx ret %d", 1233 hwctx->name, heap->mem.size, ret); 1234 } 1235 1236 amdxdna_pm_suspend_put(xdna); 1237 1238 return ret; 1239 } 1240