1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_file.h> 10 #include <drm/drm_gem.h> 11 #include <drm/drm_gem_shmem_helper.h> 12 #include <drm/drm_print.h> 13 #include <drm/gpu_scheduler.h> 14 #include <linux/xarray.h> 15 #include <trace/events/amdxdna.h> 16 17 #include "amdxdna_ctx.h" 18 #include "amdxdna_gem.h" 19 #include "amdxdna_pci_drv.h" 20 #include "amdxdna_pm.h" 21 22 #define MAX_HWCTX_ID 255 23 #define MAX_ARG_COUNT 4095 24 25 struct amdxdna_fence { 26 struct dma_fence base; 27 spinlock_t lock; /* for base */ 28 struct amdxdna_hwctx *hwctx; 29 }; 30 31 static const char *amdxdna_fence_get_driver_name(struct dma_fence *fence) 32 { 33 return KBUILD_MODNAME; 34 } 35 36 static const char *amdxdna_fence_get_timeline_name(struct dma_fence *fence) 37 { 38 struct amdxdna_fence *xdna_fence; 39 40 xdna_fence = container_of(fence, struct amdxdna_fence, base); 41 42 return xdna_fence->hwctx->name; 43 } 44 45 static const struct dma_fence_ops fence_ops = { 46 .get_driver_name = amdxdna_fence_get_driver_name, 47 .get_timeline_name = amdxdna_fence_get_timeline_name, 48 }; 49 50 static struct dma_fence *amdxdna_fence_create(struct amdxdna_hwctx *hwctx) 51 { 52 struct amdxdna_fence *fence; 53 54 fence = kzalloc_obj(*fence); 55 if (!fence) 56 return NULL; 57 58 fence->hwctx = hwctx; 59 spin_lock_init(&fence->lock); 60 dma_fence_init(&fence->base, &fence_ops, &fence->lock, hwctx->id, 0); 61 return &fence->base; 62 } 63 64 static void amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx *hwctx, 65 struct srcu_struct *ss) 66 { 67 struct amdxdna_dev *xdna = hwctx->client->xdna; 68 69 synchronize_srcu(ss); 70 71 /* At this point, user is not able to submit new commands */ 72 xdna->dev_info->ops->hwctx_fini(hwctx); 73 74 kfree(hwctx->name); 75 kfree(hwctx); 76 } 77 78 int amdxdna_hwctx_walk(struct amdxdna_client *client, void *arg, 79 int (*walk)(struct amdxdna_hwctx *hwctx, void *arg)) 80 { 81 struct amdxdna_hwctx *hwctx; 82 unsigned long hwctx_id; 83 int ret = 0, idx; 84 85 idx = srcu_read_lock(&client->hwctx_srcu); 86 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { 87 ret = walk(hwctx, arg); 88 if (ret) 89 break; 90 } 91 srcu_read_unlock(&client->hwctx_srcu, idx); 92 93 return ret; 94 } 95 96 void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size) 97 { 98 struct amdxdna_cmd *cmd = abo->mem.kva; 99 u32 num_masks, count; 100 101 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) 102 num_masks = 0; 103 else 104 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header); 105 106 if (size) { 107 count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header); 108 if (unlikely(count <= num_masks || 109 count * sizeof(u32) + 110 offsetof(struct amdxdna_cmd, data[0]) > 111 abo->mem.size)) { 112 *size = 0; 113 return NULL; 114 } 115 *size = (count - num_masks) * sizeof(u32); 116 } 117 return &cmd->data[num_masks]; 118 } 119 120 u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo) 121 { 122 struct amdxdna_cmd *cmd = abo->mem.kva; 123 u32 num_masks, i; 124 u32 *cu_mask; 125 126 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) 127 return INVALID_CU_IDX; 128 129 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header); 130 cu_mask = cmd->data; 131 for (i = 0; i < num_masks; i++) { 132 if (cu_mask[i]) 133 return ffs(cu_mask[i]) - 1; 134 } 135 136 return INVALID_CU_IDX; 137 } 138 139 int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo, 140 struct amdxdna_sched_job *job, u32 cmd_idx, 141 enum ert_cmd_state error_state) 142 { 143 struct amdxdna_client *client = job->hwctx->client; 144 struct amdxdna_cmd *cmd = abo->mem.kva; 145 struct amdxdna_cmd_chain *cc = NULL; 146 147 cmd->header &= ~AMDXDNA_CMD_STATE; 148 cmd->header |= FIELD_PREP(AMDXDNA_CMD_STATE, error_state); 149 150 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) { 151 cc = amdxdna_cmd_get_payload(abo, NULL); 152 cc->error_index = (cmd_idx < cc->command_count) ? cmd_idx : 0; 153 abo = amdxdna_gem_get_obj(client, cc->data[0], AMDXDNA_BO_CMD); 154 if (!abo) 155 return -EINVAL; 156 cmd = abo->mem.kva; 157 } 158 159 memset(cmd->data, 0xff, abo->mem.size - sizeof(*cmd)); 160 if (cc) 161 amdxdna_gem_put_obj(abo); 162 163 return 0; 164 } 165 166 /* 167 * This should be called in close() and remove(). DO NOT call in other syscalls. 168 * This guarantee that when hwctx and resources will be released, if user 169 * doesn't call amdxdna_drm_destroy_hwctx_ioctl. 170 */ 171 void amdxdna_hwctx_remove_all(struct amdxdna_client *client) 172 { 173 struct amdxdna_hwctx *hwctx; 174 unsigned long hwctx_id; 175 176 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { 177 XDNA_DBG(client->xdna, "PID %d close HW context %d", 178 client->pid, hwctx->id); 179 xa_erase(&client->hwctx_xa, hwctx->id); 180 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu); 181 } 182 } 183 184 int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 185 { 186 struct amdxdna_client *client = filp->driver_priv; 187 struct amdxdna_drm_create_hwctx *args = data; 188 struct amdxdna_dev *xdna = to_xdna_dev(dev); 189 struct amdxdna_hwctx *hwctx; 190 int ret, idx; 191 192 if (args->ext || args->ext_flags) 193 return -EINVAL; 194 195 hwctx = kzalloc_obj(*hwctx); 196 if (!hwctx) 197 return -ENOMEM; 198 199 if (copy_from_user(&hwctx->qos, u64_to_user_ptr(args->qos_p), sizeof(hwctx->qos))) { 200 XDNA_ERR(xdna, "Access QoS info failed"); 201 kfree(hwctx); 202 return -EFAULT; 203 } 204 205 hwctx->client = client; 206 hwctx->fw_ctx_id = -1; 207 hwctx->num_tiles = args->num_tiles; 208 hwctx->mem_size = args->mem_size; 209 hwctx->max_opc = args->max_opc; 210 211 guard(mutex)(&xdna->dev_lock); 212 213 if (!drm_dev_enter(dev, &idx)) { 214 ret = -ENODEV; 215 goto free_hwctx; 216 } 217 218 ret = xdna->dev_info->ops->hwctx_init(hwctx); 219 if (ret) { 220 XDNA_ERR(xdna, "Init hwctx failed, ret %d", ret); 221 goto dev_exit; 222 } 223 224 hwctx->name = kasprintf(GFP_KERNEL, "hwctx.%d.%d", client->pid, hwctx->fw_ctx_id); 225 if (!hwctx->name) { 226 ret = -ENOMEM; 227 goto fini_hwctx; 228 } 229 230 ret = xa_alloc_cyclic(&client->hwctx_xa, &hwctx->id, hwctx, 231 XA_LIMIT(AMDXDNA_INVALID_CTX_HANDLE + 1, MAX_HWCTX_ID), 232 &client->next_hwctxid, GFP_KERNEL); 233 if (ret < 0) { 234 XDNA_ERR(xdna, "Allocate hwctx ID failed, ret %d", ret); 235 goto free_name; 236 } 237 238 args->handle = hwctx->id; 239 args->syncobj_handle = hwctx->syncobj_hdl; 240 241 atomic64_set(&hwctx->job_submit_cnt, 0); 242 atomic64_set(&hwctx->job_free_cnt, 0); 243 XDNA_DBG(xdna, "PID %d create HW context %d, ret %d", client->pid, args->handle, ret); 244 drm_dev_exit(idx); 245 return 0; 246 247 free_name: 248 kfree(hwctx->name); 249 fini_hwctx: 250 xdna->dev_info->ops->hwctx_fini(hwctx); 251 dev_exit: 252 drm_dev_exit(idx); 253 free_hwctx: 254 kfree(hwctx); 255 return ret; 256 } 257 258 int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 259 { 260 struct amdxdna_client *client = filp->driver_priv; 261 struct amdxdna_drm_destroy_hwctx *args = data; 262 struct amdxdna_dev *xdna = to_xdna_dev(dev); 263 struct amdxdna_hwctx *hwctx; 264 int ret = 0, idx; 265 266 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad))) 267 return -EINVAL; 268 269 if (!drm_dev_enter(dev, &idx)) 270 return -ENODEV; 271 272 mutex_lock(&xdna->dev_lock); 273 hwctx = xa_erase(&client->hwctx_xa, args->handle); 274 if (!hwctx) { 275 ret = -EINVAL; 276 XDNA_DBG(xdna, "PID %d HW context %d not exist", 277 client->pid, args->handle); 278 goto out; 279 } 280 281 /* 282 * The pushed jobs are handled by DRM scheduler during destroy. 283 * SRCU to synchronize with exec command ioctls. 284 */ 285 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu); 286 287 XDNA_DBG(xdna, "PID %d destroyed HW context %d", client->pid, args->handle); 288 out: 289 mutex_unlock(&xdna->dev_lock); 290 drm_dev_exit(idx); 291 return ret; 292 } 293 294 int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 295 { 296 struct amdxdna_client *client = filp->driver_priv; 297 struct amdxdna_drm_config_hwctx *args = data; 298 struct amdxdna_dev *xdna = to_xdna_dev(dev); 299 struct amdxdna_hwctx *hwctx; 300 u32 buf_size; 301 void *buf; 302 int ret; 303 u64 val; 304 305 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad))) 306 return -EINVAL; 307 308 if (!xdna->dev_info->ops->hwctx_config) 309 return -EOPNOTSUPP; 310 311 val = args->param_val; 312 buf_size = args->param_val_size; 313 314 switch (args->param_type) { 315 case DRM_AMDXDNA_HWCTX_CONFIG_CU: 316 /* For those types that param_val is pointer */ 317 if (buf_size > PAGE_SIZE) { 318 XDNA_ERR(xdna, "Config CU param buffer too large"); 319 return -E2BIG; 320 } 321 322 /* Hwctx needs to keep buf */ 323 buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 324 if (!buf) 325 return -ENOMEM; 326 327 if (copy_from_user(buf, u64_to_user_ptr(val), buf_size)) { 328 kfree(buf); 329 return -EFAULT; 330 } 331 332 break; 333 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF: 334 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF: 335 /* For those types that param_val is a value */ 336 buf = NULL; 337 buf_size = 0; 338 break; 339 default: 340 XDNA_DBG(xdna, "Unknown HW context config type %d", args->param_type); 341 return -EINVAL; 342 } 343 344 guard(mutex)(&xdna->dev_lock); 345 hwctx = xa_load(&client->hwctx_xa, args->handle); 346 if (!hwctx) { 347 XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle); 348 ret = -EINVAL; 349 goto free_buf; 350 } 351 352 ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size); 353 354 free_buf: 355 kfree(buf); 356 return ret; 357 } 358 359 int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl) 360 { 361 struct amdxdna_dev *xdna = client->xdna; 362 struct amdxdna_hwctx *hwctx; 363 struct amdxdna_gem_obj *abo; 364 struct drm_gem_object *gobj; 365 int ret; 366 367 if (!xdna->dev_info->ops->hwctx_sync_debug_bo) 368 return -EOPNOTSUPP; 369 370 gobj = drm_gem_object_lookup(client->filp, debug_bo_hdl); 371 if (!gobj) 372 return -EINVAL; 373 374 abo = to_xdna_obj(gobj); 375 guard(mutex)(&xdna->dev_lock); 376 hwctx = xa_load(&client->hwctx_xa, abo->assigned_hwctx); 377 if (!hwctx) { 378 ret = -EINVAL; 379 goto put_obj; 380 } 381 382 ret = xdna->dev_info->ops->hwctx_sync_debug_bo(hwctx, debug_bo_hdl); 383 384 put_obj: 385 drm_gem_object_put(gobj); 386 return ret; 387 } 388 389 static void 390 amdxdna_arg_bos_put(struct amdxdna_sched_job *job) 391 { 392 int i; 393 394 for (i = 0; i < job->bo_cnt; i++) { 395 if (!job->bos[i]) 396 break; 397 drm_gem_object_put(job->bos[i]); 398 } 399 } 400 401 static int 402 amdxdna_arg_bos_lookup(struct amdxdna_client *client, 403 struct amdxdna_sched_job *job, 404 u32 *bo_hdls, u32 bo_cnt) 405 { 406 struct drm_gem_object *gobj; 407 int i, ret; 408 409 job->bo_cnt = bo_cnt; 410 for (i = 0; i < job->bo_cnt; i++) { 411 struct amdxdna_gem_obj *abo; 412 413 gobj = drm_gem_object_lookup(client->filp, bo_hdls[i]); 414 if (!gobj) { 415 ret = -ENOENT; 416 goto put_shmem_bo; 417 } 418 abo = to_xdna_obj(gobj); 419 420 mutex_lock(&abo->lock); 421 if (abo->pinned) { 422 mutex_unlock(&abo->lock); 423 job->bos[i] = gobj; 424 continue; 425 } 426 427 ret = amdxdna_gem_pin_nolock(abo); 428 if (ret) { 429 mutex_unlock(&abo->lock); 430 drm_gem_object_put(gobj); 431 goto put_shmem_bo; 432 } 433 abo->pinned = true; 434 mutex_unlock(&abo->lock); 435 436 job->bos[i] = gobj; 437 } 438 439 return 0; 440 441 put_shmem_bo: 442 amdxdna_arg_bos_put(job); 443 return ret; 444 } 445 446 void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job) 447 { 448 trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release"); 449 amdxdna_pm_suspend_put(job->hwctx->client->xdna); 450 amdxdna_arg_bos_put(job); 451 amdxdna_gem_put_obj(job->cmd_bo); 452 dma_fence_put(job->fence); 453 } 454 455 int amdxdna_cmd_submit(struct amdxdna_client *client, 456 struct amdxdna_drv_cmd *drv_cmd, 457 u32 cmd_bo_hdl, u32 *arg_bo_hdls, u32 arg_bo_cnt, 458 u32 hwctx_hdl, u64 *seq) 459 { 460 struct amdxdna_dev *xdna = client->xdna; 461 struct amdxdna_sched_job *job; 462 struct amdxdna_hwctx *hwctx; 463 int ret, idx; 464 465 XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt); 466 job = kzalloc_flex(*job, bos, arg_bo_cnt); 467 if (!job) 468 return -ENOMEM; 469 470 job->drv_cmd = drv_cmd; 471 472 if (cmd_bo_hdl != AMDXDNA_INVALID_BO_HANDLE) { 473 job->cmd_bo = amdxdna_gem_get_obj(client, cmd_bo_hdl, AMDXDNA_BO_CMD); 474 if (!job->cmd_bo) { 475 XDNA_ERR(xdna, "Failed to get cmd bo from %d", cmd_bo_hdl); 476 ret = -EINVAL; 477 goto free_job; 478 } 479 } 480 481 ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt); 482 if (ret) { 483 XDNA_ERR(xdna, "Argument BOs lookup failed, ret %d", ret); 484 goto cmd_put; 485 } 486 487 ret = amdxdna_pm_resume_get(xdna); 488 if (ret) { 489 XDNA_ERR(xdna, "Resume failed, ret %d", ret); 490 goto put_bos; 491 } 492 493 idx = srcu_read_lock(&client->hwctx_srcu); 494 hwctx = xa_load(&client->hwctx_xa, hwctx_hdl); 495 if (!hwctx) { 496 XDNA_DBG(xdna, "PID %d failed to get hwctx %d", 497 client->pid, hwctx_hdl); 498 ret = -EINVAL; 499 goto unlock_srcu; 500 } 501 502 503 job->hwctx = hwctx; 504 job->mm = current->mm; 505 506 job->fence = amdxdna_fence_create(hwctx); 507 if (!job->fence) { 508 XDNA_ERR(xdna, "Failed to create fence"); 509 ret = -ENOMEM; 510 goto unlock_srcu; 511 } 512 kref_init(&job->refcnt); 513 514 ret = xdna->dev_info->ops->cmd_submit(hwctx, job, seq); 515 if (ret) 516 goto put_fence; 517 518 /* 519 * The amdxdna_hwctx_destroy_rcu() will release hwctx and associated 520 * resource after synchronize_srcu(). The submitted jobs should be 521 * handled by the queue, for example DRM scheduler, in device layer. 522 * For here we can unlock SRCU. 523 */ 524 srcu_read_unlock(&client->hwctx_srcu, idx); 525 trace_amdxdna_debug_point(hwctx->name, *seq, "job pushed"); 526 527 return 0; 528 529 put_fence: 530 dma_fence_put(job->fence); 531 unlock_srcu: 532 srcu_read_unlock(&client->hwctx_srcu, idx); 533 amdxdna_pm_suspend_put(xdna); 534 put_bos: 535 amdxdna_arg_bos_put(job); 536 cmd_put: 537 amdxdna_gem_put_obj(job->cmd_bo); 538 free_job: 539 kfree(job); 540 return ret; 541 } 542 543 /* 544 * The submit command ioctl submits a command to firmware. One firmware command 545 * may contain multiple command BOs for processing as a whole. 546 * The command sequence number is returned which can be used for wait command ioctl. 547 */ 548 static int amdxdna_drm_submit_execbuf(struct amdxdna_client *client, 549 struct amdxdna_drm_exec_cmd *args) 550 { 551 struct amdxdna_dev *xdna = client->xdna; 552 u32 *arg_bo_hdls = NULL; 553 u32 cmd_bo_hdl; 554 int ret; 555 556 if (args->arg_count > MAX_ARG_COUNT) { 557 XDNA_ERR(xdna, "Invalid arg bo count %d", args->arg_count); 558 return -EINVAL; 559 } 560 561 /* Only support single command for now. */ 562 if (args->cmd_count != 1) { 563 XDNA_ERR(xdna, "Invalid cmd bo count %d", args->cmd_count); 564 return -EINVAL; 565 } 566 567 cmd_bo_hdl = (u32)args->cmd_handles; 568 if (args->arg_count) { 569 arg_bo_hdls = kcalloc(args->arg_count, sizeof(u32), GFP_KERNEL); 570 if (!arg_bo_hdls) 571 return -ENOMEM; 572 ret = copy_from_user(arg_bo_hdls, u64_to_user_ptr(args->args), 573 args->arg_count * sizeof(u32)); 574 if (ret) { 575 ret = -EFAULT; 576 goto free_cmd_bo_hdls; 577 } 578 } 579 580 ret = amdxdna_cmd_submit(client, NULL, cmd_bo_hdl, arg_bo_hdls, 581 args->arg_count, args->hwctx, &args->seq); 582 if (ret) 583 XDNA_DBG(xdna, "Submit cmds failed, ret %d", ret); 584 585 free_cmd_bo_hdls: 586 kfree(arg_bo_hdls); 587 if (!ret) 588 XDNA_DBG(xdna, "Pushed cmd %lld to scheduler", args->seq); 589 return ret; 590 } 591 592 int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 593 { 594 struct amdxdna_client *client = filp->driver_priv; 595 struct amdxdna_drm_exec_cmd *args = data; 596 597 if (args->ext || args->ext_flags) 598 return -EINVAL; 599 600 switch (args->type) { 601 case AMDXDNA_CMD_SUBMIT_EXEC_BUF: 602 return amdxdna_drm_submit_execbuf(client, args); 603 } 604 605 XDNA_ERR(client->xdna, "Invalid command type %d", args->type); 606 return -EINVAL; 607 } 608