1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_file.h> 10 #include <drm/drm_gem.h> 11 #include <drm/drm_gem_shmem_helper.h> 12 #include <drm/drm_print.h> 13 #include <drm/gpu_scheduler.h> 14 #include <linux/xarray.h> 15 #include <trace/events/amdxdna.h> 16 17 #include "amdxdna_ctx.h" 18 #include "amdxdna_gem.h" 19 #include "amdxdna_pci_drv.h" 20 #include "amdxdna_pm.h" 21 22 #define MAX_HWCTX_ID 255 23 #define MAX_ARG_COUNT 4095 24 25 struct amdxdna_fence { 26 struct dma_fence base; 27 spinlock_t lock; /* for base */ 28 struct amdxdna_hwctx *hwctx; 29 }; 30 31 static const char *amdxdna_fence_get_driver_name(struct dma_fence *fence) 32 { 33 return KBUILD_MODNAME; 34 } 35 36 static const char *amdxdna_fence_get_timeline_name(struct dma_fence *fence) 37 { 38 struct amdxdna_fence *xdna_fence; 39 40 xdna_fence = container_of(fence, struct amdxdna_fence, base); 41 42 return xdna_fence->hwctx->name; 43 } 44 45 static const struct dma_fence_ops fence_ops = { 46 .get_driver_name = amdxdna_fence_get_driver_name, 47 .get_timeline_name = amdxdna_fence_get_timeline_name, 48 }; 49 50 static struct dma_fence *amdxdna_fence_create(struct amdxdna_hwctx *hwctx) 51 { 52 struct amdxdna_fence *fence; 53 54 fence = kzalloc_obj(*fence); 55 if (!fence) 56 return NULL; 57 58 fence->hwctx = hwctx; 59 spin_lock_init(&fence->lock); 60 dma_fence_init(&fence->base, &fence_ops, &fence->lock, hwctx->id, 0); 61 return &fence->base; 62 } 63 64 static void amdxdna_hwctx_release_expanded_heap(struct amdxdna_hwctx *hwctx) 65 { 66 struct amdxdna_client *client = hwctx->client; 67 struct amdxdna_gem_obj *heap; 68 unsigned long heap_id; 69 70 mutex_lock(&client->mm_lock); 71 if (hwctx->last_attached_heap) { 72 xa_for_each_range(&client->dev_heap_xa, heap_id, heap, 1, 73 hwctx->last_attached_heap) { 74 amdxdna_gem_unpin(heap); 75 drm_gem_object_put(to_gobj(heap)); 76 } 77 } 78 mutex_unlock(&client->mm_lock); 79 } 80 81 static void amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx *hwctx, 82 struct srcu_struct *ss) 83 { 84 struct amdxdna_client *client = hwctx->client; 85 struct amdxdna_dev *xdna = client->xdna; 86 87 synchronize_srcu(ss); 88 89 /* At this point, user is not able to submit new commands */ 90 xdna->dev_info->ops->hwctx_fini(hwctx); 91 92 amdxdna_hwctx_release_expanded_heap(hwctx); 93 kfree(hwctx->name); 94 kfree(hwctx); 95 } 96 97 int amdxdna_hwctx_walk(struct amdxdna_client *client, void *arg, 98 int (*walk)(struct amdxdna_hwctx *hwctx, void *arg)) 99 { 100 struct amdxdna_hwctx *hwctx; 101 unsigned long hwctx_id; 102 int ret = 0, idx; 103 104 idx = srcu_read_lock(&client->hwctx_srcu); 105 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { 106 ret = walk(hwctx, arg); 107 if (ret) 108 break; 109 } 110 srcu_read_unlock(&client->hwctx_srcu, idx); 111 112 return ret; 113 } 114 115 void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size) 116 { 117 struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo); 118 u32 num_masks, count; 119 120 if (!cmd) 121 return NULL; 122 123 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) 124 num_masks = 0; 125 else 126 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header); 127 128 if (size) { 129 count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header); 130 if (unlikely(count <= num_masks || 131 count * sizeof(u32) + 132 offsetof(struct amdxdna_cmd, data[0]) > 133 abo->mem.size)) { 134 *size = 0; 135 return NULL; 136 } 137 *size = (count - num_masks) * sizeof(u32); 138 } 139 return &cmd->data[num_masks]; 140 } 141 142 u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo) 143 { 144 struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo); 145 u32 num_masks, i; 146 u32 *cu_mask; 147 148 if (!cmd) 149 return INVALID_CU_IDX; 150 151 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) 152 return INVALID_CU_IDX; 153 154 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header); 155 cu_mask = cmd->data; 156 for (i = 0; i < num_masks; i++) { 157 if (cu_mask[i]) 158 return ffs(cu_mask[i]) - 1; 159 } 160 161 return INVALID_CU_IDX; 162 } 163 164 int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo, 165 struct amdxdna_sched_job *job, u32 cmd_idx, 166 enum ert_cmd_state error_state, 167 void *err_data, size_t size) 168 { 169 struct amdxdna_client *client = job->hwctx->client; 170 struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo); 171 struct amdxdna_cmd_chain *cc = NULL; 172 173 if (!cmd) 174 return -ENOMEM; 175 176 cmd->header &= ~AMDXDNA_CMD_STATE; 177 cmd->header |= FIELD_PREP(AMDXDNA_CMD_STATE, error_state); 178 179 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) { 180 cc = amdxdna_cmd_get_payload(abo, NULL); 181 cc->error_index = (cmd_idx < cc->command_count) ? cmd_idx : 0; 182 abo = amdxdna_gem_get_obj(client, cc->data[0], AMDXDNA_BO_SHARE); 183 if (!abo) 184 return -EINVAL; 185 cmd = amdxdna_gem_vmap(abo); 186 if (!cmd) 187 return -ENOMEM; 188 } 189 190 memset(cmd->data, 0xff, abo->mem.size - sizeof(*cmd)); 191 if (err_data) 192 memcpy(cmd->data, err_data, min(size, abo->mem.size - sizeof(*cmd))); 193 194 if (cc) 195 amdxdna_gem_put_obj(abo); 196 197 return 0; 198 } 199 200 /* 201 * This should be called in close() and remove(). DO NOT call in other syscalls. 202 * This guarantee that when hwctx and resources will be released, if user 203 * doesn't call amdxdna_drm_destroy_hwctx_ioctl. 204 */ 205 void amdxdna_hwctx_remove_all(struct amdxdna_client *client) 206 { 207 struct amdxdna_hwctx *hwctx; 208 unsigned long hwctx_id; 209 210 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { 211 XDNA_DBG(client->xdna, "PID %d close HW context %d", 212 client->pid, hwctx->id); 213 xa_erase(&client->hwctx_xa, hwctx->id); 214 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu); 215 } 216 } 217 218 int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 219 { 220 struct amdxdna_client *client = filp->driver_priv; 221 struct amdxdna_drm_create_hwctx *args = data; 222 struct amdxdna_dev *xdna = to_xdna_dev(dev); 223 struct amdxdna_hwctx *hwctx; 224 int ret, idx; 225 226 if (args->ext || args->ext_flags) 227 return -EINVAL; 228 229 if (!xdna->dev_info->ops->hwctx_init) 230 return -EOPNOTSUPP; 231 232 hwctx = kzalloc_obj(*hwctx); 233 if (!hwctx) 234 return -ENOMEM; 235 236 if (copy_from_user(&hwctx->qos, u64_to_user_ptr(args->qos_p), sizeof(hwctx->qos))) { 237 XDNA_ERR(xdna, "Access QoS info failed"); 238 kfree(hwctx); 239 return -EFAULT; 240 } 241 242 hwctx->client = client; 243 hwctx->fw_ctx_id = -1; 244 hwctx->num_tiles = args->num_tiles; 245 hwctx->umq_bo_hdl = args->umq_bo; 246 hwctx->doorbell_offset = AMDXDNA_INVALID_DOORBELL_OFFSET; 247 hwctx->mem_size = args->mem_size; 248 hwctx->max_opc = args->max_opc; 249 250 guard(mutex)(&xdna->dev_lock); 251 252 if (!drm_dev_enter(dev, &idx)) { 253 ret = -ENODEV; 254 goto free_hwctx; 255 } 256 257 ret = xdna->dev_info->ops->hwctx_init(hwctx); 258 if (ret) { 259 XDNA_ERR(xdna, "Init hwctx failed, ret %d", ret); 260 goto release_expanded_heap; 261 } 262 263 hwctx->name = kasprintf(GFP_KERNEL, "hwctx.%d.%d", client->pid, hwctx->fw_ctx_id); 264 if (!hwctx->name) { 265 ret = -ENOMEM; 266 goto fini_hwctx; 267 } 268 269 ret = xa_alloc_cyclic(&client->hwctx_xa, &hwctx->id, hwctx, 270 XA_LIMIT(AMDXDNA_INVALID_CTX_HANDLE + 1, MAX_HWCTX_ID), 271 &client->next_hwctxid, GFP_KERNEL); 272 if (ret < 0) { 273 XDNA_ERR(xdna, "Allocate hwctx ID failed, ret %d", ret); 274 goto free_name; 275 } 276 277 args->handle = hwctx->id; 278 args->syncobj_handle = hwctx->syncobj_hdl; 279 args->umq_doorbell = hwctx->doorbell_offset; 280 281 atomic64_set(&hwctx->job_submit_cnt, 0); 282 atomic64_set(&hwctx->job_free_cnt, 0); 283 XDNA_DBG(xdna, "PID %d create HW context %d, ret %d", client->pid, args->handle, ret); 284 drm_dev_exit(idx); 285 return 0; 286 287 free_name: 288 kfree(hwctx->name); 289 fini_hwctx: 290 xdna->dev_info->ops->hwctx_fini(hwctx); 291 release_expanded_heap: 292 amdxdna_hwctx_release_expanded_heap(hwctx); 293 drm_dev_exit(idx); 294 free_hwctx: 295 kfree(hwctx); 296 return ret; 297 } 298 299 int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 300 { 301 struct amdxdna_client *client = filp->driver_priv; 302 struct amdxdna_drm_destroy_hwctx *args = data; 303 struct amdxdna_dev *xdna = to_xdna_dev(dev); 304 struct amdxdna_hwctx *hwctx; 305 int ret = 0, idx; 306 307 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad))) 308 return -EINVAL; 309 310 if (!drm_dev_enter(dev, &idx)) 311 return -ENODEV; 312 313 mutex_lock(&xdna->client_lock); 314 mutex_lock(&xdna->dev_lock); 315 hwctx = xa_erase(&client->hwctx_xa, args->handle); 316 if (!hwctx) { 317 ret = -EINVAL; 318 XDNA_DBG(xdna, "PID %d HW context %d not exist", 319 client->pid, args->handle); 320 goto out; 321 } 322 323 /* 324 * The pushed jobs are handled by DRM scheduler during destroy. 325 * SRCU to synchronize with exec command ioctls. 326 */ 327 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu); 328 329 XDNA_DBG(xdna, "PID %d destroyed HW context %d", client->pid, args->handle); 330 out: 331 mutex_unlock(&xdna->dev_lock); 332 mutex_unlock(&xdna->client_lock); 333 drm_dev_exit(idx); 334 return ret; 335 } 336 337 int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 338 { 339 struct amdxdna_client *client = filp->driver_priv; 340 struct amdxdna_drm_config_hwctx *args = data; 341 struct amdxdna_dev *xdna = to_xdna_dev(dev); 342 struct amdxdna_hwctx *hwctx; 343 u32 buf_size; 344 void *buf; 345 int ret; 346 u64 val; 347 348 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad))) 349 return -EINVAL; 350 351 if (!xdna->dev_info->ops->hwctx_config) 352 return -EOPNOTSUPP; 353 354 val = args->param_val; 355 buf_size = args->param_val_size; 356 357 switch (args->param_type) { 358 case DRM_AMDXDNA_HWCTX_CONFIG_CU: 359 /* For those types that param_val is pointer */ 360 if (buf_size > PAGE_SIZE) { 361 XDNA_ERR(xdna, "Config CU param buffer too large"); 362 return -E2BIG; 363 } 364 365 /* Hwctx needs to keep buf */ 366 buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 367 if (!buf) 368 return -ENOMEM; 369 370 if (copy_from_user(buf, u64_to_user_ptr(val), buf_size)) { 371 kfree(buf); 372 return -EFAULT; 373 } 374 375 break; 376 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF: 377 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF: 378 /* For those types that param_val is a value */ 379 buf = NULL; 380 buf_size = 0; 381 break; 382 default: 383 XDNA_DBG(xdna, "Unknown HW context config type %d", args->param_type); 384 return -EINVAL; 385 } 386 387 ret = amdxdna_pm_resume_get(xdna); 388 if (ret) { 389 XDNA_ERR(xdna, "Resume failed, ret %d", ret); 390 goto free_buf; 391 } 392 393 mutex_lock(&xdna->client_lock); 394 mutex_lock(&xdna->dev_lock); 395 hwctx = xa_load(&client->hwctx_xa, args->handle); 396 if (!hwctx) { 397 XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle); 398 ret = -EINVAL; 399 goto unlock; 400 } 401 402 ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size); 403 404 unlock: 405 mutex_unlock(&xdna->dev_lock); 406 mutex_unlock(&xdna->client_lock); 407 amdxdna_pm_suspend_put(xdna); 408 free_buf: 409 kfree(buf); 410 return ret; 411 } 412 413 int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl) 414 { 415 struct amdxdna_dev *xdna = client->xdna; 416 struct amdxdna_hwctx *hwctx; 417 struct amdxdna_gem_obj *abo; 418 struct drm_gem_object *gobj; 419 int ret; 420 421 if (!xdna->dev_info->ops->hwctx_sync_debug_bo) 422 return -EOPNOTSUPP; 423 424 gobj = drm_gem_object_lookup(client->filp, debug_bo_hdl); 425 if (!gobj) 426 return -EINVAL; 427 428 ret = amdxdna_pm_resume_get(xdna); 429 if (ret) { 430 XDNA_ERR(xdna, "Resume failed, ret %d", ret); 431 goto put_obj; 432 } 433 434 abo = to_xdna_obj(gobj); 435 mutex_lock(&xdna->client_lock); 436 mutex_lock(&xdna->dev_lock); 437 hwctx = xa_load(&client->hwctx_xa, abo->assigned_hwctx); 438 if (!hwctx) { 439 ret = -EINVAL; 440 goto unlock; 441 } 442 443 ret = xdna->dev_info->ops->hwctx_sync_debug_bo(hwctx, debug_bo_hdl); 444 445 unlock: 446 mutex_unlock(&xdna->dev_lock); 447 mutex_unlock(&xdna->client_lock); 448 amdxdna_pm_suspend_put(xdna); 449 put_obj: 450 drm_gem_object_put(gobj); 451 return ret; 452 } 453 454 static int amdxdna_hwctx_expand_heap(struct amdxdna_hwctx *hwctx) 455 { 456 struct amdxdna_client *client = hwctx->client; 457 struct amdxdna_dev *xdna = client->xdna; 458 struct amdxdna_gem_obj *heap; 459 unsigned long heap_id, nid; 460 int ret = 0; 461 462 nid = hwctx->last_attached_heap + 1; 463 if (nid == client->dev_heap_nid) 464 goto out; 465 466 xa_for_each_range(&client->dev_heap_xa, heap_id, heap, 467 nid, client->dev_heap_nid) { 468 drm_gem_object_get(to_gobj(heap)); 469 ret = amdxdna_gem_pin(heap); 470 if (ret) { 471 drm_gem_object_put(to_gobj(heap)); 472 break; 473 } 474 475 ret = xdna->dev_info->ops->hwctx_heap_expand(hwctx, heap); 476 if (ret) { 477 amdxdna_gem_unpin(heap); 478 drm_gem_object_put(to_gobj(heap)); 479 break; 480 } 481 482 hwctx->last_attached_heap = heap_id; 483 } 484 485 out: 486 return ret; 487 } 488 489 int amdxdna_update_heap(struct amdxdna_client *client, struct amdxdna_hwctx *hwctx) 490 { 491 unsigned long hwctx_id; 492 int ret; 493 494 ret = amdxdna_pm_resume_get_locked(client->xdna); 495 if (ret) 496 return ret; 497 498 mutex_lock(&client->mm_lock); 499 500 if (hwctx) { 501 ret = amdxdna_hwctx_expand_heap(hwctx); 502 } else { 503 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { 504 ret = amdxdna_hwctx_expand_heap(hwctx); 505 if (ret) 506 break; 507 } 508 } 509 mutex_unlock(&client->mm_lock); 510 511 amdxdna_pm_suspend_put(client->xdna); 512 513 return ret; 514 } 515 516 static void 517 amdxdna_arg_bos_put(struct amdxdna_sched_job *job) 518 { 519 int i; 520 521 for (i = 0; i < job->bo_cnt; i++) { 522 if (!job->bos[i]) 523 break; 524 drm_gem_object_put(job->bos[i]); 525 } 526 } 527 528 static int 529 amdxdna_arg_bos_lookup(struct amdxdna_client *client, 530 struct amdxdna_sched_job *job, 531 u32 *bo_hdls, u32 bo_cnt) 532 { 533 struct drm_gem_object *gobj; 534 int i, ret; 535 536 job->bo_cnt = bo_cnt; 537 for (i = 0; i < job->bo_cnt; i++) { 538 struct amdxdna_gem_obj *abo; 539 540 gobj = drm_gem_object_lookup(client->filp, bo_hdls[i]); 541 if (!gobj) { 542 ret = -ENOENT; 543 goto put_shmem_bo; 544 } 545 abo = to_xdna_obj(gobj); 546 547 mutex_lock(&abo->lock); 548 if (abo->pinned) { 549 mutex_unlock(&abo->lock); 550 job->bos[i] = gobj; 551 continue; 552 } 553 554 ret = amdxdna_gem_pin_nolock(abo); 555 if (ret) { 556 mutex_unlock(&abo->lock); 557 drm_gem_object_put(gobj); 558 goto put_shmem_bo; 559 } 560 abo->pinned = true; 561 mutex_unlock(&abo->lock); 562 563 job->bos[i] = gobj; 564 } 565 566 return 0; 567 568 put_shmem_bo: 569 amdxdna_arg_bos_put(job); 570 return ret; 571 } 572 573 void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job) 574 { 575 trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release"); 576 amdxdna_pm_suspend_put(job->hwctx->client->xdna); 577 amdxdna_arg_bos_put(job); 578 amdxdna_gem_put_obj(job->cmd_bo); 579 dma_fence_put(job->fence); 580 mmdrop(job->mm); 581 } 582 583 int amdxdna_cmd_submit(struct amdxdna_client *client, 584 struct amdxdna_drv_cmd *drv_cmd, 585 u32 cmd_bo_hdl, u32 *arg_bo_hdls, u32 arg_bo_cnt, 586 u32 hwctx_hdl, u64 *seq) 587 { 588 struct amdxdna_dev *xdna = client->xdna; 589 struct amdxdna_sched_job *job; 590 struct amdxdna_hwctx *hwctx; 591 int ret, idx; 592 593 XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt); 594 595 if (!xdna->dev_info->ops->cmd_submit) 596 return -EOPNOTSUPP; 597 598 job = kzalloc_flex(*job, bos, arg_bo_cnt); 599 if (!job) 600 return -ENOMEM; 601 602 job->drv_cmd = drv_cmd; 603 604 if (cmd_bo_hdl != AMDXDNA_INVALID_BO_HANDLE) { 605 job->cmd_bo = amdxdna_gem_get_obj(client, cmd_bo_hdl, AMDXDNA_BO_SHARE); 606 if (!job->cmd_bo) { 607 XDNA_ERR(xdna, "Failed to get cmd bo from %d", cmd_bo_hdl); 608 ret = -EINVAL; 609 goto free_job; 610 } 611 } else if (!drv_cmd) { 612 /* 613 * Only internal driver commands (drv_cmd != NULL) may omit a 614 * command BO. A user command submission with the invalid handle 615 * would leave job->cmd_bo NULL and later fault when the scheduler 616 * dereferences it in amdxdna_cmd_set_state(). 617 */ 618 XDNA_DBG(xdna, "Command BO handle required for user submission"); 619 ret = -EINVAL; 620 goto free_job; 621 } 622 623 ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt); 624 if (ret) { 625 XDNA_ERR(xdna, "Argument BOs lookup failed, ret %d", ret); 626 goto cmd_put; 627 } 628 629 ret = amdxdna_pm_resume_get(xdna); 630 if (ret) { 631 XDNA_ERR(xdna, "Resume failed, ret %d", ret); 632 goto put_bos; 633 } 634 635 idx = srcu_read_lock(&client->hwctx_srcu); 636 hwctx = xa_load(&client->hwctx_xa, hwctx_hdl); 637 if (!hwctx) { 638 XDNA_DBG(xdna, "PID %d failed to get hwctx %d", 639 client->pid, hwctx_hdl); 640 ret = -EINVAL; 641 goto unlock_srcu; 642 } 643 644 job->hwctx = hwctx; 645 job->mm = current->mm; 646 mmgrab(job->mm); 647 648 job->fence = amdxdna_fence_create(hwctx); 649 if (!job->fence) { 650 XDNA_ERR(xdna, "Failed to create fence"); 651 ret = -ENOMEM; 652 goto unlock_srcu; 653 } 654 kref_init(&job->refcnt); 655 656 ret = xdna->dev_info->ops->cmd_submit(hwctx, job, seq); 657 if (ret) 658 goto put_fence; 659 660 /* 661 * The amdxdna_hwctx_destroy_rcu() will release hwctx and associated 662 * resource after synchronize_srcu(). The submitted jobs should be 663 * handled by the queue, for example DRM scheduler, in device layer. 664 * For here we can unlock SRCU. 665 */ 666 srcu_read_unlock(&client->hwctx_srcu, idx); 667 trace_amdxdna_debug_point(hwctx->name, *seq, "job pushed"); 668 669 return 0; 670 671 put_fence: 672 dma_fence_put(job->fence); 673 unlock_srcu: 674 srcu_read_unlock(&client->hwctx_srcu, idx); 675 amdxdna_pm_suspend_put(xdna); 676 put_bos: 677 amdxdna_arg_bos_put(job); 678 cmd_put: 679 amdxdna_gem_put_obj(job->cmd_bo); 680 free_job: 681 if (job->mm) 682 mmdrop(job->mm); 683 kfree(job); 684 return ret; 685 } 686 687 /* 688 * The submit command ioctl submits a command to firmware. One firmware command 689 * may contain multiple command BOs for processing as a whole. 690 * The command sequence number is returned which can be used for wait command ioctl. 691 */ 692 static int amdxdna_drm_submit_execbuf(struct amdxdna_client *client, 693 struct amdxdna_drm_exec_cmd *args) 694 { 695 struct amdxdna_dev *xdna = client->xdna; 696 u32 *arg_bo_hdls = NULL; 697 u32 cmd_bo_hdl; 698 int ret; 699 700 if (args->arg_count > MAX_ARG_COUNT) { 701 XDNA_ERR(xdna, "Invalid arg bo count %d", args->arg_count); 702 return -EINVAL; 703 } 704 705 /* Only support single command for now. */ 706 if (args->cmd_count != 1) { 707 XDNA_ERR(xdna, "Invalid cmd bo count %d", args->cmd_count); 708 return -EINVAL; 709 } 710 711 cmd_bo_hdl = (u32)args->cmd_handles; 712 if (args->arg_count) { 713 arg_bo_hdls = kcalloc(args->arg_count, sizeof(u32), GFP_KERNEL); 714 if (!arg_bo_hdls) 715 return -ENOMEM; 716 ret = copy_from_user(arg_bo_hdls, u64_to_user_ptr(args->args), 717 args->arg_count * sizeof(u32)); 718 if (ret) { 719 ret = -EFAULT; 720 goto free_cmd_bo_hdls; 721 } 722 } 723 724 ret = amdxdna_cmd_submit(client, NULL, cmd_bo_hdl, arg_bo_hdls, 725 args->arg_count, args->hwctx, &args->seq); 726 if (ret) 727 XDNA_DBG(xdna, "Submit cmds failed, ret %d", ret); 728 729 free_cmd_bo_hdls: 730 kfree(arg_bo_hdls); 731 if (!ret) 732 XDNA_DBG(xdna, "Pushed cmd %lld to scheduler", args->seq); 733 return ret; 734 } 735 736 int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 737 { 738 struct amdxdna_client *client = filp->driver_priv; 739 struct amdxdna_drm_exec_cmd *args = data; 740 741 if (args->ext || args->ext_flags) 742 return -EINVAL; 743 744 trace_amdxdna_debug_point(current->comm, args->type, "job received"); 745 746 switch (args->type) { 747 case AMDXDNA_CMD_SUBMIT_EXEC_BUF: 748 return amdxdna_drm_submit_execbuf(client, args); 749 } 750 751 XDNA_ERR(client->xdna, "Invalid command type %d", args->type); 752 return -EINVAL; 753 } 754 755 int amdxdna_drm_wait_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 756 { 757 struct amdxdna_client *client = filp->driver_priv; 758 struct amdxdna_dev *xdna = to_xdna_dev(dev); 759 struct amdxdna_drm_wait_cmd *args = data; 760 struct amdxdna_hwctx *hwctx; 761 int ret, idx; 762 763 XDNA_DBG(xdna, "PID %d ctx %d timeout set %d ms for cmd %llu", 764 client->pid, args->hwctx, args->timeout, args->seq); 765 766 if (!xdna->dev_info->ops->cmd_wait) 767 return -EOPNOTSUPP; 768 769 idx = srcu_read_lock(&client->hwctx_srcu); 770 hwctx = xa_load(&client->hwctx_xa, args->hwctx); 771 if (!hwctx) { 772 XDNA_DBG(xdna, "PID %d failed to get ctx %d", client->pid, args->hwctx); 773 ret = -EINVAL; 774 goto unlock_ctx_srcu; 775 } 776 777 ret = xdna->dev_info->ops->cmd_wait(hwctx, args->seq, args->timeout); 778 779 XDNA_DBG(xdna, "PID %d ctx %d cmd %lld wait finished, ret %d", 780 client->pid, args->hwctx, args->seq, ret); 781 782 trace_amdxdna_debug_point(current->comm, args->seq, "job returned to user"); 783 784 unlock_ctx_srcu: 785 srcu_read_unlock(&client->hwctx_srcu, idx); 786 return ret; 787 } 788