1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_device.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_file.h> 10 #include <drm/drm_gem.h> 11 #include <drm/drm_gem_shmem_helper.h> 12 #include <drm/drm_print.h> 13 #include <drm/gpu_scheduler.h> 14 #include <linux/xarray.h> 15 #include <trace/events/amdxdna.h> 16 17 #include "amdxdna_ctx.h" 18 #include "amdxdna_gem.h" 19 #include "amdxdna_pci_drv.h" 20 21 #define MAX_HWCTX_ID 255 22 #define MAX_ARG_COUNT 4095 23 24 struct amdxdna_fence { 25 struct dma_fence base; 26 spinlock_t lock; /* for base */ 27 struct amdxdna_hwctx *hwctx; 28 }; 29 30 static const char *amdxdna_fence_get_driver_name(struct dma_fence *fence) 31 { 32 return KBUILD_MODNAME; 33 } 34 35 static const char *amdxdna_fence_get_timeline_name(struct dma_fence *fence) 36 { 37 struct amdxdna_fence *xdna_fence; 38 39 xdna_fence = container_of(fence, struct amdxdna_fence, base); 40 41 return xdna_fence->hwctx->name; 42 } 43 44 static const struct dma_fence_ops fence_ops = { 45 .get_driver_name = amdxdna_fence_get_driver_name, 46 .get_timeline_name = amdxdna_fence_get_timeline_name, 47 }; 48 49 static struct dma_fence *amdxdna_fence_create(struct amdxdna_hwctx *hwctx) 50 { 51 struct amdxdna_fence *fence; 52 53 fence = kzalloc_obj(*fence); 54 if (!fence) 55 return NULL; 56 57 fence->hwctx = hwctx; 58 spin_lock_init(&fence->lock); 59 dma_fence_init(&fence->base, &fence_ops, &fence->lock, hwctx->id, 0); 60 return &fence->base; 61 } 62 63 static void amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx *hwctx, 64 struct srcu_struct *ss) 65 { 66 struct amdxdna_dev *xdna = hwctx->client->xdna; 67 68 synchronize_srcu(ss); 69 70 /* At this point, user is not able to submit new commands */ 71 xdna->dev_info->ops->hwctx_fini(hwctx); 72 73 kfree(hwctx->name); 74 kfree(hwctx); 75 } 76 77 int amdxdna_hwctx_walk(struct amdxdna_client *client, void *arg, 78 int (*walk)(struct amdxdna_hwctx *hwctx, void *arg)) 79 { 80 struct amdxdna_hwctx *hwctx; 81 unsigned long hwctx_id; 82 int ret = 0, idx; 83 84 idx = srcu_read_lock(&client->hwctx_srcu); 85 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { 86 ret = walk(hwctx, arg); 87 if (ret) 88 break; 89 } 90 srcu_read_unlock(&client->hwctx_srcu, idx); 91 92 return ret; 93 } 94 95 void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size) 96 { 97 struct amdxdna_cmd *cmd = abo->mem.kva; 98 u32 num_masks, count; 99 100 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) 101 num_masks = 0; 102 else 103 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header); 104 105 if (size) { 106 count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header); 107 if (unlikely(count <= num_masks || 108 count * sizeof(u32) + 109 offsetof(struct amdxdna_cmd, data[0]) > 110 abo->mem.size)) { 111 *size = 0; 112 return NULL; 113 } 114 *size = (count - num_masks) * sizeof(u32); 115 } 116 return &cmd->data[num_masks]; 117 } 118 119 u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo) 120 { 121 struct amdxdna_cmd *cmd = abo->mem.kva; 122 u32 num_masks, i; 123 u32 *cu_mask; 124 125 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) 126 return INVALID_CU_IDX; 127 128 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header); 129 cu_mask = cmd->data; 130 for (i = 0; i < num_masks; i++) { 131 if (cu_mask[i]) 132 return ffs(cu_mask[i]) - 1; 133 } 134 135 return INVALID_CU_IDX; 136 } 137 138 int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo, 139 struct amdxdna_sched_job *job, u32 cmd_idx, 140 enum ert_cmd_state error_state) 141 { 142 struct amdxdna_client *client = job->hwctx->client; 143 struct amdxdna_cmd *cmd = abo->mem.kva; 144 struct amdxdna_cmd_chain *cc = NULL; 145 146 cmd->header &= ~AMDXDNA_CMD_STATE; 147 cmd->header |= FIELD_PREP(AMDXDNA_CMD_STATE, error_state); 148 149 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) { 150 cc = amdxdna_cmd_get_payload(abo, NULL); 151 cc->error_index = (cmd_idx < cc->command_count) ? cmd_idx : 0; 152 abo = amdxdna_gem_get_obj(client, cc->data[0], AMDXDNA_BO_CMD); 153 if (!abo) 154 return -EINVAL; 155 cmd = abo->mem.kva; 156 } 157 158 memset(cmd->data, 0xff, abo->mem.size - sizeof(*cmd)); 159 if (cc) 160 amdxdna_gem_put_obj(abo); 161 162 return 0; 163 } 164 165 /* 166 * This should be called in close() and remove(). DO NOT call in other syscalls. 167 * This guarantee that when hwctx and resources will be released, if user 168 * doesn't call amdxdna_drm_destroy_hwctx_ioctl. 169 */ 170 void amdxdna_hwctx_remove_all(struct amdxdna_client *client) 171 { 172 struct amdxdna_hwctx *hwctx; 173 unsigned long hwctx_id; 174 175 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) { 176 XDNA_DBG(client->xdna, "PID %d close HW context %d", 177 client->pid, hwctx->id); 178 xa_erase(&client->hwctx_xa, hwctx->id); 179 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu); 180 } 181 } 182 183 int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 184 { 185 struct amdxdna_client *client = filp->driver_priv; 186 struct amdxdna_drm_create_hwctx *args = data; 187 struct amdxdna_dev *xdna = to_xdna_dev(dev); 188 struct amdxdna_hwctx *hwctx; 189 int ret, idx; 190 191 if (args->ext || args->ext_flags) 192 return -EINVAL; 193 194 hwctx = kzalloc_obj(*hwctx); 195 if (!hwctx) 196 return -ENOMEM; 197 198 if (copy_from_user(&hwctx->qos, u64_to_user_ptr(args->qos_p), sizeof(hwctx->qos))) { 199 XDNA_ERR(xdna, "Access QoS info failed"); 200 kfree(hwctx); 201 return -EFAULT; 202 } 203 204 hwctx->client = client; 205 hwctx->fw_ctx_id = -1; 206 hwctx->num_tiles = args->num_tiles; 207 hwctx->mem_size = args->mem_size; 208 hwctx->max_opc = args->max_opc; 209 210 guard(mutex)(&xdna->dev_lock); 211 212 if (!drm_dev_enter(dev, &idx)) { 213 ret = -ENODEV; 214 goto free_hwctx; 215 } 216 217 ret = xdna->dev_info->ops->hwctx_init(hwctx); 218 if (ret) { 219 XDNA_ERR(xdna, "Init hwctx failed, ret %d", ret); 220 goto dev_exit; 221 } 222 223 hwctx->name = kasprintf(GFP_KERNEL, "hwctx.%d.%d", client->pid, hwctx->fw_ctx_id); 224 if (!hwctx->name) { 225 ret = -ENOMEM; 226 goto fini_hwctx; 227 } 228 229 ret = xa_alloc_cyclic(&client->hwctx_xa, &hwctx->id, hwctx, 230 XA_LIMIT(AMDXDNA_INVALID_CTX_HANDLE + 1, MAX_HWCTX_ID), 231 &client->next_hwctxid, GFP_KERNEL); 232 if (ret < 0) { 233 XDNA_ERR(xdna, "Allocate hwctx ID failed, ret %d", ret); 234 goto free_name; 235 } 236 237 args->handle = hwctx->id; 238 args->syncobj_handle = hwctx->syncobj_hdl; 239 240 atomic64_set(&hwctx->job_submit_cnt, 0); 241 atomic64_set(&hwctx->job_free_cnt, 0); 242 XDNA_DBG(xdna, "PID %d create HW context %d, ret %d", client->pid, args->handle, ret); 243 drm_dev_exit(idx); 244 return 0; 245 246 free_name: 247 kfree(hwctx->name); 248 fini_hwctx: 249 xdna->dev_info->ops->hwctx_fini(hwctx); 250 dev_exit: 251 drm_dev_exit(idx); 252 free_hwctx: 253 kfree(hwctx); 254 return ret; 255 } 256 257 int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 258 { 259 struct amdxdna_client *client = filp->driver_priv; 260 struct amdxdna_drm_destroy_hwctx *args = data; 261 struct amdxdna_dev *xdna = to_xdna_dev(dev); 262 struct amdxdna_hwctx *hwctx; 263 int ret = 0, idx; 264 265 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad))) 266 return -EINVAL; 267 268 if (!drm_dev_enter(dev, &idx)) 269 return -ENODEV; 270 271 mutex_lock(&xdna->dev_lock); 272 hwctx = xa_erase(&client->hwctx_xa, args->handle); 273 if (!hwctx) { 274 ret = -EINVAL; 275 XDNA_DBG(xdna, "PID %d HW context %d not exist", 276 client->pid, args->handle); 277 goto out; 278 } 279 280 /* 281 * The pushed jobs are handled by DRM scheduler during destroy. 282 * SRCU to synchronize with exec command ioctls. 283 */ 284 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu); 285 286 XDNA_DBG(xdna, "PID %d destroyed HW context %d", client->pid, args->handle); 287 out: 288 mutex_unlock(&xdna->dev_lock); 289 drm_dev_exit(idx); 290 return ret; 291 } 292 293 int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 294 { 295 struct amdxdna_client *client = filp->driver_priv; 296 struct amdxdna_drm_config_hwctx *args = data; 297 struct amdxdna_dev *xdna = to_xdna_dev(dev); 298 struct amdxdna_hwctx *hwctx; 299 u32 buf_size; 300 void *buf; 301 int ret; 302 u64 val; 303 304 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad))) 305 return -EINVAL; 306 307 if (!xdna->dev_info->ops->hwctx_config) 308 return -EOPNOTSUPP; 309 310 val = args->param_val; 311 buf_size = args->param_val_size; 312 313 switch (args->param_type) { 314 case DRM_AMDXDNA_HWCTX_CONFIG_CU: 315 /* For those types that param_val is pointer */ 316 if (buf_size > PAGE_SIZE) { 317 XDNA_ERR(xdna, "Config CU param buffer too large"); 318 return -E2BIG; 319 } 320 321 /* Hwctx needs to keep buf */ 322 buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 323 if (!buf) 324 return -ENOMEM; 325 326 if (copy_from_user(buf, u64_to_user_ptr(val), buf_size)) { 327 kfree(buf); 328 return -EFAULT; 329 } 330 331 break; 332 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF: 333 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF: 334 /* For those types that param_val is a value */ 335 buf = NULL; 336 buf_size = 0; 337 break; 338 default: 339 XDNA_DBG(xdna, "Unknown HW context config type %d", args->param_type); 340 return -EINVAL; 341 } 342 343 guard(mutex)(&xdna->dev_lock); 344 hwctx = xa_load(&client->hwctx_xa, args->handle); 345 if (!hwctx) { 346 XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle); 347 ret = -EINVAL; 348 goto free_buf; 349 } 350 351 ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size); 352 353 free_buf: 354 kfree(buf); 355 return ret; 356 } 357 358 int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl) 359 { 360 struct amdxdna_dev *xdna = client->xdna; 361 struct amdxdna_hwctx *hwctx; 362 struct amdxdna_gem_obj *abo; 363 struct drm_gem_object *gobj; 364 int ret; 365 366 if (!xdna->dev_info->ops->hwctx_sync_debug_bo) 367 return -EOPNOTSUPP; 368 369 gobj = drm_gem_object_lookup(client->filp, debug_bo_hdl); 370 if (!gobj) 371 return -EINVAL; 372 373 abo = to_xdna_obj(gobj); 374 guard(mutex)(&xdna->dev_lock); 375 hwctx = xa_load(&client->hwctx_xa, abo->assigned_hwctx); 376 if (!hwctx) { 377 ret = -EINVAL; 378 goto put_obj; 379 } 380 381 ret = xdna->dev_info->ops->hwctx_sync_debug_bo(hwctx, debug_bo_hdl); 382 383 put_obj: 384 drm_gem_object_put(gobj); 385 return ret; 386 } 387 388 static void 389 amdxdna_arg_bos_put(struct amdxdna_sched_job *job) 390 { 391 int i; 392 393 for (i = 0; i < job->bo_cnt; i++) { 394 if (!job->bos[i]) 395 break; 396 drm_gem_object_put(job->bos[i]); 397 } 398 } 399 400 static int 401 amdxdna_arg_bos_lookup(struct amdxdna_client *client, 402 struct amdxdna_sched_job *job, 403 u32 *bo_hdls, u32 bo_cnt) 404 { 405 struct drm_gem_object *gobj; 406 int i, ret; 407 408 job->bo_cnt = bo_cnt; 409 for (i = 0; i < job->bo_cnt; i++) { 410 struct amdxdna_gem_obj *abo; 411 412 gobj = drm_gem_object_lookup(client->filp, bo_hdls[i]); 413 if (!gobj) { 414 ret = -ENOENT; 415 goto put_shmem_bo; 416 } 417 abo = to_xdna_obj(gobj); 418 419 mutex_lock(&abo->lock); 420 if (abo->pinned) { 421 mutex_unlock(&abo->lock); 422 job->bos[i] = gobj; 423 continue; 424 } 425 426 ret = amdxdna_gem_pin_nolock(abo); 427 if (ret) { 428 mutex_unlock(&abo->lock); 429 drm_gem_object_put(gobj); 430 goto put_shmem_bo; 431 } 432 abo->pinned = true; 433 mutex_unlock(&abo->lock); 434 435 job->bos[i] = gobj; 436 } 437 438 return 0; 439 440 put_shmem_bo: 441 amdxdna_arg_bos_put(job); 442 return ret; 443 } 444 445 void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job) 446 { 447 trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release"); 448 amdxdna_arg_bos_put(job); 449 amdxdna_gem_put_obj(job->cmd_bo); 450 dma_fence_put(job->fence); 451 } 452 453 int amdxdna_cmd_submit(struct amdxdna_client *client, 454 struct amdxdna_drv_cmd *drv_cmd, 455 u32 cmd_bo_hdl, u32 *arg_bo_hdls, u32 arg_bo_cnt, 456 u32 hwctx_hdl, u64 *seq) 457 { 458 struct amdxdna_dev *xdna = client->xdna; 459 struct amdxdna_sched_job *job; 460 struct amdxdna_hwctx *hwctx; 461 int ret, idx; 462 463 XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt); 464 job = kzalloc_flex(*job, bos, arg_bo_cnt); 465 if (!job) 466 return -ENOMEM; 467 468 job->drv_cmd = drv_cmd; 469 470 if (cmd_bo_hdl != AMDXDNA_INVALID_BO_HANDLE) { 471 job->cmd_bo = amdxdna_gem_get_obj(client, cmd_bo_hdl, AMDXDNA_BO_CMD); 472 if (!job->cmd_bo) { 473 XDNA_ERR(xdna, "Failed to get cmd bo from %d", cmd_bo_hdl); 474 ret = -EINVAL; 475 goto free_job; 476 } 477 } 478 479 ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt); 480 if (ret) { 481 XDNA_ERR(xdna, "Argument BOs lookup failed, ret %d", ret); 482 goto cmd_put; 483 } 484 485 idx = srcu_read_lock(&client->hwctx_srcu); 486 hwctx = xa_load(&client->hwctx_xa, hwctx_hdl); 487 if (!hwctx) { 488 XDNA_DBG(xdna, "PID %d failed to get hwctx %d", 489 client->pid, hwctx_hdl); 490 ret = -EINVAL; 491 goto unlock_srcu; 492 } 493 494 495 job->hwctx = hwctx; 496 job->mm = current->mm; 497 498 job->fence = amdxdna_fence_create(hwctx); 499 if (!job->fence) { 500 XDNA_ERR(xdna, "Failed to create fence"); 501 ret = -ENOMEM; 502 goto unlock_srcu; 503 } 504 kref_init(&job->refcnt); 505 506 ret = xdna->dev_info->ops->cmd_submit(hwctx, job, seq); 507 if (ret) 508 goto put_fence; 509 510 /* 511 * The amdxdna_hwctx_destroy_rcu() will release hwctx and associated 512 * resource after synchronize_srcu(). The submitted jobs should be 513 * handled by the queue, for example DRM scheduler, in device layer. 514 * For here we can unlock SRCU. 515 */ 516 srcu_read_unlock(&client->hwctx_srcu, idx); 517 trace_amdxdna_debug_point(hwctx->name, *seq, "job pushed"); 518 519 return 0; 520 521 put_fence: 522 dma_fence_put(job->fence); 523 unlock_srcu: 524 srcu_read_unlock(&client->hwctx_srcu, idx); 525 amdxdna_arg_bos_put(job); 526 cmd_put: 527 amdxdna_gem_put_obj(job->cmd_bo); 528 free_job: 529 kfree(job); 530 return ret; 531 } 532 533 /* 534 * The submit command ioctl submits a command to firmware. One firmware command 535 * may contain multiple command BOs for processing as a whole. 536 * The command sequence number is returned which can be used for wait command ioctl. 537 */ 538 static int amdxdna_drm_submit_execbuf(struct amdxdna_client *client, 539 struct amdxdna_drm_exec_cmd *args) 540 { 541 struct amdxdna_dev *xdna = client->xdna; 542 u32 *arg_bo_hdls = NULL; 543 u32 cmd_bo_hdl; 544 int ret; 545 546 if (args->arg_count > MAX_ARG_COUNT) { 547 XDNA_ERR(xdna, "Invalid arg bo count %d", args->arg_count); 548 return -EINVAL; 549 } 550 551 /* Only support single command for now. */ 552 if (args->cmd_count != 1) { 553 XDNA_ERR(xdna, "Invalid cmd bo count %d", args->cmd_count); 554 return -EINVAL; 555 } 556 557 cmd_bo_hdl = (u32)args->cmd_handles; 558 if (args->arg_count) { 559 arg_bo_hdls = kcalloc(args->arg_count, sizeof(u32), GFP_KERNEL); 560 if (!arg_bo_hdls) 561 return -ENOMEM; 562 ret = copy_from_user(arg_bo_hdls, u64_to_user_ptr(args->args), 563 args->arg_count * sizeof(u32)); 564 if (ret) { 565 ret = -EFAULT; 566 goto free_cmd_bo_hdls; 567 } 568 } 569 570 ret = amdxdna_cmd_submit(client, NULL, cmd_bo_hdl, arg_bo_hdls, 571 args->arg_count, args->hwctx, &args->seq); 572 if (ret) 573 XDNA_DBG(xdna, "Submit cmds failed, ret %d", ret); 574 575 free_cmd_bo_hdls: 576 kfree(arg_bo_hdls); 577 if (!ret) 578 XDNA_DBG(xdna, "Pushed cmd %lld to scheduler", args->seq); 579 return ret; 580 } 581 582 int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 583 { 584 struct amdxdna_client *client = filp->driver_priv; 585 struct amdxdna_drm_exec_cmd *args = data; 586 587 if (args->ext || args->ext_flags) 588 return -EINVAL; 589 590 switch (args->type) { 591 case AMDXDNA_CMD_SUBMIT_EXEC_BUF: 592 return amdxdna_drm_submit_execbuf(client, args); 593 } 594 595 XDNA_ERR(client->xdna, "Invalid command type %d", args->type); 596 return -EINVAL; 597 } 598